diff --git a/routes/model_routes.py b/routes/model_routes.py index 3f2cb7d4e..4054856f6 100644 --- a/routes/model_routes.py +++ b/routes/model_routes.py @@ -472,7 +472,11 @@ def _endpoint_kind(ep: Any) -> str: def _endpoint_refresh_mode(ep: Any, endpoint_kind: str | None = None) -> str: - return _normalize_refresh_mode(getattr(ep, "model_refresh_mode", None), endpoint_kind or _endpoint_kind(ep)) + return _normalize_endpoint_refresh_mode( + getattr(ep, "model_refresh_mode", None), + endpoint_kind or _endpoint_kind(ep), + getattr(ep, "base_url", ""), + ) def _endpoint_refresh_interval(ep: Any, category: str) -> float: @@ -851,6 +855,99 @@ def _ollama_model_names(data: Any) -> List[str]: return out +def _is_google_api_base(base_url: str) -> bool: + try: + return (urlparse(base_url).hostname or "").lower() == "generativelanguage.googleapis.com" + except Exception: + return False + + +def _normalize_endpoint_refresh_mode(value: Any, endpoint_kind: str = "auto", base_url: str = "") -> str: + if not str(value or "").strip() and _is_google_api_base(base_url): + return "manual" + return _normalize_refresh_mode(value, endpoint_kind) + + +def _google_native_root(base_url: str) -> str: + """Return the Gemini native API root for a Google endpoint. + + Chat calls may be configured against Google's OpenAI-compatible + `/openai` path, but model catalog reads should use the native Models API + so we get Google's current Model resource shape. + """ + try: + parsed = urlparse(base_url) + except Exception: + return "https://generativelanguage.googleapis.com/v1beta" + path = (parsed.path or "").rstrip("/") + if path.endswith("/openai"): + path = path[: -len("/openai")].rstrip("/") + if not path: + path = "/v1beta" + return urlunparse(parsed._replace(path=path, query="", fragment="")).rstrip("/") + + +def _google_native_models_url(base_url: str) -> str: + return _google_native_root(base_url) + "/models" + + +def _google_model_id_from_item(item: Any) -> str: + if not isinstance(item, dict): + return "" + value = item.get("baseModelId") or item.get("name") or item.get("model") or "" + return str(value or "").strip().removeprefix("models/") + + +def _google_model_supports_chat(item: Any) -> bool: + """Return whether a native Google Model resource supports chat generation.""" + if not isinstance(item, dict): + return False + methods = item.get("supportedGenerationMethods") + if not isinstance(methods, list): + return False + chat_methods = {"generateContent", "generateMessage", "generateText", "generateAnswer"} + return any(method in chat_methods for method in methods) + + +def _probe_google_models(base_url: str, api_key: str = None, timeout: int = 5, page_size: int = 1000) -> List[str]: + """Read Google's native paginated Models API. + + This intentionally returns only provider-reported model IDs. Capability + mapping is handled by the model capability reader and must not infer from + names here. + """ + url = _google_native_models_url(base_url) + try: + page_size = min(max(int(page_size or 1000), 1), 1000) + except Exception: + page_size = 1000 + headers = {"Accept": "application/json"} + if api_key: + headers["x-goog-api-key"] = api_key + params: Dict[str, Any] = {"pageSize": page_size} + models: List[str] = [] + seen = set() + page_token = "" + for _ in range(20): + request_params = dict(params) + if page_token: + request_params["pageToken"] = page_token + r = httpx.get(url, headers=headers, params=request_params, timeout=timeout, verify=llm_verify()) + r.raise_for_status() + data = r.json() + for item in data.get("models") or []: + if not _google_model_supports_chat(item): + continue + model_id = _google_model_id_from_item(item) + if model_id and model_id not in seen: + seen.add(model_id) + models.append(model_id) + page_token = str(data.get("nextPageToken") or "").strip() + if not page_token: + break + return models + + def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> List[str]: """Probe a base URL's /models endpoint and return list of model IDs. For Anthropic, queries their /v1/models API, falling back to hardcoded list.""" @@ -863,6 +960,17 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis if api_key: return fetch_available_models(api_key, timeout=timeout) return [] + if _is_google_api_base(base): + try: + models = _probe_google_models(base, api_key, timeout=timeout) + if models: + return models + except httpx.HTTPStatusError as e: + status = e.response.status_code if e.response is not None else "unknown" + logger.warning(f"Google native models probe failed: HTTP {status}") + except Exception as e: + logger.warning(f"Google native models probe failed: {e}") + return [] if provider == "anthropic": # Try Anthropic's /v1/models endpoint first url = _safe_build_models_url(base) @@ -1854,7 +1962,7 @@ def setup_model_routes(model_discovery): name = base_url.replace("http://", "").replace("https://", "").split("/")[0] requested_kind = _normalize_endpoint_kind(endpoint_kind) - refresh_mode = _normalize_refresh_mode(model_refresh_mode, requested_kind) + refresh_mode = _normalize_endpoint_refresh_mode(model_refresh_mode, requested_kind, base_url) refresh_interval = _parse_positive_int(model_refresh_interval, minimum=30, maximum=86400) refresh_timeout = _parse_positive_int(model_refresh_timeout, minimum=1, maximum=60) require_model_list = _truthy(require_models) @@ -2364,7 +2472,11 @@ def setup_model_routes(model_discovery): if "endpoint_kind" in body: ep.endpoint_kind = _normalize_endpoint_kind(body.get("endpoint_kind")) if "model_refresh_mode" in body: - ep.model_refresh_mode = _normalize_refresh_mode(body.get("model_refresh_mode"), _endpoint_kind(ep)) + ep.model_refresh_mode = _normalize_endpoint_refresh_mode( + body.get("model_refresh_mode"), + _endpoint_kind(ep), + ep.base_url, + ) if "model_refresh_interval" in body: interval = _parse_positive_int(body.get("model_refresh_interval"), minimum=30, maximum=86400) ep.model_refresh_interval = interval diff --git a/src/model_capabilities.py b/src/model_capabilities.py new file mode 100644 index 000000000..d5b3d8507 --- /dev/null +++ b/src/model_capabilities.py @@ -0,0 +1,934 @@ +"""Canonical model capability metadata helpers. + +This module defines shape and normalization only. It does not probe providers, +change routing, or infer authoritative capabilities from a bare model ID. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any + + +FAMILY_CHAT = "chat" +FAMILY_EMBEDDING = "embedding" +FAMILY_IMAGE = "image" +FAMILY_VIDEO = "video" +FAMILY_AUDIO = "audio" +FAMILY_RERANK = "rerank" +FAMILY_CLASSIFICATION = "classification" +FAMILY_MODERATION = "moderation" +FAMILY_UNKNOWN = "unknown" + +FAMILIES = frozenset( + { + FAMILY_CHAT, + FAMILY_EMBEDDING, + FAMILY_IMAGE, + FAMILY_VIDEO, + FAMILY_AUDIO, + FAMILY_RERANK, + FAMILY_CLASSIFICATION, + FAMILY_MODERATION, + FAMILY_UNKNOWN, + } +) + +MODALITY_TEXT = "text" +MODALITY_IMAGE = "image" +MODALITY_FILE = "file" +MODALITY_PDF = "pdf" +MODALITY_AUDIO = "audio" +MODALITY_VIDEO = "video" +MODALITY_EMBEDDING = "embedding" + +MODALITIES = frozenset( + { + MODALITY_TEXT, + MODALITY_IMAGE, + MODALITY_FILE, + MODALITY_PDF, + MODALITY_AUDIO, + MODALITY_VIDEO, + MODALITY_EMBEDDING, + } +) + +CAP_VISION = "vision" +CAP_FILES = "files" +CAP_PDF = "pdf" +CAP_AUDIO_INPUT = "audio_input" +CAP_AUDIO_OUTPUT = "audio_output" +CAP_IMAGE_GENERATION = "image_generation" +CAP_IMAGE_EDITING = "image_editing" +CAP_INPAINTING = "inpainting" +CAP_VIDEO_GENERATION = "video_generation" +CAP_REASONING = "reasoning" +CAP_TOOL_CALL = "tool_call" +CAP_STRUCTURED_OUTPUT = "structured_output" +CAP_WEB_SEARCH = "web_search" +CAP_STREAMING = "streaming" +CAP_JSON_MODE = "json_mode" +CAP_TRANSCRIPTION = "transcription" +CAP_TTS = "tts" +CAP_REALTIME = "realtime" +CAP_TEXT_RENDERING = "text_rendering" + +CAPABILITIES = frozenset( + { + CAP_VISION, + CAP_FILES, + CAP_PDF, + CAP_AUDIO_INPUT, + CAP_AUDIO_OUTPUT, + CAP_IMAGE_GENERATION, + CAP_IMAGE_EDITING, + CAP_INPAINTING, + CAP_VIDEO_GENERATION, + CAP_REASONING, + CAP_TOOL_CALL, + CAP_STRUCTURED_OUTPUT, + CAP_WEB_SEARCH, + CAP_STREAMING, + CAP_JSON_MODE, + CAP_TRANSCRIPTION, + CAP_TTS, + CAP_REALTIME, + CAP_TEXT_RENDERING, + } +) + +SOURCE_ADMIN_OVERRIDE = "admin_override" +SOURCE_ENDPOINT_CONFIG = "endpoint_config" +SOURCE_PROVIDER_READER = "provider_reader" +SOURCE_COOKBOOK_HF = "cookbook_hf" +SOURCE_MODELS_DEV_REGISTRY = "models_dev_registry" +SOURCE_PROVIDER_DOCS_REGISTRY = "provider_docs_registry" +SOURCE_HEURISTIC = "heuristic" +SOURCE_CAPABILITY_PROBE = "capability_probe" +SOURCE_UNKNOWN = "unknown" + +SOURCES = frozenset( + { + SOURCE_ADMIN_OVERRIDE, + SOURCE_ENDPOINT_CONFIG, + SOURCE_PROVIDER_READER, + SOURCE_COOKBOOK_HF, + SOURCE_MODELS_DEV_REGISTRY, + SOURCE_PROVIDER_DOCS_REGISTRY, + SOURCE_HEURISTIC, + SOURCE_CAPABILITY_PROBE, + SOURCE_UNKNOWN, + } +) + +CONFIDENCE_EXPLICIT = "explicit" +CONFIDENCE_PROVIDER_REPORTED = "provider_reported" +CONFIDENCE_REGISTRY = "registry" +CONFIDENCE_HEURISTIC = "heuristic" +CONFIDENCE_UNKNOWN = "unknown" + +CONFIDENCES = frozenset( + { + CONFIDENCE_EXPLICIT, + CONFIDENCE_PROVIDER_REPORTED, + CONFIDENCE_REGISTRY, + CONFIDENCE_HEURISTIC, + CONFIDENCE_UNKNOWN, + } +) + +ASSERTION_CLAIMED = "claimed" +ASSERTION_VERIFIED = "verified" +ASSERTION_UNSUPPORTED = "unsupported" +ASSERTION_UNKNOWN = "unknown" + +ASSERTION_STATUSES = frozenset( + { + ASSERTION_CLAIMED, + ASSERTION_VERIFIED, + ASSERTION_UNSUPPORTED, + ASSERTION_UNKNOWN, + } +) + +PROBE_PASS = "pass" +PROBE_FAIL = "fail" +PROBE_PARTIAL = "partial" + +PROBE_STATUSES = frozenset( + { + PROBE_PASS, + PROBE_FAIL, + PROBE_PARTIAL, + } +) + +CONTROL_TEMPERATURE = "temperature" +CONTROL_TOP_P = "top_p" +CONTROL_TOP_K = "top_k" +CONTROL_SEED = "seed" +CONTROL_MODEL_VERSION_PIN = "model_version_pin" +CONTROL_STRICT_SCHEMA = "strict_schema" +CONTROL_TOOL_CHOICE = "tool_choice" +CONTROL_SYSTEM_PROMPT = "system_prompt" +CONTROL_PROMPT_CACHING = "prompt_caching" +CONTROL_BATCH = "batch" +CONTROL_REQUEST_HASH_CACHE = "request_hash_cache" +CONTROL_SYSTEM_FINGERPRINT = "system_fingerprint" + +# Canonical reasoning control mechanisms describe how a serving path accepts +# reasoning controls. They are provider/engine evidence, not user preferences. +REASONING_CONTROL_MESSAGE_DIRECTIVE = "reasoning_message_directive" # User-message soft switch, e.g. /think or /no_think. +REASONING_CONTROL_SYSTEM_DIRECTIVE = "reasoning_system_directive" # System prompt instruction, e.g. "detailed thinking on/off". +REASONING_CONTROL_TEMPLATE_KWARG = "reasoning_template_kwarg" # Chat-template kwarg, e.g. chat_template_kwargs.enable_thinking. +REASONING_CONTROL_NATIVE_BOOL = "reasoning_native_bool" # Direct API boolean, e.g. think: true/false. +REASONING_CONTROL_STRUCTURED_OBJECT = "reasoning_structured_object" # Structured API object, e.g. thinking: {type: "..."}. +REASONING_CONTROL_BUDGET = "reasoning_budget" # Token budget control, e.g. thinkingBudget: 0/-1/N. +REASONING_CONTROL_EFFORT = "reasoning_effort" # Graded effort control, e.g. low/medium/high. + +# Canonical reasoning control values describe what the provider control accepts. +# Odysseus runtime preferences can also use auto/on/off, but that is a separate +# layer that later code resolves into these provider-specific controls. +REASONING_CONTROL_VALUE_ON = "on" # Provider supports explicitly requesting reasoning on. +REASONING_CONTROL_VALUE_OFF = "off" # Provider supports explicitly requesting reasoning off. +REASONING_CONTROL_VALUE_AUTO = "auto" # Provider supports adaptive/dynamic/vendor-decided reasoning. + +REASONING_CONTROL_MECHANISMS = frozenset( + { + REASONING_CONTROL_MESSAGE_DIRECTIVE, + REASONING_CONTROL_SYSTEM_DIRECTIVE, + REASONING_CONTROL_TEMPLATE_KWARG, + REASONING_CONTROL_NATIVE_BOOL, + REASONING_CONTROL_STRUCTURED_OBJECT, + REASONING_CONTROL_BUDGET, + REASONING_CONTROL_EFFORT, + } +) + +REASONING_CONTROL_VALUES = frozenset( + { + REASONING_CONTROL_VALUE_ON, + REASONING_CONTROL_VALUE_OFF, + REASONING_CONTROL_VALUE_AUTO, + } +) + +DETERMINISTIC_CONTROLS = frozenset( + { + CONTROL_TEMPERATURE, + CONTROL_TOP_P, + CONTROL_TOP_K, + CONTROL_SEED, + CONTROL_MODEL_VERSION_PIN, + CONTROL_STRICT_SCHEMA, + CONTROL_TOOL_CHOICE, + CONTROL_SYSTEM_PROMPT, + CONTROL_PROMPT_CACHING, + CONTROL_BATCH, + CONTROL_REQUEST_HASH_CACHE, + CONTROL_SYSTEM_FINGERPRINT, + } +) + +TASK_CHAT_COMPLETIONS = "chat.completions" +TASK_EMBEDDINGS_CREATE = "embeddings.create" +TASK_IMAGE_GENERATE = "image.generate" +TASK_IMAGE_EDIT = "image.edit" +TASK_VIDEO_GENERATE = "video.generate" +TASK_AUDIO_TRANSCRIBE = "audio.transcribe" +TASK_AUDIO_SYNTHESIZE = "audio.synthesize" +TASK_RERANK = "rerank.score" +TASK_CLASSIFY = "classification.classify" +TASK_MODERATE = "moderation.moderate" +TASK_UNKNOWN = "unknown" + +_FAMILY_ALIASES = { + "llm": FAMILY_CHAT, + "text": FAMILY_CHAT, + "text2text": FAMILY_CHAT, + "chat_completion": FAMILY_CHAT, + "chat_completions": FAMILY_CHAT, + "embeddings": FAMILY_EMBEDDING, + "embed": FAMILY_EMBEDDING, + "image_generation": FAMILY_IMAGE, + "image_editing": FAMILY_IMAGE, + "video_generation": FAMILY_VIDEO, + "speech": FAMILY_AUDIO, + "stt": FAMILY_AUDIO, + "tts": FAMILY_AUDIO, + "safety": FAMILY_MODERATION, +} + +_MODALITY_ALIASES = { + "images": MODALITY_IMAGE, + "img": MODALITY_IMAGE, + "document": MODALITY_FILE, + "documents": MODALITY_FILE, + "files": MODALITY_FILE, + "docs": MODALITY_FILE, + "voice": MODALITY_AUDIO, + "sound": MODALITY_AUDIO, + "embeddings": MODALITY_EMBEDDING, +} + +_CAPABILITY_ALIASES = { + "tools": CAP_TOOL_CALL, + "tool_calls": CAP_TOOL_CALL, + "function_calling": CAP_TOOL_CALL, + "functions": CAP_TOOL_CALL, + "image_generate": CAP_IMAGE_GENERATION, + "text_to_image": CAP_IMAGE_GENERATION, + "text-to-image": CAP_IMAGE_GENERATION, + "img2img": CAP_IMAGE_EDITING, + "image_edit": CAP_IMAGE_EDITING, + "image-editing": CAP_IMAGE_EDITING, + "text_rendering": CAP_TEXT_RENDERING, + "reasoning_effort": CAP_REASONING, + "thinking": CAP_REASONING, + "json": CAP_JSON_MODE, + "structured_outputs": CAP_STRUCTURED_OUTPUT, + "search": CAP_WEB_SEARCH, +} + +_DETERMINISTIC_CONTROL_ALIASES = { + "temp": CONTROL_TEMPERATURE, + "topp": CONTROL_TOP_P, + "top-p": CONTROL_TOP_P, + "topk": CONTROL_TOP_K, + "top-k": CONTROL_TOP_K, + "version_pin": CONTROL_MODEL_VERSION_PIN, + "model_pin": CONTROL_MODEL_VERSION_PIN, + "strict_tool_schema": CONTROL_STRICT_SCHEMA, + "json_schema": CONTROL_STRICT_SCHEMA, + "tool_choice_required": CONTROL_TOOL_CHOICE, + "system": CONTROL_SYSTEM_PROMPT, + "system_message": CONTROL_SYSTEM_PROMPT, + "cache": CONTROL_REQUEST_HASH_CACHE, + "fingerprint": CONTROL_SYSTEM_FINGERPRINT, +} + +_REASONING_CONTROL_ALIASES = { + "message_directive": REASONING_CONTROL_MESSAGE_DIRECTIVE, + "user_message_directive": REASONING_CONTROL_MESSAGE_DIRECTIVE, + "think_directive": REASONING_CONTROL_MESSAGE_DIRECTIVE, + "slash_think": REASONING_CONTROL_MESSAGE_DIRECTIVE, + "system_directive": REASONING_CONTROL_SYSTEM_DIRECTIVE, + "system_prompt_directive": REASONING_CONTROL_SYSTEM_DIRECTIVE, + "template_kwarg": REASONING_CONTROL_TEMPLATE_KWARG, + "chat_template_kwarg": REASONING_CONTROL_TEMPLATE_KWARG, + "chat_template_kwargs": REASONING_CONTROL_TEMPLATE_KWARG, + "enable_thinking": REASONING_CONTROL_TEMPLATE_KWARG, + "native_bool": REASONING_CONTROL_NATIVE_BOOL, + "think_bool": REASONING_CONTROL_NATIVE_BOOL, + "thinking_bool": REASONING_CONTROL_NATIVE_BOOL, + "structured_object": REASONING_CONTROL_STRUCTURED_OBJECT, + "reasoning_object": REASONING_CONTROL_STRUCTURED_OBJECT, + "thinking_budget": REASONING_CONTROL_BUDGET, + "budget": REASONING_CONTROL_BUDGET, + "effort": REASONING_CONTROL_EFFORT, +} + +_REASONING_CONTROL_VALUE_ALIASES = { + "enabled": REASONING_CONTROL_VALUE_ON, + "enable": REASONING_CONTROL_VALUE_ON, + "true": REASONING_CONTROL_VALUE_ON, + "disabled": REASONING_CONTROL_VALUE_OFF, + "disable": REASONING_CONTROL_VALUE_OFF, + "false": REASONING_CONTROL_VALUE_OFF, + "adaptive": REASONING_CONTROL_VALUE_AUTO, + "automatic": REASONING_CONTROL_VALUE_AUTO, + "dynamic": REASONING_CONTROL_VALUE_AUTO, + "provider_auto": REASONING_CONTROL_VALUE_AUTO, + "vendor_auto": REASONING_CONTROL_VALUE_AUTO, +} + +_DEFAULT_TASK_BY_FAMILY = { + FAMILY_CHAT: TASK_CHAT_COMPLETIONS, + FAMILY_EMBEDDING: TASK_EMBEDDINGS_CREATE, + FAMILY_IMAGE: TASK_IMAGE_GENERATE, + FAMILY_VIDEO: TASK_VIDEO_GENERATE, + FAMILY_AUDIO: TASK_AUDIO_TRANSCRIBE, + FAMILY_RERANK: TASK_RERANK, + FAMILY_CLASSIFICATION: TASK_CLASSIFY, + FAMILY_MODERATION: TASK_MODERATE, + FAMILY_UNKNOWN: TASK_UNKNOWN, +} + +_DEFAULT_MODALITIES_BY_FAMILY = { + FAMILY_CHAT: ((MODALITY_TEXT,), (MODALITY_TEXT,)), + FAMILY_EMBEDDING: ((MODALITY_TEXT,), (MODALITY_EMBEDDING,)), + FAMILY_IMAGE: ((MODALITY_TEXT,), (MODALITY_IMAGE,)), + FAMILY_VIDEO: ((MODALITY_TEXT,), (MODALITY_VIDEO,)), + FAMILY_AUDIO: ((MODALITY_TEXT,), (MODALITY_AUDIO,)), + FAMILY_RERANK: ((MODALITY_TEXT,), (MODALITY_TEXT,)), + FAMILY_CLASSIFICATION: ((MODALITY_TEXT,), (MODALITY_TEXT,)), + FAMILY_MODERATION: ((MODALITY_TEXT,), (MODALITY_TEXT,)), + FAMILY_UNKNOWN: ((), ()), +} + +_DEFAULT_CAPABILITIES_BY_FAMILY = { + FAMILY_IMAGE: (CAP_IMAGE_GENERATION,), + FAMILY_VIDEO: (CAP_VIDEO_GENERATION,), +} + + +def _clean_token(value: Any) -> str: + return str(value or "").strip().lower().replace("-", "_").replace(" ", "_") + + +def _normalize_choice(value: Any, allowed: frozenset[str], aliases: Mapping[str, str], default: str) -> str: + token = _clean_token(value) + token = aliases.get(token, token) + return token if token in allowed else default + + +def normalize_family(value: Any) -> str: + return _normalize_choice(value, FAMILIES, _FAMILY_ALIASES, FAMILY_UNKNOWN) + + +def normalize_source(value: Any) -> str: + return _normalize_choice(value, SOURCES, {}, SOURCE_UNKNOWN) + + +def normalize_confidence(value: Any) -> str: + return _normalize_choice(value, CONFIDENCES, {}, CONFIDENCE_UNKNOWN) + + +def normalize_modality(value: Any) -> str: + return _normalize_choice(value, MODALITIES, _MODALITY_ALIASES, "") + + +def normalize_capability(value: Any) -> str: + token = _clean_token(value) + token = _CAPABILITY_ALIASES.get(token, token) + return token if token in CAPABILITIES else "" + + +def normalize_assertion_status(value: Any) -> str: + return _normalize_choice(value, ASSERTION_STATUSES, {}, ASSERTION_UNKNOWN) + + +def normalize_probe_status(value: Any) -> str: + return _normalize_choice(value, PROBE_STATUSES, {}, "") + + +def normalize_deterministic_control(value: Any) -> str: + token = _clean_token(value) + token = _DETERMINISTIC_CONTROL_ALIASES.get(token, token) + return token if token in DETERMINISTIC_CONTROLS else "" + + +def normalize_reasoning_control_mechanism(value: Any) -> str: + token = _clean_token(value) + token = _REASONING_CONTROL_ALIASES.get(token, token) + return token if token in REASONING_CONTROL_MECHANISMS else "" + + +def normalize_reasoning_control_value(value: Any) -> str: + token = _clean_token(value) + token = _REASONING_CONTROL_VALUE_ALIASES.get(token, token) + return token if token in REASONING_CONTROL_VALUES else "" + + +def _normalize_tokens(values: Any, normalizer) -> tuple[str, ...]: + if values is None: + return () + if isinstance(values, Mapping): + values = [key for key, enabled in values.items() if enabled] + elif isinstance(values, str) or not isinstance(values, Iterable): + values = [values] + out: list[str] = [] + for value in values: + token = normalizer(value) + if token and token not in out: + out.append(token) + return tuple(out) + + +def _normalize_limits(limits: Mapping[str, Any] | None) -> tuple[tuple[str, Any], ...]: + if not isinstance(limits, Mapping): + return () + return tuple(sorted((str(k), v) for k, v in limits.items() if str(k).strip())) + + +@dataclass(frozen=True) +class Modalities: + input: tuple[str, ...] = () + output: tuple[str, ...] = () + + @classmethod + def from_values(cls, input: Any = None, output: Any = None) -> "Modalities": + return cls( + input=_normalize_tokens(input, normalize_modality), + output=_normalize_tokens(output, normalize_modality), + ) + + def to_dict(self) -> dict[str, list[str]]: + return { + "input": list(self.input), + "output": list(self.output), + } + + +@dataclass(frozen=True) +class ModelCapability: + family: str = FAMILY_UNKNOWN + primary_task: str = TASK_UNKNOWN + modalities: Modalities = field(default_factory=Modalities) + capabilities: tuple[str, ...] = () + limits: tuple[tuple[str, Any], ...] = () + source: str = SOURCE_UNKNOWN + confidence: str = CONFIDENCE_UNKNOWN + + @classmethod + def build( + cls, + *, + family: Any = FAMILY_UNKNOWN, + primary_task: str | None = None, + input_modalities: Any = None, + output_modalities: Any = None, + capabilities: Any = None, + limits: Mapping[str, Any] | None = None, + source: Any = SOURCE_UNKNOWN, + confidence: Any = CONFIDENCE_UNKNOWN, + ) -> "ModelCapability": + normalized_family = normalize_family(family) + default_input, default_output = _DEFAULT_MODALITIES_BY_FAMILY[normalized_family] + return cls( + family=normalized_family, + primary_task=str(primary_task or _DEFAULT_TASK_BY_FAMILY[normalized_family]).strip() or TASK_UNKNOWN, + modalities=Modalities.from_values( + input_modalities if input_modalities is not None else default_input, + output_modalities if output_modalities is not None else default_output, + ), + capabilities=_normalize_tokens( + capabilities if capabilities is not None else _DEFAULT_CAPABILITIES_BY_FAMILY.get(normalized_family, ()), + normalize_capability, + ), + limits=_normalize_limits(limits), + source=normalize_source(source), + confidence=normalize_confidence(confidence), + ) + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "ModelCapability": + if not isinstance(value, Mapping): + return unknown_capability() + modalities = value.get("modalities") + if not isinstance(modalities, Mapping): + modalities = {} + return cls.build( + family=value.get("family"), + primary_task=value.get("primary_task"), + input_modalities=modalities.get("input"), + output_modalities=modalities.get("output"), + capabilities=value.get("capabilities"), + limits=value.get("limits"), + source=value.get("source"), + confidence=value.get("confidence"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "family": self.family, + "primary_task": self.primary_task, + "modalities": self.modalities.to_dict(), + "capabilities": list(self.capabilities), + "limits": dict(self.limits), + "source": self.source, + "confidence": self.confidence, + } + + +@dataclass(frozen=True) +class CapabilityAssertion: + capability: str = "" + status: str = ASSERTION_UNKNOWN + source: str = SOURCE_UNKNOWN + confidence: str = CONFIDENCE_UNKNOWN + evidence: tuple[tuple[str, Any], ...] = () + tested_at: str = "" + + @classmethod + def build( + cls, + *, + capability: Any, + status: Any = ASSERTION_UNKNOWN, + source: Any = SOURCE_UNKNOWN, + confidence: Any = CONFIDENCE_UNKNOWN, + evidence: Mapping[str, Any] | None = None, + tested_at: Any = "", + ) -> "CapabilityAssertion": + normalized_capability = normalize_capability(capability) + normalized_status = normalize_assertion_status(status) + if not normalized_capability: + normalized_status = ASSERTION_UNKNOWN + return cls( + capability=normalized_capability, + status=normalized_status, + source=normalize_source(source), + confidence=normalize_confidence(confidence), + evidence=_normalize_limits(evidence), + tested_at=str(tested_at or "").strip(), + ) + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "CapabilityAssertion": + if not isinstance(value, Mapping): + return cls.build(capability="") + return cls.build( + capability=value.get("capability"), + status=value.get("status"), + source=value.get("source"), + confidence=value.get("confidence"), + evidence=value.get("evidence"), + tested_at=value.get("tested_at"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "capability": self.capability, + "status": self.status, + "source": self.source, + "confidence": self.confidence, + "evidence": dict(self.evidence), + "tested_at": self.tested_at, + } + + +@dataclass(frozen=True) +class DeterministicControl: + control: str = "" + status: str = ASSERTION_UNKNOWN + source: str = SOURCE_UNKNOWN + confidence: str = CONFIDENCE_UNKNOWN + evidence: tuple[tuple[str, Any], ...] = () + tested_at: str = "" + + @classmethod + def build( + cls, + *, + control: Any, + status: Any = ASSERTION_UNKNOWN, + source: Any = SOURCE_UNKNOWN, + confidence: Any = CONFIDENCE_UNKNOWN, + evidence: Mapping[str, Any] | None = None, + tested_at: Any = "", + ) -> "DeterministicControl": + normalized_control = normalize_deterministic_control(control) + normalized_status = normalize_assertion_status(status) + if not normalized_control: + normalized_status = ASSERTION_UNKNOWN + return cls( + control=normalized_control, + status=normalized_status, + source=normalize_source(source), + confidence=normalize_confidence(confidence), + evidence=_normalize_limits(evidence), + tested_at=str(tested_at or "").strip(), + ) + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "DeterministicControl": + if not isinstance(value, Mapping): + return cls.build(control="") + return cls.build( + control=value.get("control"), + status=value.get("status"), + source=value.get("source"), + confidence=value.get("confidence"), + evidence=value.get("evidence"), + tested_at=value.get("tested_at"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "control": self.control, + "status": self.status, + "source": self.source, + "confidence": self.confidence, + "evidence": dict(self.evidence), + "tested_at": self.tested_at, + } + + +@dataclass(frozen=True) +class CapabilityProbeResult: + provider: str + model_id: str + capability: str + status: str + tested_at: str = "" + endpoint_id: str = "" + stable_model_id: str = "" + request_hash: str = "" + response_id: str = "" + response_fingerprint: str = "" + evidence: tuple[tuple[str, Any], ...] = () + + @classmethod + def build( + cls, + *, + provider: Any, + model_id: Any, + capability: Any, + status: Any, + tested_at: Any = "", + endpoint_id: Any = "", + stable_model_id: Any = "", + request_hash: Any = "", + response_id: Any = "", + response_fingerprint: Any = "", + evidence: Mapping[str, Any] | None = None, + ) -> "CapabilityProbeResult": + normalized_capability = normalize_capability(capability) + normalized_status = normalize_probe_status(status) + if not normalized_capability or not normalized_status: + normalized_status = PROBE_FAIL + return cls( + provider=str(provider or "").strip(), + model_id=str(model_id or "").strip(), + capability=normalized_capability, + status=normalized_status, + tested_at=str(tested_at or "").strip(), + endpoint_id=str(endpoint_id or "").strip(), + stable_model_id=str(stable_model_id or "").strip(), + request_hash=str(request_hash or "").strip(), + response_id=str(response_id or "").strip(), + response_fingerprint=str(response_fingerprint or "").strip(), + evidence=_normalize_limits(evidence), + ) + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "CapabilityProbeResult": + if not isinstance(value, Mapping): + return cls.build(provider="", model_id="", capability="", status=PROBE_FAIL) + return cls.build( + provider=value.get("provider"), + endpoint_id=value.get("endpoint_id"), + model_id=value.get("model_id"), + stable_model_id=value.get("stable_model_id"), + capability=value.get("capability"), + status=value.get("status"), + tested_at=value.get("tested_at"), + request_hash=value.get("request_hash"), + response_id=value.get("response_id"), + response_fingerprint=value.get("response_fingerprint"), + evidence=value.get("evidence"), + ) + + def to_assertion(self) -> CapabilityAssertion: + status_map = { + PROBE_PASS: ASSERTION_VERIFIED, + PROBE_FAIL: ASSERTION_UNSUPPORTED, + PROBE_PARTIAL: ASSERTION_CLAIMED, + } + return CapabilityAssertion.build( + capability=self.capability, + status=status_map.get(self.status, ASSERTION_UNKNOWN), + source=SOURCE_CAPABILITY_PROBE, + confidence=CONFIDENCE_EXPLICIT if self.status == PROBE_PASS else CONFIDENCE_HEURISTIC, + evidence={ + "provider": self.provider, + "endpoint_id": self.endpoint_id, + "model_id": self.model_id, + "stable_model_id": self.stable_model_id, + "request_hash": self.request_hash, + "response_id": self.response_id, + "response_fingerprint": self.response_fingerprint, + **dict(self.evidence), + }, + tested_at=self.tested_at, + ) + + def to_dict(self) -> dict[str, Any]: + return { + "provider": self.provider, + "endpoint_id": self.endpoint_id, + "model_id": self.model_id, + "stable_model_id": self.stable_model_id, + "capability": self.capability, + "status": self.status, + "tested_at": self.tested_at, + "request_hash": self.request_hash, + "response_id": self.response_id, + "response_fingerprint": self.response_fingerprint, + "evidence": dict(self.evidence), + } + + +def capability_assertions_from_capability( + capability: ModelCapability, + *, + status: str = ASSERTION_CLAIMED, + source: str | None = None, + confidence: str | None = None, +) -> tuple[CapabilityAssertion, ...]: + return tuple( + CapabilityAssertion.build( + capability=cap, + status=status, + source=source or capability.source, + confidence=confidence or capability.confidence, + ) + for cap in capability.capabilities + ) + + +def deterministic_controls_from_values( + values: Any, + *, + status: str = ASSERTION_CLAIMED, + source: str = SOURCE_PROVIDER_READER, + confidence: str = CONFIDENCE_PROVIDER_REPORTED, +) -> tuple[DeterministicControl, ...]: + return tuple( + DeterministicControl.build( + control=control, + status=status, + source=source, + confidence=confidence, + ) + for control in _normalize_tokens(values, normalize_deterministic_control) + ) + + +@dataclass(frozen=True) +class CapabilityQuery: + surface: str + families: tuple[str, ...] = () + primary_tasks: tuple[str, ...] = () + input_all: tuple[str, ...] = () + input_any: tuple[str, ...] = () + output_all: tuple[str, ...] = () + output_any: tuple[str, ...] = () + modality_any: tuple[str, ...] = () + capabilities_all: tuple[str, ...] = () + capabilities_any: tuple[str, ...] = () + + def matches(self, capability: ModelCapability) -> bool: + input_set = set(capability.modalities.input) + output_set = set(capability.modalities.output) + modality_set = input_set | output_set + cap_set = set(capability.capabilities) + if self.families and capability.family not in self.families: + return False + if self.primary_tasks and capability.primary_task not in self.primary_tasks: + return False + if self.input_all and not set(self.input_all).issubset(input_set): + return False + if self.input_any and input_set.isdisjoint(self.input_any): + return False + if self.output_all and not set(self.output_all).issubset(output_set): + return False + if self.output_any and output_set.isdisjoint(self.output_any): + return False + if self.modality_any and modality_set.isdisjoint(self.modality_any): + return False + if self.capabilities_all and not set(self.capabilities_all).issubset(cap_set): + return False + if self.capabilities_any and cap_set.isdisjoint(self.capabilities_any): + return False + return True + + +DISPLAY_QUERIES = ( + CapabilityQuery( + surface="chat", + families=(FAMILY_CHAT,), + input_all=(MODALITY_TEXT,), + output_all=(MODALITY_TEXT,), + ), + CapabilityQuery( + surface="vision_chat", + families=(FAMILY_CHAT,), + input_all=(MODALITY_TEXT, MODALITY_IMAGE), + output_all=(MODALITY_TEXT,), + ), + CapabilityQuery( + surface="document_chat", + families=(FAMILY_CHAT,), + input_all=(MODALITY_TEXT,), + input_any=(MODALITY_FILE, MODALITY_PDF), + output_all=(MODALITY_TEXT,), + ), + CapabilityQuery( + surface="image_generation", + families=(FAMILY_IMAGE,), + output_all=(MODALITY_IMAGE,), + capabilities_all=(CAP_IMAGE_GENERATION,), + ), + CapabilityQuery( + surface="image_editing", + families=(FAMILY_IMAGE,), + input_all=(MODALITY_IMAGE,), + output_all=(MODALITY_IMAGE,), + capabilities_any=(CAP_IMAGE_EDITING, CAP_INPAINTING), + ), + CapabilityQuery( + surface="video_generation", + families=(FAMILY_VIDEO,), + output_all=(MODALITY_VIDEO,), + capabilities_all=(CAP_VIDEO_GENERATION,), + ), + CapabilityQuery( + surface="audio_realtime", + families=(FAMILY_AUDIO,), + modality_any=(MODALITY_AUDIO,), + capabilities_any=(CAP_AUDIO_INPUT, CAP_AUDIO_OUTPUT, CAP_TRANSCRIPTION, CAP_TTS, CAP_REALTIME), + ), + CapabilityQuery( + surface="embeddings", + families=(FAMILY_EMBEDDING,), + output_all=(MODALITY_EMBEDDING,), + ), + CapabilityQuery( + surface="rerank_scoring", + families=(FAMILY_RERANK,), + ), + CapabilityQuery( + surface="moderation_classification", + families=(FAMILY_MODERATION, FAMILY_CLASSIFICATION), + ), +) + + +def display_surfaces_for(capability: ModelCapability) -> tuple[str, ...]: + return tuple(query.surface for query in DISPLAY_QUERIES if query.matches(capability)) + + +def unknown_capability( + *, + source: str = SOURCE_UNKNOWN, + confidence: str = CONFIDENCE_UNKNOWN, +) -> ModelCapability: + return ModelCapability.build(source=source, confidence=confidence) + + +def capability_from_endpoint_type(model_type: Any) -> ModelCapability: + """Return capability metadata from an explicit endpoint model type. + + Missing or unknown endpoint types remain unknown here. Runtime compatibility + may still treat legacy rows as chat-capable, but this schema layer should + not turn absence of evidence into model capability truth. + """ + token = _clean_token(model_type) + if token == "llm": + return ModelCapability.build( + family=FAMILY_CHAT, + source=SOURCE_ENDPOINT_CONFIG, + confidence=CONFIDENCE_EXPLICIT, + ) + if token == "image": + return ModelCapability.build( + family=FAMILY_IMAGE, + source=SOURCE_ENDPOINT_CONFIG, + confidence=CONFIDENCE_EXPLICIT, + ) + return unknown_capability(source=SOURCE_ENDPOINT_CONFIG) diff --git a/src/model_capability_readers/__init__.py b/src/model_capability_readers/__init__.py new file mode 100644 index 000000000..74448281f --- /dev/null +++ b/src/model_capability_readers/__init__.py @@ -0,0 +1,95 @@ +"""Vendor-specific model capability reader registry.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src.model_capability_readers import generic_openai, google, llamacpp, lmstudio, ollama, openai, openrouter +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_ANTHROPIC, + VENDOR_GENERIC_OPENAI, + VENDOR_GOOGLE, + VENDOR_HUGGINGFACE, + VENDOR_LLAMACPP, + VENDOR_LMSTUDIO, + VENDOR_OLLAMA, + VENDOR_OPENAI, + VENDOR_OPENROUTER, + VENDOR_SGLANG, + VENDOR_UNKNOWN, + VENDOR_VLLM, + detect_vendor, + stable_model_id_for, +) + + +READER_MODULES = { + VENDOR_GENERIC_OPENAI: generic_openai, + VENDOR_OPENAI: openai, + VENDOR_OPENROUTER: openrouter, + VENDOR_GOOGLE: google, + VENDOR_LLAMACPP: llamacpp, + VENDOR_OLLAMA: ollama, + VENDOR_LMSTUDIO: lmstudio, +} + + +PLACEHOLDER_VENDOR_IDS = frozenset( + { + VENDOR_ANTHROPIC, + VENDOR_HUGGINGFACE, + VENDOR_SGLANG, + VENDOR_VLLM, + } +) + + +def reader_for_vendor(vendor: Any): + vendor_id = str(vendor or "").strip().lower().replace("-", "_") + return READER_MODULES.get(vendor_id, generic_openai) + + +def records_from_payload( + payload: Mapping[str, Any], + *, + vendor: str | None = None, + base_url: str = "", + endpoint_kind: str = "", + endpoint_id: str = "", +) -> tuple[ModelCapabilityRecord, ...]: + vendor_id = vendor or detect_vendor(base_url, endpoint_kind) + reader = reader_for_vendor(vendor_id) + if reader is generic_openai: + record_vendor = vendor_id if vendor_id not in {VENDOR_UNKNOWN, ""} else VENDOR_GENERIC_OPENAI + return reader.records_from_payload( + payload, + vendor_id=record_vendor, + endpoint_id=endpoint_id, + base_url=base_url, + ) + return reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url) + + +__all__ = [ + "ModelCapabilityRecord", + "PLACEHOLDER_VENDOR_IDS", + "READER_MODULES", + "VENDOR_ANTHROPIC", + "VENDOR_GENERIC_OPENAI", + "VENDOR_GOOGLE", + "VENDOR_HUGGINGFACE", + "VENDOR_LLAMACPP", + "VENDOR_LMSTUDIO", + "VENDOR_OLLAMA", + "VENDOR_OPENAI", + "VENDOR_OPENROUTER", + "VENDOR_SGLANG", + "VENDOR_UNKNOWN", + "VENDOR_VLLM", + "detect_vendor", + "reader_for_vendor", + "records_from_payload", + "stable_model_id_for", +] diff --git a/src/model_capability_readers/base.py b/src/model_capability_readers/base.py new file mode 100644 index 000000000..ee17650a6 --- /dev/null +++ b/src/model_capability_readers/base.py @@ -0,0 +1,311 @@ +"""Shared helpers for vendor-specific model capability readers. + +Readers in this package normalize already-fetched provider payload shapes and +explicit provider fields. They do not perform network I/O and must not infer +authoritative capability from model IDs, names, display names, or ownership +labels. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any, Protocol +from urllib.parse import urlparse + +from src import model_capabilities as mc + + +VENDOR_GENERIC_OPENAI = "generic_openai" +VENDOR_OPENAI = "openai" +VENDOR_OPENROUTER = "openrouter" +VENDOR_GOOGLE = "google" +VENDOR_ANTHROPIC = "anthropic" +VENDOR_OLLAMA = "ollama" +VENDOR_LMSTUDIO = "lmstudio" +VENDOR_LLAMACPP = "llamacpp" +VENDOR_VLLM = "vllm" +VENDOR_SGLANG = "sglang" +VENDOR_HUGGINGFACE = "huggingface" +VENDOR_UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class ModelCapabilityRecord: + vendor: str + model_id: str + capability: mc.ModelCapability + display_name: str = "" + stable_model_id: str = "" + capability_assertions: tuple[mc.CapabilityAssertion, ...] = () + deterministic_controls: tuple[mc.DeterministicControl, ...] = () + raw: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.stable_model_id: + object.__setattr__(self, "stable_model_id", stable_model_id_for(self.vendor, self.model_id)) + if not self.capability_assertions and self.capability.capabilities: + object.__setattr__( + self, + "capability_assertions", + mc.capability_assertions_from_capability( + self.capability, + status=mc.ASSERTION_CLAIMED, + source=self.capability.source, + confidence=self.capability.confidence, + ), + ) + + def to_dict(self, *, include_raw: bool = False) -> dict[str, Any]: + data = { + "vendor": self.vendor, + "model_id": self.model_id, + "stable_model_id": self.stable_model_id, + "display_name": self.display_name, + "capability": self.capability.to_dict(), + "capability_assertions": [assertion.to_dict() for assertion in self.capability_assertions], + "deterministic_controls": [control.to_dict() for control in self.deterministic_controls], + } + if include_raw: + data["raw"] = dict(self.raw) + return data + + +class CapabilityReader(Protocol): + vendor: str + + def records_from_payload( + self, + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", + ) -> tuple[ModelCapabilityRecord, ...]: + """Normalize a provider model-list payload into capability records.""" + + +def as_mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def as_list(value: Any) -> list[Any]: + if value is None: + return [] + if isinstance(value, list): + return value + if isinstance(value, tuple): + return list(value) + return [value] + + +def compact_str(value: Any) -> str: + return str(value or "").strip() + + +def _identity_part(value: Any) -> str: + text = compact_str(value).lower() + out = [] + for char in text: + out.append(char if char.isalnum() or char in {"-", "_", ".", "/", ":"} else "_") + return "".join(out).strip("_") or "unknown" + + +def _base_url_scope(base_url: Any) -> str: + parsed = urlparse(compact_str(base_url)) + if not parsed.hostname: + return "" + port = f":{parsed.port}" if parsed.port else "" + path = parsed.path.rstrip("/") + normalized = f"{parsed.scheme or 'http'}://{parsed.hostname.lower()}{port}{path}" + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:12] + return f"url:{digest}" + + +def stable_model_id_for(vendor: Any, model_id: Any, *, endpoint_id: Any = "", base_url: Any = "") -> str: + vendor_part = _identity_part(vendor or VENDOR_UNKNOWN) + model_part = _identity_part(model_id) + endpoint = compact_str(endpoint_id) + if endpoint: + scope = f"endpoint:{_identity_part(endpoint)}" + else: + scope = _base_url_scope(base_url) or "global" + return f"{vendor_part}|{scope}|{model_part}" + + +def model_id_from(raw: Mapping[str, Any], *keys: str) -> str: + for key in keys: + value = compact_str(raw.get(key)) + if value: + return value.removeprefix("models/") + return "" + + +def int_limit(value: Any) -> int | None: + try: + limit = int(value) + except (TypeError, ValueError): + return None + return limit if limit > 0 else None + + +def merge_unique(*groups: Iterable[str]) -> tuple[str, ...]: + out: list[str] = [] + for group in groups: + for value in group: + token = compact_str(value) + if token and token not in out: + out.append(token) + return tuple(out) + + +def deterministic_controls_from_supported_parameters(values: Any) -> tuple[mc.DeterministicControl, ...]: + return mc.deterministic_controls_from_values( + values, + status=mc.ASSERTION_CLAIMED, + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_PROVIDER_REPORTED, + ) + + +def openai_model_items(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]: + payload = as_mapping(payload) + data = payload.get("data") + if data is None: + data = payload.get("models") + return tuple(item for item in as_list(data) if isinstance(item, Mapping)) + + +def normalize_modality_token(value: Any) -> str: + token = compact_str(value).lower().replace("-", "_").replace(" ", "_") + aliases = { + "txt": mc.MODALITY_TEXT, + "textual": mc.MODALITY_TEXT, + "image_url": mc.MODALITY_IMAGE, + "images": mc.MODALITY_IMAGE, + "img": mc.MODALITY_IMAGE, + "audio_url": mc.MODALITY_AUDIO, + "speech": mc.MODALITY_AUDIO, + "documents": mc.MODALITY_FILE, + "document": mc.MODALITY_FILE, + "files": mc.MODALITY_FILE, + "file_search": mc.MODALITY_FILE, + "pdfs": mc.MODALITY_PDF, + "embeddings": mc.MODALITY_EMBEDDING, + } + token = aliases.get(token, token) + return mc.normalize_modality(token) + + +def modalities_from_value(value: Any) -> tuple[str, ...]: + if isinstance(value, str): + parts = value.replace(",", "+").replace("/", "+").split("+") + else: + parts = as_list(value) + out: list[str] = [] + for part in parts: + token = normalize_modality_token(part) + if token and token not in out: + out.append(token) + return tuple(out) + + +def split_modality_arrow(value: Any) -> tuple[tuple[str, ...], tuple[str, ...]]: + text = compact_str(value).lower() + if not text: + return (), () + for arrow in ("->", "=>", "to"): + if arrow in text: + left, right = text.split(arrow, 1) + return modalities_from_value(left), modalities_from_value(right) + return modalities_from_value(text), () + + +def family_from_modalities(input_modalities: Iterable[str], output_modalities: Iterable[str]) -> str: + output_set = set(output_modalities) + if mc.MODALITY_EMBEDDING in output_set: + return mc.FAMILY_EMBEDDING + if mc.MODALITY_IMAGE in output_set: + return mc.FAMILY_IMAGE + if mc.MODALITY_VIDEO in output_set: + return mc.FAMILY_VIDEO + if mc.MODALITY_AUDIO in output_set: + return mc.FAMILY_AUDIO + if mc.MODALITY_TEXT in output_set: + return mc.FAMILY_CHAT + return mc.FAMILY_UNKNOWN + + +def primary_task_for_family(family: str, capabilities: Iterable[str] = ()) -> str | None: + caps = set(capabilities) + if family == mc.FAMILY_IMAGE and (mc.CAP_IMAGE_EDITING in caps or mc.CAP_INPAINTING in caps): + return mc.TASK_IMAGE_EDIT + if family == mc.FAMILY_AUDIO and mc.CAP_TTS in caps: + return mc.TASK_AUDIO_SYNTHESIZE + if family == mc.FAMILY_AUDIO and mc.CAP_TRANSCRIPTION in caps: + return mc.TASK_AUDIO_TRANSCRIBE + return None + + +def build_capability( + *, + family: str, + input_modalities: Iterable[str] = (), + output_modalities: Iterable[str] = (), + capabilities: Iterable[str] = (), + limits: Mapping[str, Any] | None = None, + confidence: str = mc.CONFIDENCE_PROVIDER_REPORTED, +) -> mc.ModelCapability: + return mc.ModelCapability.build( + family=family, + primary_task=primary_task_for_family(family, capabilities), + input_modalities=tuple(input_modalities), + output_modalities=tuple(output_modalities), + capabilities=tuple(capabilities), + limits=limits, + source=mc.SOURCE_PROVIDER_READER, + confidence=confidence, + ) + + +def detect_vendor(base_url: Any = "", endpoint_kind: Any = "") -> str: + kind = compact_str(endpoint_kind).lower().replace("-", "_") + kind_map = { + "openai": VENDOR_OPENAI, + "openrouter": VENDOR_OPENROUTER, + "google": VENDOR_GOOGLE, + "gemini": VENDOR_GOOGLE, + "anthropic": VENDOR_ANTHROPIC, + "ollama": VENDOR_OLLAMA, + "lmstudio": VENDOR_LMSTUDIO, + "lm_studio": VENDOR_LMSTUDIO, + "llamacpp": VENDOR_LLAMACPP, + "llama_cpp": VENDOR_LLAMACPP, + "vllm": VENDOR_VLLM, + "sglang": VENDOR_SGLANG, + "huggingface": VENDOR_HUGGINGFACE, + "hf": VENDOR_HUGGINGFACE, + } + if kind in kind_map: + return kind_map[kind] + + parsed = urlparse(compact_str(base_url)) + host = (parsed.hostname or "").lower() + port = parsed.port + if host.endswith("openrouter.ai"): + return VENDOR_OPENROUTER + if host.endswith("openai.com"): + return VENDOR_OPENAI + if host.endswith("anthropic.com"): + return VENDOR_ANTHROPIC + if host.endswith("googleapis.com"): + return VENDOR_GOOGLE + if host.endswith("ollama.com") or port == 11434: + return VENDOR_OLLAMA + if port == 1234: + return VENDOR_LMSTUDIO + if port == 8000: + return VENDOR_VLLM + if port == 30000: + return VENDOR_SGLANG + return VENDOR_GENERIC_OPENAI if host else VENDOR_UNKNOWN diff --git a/src/model_capability_readers/generic_openai.py b/src/model_capability_readers/generic_openai.py new file mode 100644 index 000000000..edff3ad33 --- /dev/null +++ b/src/model_capability_readers/generic_openai.py @@ -0,0 +1,58 @@ +"""Reader for bare OpenAI-compatible model-list payloads.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_GENERIC_OPENAI, + compact_str, + model_id_from, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_GENERIC_OPENAI + + +def record_from_model( + raw: Mapping[str, Any], + *, + vendor_id: str = VENDOR_GENERIC_OPENAI, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = model_id_from(raw, "id", "name", "model") + if not model_id: + return None + capability = mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ) + return ModelCapabilityRecord( + vendor=vendor_id, + model_id=model_id, + stable_model_id=stable_model_id_for(vendor_id, model_id, endpoint_id=endpoint_id, base_url=base_url), + display_name=compact_str(raw.get("display_name") or raw.get("name")), + capability=capability, + raw=raw, + ) + + +def records_from_payload( + payload: Mapping[str, Any], + *, + vendor_id: str = VENDOR_GENERIC_OPENAI, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = record_from_model(item, vendor_id=vendor_id, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + return tuple(records) diff --git a/src/model_capability_readers/google.py b/src/model_capability_readers/google.py new file mode 100644 index 000000000..9edb57bdb --- /dev/null +++ b/src/model_capability_readers/google.py @@ -0,0 +1,60 @@ +"""Google Gemini model metadata reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src.model_capability_readers import google_ai_studio_mapping as ai_studio +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_GOOGLE, + as_list, + compact_str, + stable_model_id_for, +) + + +vendor = VENDOR_GOOGLE + + +def _model_items(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]: + models = payload.get("models") if isinstance(payload, Mapping) else None + if models is None and isinstance(payload, Mapping) and payload.get("name"): + models = [payload] + return tuple(item for item in as_list(models) if isinstance(item, Mapping)) + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = ai_studio.google_model_id(raw) + if not model_id: + return None + + return ModelCapabilityRecord( + vendor=VENDOR_GOOGLE, + model_id=model_id, + stable_model_id=stable_model_id_for(VENDOR_GOOGLE, model_id, endpoint_id=endpoint_id, base_url=base_url), + display_name=compact_str(raw.get("displayName")) or model_id, + capability=ai_studio.capability_from_model(raw), + deterministic_controls=ai_studio.deterministic_controls_from_model(raw), + raw=raw, + ) + + +def records_from_payload( + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + records: list[ModelCapabilityRecord] = [] + for item in _model_items(payload): + record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + return tuple(records) diff --git a/src/model_capability_readers/google_ai_studio_mapping.py b/src/model_capability_readers/google_ai_studio_mapping.py new file mode 100644 index 000000000..a6f5dec19 --- /dev/null +++ b/src/model_capability_readers/google_ai_studio_mapping.py @@ -0,0 +1,162 @@ +"""Google AI Studio / Gemini native Models API capability mapping. + +This module maps already-fetched `models.list` and `models.get` payloads into +Odysseus' canonical model capability shape. It performs no network I/O and +does not infer model capabilities from model IDs, display names, or product +families. Only fields explicitly returned by Google's Model resource are +mapped here. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import as_list, compact_str, int_limit + + +METHOD_GENERATE_CONTENT = "generateContent" +METHOD_GENERATE_MESSAGE = "generateMessage" +METHOD_GENERATE_TEXT = "generateText" +METHOD_GENERATE_ANSWER = "generateAnswer" +METHOD_EMBED_CONTENT = "embedContent" +METHOD_ASYNC_BATCH_EMBED = "asyncBatchEmbedContent" +METHOD_PREDICT = "predict" +METHOD_PREDICT_LONG_RUNNING = "predictLongRunning" +METHOD_BATCH_GENERATE = "batchGenerateContent" +METHOD_CREATE_CACHED_CONTENT = "createCachedContent" + +TEXT_GENERATION_METHODS = frozenset( + { + METHOD_GENERATE_CONTENT, + METHOD_GENERATE_MESSAGE, + METHOD_GENERATE_TEXT, + METHOD_GENERATE_ANSWER, + } +) +EMBEDDING_METHODS = frozenset({METHOD_EMBED_CONTENT, METHOD_ASYNC_BATCH_EMBED}) +BATCH_METHODS = frozenset({METHOD_BATCH_GENERATE, METHOD_ASYNC_BATCH_EMBED}) + +MODEL_FIELD_MAP = { + "name": "vendor resource name", + "baseModelId": "vendor model id", + "displayName": "display name", + "description": "display description only", + "inputTokenLimit": "limits.input_tokens and limits.context_tokens", + "outputTokenLimit": "limits.output_tokens", + "supportedGenerationMethods": "provider method support signal", + "thinking": "capabilities.reasoning when true", + "temperature": "deterministic_controls.temperature when present", + "maxTemperature": "deterministic_controls.temperature when present", + "topP": "deterministic_controls.top_p when present", + "topK": "deterministic_controls.top_k when present", +} + + +def google_model_id(raw: Mapping[str, Any]) -> str: + value = compact_str(raw.get("baseModelId")) or compact_str(raw.get("name")) + return value.removeprefix("models/") + + +def supported_methods(raw: Mapping[str, Any]) -> frozenset[str]: + return frozenset(compact_str(method) for method in as_list(raw.get("supportedGenerationMethods")) if method) + + +def limits_from_model(raw: Mapping[str, Any]) -> dict[str, Any]: + limits: dict[str, Any] = {} + input_limit = int_limit(raw.get("inputTokenLimit")) + output_limit = int_limit(raw.get("outputTokenLimit")) + if input_limit: + limits["input_tokens"] = input_limit + limits["context_tokens"] = input_limit + if output_limit: + limits["output_tokens"] = output_limit + return limits + + +def _capability( + *, + family: str, + input_modalities: tuple[str, ...], + output_modalities: tuple[str, ...], + capabilities: tuple[str, ...] = (), + limits: Mapping[str, Any] | None = None, + primary_task: str | None = None, + source: str = mc.SOURCE_PROVIDER_READER, + confidence: str = mc.CONFIDENCE_PROVIDER_REPORTED, +) -> mc.ModelCapability: + return mc.ModelCapability.build( + family=family, + primary_task=primary_task, + input_modalities=input_modalities, + output_modalities=output_modalities, + capabilities=capabilities, + limits=limits, + source=source, + confidence=confidence, + ) + + +def capability_from_model(raw: Mapping[str, Any]) -> mc.ModelCapability: + methods = supported_methods(raw) + capabilities: list[str] = [] + if raw.get("thinking") is True: + capabilities.append(mc.CAP_REASONING) + + if methods & EMBEDDING_METHODS and not methods & TEXT_GENERATION_METHODS: + return _capability( + family=mc.FAMILY_EMBEDDING, + input_modalities=(mc.MODALITY_TEXT,), + output_modalities=(mc.MODALITY_EMBEDDING,), + capabilities=tuple(capabilities), + limits=limits_from_model(raw), + ) + + # `generateContent` proves the model supports Google's content generation + # method, but the Model resource does not expose input/output modalities. + # Keep the model unknown instead of guessing chat/image/audio/video from ID. + if methods & TEXT_GENERATION_METHODS: + return _capability( + family=mc.FAMILY_UNKNOWN, + input_modalities=(), + output_modalities=(), + capabilities=tuple(capabilities), + limits=limits_from_model(raw), + ) + + capability = mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ) + limits = limits_from_model(raw) + if limits or capabilities: + return _capability( + family=mc.FAMILY_UNKNOWN, + input_modalities=(), + output_modalities=(), + capabilities=tuple(capabilities), + limits=limits, + ) + return capability + + +def deterministic_controls_from_model(raw: Mapping[str, Any]) -> tuple[mc.DeterministicControl, ...]: + methods = supported_methods(raw) + controls: list[str] = [] + if "temperature" in raw or "maxTemperature" in raw: + controls.append(mc.CONTROL_TEMPERATURE) + if "topP" in raw: + controls.append(mc.CONTROL_TOP_P) + if raw.get("topK") not in (None, ""): + controls.append(mc.CONTROL_TOP_K) + if METHOD_CREATE_CACHED_CONTENT in methods: + controls.append(mc.CONTROL_PROMPT_CACHING) + if methods & BATCH_METHODS: + controls.append(mc.CONTROL_BATCH) + return mc.deterministic_controls_from_values( + controls, + status=mc.ASSERTION_CLAIMED, + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_PROVIDER_REPORTED, + ) diff --git a/src/model_capability_readers/llamacpp.py b/src/model_capability_readers/llamacpp.py new file mode 100644 index 000000000..9c3beb5c0 --- /dev/null +++ b/src/model_capability_readers/llamacpp.py @@ -0,0 +1,428 @@ +"""llama.cpp server capability reader. + +llama-server exposes OpenAI-compatible model IDs through /v1/models, but its +useful runtime metadata lives in native endpoints such as /props and /slots. +This reader can normalize each payload independently and can merge the three +payloads when the probe script has them all. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import PurePosixPath +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers import generic_openai +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_LLAMACPP, + as_list, + as_mapping, + build_capability, + compact_str, + deterministic_controls_from_supported_parameters, + int_limit, + merge_unique, + model_id_from, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_LLAMACPP + + +_SAMPLER_CONTROL_MAP = { + "temperature": mc.CONTROL_TEMPERATURE, + "top_p": mc.CONTROL_TOP_P, +} + + +def _model_entries(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]: + payload = as_mapping(payload) + data_items = openai_model_items(payload) + if data_items: + return data_items + return tuple(item for item in as_list(payload.get("models")) if isinstance(item, Mapping)) + + +def _server_model_entries(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]: + return tuple(item for item in as_list(as_mapping(payload).get("models")) if isinstance(item, Mapping)) + + +def _model_id_from_props(payload: Mapping[str, Any]) -> str: + payload = as_mapping(payload) + model_alias = compact_str(payload.get("model_alias")) + if model_alias: + return model_alias + model_path = compact_str(payload.get("model_path")) + if model_path: + return PurePosixPath(model_path).name + return "" + + +def _capability_tokens_from_server_model(raw: Mapping[str, Any]) -> tuple[str, ...]: + out: list[str] = [] + for value in as_list(raw.get("capabilities")): + token = compact_str(value).lower().replace("-", "_") + if token in {"embedding", "embeddings"}: + continue + if token in {"rerank", "reranking"}: + continue + if token in {"completion", "completions", "chat"}: + continue + cap = mc.normalize_capability(token) + if cap and cap not in out: + out.append(cap) + return tuple(out) + + +def _family_from_server_model(raw: Mapping[str, Any]) -> str: + capabilities = {compact_str(value).lower().replace("-", "_") for value in as_list(raw.get("capabilities"))} + if "embedding" in capabilities or "embeddings" in capabilities: + return mc.FAMILY_EMBEDDING + if "rerank" in capabilities or "reranking" in capabilities: + return mc.FAMILY_RERANK + if "completion" in capabilities or "completions" in capabilities or "chat" in capabilities: + return mc.FAMILY_CHAT + return mc.FAMILY_UNKNOWN + + +def _matching_server_model(payload: Mapping[str, Any], model_id: str) -> Mapping[str, Any]: + for item in _server_model_entries(payload): + if model_id in { + model_id_from(item, "id", "name", "model"), + compact_str(item.get("name")), + compact_str(item.get("model")), + }: + return item + return {} + + +def _limits_from_model_entry(raw: Mapping[str, Any]) -> dict[str, Any]: + meta = as_mapping(raw.get("meta")) + limits: dict[str, Any] = {} + n_ctx_train = int_limit(raw.get("n_ctx_train") or meta.get("n_ctx_train")) + n_params = int_limit(raw.get("n_params") or meta.get("n_params")) + size = int_limit(raw.get("size") or meta.get("size")) + if n_ctx_train: + limits["training_context_tokens"] = n_ctx_train + if n_params: + limits["parameters"] = n_params + if size: + limits["model_bytes"] = size + return limits + + +def _props_params(payload: Mapping[str, Any]) -> Mapping[str, Any]: + return as_mapping(as_mapping(payload.get("default_generation_settings")).get("params")) + + +def _limits_from_props(payload: Mapping[str, Any], slots_payload: Any = None) -> dict[str, Any]: + default_settings = as_mapping(payload.get("default_generation_settings")) + limits: dict[str, Any] = {} + n_ctx = int_limit(default_settings.get("n_ctx")) + total_slots = int_limit(payload.get("total_slots")) + if not n_ctx and isinstance(slots_payload, list): + slot_contexts = [int_limit(as_mapping(slot).get("n_ctx")) for slot in slots_payload] + slot_contexts = [value for value in slot_contexts if value] + if slot_contexts: + n_ctx = min(slot_contexts) + if n_ctx: + limits["context_tokens"] = n_ctx + if total_slots: + limits["parallel_slots"] = total_slots + elif isinstance(slots_payload, list) and slots_payload: + limits["parallel_slots"] = len(slots_payload) + return limits + + +def _modalities_from_props(payload: Mapping[str, Any]) -> tuple[tuple[str, ...], tuple[str, ...]]: + modalities = as_mapping(payload.get("modalities")) + input_modalities = [mc.MODALITY_TEXT] + output_modalities = [mc.MODALITY_TEXT] + if modalities.get("vision") is True: + input_modalities.append(mc.MODALITY_IMAGE) + if modalities.get("audio") is True: + input_modalities.append(mc.MODALITY_AUDIO) + return tuple(input_modalities), tuple(output_modalities) + + +def _capabilities_from_props(payload: Mapping[str, Any]) -> tuple[str, ...]: + caps = as_mapping(payload.get("chat_template_caps")) + params = _props_params(payload) + out: list[str] = [] + if caps.get("supports_tools") is True or caps.get("supports_tool_calls") is True: + out.append(mc.CAP_TOOL_CALL) + if params.get("stream") is not None: + out.append(mc.CAP_STREAMING) + if as_mapping(payload.get("modalities")).get("vision") is True: + out.append(mc.CAP_VISION) + if as_mapping(payload.get("modalities")).get("audio") is True: + out.append(mc.CAP_AUDIO_INPUT) + return tuple(out) + + +def _unsupported_assertions_from_props(payload: Mapping[str, Any]) -> tuple[mc.CapabilityAssertion, ...]: + modalities = as_mapping(payload.get("modalities")) + assertions: list[mc.CapabilityAssertion] = [] + if modalities.get("vision") is False: + assertions.append( + mc.CapabilityAssertion.build( + capability=mc.CAP_VISION, + status=mc.ASSERTION_UNSUPPORTED, + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_PROVIDER_REPORTED, + evidence={"field": "modalities.vision"}, + ) + ) + if modalities.get("audio") is False: + assertions.append( + mc.CapabilityAssertion.build( + capability=mc.CAP_AUDIO_INPUT, + status=mc.ASSERTION_UNSUPPORTED, + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_PROVIDER_REPORTED, + evidence={"field": "modalities.audio"}, + ) + ) + return tuple(assertions) + + +def _deterministic_controls_from_props(payload: Mapping[str, Any]) -> tuple[mc.DeterministicControl, ...]: + controls: list[str] = [] + params = _props_params(payload) + for key in ("temperature", "top_p", "seed"): + if key in params: + controls.append(key) + for sampler in as_list(params.get("samplers")): + control = _SAMPLER_CONTROL_MAP.get(compact_str(sampler).lower()) + if control: + controls.append(control) + template_caps = as_mapping(payload.get("chat_template_caps")) + if template_caps.get("supports_system_role") is True: + controls.append(mc.CONTROL_SYSTEM_PROMPT) + if template_caps.get("supports_tools") is True or template_caps.get("supports_tool_calls") is True: + controls.append(mc.CONTROL_TOOL_CHOICE) + return deterministic_controls_from_supported_parameters(merge_unique(controls)) + + +def _capability_for_family( + family: str, + *, + capabilities: tuple[str, ...] = (), + limits: Mapping[str, Any] | None = None, + props_payload: Mapping[str, Any] | None = None, +) -> mc.ModelCapability: + if family == mc.FAMILY_EMBEDDING: + return build_capability( + family=mc.FAMILY_EMBEDDING, + input_modalities=(mc.MODALITY_TEXT,), + output_modalities=(mc.MODALITY_EMBEDDING,), + capabilities=capabilities, + limits=limits, + ) + if family == mc.FAMILY_RERANK: + return build_capability( + family=mc.FAMILY_RERANK, + input_modalities=(mc.MODALITY_TEXT,), + output_modalities=(mc.MODALITY_TEXT,), + capabilities=capabilities, + limits=limits, + ) + if props_payload: + input_modalities, output_modalities = _modalities_from_props(props_payload) + else: + input_modalities, output_modalities = (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) + return build_capability( + family=mc.FAMILY_CHAT, + input_modalities=input_modalities, + output_modalities=output_modalities, + capabilities=capabilities, + limits=limits, + ) + + +def _record( + *, + model_id: str, + family: str, + capabilities: tuple[str, ...] = (), + limits: Mapping[str, Any] | None = None, + props_payload: Mapping[str, Any] | None = None, + deterministic_controls: tuple[mc.DeterministicControl, ...] = (), + extra_assertions: tuple[mc.CapabilityAssertion, ...] = (), + raw: Mapping[str, Any] | None = None, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord: + capability = _capability_for_family( + family, + capabilities=capabilities, + limits=limits, + props_payload=props_payload, + ) + return ModelCapabilityRecord( + vendor=VENDOR_LLAMACPP, + model_id=model_id, + stable_model_id=stable_model_id_for(VENDOR_LLAMACPP, model_id, endpoint_id=endpoint_id, base_url=base_url), + display_name=model_id, + capability=capability, + capability_assertions=( + mc.capability_assertions_from_capability( + capability, + status=mc.ASSERTION_CLAIMED, + source=capability.source, + confidence=capability.confidence, + ) + + extra_assertions + ), + deterministic_controls=deterministic_controls, + raw=raw or {}, + ) + + +def record_from_model_payload( + raw: Mapping[str, Any], + *, + server_model: Mapping[str, Any] | None = None, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = model_id_from(raw, "id", "name", "model") + if not model_id: + return None + server_model = as_mapping(server_model) + family = _family_from_server_model(server_model) if server_model else mc.FAMILY_UNKNOWN + if family == mc.FAMILY_UNKNOWN: + return generic_openai.record_from_model( + raw, + vendor_id=VENDOR_LLAMACPP, + endpoint_id=endpoint_id, + base_url=base_url, + ) + capabilities = _capability_tokens_from_server_model(server_model) + return _record( + model_id=model_id, + family=family, + capabilities=capabilities, + limits=_limits_from_model_entry(raw), + raw=raw, + endpoint_id=endpoint_id, + base_url=base_url, + ) + + +def record_from_props_payload( + payload: Mapping[str, Any], + *, + slots_payload: Any = None, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + payload = as_mapping(payload) + model_id = _model_id_from_props(payload) + if not model_id: + return None + return _record( + model_id=model_id, + family=mc.FAMILY_CHAT, + capabilities=_capabilities_from_props(payload), + limits=_limits_from_props(payload, slots_payload), + props_payload=payload, + deterministic_controls=_deterministic_controls_from_props(payload), + extra_assertions=_unsupported_assertions_from_props(payload), + raw=payload, + endpoint_id=endpoint_id, + base_url=base_url, + ) + + +def records_from_payloads( + *, + models_payload: Mapping[str, Any] | None = None, + props_payload: Mapping[str, Any] | None = None, + slots_payload: Any = None, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + props_payload = as_mapping(props_payload) + models_payload = as_mapping(models_payload) + props_record = ( + record_from_props_payload(props_payload, slots_payload=slots_payload, endpoint_id=endpoint_id, base_url=base_url) + if props_payload + else None + ) + if not models_payload: + return (props_record,) if props_record else () + + records: list[ModelCapabilityRecord] = [] + for item in _model_entries(models_payload): + model_id = model_id_from(item, "id", "name", "model") + if not model_id: + continue + server_model = _matching_server_model(models_payload, model_id) + model_record = record_from_model_payload( + item, + server_model=server_model, + endpoint_id=endpoint_id, + base_url=base_url, + ) + if not model_record: + continue + if props_record and props_record.model_id == model_id: + limits = {**dict(model_record.capability.limits), **dict(props_record.capability.limits)} + capability = _capability_for_family( + props_record.capability.family, + capabilities=merge_unique(model_record.capability.capabilities, props_record.capability.capabilities), + limits=limits, + props_payload=props_payload, + ) + records.append( + ModelCapabilityRecord( + vendor=VENDOR_LLAMACPP, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_LLAMACPP, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=model_id, + capability=capability, + capability_assertions=( + mc.capability_assertions_from_capability( + capability, + status=mc.ASSERTION_CLAIMED, + source=capability.source, + confidence=capability.confidence, + ) + + _unsupported_assertions_from_props(props_payload) + ), + deterministic_controls=props_record.deterministic_controls, + raw={"models": item, "props": props_payload, "slots": slots_payload or []}, + ) + ) + else: + records.append(model_record) + if not records and props_record: + records.append(props_record) + return tuple(records) + + +def records_from_payload( + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + payload = as_mapping(payload) + if not payload: + return () + if "default_generation_settings" in payload or "chat_template_caps" in payload: + record = record_from_props_payload(payload, endpoint_id=endpoint_id, base_url=base_url) + return (record,) if record else () + if "models" in payload or "data" in payload: + return records_from_payloads(models_payload=payload, endpoint_id=endpoint_id, base_url=base_url) + return () diff --git a/src/model_capability_readers/lmstudio.py b/src/model_capability_readers/lmstudio.py new file mode 100644 index 000000000..960649959 --- /dev/null +++ b/src/model_capability_readers/lmstudio.py @@ -0,0 +1,186 @@ +"""LM Studio native model metadata reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers import generic_openai +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_LMSTUDIO, + as_list, + as_mapping, + build_capability, + compact_str, + int_limit, + merge_unique, + model_id_from, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_LMSTUDIO + + +def _loaded_instance_contexts(raw: Mapping[str, Any]) -> tuple[int, ...]: + contexts: list[int] = [] + for instance in as_list(raw.get("loaded_instances")): + instance_payload = as_mapping(instance) + config = as_mapping(instance_payload.get("config")) + value = int_limit(instance_payload.get("context_length")) or int_limit( + config.get("context_length") + ) + if value: + contexts.append(value) + return tuple(contexts) + + +def _limits_from_model(raw: Mapping[str, Any]) -> dict[str, Any]: + limits: dict[str, Any] = {} + loaded_contexts = _loaded_instance_contexts(raw) + loaded_context = int_limit(raw.get("loaded_context_length")) or ( + min(loaded_contexts) if loaded_contexts else None + ) + configured_context = int_limit(raw.get("context_length")) or int_limit(raw.get("contextLength")) + max_context = int_limit(raw.get("max_context_length")) or int_limit(raw.get("maxContextLength")) + context_tokens = loaded_context or configured_context or max_context + if context_tokens: + limits["context_tokens"] = context_tokens + if max_context and max_context != context_tokens: + limits["max_context_tokens"] = max_context + return limits + + +def _family_from_type(raw: Mapping[str, Any]) -> str: + kind = compact_str(raw.get("type") or raw.get("model_type") or raw.get("task")).lower().replace("-", "_") + if kind in {"embedding", "embeddings", "text_embedding", "text_embeddings"}: + return mc.FAMILY_EMBEDDING + if kind in {"llm", "chat", "vlm", "vision", "text_generation"}: + return mc.FAMILY_CHAT + return mc.FAMILY_UNKNOWN + + +def _capabilities_from_native_payload(raw: Mapping[str, Any]) -> tuple[str, ...]: + capabilities_payload = as_mapping(raw.get("capabilities")) + capabilities: list[str] = [] + if capabilities_payload.get("vision") is True: + capabilities.append(mc.CAP_VISION) + if ( + capabilities_payload.get("trained_for_tool_use") is True + or capabilities_payload.get("tools") is True + or capabilities_payload.get("tool_use") is True + ): + capabilities.append(mc.CAP_TOOL_CALL) + if capabilities_payload.get("reasoning"): + capabilities.append(mc.CAP_REASONING) + return merge_unique(capabilities) + + +def _unknown_record( + raw: Mapping[str, Any], + model_id: str, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord: + return ModelCapabilityRecord( + vendor=VENDOR_LMSTUDIO, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_LMSTUDIO, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=compact_str(raw.get("display_name") or raw.get("name")) or model_id, + capability=mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ), + raw=raw, + ) + + +def record_from_native_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = model_id_from(raw, "key", "id", "model", "name") + if not model_id: + return None + + family = _family_from_type(raw) + capabilities = _capabilities_from_native_payload(raw) + + if family == mc.FAMILY_UNKNOWN and capabilities: + family = mc.FAMILY_CHAT + + if family == mc.FAMILY_EMBEDDING: + input_modalities = (mc.MODALITY_TEXT,) + output_modalities = (mc.MODALITY_EMBEDDING,) + elif family == mc.FAMILY_CHAT and mc.CAP_VISION in capabilities: + input_modalities = (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) + output_modalities = (mc.MODALITY_TEXT,) + elif family == mc.FAMILY_CHAT: + input_modalities = (mc.MODALITY_TEXT,) + output_modalities = (mc.MODALITY_TEXT,) + else: + return generic_openai.record_from_model( + raw, + vendor_id=VENDOR_LMSTUDIO, + endpoint_id=endpoint_id, + base_url=base_url, + ) or _unknown_record( + raw, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ) + + capability = build_capability( + family=family, + input_modalities=input_modalities, + output_modalities=output_modalities, + capabilities=capabilities, + limits=_limits_from_model(raw), + ) + return ModelCapabilityRecord( + vendor=VENDOR_LMSTUDIO, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_LMSTUDIO, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=compact_str(raw.get("display_name") or raw.get("name")) or model_id, + capability=capability, + raw=raw, + ) + + +def records_from_payload( + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = record_from_native_model(item, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + if records: + return tuple(records) + for item in as_list(as_mapping(payload).get("models")): + if not isinstance(item, Mapping): + continue + record = record_from_native_model(item, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + return tuple(records) diff --git a/src/model_capability_readers/ollama.py b/src/model_capability_readers/ollama.py new file mode 100644 index 000000000..8ad7a5e54 --- /dev/null +++ b/src/model_capability_readers/ollama.py @@ -0,0 +1,204 @@ +"""Ollama native API capability reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_OLLAMA, + as_list, + as_mapping, + build_capability, + compact_str, + int_limit, + merge_unique, + model_id_from, + stable_model_id_for, +) + + +vendor = VENDOR_OLLAMA + + +_CAPABILITY_MAP = { + "completion": None, + "completions": None, + "chat": None, + "thinking": mc.CAP_REASONING, + "reasoning": mc.CAP_REASONING, + "vision": mc.CAP_VISION, + "tools": mc.CAP_TOOL_CALL, + "tool": mc.CAP_TOOL_CALL, + "embedding": None, + "embeddings": None, +} + + +def _capability_tokens(values: Any) -> tuple[str, ...]: + out: list[str] = [] + for value in as_list(values): + token = compact_str(value).lower().replace("-", "_") + cap = _CAPABILITY_MAP.get(token) + if cap and cap not in out: + out.append(cap) + return tuple(out) + + +def _family_from_ollama_capabilities(values: Any) -> str: + tokens = {compact_str(value).lower().replace("-", "_") for value in as_list(values)} + if tokens and tokens.issubset({"embedding", "embeddings"}): + return mc.FAMILY_EMBEDDING + if "embedding" in tokens or "embeddings" in tokens: + return mc.FAMILY_EMBEDDING + if tokens.intersection({"completion", "completions", "chat", "thinking", "reasoning", "tools", "tool", "vision"}): + return mc.FAMILY_CHAT + return mc.FAMILY_UNKNOWN + + +def _parameters_mapping(value: Any) -> Mapping[str, Any]: + if isinstance(value, Mapping): + return value + text = compact_str(value) + if not text: + return {} + parsed: dict[str, str] = {} + for line in text.splitlines(): + parts = line.strip().split(None, 1) + if len(parts) == 2: + parsed[parts[0]] = parts[1] + return parsed + + +def _modalities_for_family(family: str, capabilities: tuple[str, ...]) -> tuple[tuple[str, ...], tuple[str, ...]]: + if family == mc.FAMILY_EMBEDDING: + return (mc.MODALITY_TEXT,), (mc.MODALITY_EMBEDDING,) + if family == mc.FAMILY_CHAT and mc.CAP_VISION in capabilities: + return (mc.MODALITY_TEXT, mc.MODALITY_IMAGE), (mc.MODALITY_TEXT,) + if family == mc.FAMILY_CHAT: + return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) + return (), () + + +def _first_int_by_key_shape(*mappings: Mapping[str, Any], exact_keys: tuple[str, ...] = ()) -> int | None: + for key in exact_keys: + for mapping in mappings: + value = int_limit(mapping.get(key)) + if value: + return value + for mapping in mappings: + for key, value in mapping.items(): + key_text = compact_str(key).lower() + if key_text == "context_length" or key_text.endswith(".context_length"): + limit = int_limit(value) + if limit: + return limit + return None + + +def _limits_from_show(raw: Mapping[str, Any]) -> dict[str, Any]: + model_info = as_mapping(raw.get("model_info")) + parameters = _parameters_mapping(raw.get("parameters")) + details = as_mapping(raw.get("details")) + limits: dict[str, Any] = {} + context_tokens = _first_int_by_key_shape( + raw, + model_info, + parameters, + details, + exact_keys=("context_length", "num_ctx"), + ) + if context_tokens: + limits["context_tokens"] = context_tokens + return limits + + +def record_from_show_payload( + model_id: str, + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = compact_str(model_id) or model_id_from(payload, "model", "name") + if not model_id: + return None + capability_values = payload.get("capabilities") + capabilities = _capability_tokens(capability_values) + family = _family_from_ollama_capabilities(capability_values) + if family == mc.FAMILY_UNKNOWN: + capability = mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ) + else: + input_modalities, output_modalities = _modalities_for_family(family, capabilities) + capability = build_capability( + family=family, + input_modalities=input_modalities, + output_modalities=output_modalities, + capabilities=merge_unique(capabilities), + limits=_limits_from_show(payload), + ) + return ModelCapabilityRecord( + vendor=VENDOR_OLLAMA, + model_id=model_id, + stable_model_id=stable_model_id_for(VENDOR_OLLAMA, model_id, endpoint_id=endpoint_id, base_url=base_url), + display_name=model_id, + capability=capability, + raw=payload, + ) + + +def records_from_tags_payload( + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + records: list[ModelCapabilityRecord] = [] + for item in as_list(as_mapping(payload).get("models")): + if not isinstance(item, Mapping): + continue + model_id = model_id_from(item, "model", "name") + if not model_id: + continue + records.append( + ModelCapabilityRecord( + vendor=VENDOR_OLLAMA, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_OLLAMA, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=model_id, + capability=mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ), + raw=item, + ) + ) + return tuple(records) + + +def records_from_payload( + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + payload = as_mapping(payload) + if "models" in payload: + return records_from_tags_payload(payload, endpoint_id=endpoint_id, base_url=base_url) + record = record_from_show_payload( + model_id_from(payload, "model", "name"), + payload, + endpoint_id=endpoint_id, + base_url=base_url, + ) + return (record,) if record else () diff --git a/src/model_capability_readers/openai.py b/src/model_capability_readers/openai.py new file mode 100644 index 000000000..b7cc6ec9b --- /dev/null +++ b/src/model_capability_readers/openai.py @@ -0,0 +1,65 @@ +"""OpenAI Models API capability reader. + +OpenAI's `/v1/models` list/retrieve shape currently provides model identity +metadata only: `id`, `object`, `created`, and `owned_by`. Those fields prove +availability, not model capabilities, so this reader keeps capabilities +unknown unless OpenAI adds explicit capability fields to the API shape later. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_OPENAI, + compact_str, + model_id_from, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_OPENAI + + +OFFICIAL_MODEL_FIELDS = frozenset({"id", "object", "created", "owned_by"}) + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = model_id_from(raw, "id") + if not model_id: + return None + + return ModelCapabilityRecord( + vendor=VENDOR_OPENAI, + model_id=model_id, + stable_model_id=stable_model_id_for(VENDOR_OPENAI, model_id, endpoint_id=endpoint_id, base_url=base_url), + display_name=compact_str(raw.get("name") or raw.get("display_name")), + capability=mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ), + raw=raw, + ) + + +def records_from_payload( + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + return tuple(records) diff --git a/src/model_capability_readers/openrouter.py b/src/model_capability_readers/openrouter.py new file mode 100644 index 000000000..68c38a15e --- /dev/null +++ b/src/model_capability_readers/openrouter.py @@ -0,0 +1,200 @@ +"""OpenRouter model catalog capability reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers import generic_openai +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_OPENROUTER, + as_list, + as_mapping, + build_capability, + compact_str, + deterministic_controls_from_supported_parameters, + family_from_modalities, + int_limit, + merge_unique, + model_id_from, + modalities_from_value, + openai_model_items, + split_modality_arrow, + stable_model_id_for, +) + + +vendor = VENDOR_OPENROUTER + + +_SUPPORTED_PARAMETER_CAPS = { + "tools": mc.CAP_TOOL_CALL, + "tool_choice": mc.CAP_TOOL_CALL, + "function_calling": mc.CAP_TOOL_CALL, + "parallel_tool_calls": mc.CAP_TOOL_CALL, + "response_format": mc.CAP_JSON_MODE, + "structured_outputs": mc.CAP_STRUCTURED_OUTPUT, + "structured_output": mc.CAP_STRUCTURED_OUTPUT, + "reasoning": mc.CAP_REASONING, + "reasoning_effort": mc.CAP_REASONING, + "include_reasoning": mc.CAP_REASONING, + "web_search": mc.CAP_WEB_SEARCH, + "web_search_options": mc.CAP_WEB_SEARCH, +} + + +def _capabilities_from_supported_parameters(values: Any) -> tuple[str, ...]: + iterable = values if isinstance(values, list) else () + out: list[str] = [] + for value in iterable: + cap = _SUPPORTED_PARAMETER_CAPS.get(compact_str(value).lower().replace("-", "_")) + if cap and cap not in out: + out.append(cap) + return tuple(out) + + +def _limits_from_model(raw: Mapping[str, Any]) -> dict[str, Any]: + architecture = as_mapping(raw.get("architecture")) + top_provider = as_mapping(raw.get("top_provider")) + per_request_limits = as_mapping(raw.get("per_request_limits")) + limits: dict[str, Any] = {} + for key, canonical in ( + ("context_length", "context_tokens"), + ("max_context_length", "context_tokens"), + ("input_token_limit", "input_tokens"), + ("output_token_limit", "output_tokens"), + ("max_completion_tokens", "output_tokens"), + ): + value = int_limit(raw.get(key) or architecture.get(key) or top_provider.get(key)) + if value: + limits[canonical] = value + for key, value in per_request_limits.items(): + limit = int_limit(value) + if limit: + limits[f"per_request_{key}"] = limit + return limits + + +def _has_supported_voices(value: Any) -> bool: + return any(compact_str(item) for item in as_list(value)) + + +def _capabilities_from_modalities( + input_modalities: tuple[str, ...], + output_modalities: tuple[str, ...], + *, + supported_voices: Any = None, +) -> tuple[str, ...]: + input_set = set(input_modalities) + output_set = set(output_modalities) + capabilities: list[str] = [] + if mc.MODALITY_IMAGE in input_set and mc.MODALITY_TEXT in output_set: + capabilities.append(mc.CAP_VISION) + if mc.MODALITY_FILE in input_set: + capabilities.append(mc.CAP_FILES) + if mc.MODALITY_PDF in input_set: + capabilities.append(mc.CAP_PDF) + if mc.MODALITY_AUDIO in input_set: + capabilities.append(mc.CAP_AUDIO_INPUT) + if mc.MODALITY_AUDIO in output_set: + capabilities.append(mc.CAP_AUDIO_OUTPUT) + if _has_supported_voices(supported_voices): + capabilities.append(mc.CAP_TTS) + if mc.MODALITY_IMAGE in output_set: + capabilities.append(mc.CAP_IMAGE_GENERATION) + if mc.MODALITY_IMAGE in input_set: + capabilities.append(mc.CAP_IMAGE_EDITING) + if mc.MODALITY_VIDEO in output_set: + capabilities.append(mc.CAP_VIDEO_GENERATION) + return tuple(capabilities) + + +def _default_parameter_controls(raw: Mapping[str, Any]) -> tuple[str, ...]: + defaults = as_mapping(raw.get("default_parameters")) + return tuple(key for key, value in defaults.items() if value is not None) + + +def _deterministic_controls_from_model(raw: Mapping[str, Any]) -> tuple[mc.DeterministicControl, ...]: + return deterministic_controls_from_supported_parameters( + merge_unique( + as_list(raw.get("supported_parameters")), + _default_parameter_controls(raw), + ) + ) + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = model_id_from(raw, "id", "name") + if not model_id: + return None + + architecture = as_mapping(raw.get("architecture")) + input_modalities = modalities_from_value( + raw.get("input_modalities") or architecture.get("input_modalities") + ) + output_modalities = modalities_from_value( + raw.get("output_modalities") or architecture.get("output_modalities") + ) + if not input_modalities or not output_modalities: + arrow_input, arrow_output = split_modality_arrow( + raw.get("modality") or architecture.get("modality") + ) + input_modalities = input_modalities or arrow_input + output_modalities = output_modalities or arrow_output + + capabilities = list(_capabilities_from_supported_parameters(raw.get("supported_parameters"))) + capabilities.extend( + _capabilities_from_modalities( + input_modalities, + output_modalities, + supported_voices=raw.get("supported_voices"), + ) + ) + + family = family_from_modalities(input_modalities, output_modalities) + if family == mc.FAMILY_UNKNOWN: + fallback = generic_openai.record_from_model( + raw, + vendor_id=VENDOR_OPENROUTER, + endpoint_id=endpoint_id, + base_url=base_url, + ) + return fallback + + capability = build_capability( + family=family, + input_modalities=input_modalities, + output_modalities=output_modalities, + capabilities=merge_unique(capabilities), + limits=_limits_from_model(raw), + ) + return ModelCapabilityRecord( + vendor=VENDOR_OPENROUTER, + model_id=model_id, + stable_model_id=stable_model_id_for(VENDOR_OPENROUTER, model_id, endpoint_id=endpoint_id, base_url=base_url), + display_name=compact_str(raw.get("name")) or model_id, + capability=capability, + deterministic_controls=_deterministic_controls_from_model(raw), + raw=raw, + ) + + +def records_from_payload( + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + return tuple(records) diff --git a/static/js/admin.js b/static/js/admin.js index d409614a8..7ea8458c8 100644 --- a/static/js/admin.js +++ b/static/js/admin.js @@ -934,6 +934,13 @@ function initEndpointForm() { function _apiEndpointKind() { return (kindSel && kindSel.value) ? kindSel.value : 'api'; } + function _modelRefreshModeForApiEndpoint(url, endpointKind) { + if (endpointKind === 'proxy') return 'manual'; + try { + if ((new URL(url)).hostname.toLowerCase() === 'generativelanguage.googleapis.com') return ''; + } catch (_) {} + return 'auto'; + } function _normalizeBaseUrl(raw) { let u = raw.trim(); // Fix common protocol typos @@ -1081,7 +1088,8 @@ function initEndpointForm() { fd.append('base_url', url); const endpointKind = _apiEndpointKind(); fd.append('endpoint_kind', endpointKind); - fd.append('model_refresh_mode', endpointKind === 'proxy' ? 'manual' : 'auto'); + const refreshMode = _modelRefreshModeForApiEndpoint(url, endpointKind); + if (refreshMode) fd.append('model_refresh_mode', refreshMode); fd.append('model_refresh_timeout', '30'); if (apiKey) fd.append('api_key', apiKey); if (provider.value && provider.selectedOptions && provider.selectedOptions[0]) { diff --git a/tests/test_admin_device_flow_static.py b/tests/test_admin_device_flow_static.py index 94f837340..d6becde0d 100644 --- a/tests/test_admin_device_flow_static.py +++ b/tests/test_admin_device_flow_static.py @@ -28,6 +28,24 @@ def test_provider_selection_is_inert_and_add_button_starts_device_flow(): assert "_startProviderDeviceAuth(deviceAuthProvider" in add_block +def test_google_add_omits_auto_refresh_mode_for_backend_manual_default(): + refresh_helper = _between( + _ADMIN, + "function _modelRefreshModeForApiEndpoint", + "function _normalizeBaseUrl", + ) + add_block = _between( + _ADMIN, + "el('adm-epAddBtn').addEventListener('click'", + "async function _startProviderDeviceAuth", + ) + + assert "generativelanguage.googleapis.com" in refresh_helper + assert "return '';" in refresh_helper + assert "_modelRefreshModeForApiEndpoint(url, endpointKind)" in add_block + assert "if (refreshMode) fd.append('model_refresh_mode', refreshMode)" in add_block + + def test_device_auth_selection_disables_and_dims_api_test_button(): form_block = _between(_ADMIN, "function _setApiFormForProvider()", "function _renderPickerMenu()") diff --git a/tests/test_model_capabilities.py b/tests/test_model_capabilities.py new file mode 100644 index 000000000..bead66b5b --- /dev/null +++ b/tests/test_model_capabilities.py @@ -0,0 +1,248 @@ +import src.model_capabilities as mc + + +def surfaces(capability): + return set(mc.display_surfaces_for(capability)) + + +def test_endpoint_type_llm_maps_to_explicit_chat_capability(): + capability = mc.capability_from_endpoint_type("llm") + + assert capability.family == mc.FAMILY_CHAT + assert capability.primary_task == mc.TASK_CHAT_COMPLETIONS + assert capability.modalities.input == (mc.MODALITY_TEXT,) + assert capability.modalities.output == (mc.MODALITY_TEXT,) + assert capability.source == mc.SOURCE_ENDPOINT_CONFIG + assert capability.confidence == mc.CONFIDENCE_EXPLICIT + assert surfaces(capability) == {"chat"} + + +def test_endpoint_type_image_maps_to_explicit_image_generation_capability(): + capability = mc.capability_from_endpoint_type("image") + + assert capability.family == mc.FAMILY_IMAGE + assert capability.primary_task == mc.TASK_IMAGE_GENERATE + assert capability.modalities.output == (mc.MODALITY_IMAGE,) + assert capability.capabilities == (mc.CAP_IMAGE_GENERATION,) + assert capability.source == mc.SOURCE_ENDPOINT_CONFIG + assert capability.confidence == mc.CONFIDENCE_EXPLICIT + assert surfaces(capability) == {"image_generation"} + + +def test_missing_or_unknown_endpoint_type_does_not_imply_chat(): + for model_type in (None, "", "openai-compatible", "text"): + capability = mc.capability_from_endpoint_type(model_type) + + assert capability.family == mc.FAMILY_UNKNOWN + assert capability.primary_task == mc.TASK_UNKNOWN + assert capability.source == mc.SOURCE_ENDPOINT_CONFIG + assert mc.display_surfaces_for(capability) == () + + +def test_provider_record_normalizes_aliases_and_boolean_capability_maps(): + capability = mc.ModelCapability.from_dict( + { + "family": "llm", + "modalities": { + "input": ["text", "images", "docs", "images"], + "output": "text", + }, + "capabilities": { + "tools": True, + "unknown_vendor_flag": True, + "vision": True, + "tts": False, + }, + "limits": {"max_context_tokens": 32768, "": "ignored"}, + "source": "provider_reader", + "confidence": "provider_reported", + } + ) + + assert capability.family == mc.FAMILY_CHAT + assert capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE, mc.MODALITY_FILE) + assert capability.modalities.output == (mc.MODALITY_TEXT,) + assert capability.capabilities == (mc.CAP_TOOL_CALL, mc.CAP_VISION) + assert capability.limits == (("max_context_tokens", 32768),) + assert surfaces(capability) == {"chat", "vision_chat", "document_chat"} + assert capability.to_dict() == { + "family": mc.FAMILY_CHAT, + "primary_task": mc.TASK_CHAT_COMPLETIONS, + "modalities": { + "input": [mc.MODALITY_TEXT, mc.MODALITY_IMAGE, mc.MODALITY_FILE], + "output": [mc.MODALITY_TEXT], + }, + "capabilities": [mc.CAP_TOOL_CALL, mc.CAP_VISION], + "limits": {"max_context_tokens": 32768}, + "source": mc.SOURCE_PROVIDER_READER, + "confidence": mc.CONFIDENCE_PROVIDER_REPORTED, + } + + +def test_unknown_or_malformed_capability_record_stays_unknown(): + assert mc.ModelCapability.from_dict(None).to_dict() == mc.unknown_capability().to_dict() + + capability = mc.ModelCapability.build( + family="not-real", + primary_task=1234, + input_modalities=object(), + output_modalities=["text", "not-real"], + capabilities=["vision", "not-real"], + source="not-real", + confidence="not-real", + ) + + assert capability.family == mc.FAMILY_UNKNOWN + assert capability.primary_task == "1234" + assert capability.modalities.input == () + assert capability.modalities.output == (mc.MODALITY_TEXT,) + assert capability.capabilities == (mc.CAP_VISION,) + assert capability.source == mc.SOURCE_UNKNOWN + assert capability.confidence == mc.CONFIDENCE_UNKNOWN + assert mc.display_surfaces_for(capability) == () + + +def test_display_surface_queries_cover_core_model_categories(): + assert surfaces( + mc.ModelCapability.build( + family=mc.FAMILY_IMAGE, + input_modalities=[mc.MODALITY_IMAGE], + output_modalities=[mc.MODALITY_IMAGE], + capabilities=[mc.CAP_INPAINTING], + ) + ) == {"image_editing"} + + assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_EMBEDDING)) == {"embeddings"} + assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_RERANK)) == {"rerank_scoring"} + assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_MODERATION)) == {"moderation_classification"} + assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_CLASSIFICATION)) == {"moderation_classification"} + + +def test_audio_surface_matches_audio_input_or_output_when_capability_is_known(): + transcription = mc.ModelCapability.build( + family=mc.FAMILY_AUDIO, + primary_task=mc.TASK_AUDIO_TRANSCRIBE, + input_modalities=[mc.MODALITY_AUDIO], + output_modalities=[mc.MODALITY_TEXT], + capabilities=[mc.CAP_TRANSCRIPTION], + ) + synthesis = mc.ModelCapability.build( + family=mc.FAMILY_AUDIO, + primary_task=mc.TASK_AUDIO_SYNTHESIZE, + input_modalities=[mc.MODALITY_TEXT], + output_modalities=[mc.MODALITY_AUDIO], + capabilities=[mc.CAP_TTS], + ) + + assert surfaces(transcription) == {"audio_realtime"} + assert surfaces(synthesis) == {"audio_realtime"} + + +def test_capability_assertion_tracks_claimed_status_separately_from_capability_metadata(): + assertion = mc.CapabilityAssertion.build( + capability="tools", + status="claimed", + source="provider_reader", + confidence="provider_reported", + evidence={"field": "supported_parameters"}, + ) + + assert assertion.capability == mc.CAP_TOOL_CALL + assert assertion.status == mc.ASSERTION_CLAIMED + assert assertion.source == mc.SOURCE_PROVIDER_READER + assert assertion.confidence == mc.CONFIDENCE_PROVIDER_REPORTED + assert assertion.to_dict() == { + "capability": mc.CAP_TOOL_CALL, + "status": mc.ASSERTION_CLAIMED, + "source": mc.SOURCE_PROVIDER_READER, + "confidence": mc.CONFIDENCE_PROVIDER_REPORTED, + "evidence": {"field": "supported_parameters"}, + "tested_at": "", + } + + +def test_capability_probe_result_converts_pass_and_fail_to_assertions(): + passed = mc.CapabilityProbeResult.build( + provider="openrouter", + endpoint_id="ep-1", + model_id="vendor/model", + stable_model_id="openrouter|endpoint:ep-1|vendor/model", + capability="tool_calls", + status="pass", + tested_at="2026-06-04T20:00:00Z", + request_hash="abc123", + response_fingerprint="fp-test", + evidence={"contract": "single_fake_tool"}, + ) + failed = mc.CapabilityProbeResult.build( + provider="openrouter", + model_id="vendor/model", + capability="vision", + status="fail", + ) + + pass_assertion = passed.to_assertion() + fail_assertion = failed.to_assertion() + + assert pass_assertion.capability == mc.CAP_TOOL_CALL + assert pass_assertion.status == mc.ASSERTION_VERIFIED + assert pass_assertion.source == mc.SOURCE_CAPABILITY_PROBE + assert pass_assertion.confidence == mc.CONFIDENCE_EXPLICIT + assert pass_assertion.tested_at == "2026-06-04T20:00:00Z" + assert dict(pass_assertion.evidence)["request_hash"] == "abc123" + assert dict(pass_assertion.evidence)["contract"] == "single_fake_tool" + + assert fail_assertion.capability == mc.CAP_VISION + assert fail_assertion.status == mc.ASSERTION_UNSUPPORTED + assert fail_assertion.source == mc.SOURCE_CAPABILITY_PROBE + + +def test_deterministic_controls_are_normalized_as_claims_not_capabilities(): + controls = mc.deterministic_controls_from_values( + ["temp", "top-p", "top-k", "seed", "unknown"], + source=mc.SOURCE_PROVIDER_READER, + ) + + assert [control.control for control in controls] == [ + mc.CONTROL_TEMPERATURE, + mc.CONTROL_TOP_P, + mc.CONTROL_TOP_K, + mc.CONTROL_SEED, + ] + assert {control.status for control in controls} == {mc.ASSERTION_CLAIMED} + assert {control.source for control in controls} == {mc.SOURCE_PROVIDER_READER} + + +def test_reasoning_control_mechanisms_normalize_known_provider_shapes(): + values = [ + "think_directive", + "system_prompt_directive", + "enable_thinking", + "think_bool", + "reasoning_object", + "thinking_budget", + "reasoning_effort", + ] + + assert [mc.normalize_reasoning_control_mechanism(value) for value in values] == [ + mc.REASONING_CONTROL_MESSAGE_DIRECTIVE, + mc.REASONING_CONTROL_SYSTEM_DIRECTIVE, + mc.REASONING_CONTROL_TEMPLATE_KWARG, + mc.REASONING_CONTROL_NATIVE_BOOL, + mc.REASONING_CONTROL_STRUCTURED_OBJECT, + mc.REASONING_CONTROL_BUDGET, + mc.REASONING_CONTROL_EFFORT, + ] + + +def test_reasoning_control_values_can_describe_provider_supported_auto(): + values = ["enabled", "disabled", "adaptive", "dynamic", "provider_auto"] + + assert [mc.normalize_reasoning_control_value(value) for value in values] == [ + mc.REASONING_CONTROL_VALUE_ON, + mc.REASONING_CONTROL_VALUE_OFF, + mc.REASONING_CONTROL_VALUE_AUTO, + mc.REASONING_CONTROL_VALUE_AUTO, + mc.REASONING_CONTROL_VALUE_AUTO, + ] + assert mc.normalize_reasoning_control_value("message_directive") == "" diff --git a/tests/test_model_capability_readers.py b/tests/test_model_capability_readers.py new file mode 100644 index 000000000..36ef93ab1 --- /dev/null +++ b/tests/test_model_capability_readers.py @@ -0,0 +1,646 @@ +import src.model_capabilities as mc +import src.model_capability_readers as readers +from src.model_capability_readers import generic_openai, google, llamacpp, lmstudio, ollama, openai, openrouter +from src.model_capability_readers.base import ( + VENDOR_GENERIC_OPENAI, + VENDOR_GOOGLE, + VENDOR_LLAMACPP, + VENDOR_LMSTUDIO, + VENDOR_OLLAMA, + VENDOR_OPENAI, + VENDOR_OPENROUTER, + detect_vendor, + stable_model_id_for, +) + + +def surfaces(record): + return set(mc.display_surfaces_for(record.capability)) + + +def test_detect_vendor_uses_endpoint_kind_then_host_and_common_local_ports(): + assert detect_vendor("https://example.test/v1", endpoint_kind="ollama") == VENDOR_OLLAMA + assert detect_vendor("http://127.0.0.1:8080", endpoint_kind="llama_cpp") == VENDOR_LLAMACPP + assert detect_vendor("https://openrouter.ai/api/v1") == VENDOR_OPENROUTER + assert detect_vendor("https://api.openai.com/v1") == VENDOR_OPENAI + assert detect_vendor("https://generativelanguage.googleapis.com/v1beta/openai") == VENDOR_GOOGLE + assert detect_vendor("http://127.0.0.1:11434") == VENDOR_OLLAMA + assert detect_vendor("http://127.0.0.1:1234") == VENDOR_LMSTUDIO + assert detect_vendor("http://127.0.0.1:8080") == VENDOR_GENERIC_OPENAI + assert detect_vendor("http://localhost:7000/v1") == VENDOR_GENERIC_OPENAI + + +def test_generic_openai_reader_keeps_basic_model_payload_unknown(): + records = generic_openai.records_from_payload( + { + "object": "list", + "data": [ + {"id": "gpt-example", "object": "model", "owned_by": "vendor"}, + ], + } + ) + + assert len(records) == 1 + record = records[0] + assert record.vendor == VENDOR_GENERIC_OPENAI + assert record.model_id == "gpt-example" + assert record.capability.family == mc.FAMILY_UNKNOWN + assert record.capability.source == mc.SOURCE_PROVIDER_READER + assert record.capability.confidence == mc.CONFIDENCE_UNKNOWN + assert record.stable_model_id == "generic_openai|global|gpt-example" + assert record.capability_assertions == () + assert record.deterministic_controls == () + assert surfaces(record) == set() + + +def test_stable_model_id_is_endpoint_scoped_for_local_or_configured_servers(): + assert stable_model_id_for("ollama", "qwen:latest", endpoint_id="7") == "ollama|endpoint:7|qwen:latest" + assert stable_model_id_for("ollama", "qwen:latest", base_url="http://127.0.0.1:11434") != stable_model_id_for( + "ollama", + "qwen:latest", + base_url="http://10.0.0.12:11434", + ) + + +def test_registry_uses_openai_reader_for_openai_vendor(): + records = readers.records_from_payload({"data": [{"id": "shape-only-model"}]}, vendor=VENDOR_OPENAI) + + assert len(records) == 1 + assert records[0].vendor == VENDOR_OPENAI + assert records[0].stable_model_id == "openai|global|shape-only-model" + assert records[0].capability.family == mc.FAMILY_UNKNOWN + + +def test_openai_reader_keeps_official_model_shape_identity_only(): + records = openai.records_from_payload( + { + "object": "list", + "data": [ + { + "id": "shape-only-model", + "object": "model", + "created": 1700000000, + "owned_by": "openai", + } + ], + } + ) + + assert len(records) == 1 + record = records[0] + assert record.vendor == VENDOR_OPENAI + assert record.model_id == "shape-only-model" + assert record.capability.family == mc.FAMILY_UNKNOWN + assert record.capability.source == mc.SOURCE_PROVIDER_READER + assert record.capability.confidence == mc.CONFIDENCE_UNKNOWN + assert record.capability_assertions == () + assert record.deterministic_controls == () + assert surfaces(record) == set() + + +def test_registry_passes_endpoint_context_to_vendor_reader(): + records = readers.records_from_payload( + {"data": [{"id": "local.gguf", "owned_by": "llamacpp"}]}, + vendor=VENDOR_LLAMACPP, + base_url="http://localhost:8000", + ) + + assert len(records) == 1 + assert records[0].stable_model_id == stable_model_id_for( + VENDOR_LLAMACPP, + "local.gguf", + base_url="http://localhost:8000", + ) + + +def test_openrouter_reader_maps_rich_architecture_and_supported_parameters(): + records = openrouter.records_from_payload( + { + "data": [ + { + "id": "google/gemini-vision", + "name": "Gemini Vision", + "architecture": {"modality": "text+image->text"}, + "supported_parameters": [ + "tools", + "response_format", + "reasoning", + "include_reasoning", + "parallel_tool_calls", + "temperature", + "top_p", + "seed", + ], + "context_length": 1048576, + "top_provider": {"max_completion_tokens": 65536}, + }, + { + "id": "black-forest-labs/flux", + "architecture": {"input_modalities": ["text"], "output_modalities": ["image"]}, + }, + { + "id": "vendor/image-edit-shape", + "architecture": {"input_modalities": ["text", "image", "file"], "output_modalities": ["text", "image"]}, + "supported_parameters": ["structured_outputs", "web_search_options"], + }, + { + "id": "vendor/audio-shape", + "architecture": {"input_modalities": ["text", "audio"], "output_modalities": ["text", "audio"]}, + "supported_voices": ["alloy"], + "default_parameters": {"temperature": 0.7, "top_p": 0.9, "top_k": None}, + "per_request_limits": {"prompt_tokens": 12000, "completion_tokens": 4000, "requests": "2"}, + }, + { + "id": "vendor/embedder", + "architecture": {"modality": "text->embedding"}, + }, + ] + } + ) + + assert [record.model_id for record in records] == [ + "google/gemini-vision", + "black-forest-labs/flux", + "vendor/image-edit-shape", + "vendor/audio-shape", + "vendor/embedder", + ] + + vision = records[0] + assert vision.capability.family == mc.FAMILY_CHAT + assert vision.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) + assert vision.capability.capabilities == ( + mc.CAP_TOOL_CALL, + mc.CAP_JSON_MODE, + mc.CAP_REASONING, + mc.CAP_VISION, + ) + assert [(assertion.capability, assertion.status) for assertion in vision.capability_assertions] == [ + (mc.CAP_TOOL_CALL, mc.ASSERTION_CLAIMED), + (mc.CAP_JSON_MODE, mc.ASSERTION_CLAIMED), + (mc.CAP_REASONING, mc.ASSERTION_CLAIMED), + (mc.CAP_VISION, mc.ASSERTION_CLAIMED), + ] + assert [(control.control, control.status) for control in vision.deterministic_controls] == [ + (mc.CONTROL_TEMPERATURE, mc.ASSERTION_CLAIMED), + (mc.CONTROL_TOP_P, mc.ASSERTION_CLAIMED), + (mc.CONTROL_SEED, mc.ASSERTION_CLAIMED), + ] + assert dict(vision.capability.limits) == {"context_tokens": 1048576, "output_tokens": 65536} + assert surfaces(vision) == {"chat", "vision_chat"} + + assert records[1].capability.family == mc.FAMILY_IMAGE + assert records[1].capability.capabilities == (mc.CAP_IMAGE_GENERATION,) + assert surfaces(records[1]) == {"image_generation"} + + image_edit = records[2] + assert image_edit.capability.family == mc.FAMILY_IMAGE + assert image_edit.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE, mc.MODALITY_FILE) + assert image_edit.capability.modalities.output == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) + assert image_edit.capability.capabilities == ( + mc.CAP_STRUCTURED_OUTPUT, + mc.CAP_WEB_SEARCH, + mc.CAP_VISION, + mc.CAP_FILES, + mc.CAP_IMAGE_GENERATION, + mc.CAP_IMAGE_EDITING, + ) + assert surfaces(image_edit) == {"image_generation", "image_editing"} + + audio = records[3] + assert audio.capability.family == mc.FAMILY_AUDIO + assert audio.capability.capabilities == (mc.CAP_AUDIO_INPUT, mc.CAP_AUDIO_OUTPUT, mc.CAP_TTS) + assert dict(audio.capability.limits) == { + "per_request_completion_tokens": 4000, + "per_request_prompt_tokens": 12000, + "per_request_requests": 2, + } + assert [control.control for control in audio.deterministic_controls] == [ + mc.CONTROL_TEMPERATURE, + mc.CONTROL_TOP_P, + ] + assert surfaces(audio) == {"audio_realtime"} + + assert records[4].capability.family == mc.FAMILY_EMBEDDING + assert surfaces(records[4]) == {"embeddings"} + + +def test_google_reader_maps_provider_fields_without_claiming_unreported_modalities(): + records = google.records_from_payload( + { + "models": [ + { + "name": "models/gemini-3.1-flash-image", + "displayName": "Gemini 3.1 Flash Image", + "supportedGenerationMethods": ["generateContent"], + "inputTokenLimit": 1000000, + "outputTokenLimit": 8192, + "thinking": True, + "temperature": 1.0, + "topP": 0.95, + "topK": 40, + }, + { + "name": "models/text-embedding-example", + "supportedGenerationMethods": ["embedContent"], + }, + ] + } + ) + + assert len(records) == 2 + content = records[0] + assert content.vendor == VENDOR_GOOGLE + assert content.model_id == "gemini-3.1-flash-image" + assert content.capability.family == mc.FAMILY_UNKNOWN + assert content.capability.modalities.input == () + assert content.capability.modalities.output == () + assert content.capability.capabilities == (mc.CAP_REASONING,) + assert dict(content.capability.limits) == { + "context_tokens": 1000000, + "input_tokens": 1000000, + "output_tokens": 8192, + } + assert [control.control for control in content.deterministic_controls] == [ + mc.CONTROL_TEMPERATURE, + mc.CONTROL_TOP_P, + mc.CONTROL_TOP_K, + ] + assert surfaces(content) == set() + + embedding = records[1] + assert embedding.capability.family == mc.FAMILY_EMBEDDING + assert surfaces(embedding) == {"embeddings"} + + +def test_google_ai_studio_mapping_does_not_infer_media_from_model_names(): + records = google.records_from_payload( + { + "models": [ + { + "name": "models/imagen-4.0-generate-001", + "displayName": "Imagen 4", + "supportedGenerationMethods": ["predict"], + }, + { + "name": "models/veo-3.1-generate-preview", + "displayName": "Veo 3.1", + "supportedGenerationMethods": ["predictLongRunning"], + }, + { + "name": "models/gemini-3.1-flash-tts-preview", + "supportedGenerationMethods": ["generateContent", "countTokens", "createCachedContent", "batchGenerateContent"], + }, + { + "name": "models/lyria-3-pro-preview", + "displayName": "Lyria 3 Pro Preview", + "supportedGenerationMethods": ["generateContent", "countTokens"], + }, + ] + } + ) + + assert len(records) == 4 + assert [record.capability.family for record in records] == [ + mc.FAMILY_UNKNOWN, + mc.FAMILY_UNKNOWN, + mc.FAMILY_UNKNOWN, + mc.FAMILY_UNKNOWN, + ] + assert all(record.capability.modalities.input == () for record in records) + assert all(record.capability.modalities.output == () for record in records) + assert all(surfaces(record) == set() for record in records) + assert [control.control for control in records[2].deterministic_controls] == [ + mc.CONTROL_PROMPT_CACHING, + mc.CONTROL_BATCH, + ] + + +def test_google_ai_studio_mapping_keeps_unrecognized_predict_models_unknown(): + records = google.records_from_payload( + { + "models": [ + { + "name": "models/vendor-future-media-001", + "supportedGenerationMethods": ["predict"], + } + ] + } + ) + + assert len(records) == 1 + assert records[0].capability.family == mc.FAMILY_UNKNOWN + assert surfaces(records[0]) == set() + + +def test_ollama_reader_maps_show_capabilities_and_tags_are_unknown(): + vision = ollama.record_from_show_payload( + "llava:latest", + { + "capabilities": ["completion", "vision", "tools"], + "model_info": {"llama.context_length": 4096}, + }, + ) + embedding = ollama.record_from_show_payload( + "nomic-embed-text:latest", + {"capabilities": ["embedding"]}, + ) + tags = ollama.records_from_tags_payload({"models": [{"name": "qwen3:latest"}]}) + + assert vision is not None + assert vision.capability.family == mc.FAMILY_CHAT + assert vision.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) + assert vision.capability.capabilities == (mc.CAP_VISION, mc.CAP_TOOL_CALL) + assert dict(vision.capability.limits) == {"context_tokens": 4096} + assert surfaces(vision) == {"chat", "vision_chat"} + + assert embedding is not None + assert embedding.capability.family == mc.FAMILY_EMBEDDING + assert surfaces(embedding) == {"embeddings"} + + assert len(tags) == 1 + assert tags[0].capability.family == mc.FAMILY_UNKNOWN + assert surfaces(tags[0]) == set() + + +def test_ollama_reader_uses_show_shape_without_architecture_name_matching(): + record = ollama.record_from_show_payload( + "local:latest", + { + "capabilities": ["completion", "thinking", "tools"], + "parameters": "temperature 0.7\nnum_ctx 8192", + "model_info": { + "future_architecture.context_length": 32768, + "future_architecture.embedding_length": 4096, + }, + }, + ) + + assert record is not None + assert record.capability.family == mc.FAMILY_CHAT + assert record.capability.modalities.input == (mc.MODALITY_TEXT,) + assert record.capability.modalities.output == (mc.MODALITY_TEXT,) + assert record.capability.capabilities == (mc.CAP_REASONING, mc.CAP_TOOL_CALL) + assert dict(record.capability.limits) == {"context_tokens": 8192} + assert surfaces(record) == {"chat"} + + +def test_ollama_reader_uses_generic_model_info_context_length_when_no_num_ctx(): + record = ollama.record_from_show_payload( + "local:latest", + { + "capabilities": ["completion"], + "model_info": {"future_architecture.context_length": 32768}, + }, + ) + + assert record is not None + assert record.capability.family == mc.FAMILY_CHAT + assert dict(record.capability.limits) == {"context_tokens": 32768} + + +def test_lmstudio_reader_uses_native_v1_capabilities_when_present(): + records = lmstudio.records_from_payload( + { + "models": [ + { + "type": "llm", + "key": "google/gemma-vl", + "display_name": "Gemma VL", + "capabilities": { + "vision": True, + "trained_for_tool_use": True, + "reasoning": {"allowed_options": ["off", "on"], "default": "on"}, + }, + "loaded_instances": [ + {"config": {"context_length": 8192}}, + {"config": {"context_length": 4096}}, + ], + "max_context_length": 262144, + }, + { + "type": "embedding", + "key": "nomic/embed", + }, + {"key": "shape-without-type"}, + ] + } + ) + + assert len(records) == 3 + vision = records[0] + assert vision.vendor == VENDOR_LMSTUDIO + assert vision.model_id == "google/gemma-vl" + assert vision.display_name == "Gemma VL" + assert vision.capability.family == mc.FAMILY_CHAT + assert vision.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) + assert vision.capability.capabilities == (mc.CAP_VISION, mc.CAP_TOOL_CALL, mc.CAP_REASONING) + assert dict(vision.capability.limits) == {"context_tokens": 4096, "max_context_tokens": 262144} + assert surfaces(vision) == {"chat", "vision_chat"} + + assert records[1].capability.family == mc.FAMILY_EMBEDDING + assert surfaces(records[1]) == {"embeddings"} + + assert records[2].capability.family == mc.FAMILY_UNKNOWN + assert surfaces(records[2]) == set() + + +def test_lmstudio_reader_uses_legacy_native_v0_shape_for_family_and_limits(): + records = lmstudio.records_from_payload( + { + "data": [ + { + "id": "local-gemma", + "type": "llm", + "arch": "gemma3", + "loaded_context_length": 16384, + "max_context_length": 32768, + }, + { + "id": "text-embedding-local", + "type": "embeddings", + "max_context_length": 2048, + }, + ] + } + ) + + assert len(records) == 2 + chat = records[0] + assert chat.vendor == VENDOR_LMSTUDIO + assert chat.capability.family == mc.FAMILY_CHAT + assert chat.capability.modalities.input == (mc.MODALITY_TEXT,) + assert chat.capability.capabilities == () + assert dict(chat.capability.limits) == {"context_tokens": 16384, "max_context_tokens": 32768} + assert surfaces(chat) == {"chat"} + + assert records[1].capability.family == mc.FAMILY_EMBEDDING + assert dict(records[1].capability.limits) == {"context_tokens": 2048} + assert surfaces(records[1]) == {"embeddings"} + + +def test_lmstudio_openai_compatible_model_list_remains_identity_only(): + records = lmstudio.records_from_payload( + { + "object": "list", + "data": [ + {"id": "local-gemma-3-270m-it-qat-q4_k_m", "object": "model", "owned_by": "organization_owner"}, + {"id": "text-embedding-nomic-embed-text-v1.5", "object": "model", "owned_by": "organization_owner"}, + ], + } + ) + + assert len(records) == 2 + for record in records: + assert record.vendor == VENDOR_LMSTUDIO + assert record.capability.family == mc.FAMILY_UNKNOWN + assert record.capability_assertions == () + assert surfaces(record) == set() + + +def test_lmstudio_unexpected_native_endpoint_error_yields_no_records(): + assert lmstudio.records_from_payload({"error": "Unexpected endpoint or method. (GET /api/v1/models)"}) == () + + +def test_llamacpp_reader_merges_models_props_and_slots_payloads(): + models_payload = { + "object": "list", + "data": [ + { + "id": "gemma-4-E2B-it-Q8_0.gguf", + "owned_by": "llamacpp", + "meta": { + "n_ctx_train": 131072, + "n_params": 4647450147, + "size": 5032532108, + }, + } + ], + "models": [ + { + "name": "gemma-4-E2B-it-Q8_0.gguf", + "model": "gemma-4-E2B-it-Q8_0.gguf", + "capabilities": ["completion"], + "details": {"format": "gguf"}, + } + ], + } + props_payload = { + "model_alias": "gemma-4-E2B-it-Q8_0.gguf", + "model_path": "/models/gemma-4-E2B-it-Q8_0.gguf", + "build_info": "b1-c8ac02f", + "total_slots": 4, + "modalities": {"vision": False, "audio": False}, + "chat_template_caps": { + "supports_object_arguments": True, + "supports_parallel_tool_calls": True, + "supports_preserve_reasoning": False, + "supports_string_content": True, + "supports_system_role": True, + "supports_tool_calls": True, + "supports_tools": True, + "supports_typed_content": False, + }, + "default_generation_settings": { + "n_ctx": 16384, + "params": { + "temperature": 1.0, + "top_p": 0.95, + "seed": 4294967295, + "stream": True, + "samplers": ["top_p", "temperature"], + }, + }, + } + slots_payload = [ + {"id": 0, "n_ctx": 16384, "speculative": False, "is_processing": False}, + {"id": 1, "n_ctx": 16384, "speculative": False, "is_processing": False}, + {"id": 2, "n_ctx": 16384, "speculative": False, "is_processing": False}, + {"id": 3, "n_ctx": 16384, "speculative": False, "is_processing": False}, + ] + + records = llamacpp.records_from_payloads( + models_payload=models_payload, + props_payload=props_payload, + slots_payload=slots_payload, + base_url="http://localhost:8000", + ) + + assert len(records) == 1 + record = records[0] + assert record.vendor == VENDOR_LLAMACPP + assert record.model_id == "gemma-4-E2B-it-Q8_0.gguf" + assert record.stable_model_id == stable_model_id_for( + VENDOR_LLAMACPP, + "gemma-4-E2B-it-Q8_0.gguf", + base_url="http://localhost:8000", + ) + assert record.capability.family == mc.FAMILY_CHAT + assert record.capability.modalities.input == (mc.MODALITY_TEXT,) + assert record.capability.modalities.output == (mc.MODALITY_TEXT,) + assert record.capability.capabilities == (mc.CAP_TOOL_CALL, mc.CAP_STREAMING) + assert dict(record.capability.limits) == { + "context_tokens": 16384, + "model_bytes": 5032532108, + "parallel_slots": 4, + "parameters": 4647450147, + "training_context_tokens": 131072, + } + assert surfaces(record) == {"chat"} + + assertion_status = {assertion.capability: assertion.status for assertion in record.capability_assertions} + assert assertion_status[mc.CAP_TOOL_CALL] == mc.ASSERTION_CLAIMED + assert assertion_status[mc.CAP_STREAMING] == mc.ASSERTION_CLAIMED + assert assertion_status[mc.CAP_VISION] == mc.ASSERTION_UNSUPPORTED + assert assertion_status[mc.CAP_AUDIO_INPUT] == mc.ASSERTION_UNSUPPORTED + assert mc.CAP_REASONING not in assertion_status + + controls = {control.control: control.status for control in record.deterministic_controls} + assert controls == { + mc.CONTROL_TEMPERATURE: mc.ASSERTION_CLAIMED, + mc.CONTROL_TOP_P: mc.ASSERTION_CLAIMED, + mc.CONTROL_SEED: mc.ASSERTION_CLAIMED, + mc.CONTROL_SYSTEM_PROMPT: mc.ASSERTION_CLAIMED, + mc.CONTROL_TOOL_CHOICE: mc.ASSERTION_CLAIMED, + } + + +def test_llamacpp_openai_model_list_without_native_capability_shape_stays_unknown(): + records = llamacpp.records_from_payload( + { + "object": "list", + "data": [ + { + "id": "local-chat.gguf", + "owned_by": "llamacpp", + } + ], + } + ) + + assert len(records) == 1 + assert records[0].capability.family == mc.FAMILY_UNKNOWN + assert records[0].capability.capabilities == () + assert records[0].capability_assertions == () + assert surfaces(records[0]) == set() + + +def test_llamacpp_props_payload_reports_unsupported_modalities_without_model_list(): + records = llamacpp.records_from_payload( + { + "model_alias": "local.gguf", + "modalities": {"vision": False, "audio": False}, + "chat_template_caps": {"supports_tools": False, "supports_preserve_reasoning": False}, + "default_generation_settings": {"n_ctx": 4096, "params": {"stream": True}}, + } + ) + + assert len(records) == 1 + record = records[0] + assert record.capability.family == mc.FAMILY_CHAT + assert record.capability.capabilities == (mc.CAP_STREAMING,) + assert {a.capability: a.status for a in record.capability_assertions} == { + mc.CAP_STREAMING: mc.ASSERTION_CLAIMED, + mc.CAP_VISION: mc.ASSERTION_UNSUPPORTED, + mc.CAP_AUDIO_INPUT: mc.ASSERTION_UNSUPPORTED, + } diff --git a/tests/test_model_routes.py b/tests/test_model_routes.py index d413820c5..2794cf9f4 100644 --- a/tests/test_model_routes.py +++ b/tests/test_model_routes.py @@ -48,6 +48,9 @@ with preserve_import_state("core.database", "src.database", "core.session_manage _ping_endpoint, _parse_model_list, _normalize_refresh_mode, + _normalize_endpoint_refresh_mode, + _endpoint_refresh_mode, + _is_google_api_base, _truthy, _speech_settings_using_endpoint, _clear_speech_settings_for_endpoint, @@ -464,6 +467,28 @@ class TestClassifyEndpoint: assert _normalize_refresh_mode("manual", "proxy") == "manual" assert _normalize_refresh_mode("auto", "api") == "auto" + def test_google_refresh_mode_defaults_manual_unless_explicit(self): + base = "https://generativelanguage.googleapis.com/v1beta/openai" + assert _normalize_endpoint_refresh_mode("", "api", base) == "manual" + assert _normalize_endpoint_refresh_mode(None, "auto", base) == "manual" + assert _normalize_endpoint_refresh_mode("auto", "api", base) == "auto" + + def test_only_gemini_native_host_uses_google_models_api(self): + assert _is_google_api_base("https://generativelanguage.googleapis.com/v1beta/openai") is True + assert _is_google_api_base( + "https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/us-central1/endpoints/openapi" + ) is False + + def test_existing_google_endpoint_refresh_mode_defaults_manual(self): + ep = SimpleNamespace( + model_refresh_mode=None, + endpoint_kind="api", + base_url="https://generativelanguage.googleapis.com/v1beta/openai", + ) + assert _endpoint_refresh_mode(ep, "api") == "manual" + ep.model_refresh_mode = "auto" + assert _endpoint_refresh_mode(ep, "api") == "auto" + def test_parse_model_list_accepts_json_and_text(self): assert _parse_model_list('["a", "b", "a"]') == ["a", "b"] assert _parse_model_list("a, b\nc") == ["a", "b", "c"] @@ -568,6 +593,83 @@ class TestSetupProbeSafety: assert _probe_endpoint("https://api.groq.com/openai/v1") == _PROVIDER_CURATED["groq"] + def test_google_probe_uses_native_paginated_models_api(self, monkeypatch): + monkeypatch.setattr(endpoint_resolver, "resolve_url", lambda url: url, raising=False) + monkeypatch.setattr(model_routes, "_normalize_base", lambda url: url.rstrip("/")) + seen = [] + + def fake_get(url, headers=None, params=None, timeout=None, verify=None, **kwargs): + seen.append((url, headers, params, timeout, verify)) + request = httpx.Request("GET", url) + page_token = (params or {}).get("pageToken") + if page_token: + return httpx.Response( + 200, + request=request, + json={ + "models": [{ + "name": "models/gemini-page-two", + "supportedGenerationMethods": ["generateContent"], + }] + }, + ) + return httpx.Response( + 200, + request=request, + json={ + "models": [ + { + "name": "models/gemini-page-one", + "supportedGenerationMethods": ["generateContent"], + }, + { + "baseModelId": "gemini-base-id", + "name": "models/ignored-version", + "supportedGenerationMethods": ["generateText"], + }, + { + "name": "models/imagen-4.0-generate-001", + "supportedGenerationMethods": ["predict"], + }, + { + "name": "models/text-embedding-example", + "supportedGenerationMethods": ["embedContent"], + }, + {"name": "models/missing-method-metadata"}, + ], + "nextPageToken": "next-page", + }, + ) + + monkeypatch.setattr(model_routes.httpx, "get", fake_get) + + assert _probe_endpoint("https://generativelanguage.googleapis.com/v1beta/openai", "google-key") == [ + "gemini-page-one", + "gemini-base-id", + "gemini-page-two", + ] + assert [call[0] for call in seen] == [ + "https://generativelanguage.googleapis.com/v1beta/models", + "https://generativelanguage.googleapis.com/v1beta/models", + ] + assert seen[0][1] == {"Accept": "application/json", "x-goog-api-key": "google-key"} + assert seen[0][2] == {"pageSize": 1000} + assert seen[1][1] == {"Accept": "application/json", "x-goog-api-key": "google-key"} + assert seen[1][2] == {"pageSize": 1000, "pageToken": "next-page"} + + def test_google_probe_does_not_use_curated_fallback_on_failure(self, monkeypatch): + monkeypatch.setattr(endpoint_resolver, "resolve_url", lambda url: url, raising=False) + monkeypatch.setattr(model_routes, "_normalize_base", lambda url: url.rstrip("/")) + + def fake_get(url, headers=None, params=None, timeout=None, verify=None, **kwargs): + request = httpx.Request("GET", url) + response = httpx.Response(401, request=request) + raise httpx.HTTPStatusError("unauthorized", request=request, response=response) + + monkeypatch.setattr(model_routes.httpx, "get", fake_get) + + assert _probe_endpoint("https://generativelanguage.googleapis.com/v1beta/openai", "bad-key") == [] + def test_keyed_anthropic_probe_does_not_fallback_on_failure(self, monkeypatch): monkeypatch.setattr(endpoint_resolver, "resolve_url", lambda url: url, raising=False) monkeypatch.setattr(model_routes, "_normalize_base", lambda url: url.rstrip("/")) @@ -1101,6 +1203,26 @@ def test_post_creates_endpoint_with_pinned_models(monkeypatch): assert json.loads(db.added[0].pinned_models) == ["deploy-1", "deploy-2"] +def test_post_google_endpoint_defaults_to_manual_refresh_when_mode_omitted(monkeypatch): + db = _PinnedFakeDb([]) + _patch_create_deps(monkeypatch, db) + monkeypatch.setattr(model_routes, "_probe_endpoint", lambda *args, **kwargs: ["gemini-test"]) + create = _get_route("/api/model-endpoints", "POST") + + create( + _PinnedFakeRequest(), + base_url="https://generativelanguage.googleapis.com/v1beta/openai", + **_create_form_kwargs( + api_key="google-key", + endpoint_kind="api", + model_refresh_mode="", + ), + ) + + assert len(db.added) == 1 + assert db.added[0].model_refresh_mode == "manual" + + def test_post_dedupe_existing_merges_and_returns_pinned(monkeypatch): existing = _make_endpoint( base_url="http://host:1234/v1",