mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-07-18 13:28:02 +00:00
feat(models): define capability schema and readers (#2739)
* feat(models): define capability schema and readers * fix(models): harden Google catalog probing Restrict native catalog probing to the Gemini host, keep provider keys out of request URLs, filter non-chat model resources, and preserve the manual refresh default in the built-in Google add flow.
This commit is contained in:
+115
-3
@@ -472,7 +472,11 @@ def _endpoint_kind(ep: Any) -> str:
|
||||
|
||||
|
||||
def _endpoint_refresh_mode(ep: Any, endpoint_kind: str | None = None) -> str:
|
||||
return _normalize_refresh_mode(getattr(ep, "model_refresh_mode", None), endpoint_kind or _endpoint_kind(ep))
|
||||
return _normalize_endpoint_refresh_mode(
|
||||
getattr(ep, "model_refresh_mode", None),
|
||||
endpoint_kind or _endpoint_kind(ep),
|
||||
getattr(ep, "base_url", ""),
|
||||
)
|
||||
|
||||
|
||||
def _endpoint_refresh_interval(ep: Any, category: str) -> float:
|
||||
@@ -851,6 +855,99 @@ def _ollama_model_names(data: Any) -> List[str]:
|
||||
return out
|
||||
|
||||
|
||||
def _is_google_api_base(base_url: str) -> bool:
|
||||
try:
|
||||
return (urlparse(base_url).hostname or "").lower() == "generativelanguage.googleapis.com"
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _normalize_endpoint_refresh_mode(value: Any, endpoint_kind: str = "auto", base_url: str = "") -> str:
|
||||
if not str(value or "").strip() and _is_google_api_base(base_url):
|
||||
return "manual"
|
||||
return _normalize_refresh_mode(value, endpoint_kind)
|
||||
|
||||
|
||||
def _google_native_root(base_url: str) -> str:
|
||||
"""Return the Gemini native API root for a Google endpoint.
|
||||
|
||||
Chat calls may be configured against Google's OpenAI-compatible
|
||||
`/openai` path, but model catalog reads should use the native Models API
|
||||
so we get Google's current Model resource shape.
|
||||
"""
|
||||
try:
|
||||
parsed = urlparse(base_url)
|
||||
except Exception:
|
||||
return "https://generativelanguage.googleapis.com/v1beta"
|
||||
path = (parsed.path or "").rstrip("/")
|
||||
if path.endswith("/openai"):
|
||||
path = path[: -len("/openai")].rstrip("/")
|
||||
if not path:
|
||||
path = "/v1beta"
|
||||
return urlunparse(parsed._replace(path=path, query="", fragment="")).rstrip("/")
|
||||
|
||||
|
||||
def _google_native_models_url(base_url: str) -> str:
|
||||
return _google_native_root(base_url) + "/models"
|
||||
|
||||
|
||||
def _google_model_id_from_item(item: Any) -> str:
|
||||
if not isinstance(item, dict):
|
||||
return ""
|
||||
value = item.get("baseModelId") or item.get("name") or item.get("model") or ""
|
||||
return str(value or "").strip().removeprefix("models/")
|
||||
|
||||
|
||||
def _google_model_supports_chat(item: Any) -> bool:
|
||||
"""Return whether a native Google Model resource supports chat generation."""
|
||||
if not isinstance(item, dict):
|
||||
return False
|
||||
methods = item.get("supportedGenerationMethods")
|
||||
if not isinstance(methods, list):
|
||||
return False
|
||||
chat_methods = {"generateContent", "generateMessage", "generateText", "generateAnswer"}
|
||||
return any(method in chat_methods for method in methods)
|
||||
|
||||
|
||||
def _probe_google_models(base_url: str, api_key: str = None, timeout: int = 5, page_size: int = 1000) -> List[str]:
|
||||
"""Read Google's native paginated Models API.
|
||||
|
||||
This intentionally returns only provider-reported model IDs. Capability
|
||||
mapping is handled by the model capability reader and must not infer from
|
||||
names here.
|
||||
"""
|
||||
url = _google_native_models_url(base_url)
|
||||
try:
|
||||
page_size = min(max(int(page_size or 1000), 1), 1000)
|
||||
except Exception:
|
||||
page_size = 1000
|
||||
headers = {"Accept": "application/json"}
|
||||
if api_key:
|
||||
headers["x-goog-api-key"] = api_key
|
||||
params: Dict[str, Any] = {"pageSize": page_size}
|
||||
models: List[str] = []
|
||||
seen = set()
|
||||
page_token = ""
|
||||
for _ in range(20):
|
||||
request_params = dict(params)
|
||||
if page_token:
|
||||
request_params["pageToken"] = page_token
|
||||
r = httpx.get(url, headers=headers, params=request_params, timeout=timeout, verify=llm_verify())
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
for item in data.get("models") or []:
|
||||
if not _google_model_supports_chat(item):
|
||||
continue
|
||||
model_id = _google_model_id_from_item(item)
|
||||
if model_id and model_id not in seen:
|
||||
seen.add(model_id)
|
||||
models.append(model_id)
|
||||
page_token = str(data.get("nextPageToken") or "").strip()
|
||||
if not page_token:
|
||||
break
|
||||
return models
|
||||
|
||||
|
||||
def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> List[str]:
|
||||
"""Probe a base URL's /models endpoint and return list of model IDs.
|
||||
For Anthropic, queries their /v1/models API, falling back to hardcoded list."""
|
||||
@@ -863,6 +960,17 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
|
||||
if api_key:
|
||||
return fetch_available_models(api_key, timeout=timeout)
|
||||
return []
|
||||
if _is_google_api_base(base):
|
||||
try:
|
||||
models = _probe_google_models(base, api_key, timeout=timeout)
|
||||
if models:
|
||||
return models
|
||||
except httpx.HTTPStatusError as e:
|
||||
status = e.response.status_code if e.response is not None else "unknown"
|
||||
logger.warning(f"Google native models probe failed: HTTP {status}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Google native models probe failed: {e}")
|
||||
return []
|
||||
if provider == "anthropic":
|
||||
# Try Anthropic's /v1/models endpoint first
|
||||
url = _safe_build_models_url(base)
|
||||
@@ -1854,7 +1962,7 @@ def setup_model_routes(model_discovery):
|
||||
name = base_url.replace("http://", "").replace("https://", "").split("/")[0]
|
||||
|
||||
requested_kind = _normalize_endpoint_kind(endpoint_kind)
|
||||
refresh_mode = _normalize_refresh_mode(model_refresh_mode, requested_kind)
|
||||
refresh_mode = _normalize_endpoint_refresh_mode(model_refresh_mode, requested_kind, base_url)
|
||||
refresh_interval = _parse_positive_int(model_refresh_interval, minimum=30, maximum=86400)
|
||||
refresh_timeout = _parse_positive_int(model_refresh_timeout, minimum=1, maximum=60)
|
||||
require_model_list = _truthy(require_models)
|
||||
@@ -2364,7 +2472,11 @@ def setup_model_routes(model_discovery):
|
||||
if "endpoint_kind" in body:
|
||||
ep.endpoint_kind = _normalize_endpoint_kind(body.get("endpoint_kind"))
|
||||
if "model_refresh_mode" in body:
|
||||
ep.model_refresh_mode = _normalize_refresh_mode(body.get("model_refresh_mode"), _endpoint_kind(ep))
|
||||
ep.model_refresh_mode = _normalize_endpoint_refresh_mode(
|
||||
body.get("model_refresh_mode"),
|
||||
_endpoint_kind(ep),
|
||||
ep.base_url,
|
||||
)
|
||||
if "model_refresh_interval" in body:
|
||||
interval = _parse_positive_int(body.get("model_refresh_interval"), minimum=30, maximum=86400)
|
||||
ep.model_refresh_interval = interval
|
||||
|
||||
@@ -0,0 +1,934 @@
|
||||
"""Canonical model capability metadata helpers.
|
||||
|
||||
This module defines shape and normalization only. It does not probe providers,
|
||||
change routing, or infer authoritative capabilities from a bare model ID.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
FAMILY_CHAT = "chat"
|
||||
FAMILY_EMBEDDING = "embedding"
|
||||
FAMILY_IMAGE = "image"
|
||||
FAMILY_VIDEO = "video"
|
||||
FAMILY_AUDIO = "audio"
|
||||
FAMILY_RERANK = "rerank"
|
||||
FAMILY_CLASSIFICATION = "classification"
|
||||
FAMILY_MODERATION = "moderation"
|
||||
FAMILY_UNKNOWN = "unknown"
|
||||
|
||||
FAMILIES = frozenset(
|
||||
{
|
||||
FAMILY_CHAT,
|
||||
FAMILY_EMBEDDING,
|
||||
FAMILY_IMAGE,
|
||||
FAMILY_VIDEO,
|
||||
FAMILY_AUDIO,
|
||||
FAMILY_RERANK,
|
||||
FAMILY_CLASSIFICATION,
|
||||
FAMILY_MODERATION,
|
||||
FAMILY_UNKNOWN,
|
||||
}
|
||||
)
|
||||
|
||||
MODALITY_TEXT = "text"
|
||||
MODALITY_IMAGE = "image"
|
||||
MODALITY_FILE = "file"
|
||||
MODALITY_PDF = "pdf"
|
||||
MODALITY_AUDIO = "audio"
|
||||
MODALITY_VIDEO = "video"
|
||||
MODALITY_EMBEDDING = "embedding"
|
||||
|
||||
MODALITIES = frozenset(
|
||||
{
|
||||
MODALITY_TEXT,
|
||||
MODALITY_IMAGE,
|
||||
MODALITY_FILE,
|
||||
MODALITY_PDF,
|
||||
MODALITY_AUDIO,
|
||||
MODALITY_VIDEO,
|
||||
MODALITY_EMBEDDING,
|
||||
}
|
||||
)
|
||||
|
||||
CAP_VISION = "vision"
|
||||
CAP_FILES = "files"
|
||||
CAP_PDF = "pdf"
|
||||
CAP_AUDIO_INPUT = "audio_input"
|
||||
CAP_AUDIO_OUTPUT = "audio_output"
|
||||
CAP_IMAGE_GENERATION = "image_generation"
|
||||
CAP_IMAGE_EDITING = "image_editing"
|
||||
CAP_INPAINTING = "inpainting"
|
||||
CAP_VIDEO_GENERATION = "video_generation"
|
||||
CAP_REASONING = "reasoning"
|
||||
CAP_TOOL_CALL = "tool_call"
|
||||
CAP_STRUCTURED_OUTPUT = "structured_output"
|
||||
CAP_WEB_SEARCH = "web_search"
|
||||
CAP_STREAMING = "streaming"
|
||||
CAP_JSON_MODE = "json_mode"
|
||||
CAP_TRANSCRIPTION = "transcription"
|
||||
CAP_TTS = "tts"
|
||||
CAP_REALTIME = "realtime"
|
||||
CAP_TEXT_RENDERING = "text_rendering"
|
||||
|
||||
CAPABILITIES = frozenset(
|
||||
{
|
||||
CAP_VISION,
|
||||
CAP_FILES,
|
||||
CAP_PDF,
|
||||
CAP_AUDIO_INPUT,
|
||||
CAP_AUDIO_OUTPUT,
|
||||
CAP_IMAGE_GENERATION,
|
||||
CAP_IMAGE_EDITING,
|
||||
CAP_INPAINTING,
|
||||
CAP_VIDEO_GENERATION,
|
||||
CAP_REASONING,
|
||||
CAP_TOOL_CALL,
|
||||
CAP_STRUCTURED_OUTPUT,
|
||||
CAP_WEB_SEARCH,
|
||||
CAP_STREAMING,
|
||||
CAP_JSON_MODE,
|
||||
CAP_TRANSCRIPTION,
|
||||
CAP_TTS,
|
||||
CAP_REALTIME,
|
||||
CAP_TEXT_RENDERING,
|
||||
}
|
||||
)
|
||||
|
||||
SOURCE_ADMIN_OVERRIDE = "admin_override"
|
||||
SOURCE_ENDPOINT_CONFIG = "endpoint_config"
|
||||
SOURCE_PROVIDER_READER = "provider_reader"
|
||||
SOURCE_COOKBOOK_HF = "cookbook_hf"
|
||||
SOURCE_MODELS_DEV_REGISTRY = "models_dev_registry"
|
||||
SOURCE_PROVIDER_DOCS_REGISTRY = "provider_docs_registry"
|
||||
SOURCE_HEURISTIC = "heuristic"
|
||||
SOURCE_CAPABILITY_PROBE = "capability_probe"
|
||||
SOURCE_UNKNOWN = "unknown"
|
||||
|
||||
SOURCES = frozenset(
|
||||
{
|
||||
SOURCE_ADMIN_OVERRIDE,
|
||||
SOURCE_ENDPOINT_CONFIG,
|
||||
SOURCE_PROVIDER_READER,
|
||||
SOURCE_COOKBOOK_HF,
|
||||
SOURCE_MODELS_DEV_REGISTRY,
|
||||
SOURCE_PROVIDER_DOCS_REGISTRY,
|
||||
SOURCE_HEURISTIC,
|
||||
SOURCE_CAPABILITY_PROBE,
|
||||
SOURCE_UNKNOWN,
|
||||
}
|
||||
)
|
||||
|
||||
CONFIDENCE_EXPLICIT = "explicit"
|
||||
CONFIDENCE_PROVIDER_REPORTED = "provider_reported"
|
||||
CONFIDENCE_REGISTRY = "registry"
|
||||
CONFIDENCE_HEURISTIC = "heuristic"
|
||||
CONFIDENCE_UNKNOWN = "unknown"
|
||||
|
||||
CONFIDENCES = frozenset(
|
||||
{
|
||||
CONFIDENCE_EXPLICIT,
|
||||
CONFIDENCE_PROVIDER_REPORTED,
|
||||
CONFIDENCE_REGISTRY,
|
||||
CONFIDENCE_HEURISTIC,
|
||||
CONFIDENCE_UNKNOWN,
|
||||
}
|
||||
)
|
||||
|
||||
ASSERTION_CLAIMED = "claimed"
|
||||
ASSERTION_VERIFIED = "verified"
|
||||
ASSERTION_UNSUPPORTED = "unsupported"
|
||||
ASSERTION_UNKNOWN = "unknown"
|
||||
|
||||
ASSERTION_STATUSES = frozenset(
|
||||
{
|
||||
ASSERTION_CLAIMED,
|
||||
ASSERTION_VERIFIED,
|
||||
ASSERTION_UNSUPPORTED,
|
||||
ASSERTION_UNKNOWN,
|
||||
}
|
||||
)
|
||||
|
||||
PROBE_PASS = "pass"
|
||||
PROBE_FAIL = "fail"
|
||||
PROBE_PARTIAL = "partial"
|
||||
|
||||
PROBE_STATUSES = frozenset(
|
||||
{
|
||||
PROBE_PASS,
|
||||
PROBE_FAIL,
|
||||
PROBE_PARTIAL,
|
||||
}
|
||||
)
|
||||
|
||||
CONTROL_TEMPERATURE = "temperature"
|
||||
CONTROL_TOP_P = "top_p"
|
||||
CONTROL_TOP_K = "top_k"
|
||||
CONTROL_SEED = "seed"
|
||||
CONTROL_MODEL_VERSION_PIN = "model_version_pin"
|
||||
CONTROL_STRICT_SCHEMA = "strict_schema"
|
||||
CONTROL_TOOL_CHOICE = "tool_choice"
|
||||
CONTROL_SYSTEM_PROMPT = "system_prompt"
|
||||
CONTROL_PROMPT_CACHING = "prompt_caching"
|
||||
CONTROL_BATCH = "batch"
|
||||
CONTROL_REQUEST_HASH_CACHE = "request_hash_cache"
|
||||
CONTROL_SYSTEM_FINGERPRINT = "system_fingerprint"
|
||||
|
||||
# Canonical reasoning control mechanisms describe how a serving path accepts
|
||||
# reasoning controls. They are provider/engine evidence, not user preferences.
|
||||
REASONING_CONTROL_MESSAGE_DIRECTIVE = "reasoning_message_directive" # User-message soft switch, e.g. /think or /no_think.
|
||||
REASONING_CONTROL_SYSTEM_DIRECTIVE = "reasoning_system_directive" # System prompt instruction, e.g. "detailed thinking on/off".
|
||||
REASONING_CONTROL_TEMPLATE_KWARG = "reasoning_template_kwarg" # Chat-template kwarg, e.g. chat_template_kwargs.enable_thinking.
|
||||
REASONING_CONTROL_NATIVE_BOOL = "reasoning_native_bool" # Direct API boolean, e.g. think: true/false.
|
||||
REASONING_CONTROL_STRUCTURED_OBJECT = "reasoning_structured_object" # Structured API object, e.g. thinking: {type: "..."}.
|
||||
REASONING_CONTROL_BUDGET = "reasoning_budget" # Token budget control, e.g. thinkingBudget: 0/-1/N.
|
||||
REASONING_CONTROL_EFFORT = "reasoning_effort" # Graded effort control, e.g. low/medium/high.
|
||||
|
||||
# Canonical reasoning control values describe what the provider control accepts.
|
||||
# Odysseus runtime preferences can also use auto/on/off, but that is a separate
|
||||
# layer that later code resolves into these provider-specific controls.
|
||||
REASONING_CONTROL_VALUE_ON = "on" # Provider supports explicitly requesting reasoning on.
|
||||
REASONING_CONTROL_VALUE_OFF = "off" # Provider supports explicitly requesting reasoning off.
|
||||
REASONING_CONTROL_VALUE_AUTO = "auto" # Provider supports adaptive/dynamic/vendor-decided reasoning.
|
||||
|
||||
REASONING_CONTROL_MECHANISMS = frozenset(
|
||||
{
|
||||
REASONING_CONTROL_MESSAGE_DIRECTIVE,
|
||||
REASONING_CONTROL_SYSTEM_DIRECTIVE,
|
||||
REASONING_CONTROL_TEMPLATE_KWARG,
|
||||
REASONING_CONTROL_NATIVE_BOOL,
|
||||
REASONING_CONTROL_STRUCTURED_OBJECT,
|
||||
REASONING_CONTROL_BUDGET,
|
||||
REASONING_CONTROL_EFFORT,
|
||||
}
|
||||
)
|
||||
|
||||
REASONING_CONTROL_VALUES = frozenset(
|
||||
{
|
||||
REASONING_CONTROL_VALUE_ON,
|
||||
REASONING_CONTROL_VALUE_OFF,
|
||||
REASONING_CONTROL_VALUE_AUTO,
|
||||
}
|
||||
)
|
||||
|
||||
DETERMINISTIC_CONTROLS = frozenset(
|
||||
{
|
||||
CONTROL_TEMPERATURE,
|
||||
CONTROL_TOP_P,
|
||||
CONTROL_TOP_K,
|
||||
CONTROL_SEED,
|
||||
CONTROL_MODEL_VERSION_PIN,
|
||||
CONTROL_STRICT_SCHEMA,
|
||||
CONTROL_TOOL_CHOICE,
|
||||
CONTROL_SYSTEM_PROMPT,
|
||||
CONTROL_PROMPT_CACHING,
|
||||
CONTROL_BATCH,
|
||||
CONTROL_REQUEST_HASH_CACHE,
|
||||
CONTROL_SYSTEM_FINGERPRINT,
|
||||
}
|
||||
)
|
||||
|
||||
TASK_CHAT_COMPLETIONS = "chat.completions"
|
||||
TASK_EMBEDDINGS_CREATE = "embeddings.create"
|
||||
TASK_IMAGE_GENERATE = "image.generate"
|
||||
TASK_IMAGE_EDIT = "image.edit"
|
||||
TASK_VIDEO_GENERATE = "video.generate"
|
||||
TASK_AUDIO_TRANSCRIBE = "audio.transcribe"
|
||||
TASK_AUDIO_SYNTHESIZE = "audio.synthesize"
|
||||
TASK_RERANK = "rerank.score"
|
||||
TASK_CLASSIFY = "classification.classify"
|
||||
TASK_MODERATE = "moderation.moderate"
|
||||
TASK_UNKNOWN = "unknown"
|
||||
|
||||
_FAMILY_ALIASES = {
|
||||
"llm": FAMILY_CHAT,
|
||||
"text": FAMILY_CHAT,
|
||||
"text2text": FAMILY_CHAT,
|
||||
"chat_completion": FAMILY_CHAT,
|
||||
"chat_completions": FAMILY_CHAT,
|
||||
"embeddings": FAMILY_EMBEDDING,
|
||||
"embed": FAMILY_EMBEDDING,
|
||||
"image_generation": FAMILY_IMAGE,
|
||||
"image_editing": FAMILY_IMAGE,
|
||||
"video_generation": FAMILY_VIDEO,
|
||||
"speech": FAMILY_AUDIO,
|
||||
"stt": FAMILY_AUDIO,
|
||||
"tts": FAMILY_AUDIO,
|
||||
"safety": FAMILY_MODERATION,
|
||||
}
|
||||
|
||||
_MODALITY_ALIASES = {
|
||||
"images": MODALITY_IMAGE,
|
||||
"img": MODALITY_IMAGE,
|
||||
"document": MODALITY_FILE,
|
||||
"documents": MODALITY_FILE,
|
||||
"files": MODALITY_FILE,
|
||||
"docs": MODALITY_FILE,
|
||||
"voice": MODALITY_AUDIO,
|
||||
"sound": MODALITY_AUDIO,
|
||||
"embeddings": MODALITY_EMBEDDING,
|
||||
}
|
||||
|
||||
_CAPABILITY_ALIASES = {
|
||||
"tools": CAP_TOOL_CALL,
|
||||
"tool_calls": CAP_TOOL_CALL,
|
||||
"function_calling": CAP_TOOL_CALL,
|
||||
"functions": CAP_TOOL_CALL,
|
||||
"image_generate": CAP_IMAGE_GENERATION,
|
||||
"text_to_image": CAP_IMAGE_GENERATION,
|
||||
"text-to-image": CAP_IMAGE_GENERATION,
|
||||
"img2img": CAP_IMAGE_EDITING,
|
||||
"image_edit": CAP_IMAGE_EDITING,
|
||||
"image-editing": CAP_IMAGE_EDITING,
|
||||
"text_rendering": CAP_TEXT_RENDERING,
|
||||
"reasoning_effort": CAP_REASONING,
|
||||
"thinking": CAP_REASONING,
|
||||
"json": CAP_JSON_MODE,
|
||||
"structured_outputs": CAP_STRUCTURED_OUTPUT,
|
||||
"search": CAP_WEB_SEARCH,
|
||||
}
|
||||
|
||||
_DETERMINISTIC_CONTROL_ALIASES = {
|
||||
"temp": CONTROL_TEMPERATURE,
|
||||
"topp": CONTROL_TOP_P,
|
||||
"top-p": CONTROL_TOP_P,
|
||||
"topk": CONTROL_TOP_K,
|
||||
"top-k": CONTROL_TOP_K,
|
||||
"version_pin": CONTROL_MODEL_VERSION_PIN,
|
||||
"model_pin": CONTROL_MODEL_VERSION_PIN,
|
||||
"strict_tool_schema": CONTROL_STRICT_SCHEMA,
|
||||
"json_schema": CONTROL_STRICT_SCHEMA,
|
||||
"tool_choice_required": CONTROL_TOOL_CHOICE,
|
||||
"system": CONTROL_SYSTEM_PROMPT,
|
||||
"system_message": CONTROL_SYSTEM_PROMPT,
|
||||
"cache": CONTROL_REQUEST_HASH_CACHE,
|
||||
"fingerprint": CONTROL_SYSTEM_FINGERPRINT,
|
||||
}
|
||||
|
||||
_REASONING_CONTROL_ALIASES = {
|
||||
"message_directive": REASONING_CONTROL_MESSAGE_DIRECTIVE,
|
||||
"user_message_directive": REASONING_CONTROL_MESSAGE_DIRECTIVE,
|
||||
"think_directive": REASONING_CONTROL_MESSAGE_DIRECTIVE,
|
||||
"slash_think": REASONING_CONTROL_MESSAGE_DIRECTIVE,
|
||||
"system_directive": REASONING_CONTROL_SYSTEM_DIRECTIVE,
|
||||
"system_prompt_directive": REASONING_CONTROL_SYSTEM_DIRECTIVE,
|
||||
"template_kwarg": REASONING_CONTROL_TEMPLATE_KWARG,
|
||||
"chat_template_kwarg": REASONING_CONTROL_TEMPLATE_KWARG,
|
||||
"chat_template_kwargs": REASONING_CONTROL_TEMPLATE_KWARG,
|
||||
"enable_thinking": REASONING_CONTROL_TEMPLATE_KWARG,
|
||||
"native_bool": REASONING_CONTROL_NATIVE_BOOL,
|
||||
"think_bool": REASONING_CONTROL_NATIVE_BOOL,
|
||||
"thinking_bool": REASONING_CONTROL_NATIVE_BOOL,
|
||||
"structured_object": REASONING_CONTROL_STRUCTURED_OBJECT,
|
||||
"reasoning_object": REASONING_CONTROL_STRUCTURED_OBJECT,
|
||||
"thinking_budget": REASONING_CONTROL_BUDGET,
|
||||
"budget": REASONING_CONTROL_BUDGET,
|
||||
"effort": REASONING_CONTROL_EFFORT,
|
||||
}
|
||||
|
||||
_REASONING_CONTROL_VALUE_ALIASES = {
|
||||
"enabled": REASONING_CONTROL_VALUE_ON,
|
||||
"enable": REASONING_CONTROL_VALUE_ON,
|
||||
"true": REASONING_CONTROL_VALUE_ON,
|
||||
"disabled": REASONING_CONTROL_VALUE_OFF,
|
||||
"disable": REASONING_CONTROL_VALUE_OFF,
|
||||
"false": REASONING_CONTROL_VALUE_OFF,
|
||||
"adaptive": REASONING_CONTROL_VALUE_AUTO,
|
||||
"automatic": REASONING_CONTROL_VALUE_AUTO,
|
||||
"dynamic": REASONING_CONTROL_VALUE_AUTO,
|
||||
"provider_auto": REASONING_CONTROL_VALUE_AUTO,
|
||||
"vendor_auto": REASONING_CONTROL_VALUE_AUTO,
|
||||
}
|
||||
|
||||
_DEFAULT_TASK_BY_FAMILY = {
|
||||
FAMILY_CHAT: TASK_CHAT_COMPLETIONS,
|
||||
FAMILY_EMBEDDING: TASK_EMBEDDINGS_CREATE,
|
||||
FAMILY_IMAGE: TASK_IMAGE_GENERATE,
|
||||
FAMILY_VIDEO: TASK_VIDEO_GENERATE,
|
||||
FAMILY_AUDIO: TASK_AUDIO_TRANSCRIBE,
|
||||
FAMILY_RERANK: TASK_RERANK,
|
||||
FAMILY_CLASSIFICATION: TASK_CLASSIFY,
|
||||
FAMILY_MODERATION: TASK_MODERATE,
|
||||
FAMILY_UNKNOWN: TASK_UNKNOWN,
|
||||
}
|
||||
|
||||
_DEFAULT_MODALITIES_BY_FAMILY = {
|
||||
FAMILY_CHAT: ((MODALITY_TEXT,), (MODALITY_TEXT,)),
|
||||
FAMILY_EMBEDDING: ((MODALITY_TEXT,), (MODALITY_EMBEDDING,)),
|
||||
FAMILY_IMAGE: ((MODALITY_TEXT,), (MODALITY_IMAGE,)),
|
||||
FAMILY_VIDEO: ((MODALITY_TEXT,), (MODALITY_VIDEO,)),
|
||||
FAMILY_AUDIO: ((MODALITY_TEXT,), (MODALITY_AUDIO,)),
|
||||
FAMILY_RERANK: ((MODALITY_TEXT,), (MODALITY_TEXT,)),
|
||||
FAMILY_CLASSIFICATION: ((MODALITY_TEXT,), (MODALITY_TEXT,)),
|
||||
FAMILY_MODERATION: ((MODALITY_TEXT,), (MODALITY_TEXT,)),
|
||||
FAMILY_UNKNOWN: ((), ()),
|
||||
}
|
||||
|
||||
_DEFAULT_CAPABILITIES_BY_FAMILY = {
|
||||
FAMILY_IMAGE: (CAP_IMAGE_GENERATION,),
|
||||
FAMILY_VIDEO: (CAP_VIDEO_GENERATION,),
|
||||
}
|
||||
|
||||
|
||||
def _clean_token(value: Any) -> str:
|
||||
return str(value or "").strip().lower().replace("-", "_").replace(" ", "_")
|
||||
|
||||
|
||||
def _normalize_choice(value: Any, allowed: frozenset[str], aliases: Mapping[str, str], default: str) -> str:
|
||||
token = _clean_token(value)
|
||||
token = aliases.get(token, token)
|
||||
return token if token in allowed else default
|
||||
|
||||
|
||||
def normalize_family(value: Any) -> str:
|
||||
return _normalize_choice(value, FAMILIES, _FAMILY_ALIASES, FAMILY_UNKNOWN)
|
||||
|
||||
|
||||
def normalize_source(value: Any) -> str:
|
||||
return _normalize_choice(value, SOURCES, {}, SOURCE_UNKNOWN)
|
||||
|
||||
|
||||
def normalize_confidence(value: Any) -> str:
|
||||
return _normalize_choice(value, CONFIDENCES, {}, CONFIDENCE_UNKNOWN)
|
||||
|
||||
|
||||
def normalize_modality(value: Any) -> str:
|
||||
return _normalize_choice(value, MODALITIES, _MODALITY_ALIASES, "")
|
||||
|
||||
|
||||
def normalize_capability(value: Any) -> str:
|
||||
token = _clean_token(value)
|
||||
token = _CAPABILITY_ALIASES.get(token, token)
|
||||
return token if token in CAPABILITIES else ""
|
||||
|
||||
|
||||
def normalize_assertion_status(value: Any) -> str:
|
||||
return _normalize_choice(value, ASSERTION_STATUSES, {}, ASSERTION_UNKNOWN)
|
||||
|
||||
|
||||
def normalize_probe_status(value: Any) -> str:
|
||||
return _normalize_choice(value, PROBE_STATUSES, {}, "")
|
||||
|
||||
|
||||
def normalize_deterministic_control(value: Any) -> str:
|
||||
token = _clean_token(value)
|
||||
token = _DETERMINISTIC_CONTROL_ALIASES.get(token, token)
|
||||
return token if token in DETERMINISTIC_CONTROLS else ""
|
||||
|
||||
|
||||
def normalize_reasoning_control_mechanism(value: Any) -> str:
|
||||
token = _clean_token(value)
|
||||
token = _REASONING_CONTROL_ALIASES.get(token, token)
|
||||
return token if token in REASONING_CONTROL_MECHANISMS else ""
|
||||
|
||||
|
||||
def normalize_reasoning_control_value(value: Any) -> str:
|
||||
token = _clean_token(value)
|
||||
token = _REASONING_CONTROL_VALUE_ALIASES.get(token, token)
|
||||
return token if token in REASONING_CONTROL_VALUES else ""
|
||||
|
||||
|
||||
def _normalize_tokens(values: Any, normalizer) -> tuple[str, ...]:
|
||||
if values is None:
|
||||
return ()
|
||||
if isinstance(values, Mapping):
|
||||
values = [key for key, enabled in values.items() if enabled]
|
||||
elif isinstance(values, str) or not isinstance(values, Iterable):
|
||||
values = [values]
|
||||
out: list[str] = []
|
||||
for value in values:
|
||||
token = normalizer(value)
|
||||
if token and token not in out:
|
||||
out.append(token)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _normalize_limits(limits: Mapping[str, Any] | None) -> tuple[tuple[str, Any], ...]:
|
||||
if not isinstance(limits, Mapping):
|
||||
return ()
|
||||
return tuple(sorted((str(k), v) for k, v in limits.items() if str(k).strip()))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Modalities:
|
||||
input: tuple[str, ...] = ()
|
||||
output: tuple[str, ...] = ()
|
||||
|
||||
@classmethod
|
||||
def from_values(cls, input: Any = None, output: Any = None) -> "Modalities":
|
||||
return cls(
|
||||
input=_normalize_tokens(input, normalize_modality),
|
||||
output=_normalize_tokens(output, normalize_modality),
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, list[str]]:
|
||||
return {
|
||||
"input": list(self.input),
|
||||
"output": list(self.output),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelCapability:
|
||||
family: str = FAMILY_UNKNOWN
|
||||
primary_task: str = TASK_UNKNOWN
|
||||
modalities: Modalities = field(default_factory=Modalities)
|
||||
capabilities: tuple[str, ...] = ()
|
||||
limits: tuple[tuple[str, Any], ...] = ()
|
||||
source: str = SOURCE_UNKNOWN
|
||||
confidence: str = CONFIDENCE_UNKNOWN
|
||||
|
||||
@classmethod
|
||||
def build(
|
||||
cls,
|
||||
*,
|
||||
family: Any = FAMILY_UNKNOWN,
|
||||
primary_task: str | None = None,
|
||||
input_modalities: Any = None,
|
||||
output_modalities: Any = None,
|
||||
capabilities: Any = None,
|
||||
limits: Mapping[str, Any] | None = None,
|
||||
source: Any = SOURCE_UNKNOWN,
|
||||
confidence: Any = CONFIDENCE_UNKNOWN,
|
||||
) -> "ModelCapability":
|
||||
normalized_family = normalize_family(family)
|
||||
default_input, default_output = _DEFAULT_MODALITIES_BY_FAMILY[normalized_family]
|
||||
return cls(
|
||||
family=normalized_family,
|
||||
primary_task=str(primary_task or _DEFAULT_TASK_BY_FAMILY[normalized_family]).strip() or TASK_UNKNOWN,
|
||||
modalities=Modalities.from_values(
|
||||
input_modalities if input_modalities is not None else default_input,
|
||||
output_modalities if output_modalities is not None else default_output,
|
||||
),
|
||||
capabilities=_normalize_tokens(
|
||||
capabilities if capabilities is not None else _DEFAULT_CAPABILITIES_BY_FAMILY.get(normalized_family, ()),
|
||||
normalize_capability,
|
||||
),
|
||||
limits=_normalize_limits(limits),
|
||||
source=normalize_source(source),
|
||||
confidence=normalize_confidence(confidence),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: Mapping[str, Any]) -> "ModelCapability":
|
||||
if not isinstance(value, Mapping):
|
||||
return unknown_capability()
|
||||
modalities = value.get("modalities")
|
||||
if not isinstance(modalities, Mapping):
|
||||
modalities = {}
|
||||
return cls.build(
|
||||
family=value.get("family"),
|
||||
primary_task=value.get("primary_task"),
|
||||
input_modalities=modalities.get("input"),
|
||||
output_modalities=modalities.get("output"),
|
||||
capabilities=value.get("capabilities"),
|
||||
limits=value.get("limits"),
|
||||
source=value.get("source"),
|
||||
confidence=value.get("confidence"),
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"family": self.family,
|
||||
"primary_task": self.primary_task,
|
||||
"modalities": self.modalities.to_dict(),
|
||||
"capabilities": list(self.capabilities),
|
||||
"limits": dict(self.limits),
|
||||
"source": self.source,
|
||||
"confidence": self.confidence,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CapabilityAssertion:
|
||||
capability: str = ""
|
||||
status: str = ASSERTION_UNKNOWN
|
||||
source: str = SOURCE_UNKNOWN
|
||||
confidence: str = CONFIDENCE_UNKNOWN
|
||||
evidence: tuple[tuple[str, Any], ...] = ()
|
||||
tested_at: str = ""
|
||||
|
||||
@classmethod
|
||||
def build(
|
||||
cls,
|
||||
*,
|
||||
capability: Any,
|
||||
status: Any = ASSERTION_UNKNOWN,
|
||||
source: Any = SOURCE_UNKNOWN,
|
||||
confidence: Any = CONFIDENCE_UNKNOWN,
|
||||
evidence: Mapping[str, Any] | None = None,
|
||||
tested_at: Any = "",
|
||||
) -> "CapabilityAssertion":
|
||||
normalized_capability = normalize_capability(capability)
|
||||
normalized_status = normalize_assertion_status(status)
|
||||
if not normalized_capability:
|
||||
normalized_status = ASSERTION_UNKNOWN
|
||||
return cls(
|
||||
capability=normalized_capability,
|
||||
status=normalized_status,
|
||||
source=normalize_source(source),
|
||||
confidence=normalize_confidence(confidence),
|
||||
evidence=_normalize_limits(evidence),
|
||||
tested_at=str(tested_at or "").strip(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: Mapping[str, Any]) -> "CapabilityAssertion":
|
||||
if not isinstance(value, Mapping):
|
||||
return cls.build(capability="")
|
||||
return cls.build(
|
||||
capability=value.get("capability"),
|
||||
status=value.get("status"),
|
||||
source=value.get("source"),
|
||||
confidence=value.get("confidence"),
|
||||
evidence=value.get("evidence"),
|
||||
tested_at=value.get("tested_at"),
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"capability": self.capability,
|
||||
"status": self.status,
|
||||
"source": self.source,
|
||||
"confidence": self.confidence,
|
||||
"evidence": dict(self.evidence),
|
||||
"tested_at": self.tested_at,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeterministicControl:
|
||||
control: str = ""
|
||||
status: str = ASSERTION_UNKNOWN
|
||||
source: str = SOURCE_UNKNOWN
|
||||
confidence: str = CONFIDENCE_UNKNOWN
|
||||
evidence: tuple[tuple[str, Any], ...] = ()
|
||||
tested_at: str = ""
|
||||
|
||||
@classmethod
|
||||
def build(
|
||||
cls,
|
||||
*,
|
||||
control: Any,
|
||||
status: Any = ASSERTION_UNKNOWN,
|
||||
source: Any = SOURCE_UNKNOWN,
|
||||
confidence: Any = CONFIDENCE_UNKNOWN,
|
||||
evidence: Mapping[str, Any] | None = None,
|
||||
tested_at: Any = "",
|
||||
) -> "DeterministicControl":
|
||||
normalized_control = normalize_deterministic_control(control)
|
||||
normalized_status = normalize_assertion_status(status)
|
||||
if not normalized_control:
|
||||
normalized_status = ASSERTION_UNKNOWN
|
||||
return cls(
|
||||
control=normalized_control,
|
||||
status=normalized_status,
|
||||
source=normalize_source(source),
|
||||
confidence=normalize_confidence(confidence),
|
||||
evidence=_normalize_limits(evidence),
|
||||
tested_at=str(tested_at or "").strip(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: Mapping[str, Any]) -> "DeterministicControl":
|
||||
if not isinstance(value, Mapping):
|
||||
return cls.build(control="")
|
||||
return cls.build(
|
||||
control=value.get("control"),
|
||||
status=value.get("status"),
|
||||
source=value.get("source"),
|
||||
confidence=value.get("confidence"),
|
||||
evidence=value.get("evidence"),
|
||||
tested_at=value.get("tested_at"),
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"control": self.control,
|
||||
"status": self.status,
|
||||
"source": self.source,
|
||||
"confidence": self.confidence,
|
||||
"evidence": dict(self.evidence),
|
||||
"tested_at": self.tested_at,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CapabilityProbeResult:
|
||||
provider: str
|
||||
model_id: str
|
||||
capability: str
|
||||
status: str
|
||||
tested_at: str = ""
|
||||
endpoint_id: str = ""
|
||||
stable_model_id: str = ""
|
||||
request_hash: str = ""
|
||||
response_id: str = ""
|
||||
response_fingerprint: str = ""
|
||||
evidence: tuple[tuple[str, Any], ...] = ()
|
||||
|
||||
@classmethod
|
||||
def build(
|
||||
cls,
|
||||
*,
|
||||
provider: Any,
|
||||
model_id: Any,
|
||||
capability: Any,
|
||||
status: Any,
|
||||
tested_at: Any = "",
|
||||
endpoint_id: Any = "",
|
||||
stable_model_id: Any = "",
|
||||
request_hash: Any = "",
|
||||
response_id: Any = "",
|
||||
response_fingerprint: Any = "",
|
||||
evidence: Mapping[str, Any] | None = None,
|
||||
) -> "CapabilityProbeResult":
|
||||
normalized_capability = normalize_capability(capability)
|
||||
normalized_status = normalize_probe_status(status)
|
||||
if not normalized_capability or not normalized_status:
|
||||
normalized_status = PROBE_FAIL
|
||||
return cls(
|
||||
provider=str(provider or "").strip(),
|
||||
model_id=str(model_id or "").strip(),
|
||||
capability=normalized_capability,
|
||||
status=normalized_status,
|
||||
tested_at=str(tested_at or "").strip(),
|
||||
endpoint_id=str(endpoint_id or "").strip(),
|
||||
stable_model_id=str(stable_model_id or "").strip(),
|
||||
request_hash=str(request_hash or "").strip(),
|
||||
response_id=str(response_id or "").strip(),
|
||||
response_fingerprint=str(response_fingerprint or "").strip(),
|
||||
evidence=_normalize_limits(evidence),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: Mapping[str, Any]) -> "CapabilityProbeResult":
|
||||
if not isinstance(value, Mapping):
|
||||
return cls.build(provider="", model_id="", capability="", status=PROBE_FAIL)
|
||||
return cls.build(
|
||||
provider=value.get("provider"),
|
||||
endpoint_id=value.get("endpoint_id"),
|
||||
model_id=value.get("model_id"),
|
||||
stable_model_id=value.get("stable_model_id"),
|
||||
capability=value.get("capability"),
|
||||
status=value.get("status"),
|
||||
tested_at=value.get("tested_at"),
|
||||
request_hash=value.get("request_hash"),
|
||||
response_id=value.get("response_id"),
|
||||
response_fingerprint=value.get("response_fingerprint"),
|
||||
evidence=value.get("evidence"),
|
||||
)
|
||||
|
||||
def to_assertion(self) -> CapabilityAssertion:
|
||||
status_map = {
|
||||
PROBE_PASS: ASSERTION_VERIFIED,
|
||||
PROBE_FAIL: ASSERTION_UNSUPPORTED,
|
||||
PROBE_PARTIAL: ASSERTION_CLAIMED,
|
||||
}
|
||||
return CapabilityAssertion.build(
|
||||
capability=self.capability,
|
||||
status=status_map.get(self.status, ASSERTION_UNKNOWN),
|
||||
source=SOURCE_CAPABILITY_PROBE,
|
||||
confidence=CONFIDENCE_EXPLICIT if self.status == PROBE_PASS else CONFIDENCE_HEURISTIC,
|
||||
evidence={
|
||||
"provider": self.provider,
|
||||
"endpoint_id": self.endpoint_id,
|
||||
"model_id": self.model_id,
|
||||
"stable_model_id": self.stable_model_id,
|
||||
"request_hash": self.request_hash,
|
||||
"response_id": self.response_id,
|
||||
"response_fingerprint": self.response_fingerprint,
|
||||
**dict(self.evidence),
|
||||
},
|
||||
tested_at=self.tested_at,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"provider": self.provider,
|
||||
"endpoint_id": self.endpoint_id,
|
||||
"model_id": self.model_id,
|
||||
"stable_model_id": self.stable_model_id,
|
||||
"capability": self.capability,
|
||||
"status": self.status,
|
||||
"tested_at": self.tested_at,
|
||||
"request_hash": self.request_hash,
|
||||
"response_id": self.response_id,
|
||||
"response_fingerprint": self.response_fingerprint,
|
||||
"evidence": dict(self.evidence),
|
||||
}
|
||||
|
||||
|
||||
def capability_assertions_from_capability(
|
||||
capability: ModelCapability,
|
||||
*,
|
||||
status: str = ASSERTION_CLAIMED,
|
||||
source: str | None = None,
|
||||
confidence: str | None = None,
|
||||
) -> tuple[CapabilityAssertion, ...]:
|
||||
return tuple(
|
||||
CapabilityAssertion.build(
|
||||
capability=cap,
|
||||
status=status,
|
||||
source=source or capability.source,
|
||||
confidence=confidence or capability.confidence,
|
||||
)
|
||||
for cap in capability.capabilities
|
||||
)
|
||||
|
||||
|
||||
def deterministic_controls_from_values(
|
||||
values: Any,
|
||||
*,
|
||||
status: str = ASSERTION_CLAIMED,
|
||||
source: str = SOURCE_PROVIDER_READER,
|
||||
confidence: str = CONFIDENCE_PROVIDER_REPORTED,
|
||||
) -> tuple[DeterministicControl, ...]:
|
||||
return tuple(
|
||||
DeterministicControl.build(
|
||||
control=control,
|
||||
status=status,
|
||||
source=source,
|
||||
confidence=confidence,
|
||||
)
|
||||
for control in _normalize_tokens(values, normalize_deterministic_control)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CapabilityQuery:
|
||||
surface: str
|
||||
families: tuple[str, ...] = ()
|
||||
primary_tasks: tuple[str, ...] = ()
|
||||
input_all: tuple[str, ...] = ()
|
||||
input_any: tuple[str, ...] = ()
|
||||
output_all: tuple[str, ...] = ()
|
||||
output_any: tuple[str, ...] = ()
|
||||
modality_any: tuple[str, ...] = ()
|
||||
capabilities_all: tuple[str, ...] = ()
|
||||
capabilities_any: tuple[str, ...] = ()
|
||||
|
||||
def matches(self, capability: ModelCapability) -> bool:
|
||||
input_set = set(capability.modalities.input)
|
||||
output_set = set(capability.modalities.output)
|
||||
modality_set = input_set | output_set
|
||||
cap_set = set(capability.capabilities)
|
||||
if self.families and capability.family not in self.families:
|
||||
return False
|
||||
if self.primary_tasks and capability.primary_task not in self.primary_tasks:
|
||||
return False
|
||||
if self.input_all and not set(self.input_all).issubset(input_set):
|
||||
return False
|
||||
if self.input_any and input_set.isdisjoint(self.input_any):
|
||||
return False
|
||||
if self.output_all and not set(self.output_all).issubset(output_set):
|
||||
return False
|
||||
if self.output_any and output_set.isdisjoint(self.output_any):
|
||||
return False
|
||||
if self.modality_any and modality_set.isdisjoint(self.modality_any):
|
||||
return False
|
||||
if self.capabilities_all and not set(self.capabilities_all).issubset(cap_set):
|
||||
return False
|
||||
if self.capabilities_any and cap_set.isdisjoint(self.capabilities_any):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
DISPLAY_QUERIES = (
|
||||
CapabilityQuery(
|
||||
surface="chat",
|
||||
families=(FAMILY_CHAT,),
|
||||
input_all=(MODALITY_TEXT,),
|
||||
output_all=(MODALITY_TEXT,),
|
||||
),
|
||||
CapabilityQuery(
|
||||
surface="vision_chat",
|
||||
families=(FAMILY_CHAT,),
|
||||
input_all=(MODALITY_TEXT, MODALITY_IMAGE),
|
||||
output_all=(MODALITY_TEXT,),
|
||||
),
|
||||
CapabilityQuery(
|
||||
surface="document_chat",
|
||||
families=(FAMILY_CHAT,),
|
||||
input_all=(MODALITY_TEXT,),
|
||||
input_any=(MODALITY_FILE, MODALITY_PDF),
|
||||
output_all=(MODALITY_TEXT,),
|
||||
),
|
||||
CapabilityQuery(
|
||||
surface="image_generation",
|
||||
families=(FAMILY_IMAGE,),
|
||||
output_all=(MODALITY_IMAGE,),
|
||||
capabilities_all=(CAP_IMAGE_GENERATION,),
|
||||
),
|
||||
CapabilityQuery(
|
||||
surface="image_editing",
|
||||
families=(FAMILY_IMAGE,),
|
||||
input_all=(MODALITY_IMAGE,),
|
||||
output_all=(MODALITY_IMAGE,),
|
||||
capabilities_any=(CAP_IMAGE_EDITING, CAP_INPAINTING),
|
||||
),
|
||||
CapabilityQuery(
|
||||
surface="video_generation",
|
||||
families=(FAMILY_VIDEO,),
|
||||
output_all=(MODALITY_VIDEO,),
|
||||
capabilities_all=(CAP_VIDEO_GENERATION,),
|
||||
),
|
||||
CapabilityQuery(
|
||||
surface="audio_realtime",
|
||||
families=(FAMILY_AUDIO,),
|
||||
modality_any=(MODALITY_AUDIO,),
|
||||
capabilities_any=(CAP_AUDIO_INPUT, CAP_AUDIO_OUTPUT, CAP_TRANSCRIPTION, CAP_TTS, CAP_REALTIME),
|
||||
),
|
||||
CapabilityQuery(
|
||||
surface="embeddings",
|
||||
families=(FAMILY_EMBEDDING,),
|
||||
output_all=(MODALITY_EMBEDDING,),
|
||||
),
|
||||
CapabilityQuery(
|
||||
surface="rerank_scoring",
|
||||
families=(FAMILY_RERANK,),
|
||||
),
|
||||
CapabilityQuery(
|
||||
surface="moderation_classification",
|
||||
families=(FAMILY_MODERATION, FAMILY_CLASSIFICATION),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def display_surfaces_for(capability: ModelCapability) -> tuple[str, ...]:
|
||||
return tuple(query.surface for query in DISPLAY_QUERIES if query.matches(capability))
|
||||
|
||||
|
||||
def unknown_capability(
|
||||
*,
|
||||
source: str = SOURCE_UNKNOWN,
|
||||
confidence: str = CONFIDENCE_UNKNOWN,
|
||||
) -> ModelCapability:
|
||||
return ModelCapability.build(source=source, confidence=confidence)
|
||||
|
||||
|
||||
def capability_from_endpoint_type(model_type: Any) -> ModelCapability:
|
||||
"""Return capability metadata from an explicit endpoint model type.
|
||||
|
||||
Missing or unknown endpoint types remain unknown here. Runtime compatibility
|
||||
may still treat legacy rows as chat-capable, but this schema layer should
|
||||
not turn absence of evidence into model capability truth.
|
||||
"""
|
||||
token = _clean_token(model_type)
|
||||
if token == "llm":
|
||||
return ModelCapability.build(
|
||||
family=FAMILY_CHAT,
|
||||
source=SOURCE_ENDPOINT_CONFIG,
|
||||
confidence=CONFIDENCE_EXPLICIT,
|
||||
)
|
||||
if token == "image":
|
||||
return ModelCapability.build(
|
||||
family=FAMILY_IMAGE,
|
||||
source=SOURCE_ENDPOINT_CONFIG,
|
||||
confidence=CONFIDENCE_EXPLICIT,
|
||||
)
|
||||
return unknown_capability(source=SOURCE_ENDPOINT_CONFIG)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Vendor-specific model capability reader registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from src.model_capability_readers import generic_openai, google, llamacpp, lmstudio, ollama, openai, openrouter
|
||||
from src.model_capability_readers.base import (
|
||||
ModelCapabilityRecord,
|
||||
VENDOR_ANTHROPIC,
|
||||
VENDOR_GENERIC_OPENAI,
|
||||
VENDOR_GOOGLE,
|
||||
VENDOR_HUGGINGFACE,
|
||||
VENDOR_LLAMACPP,
|
||||
VENDOR_LMSTUDIO,
|
||||
VENDOR_OLLAMA,
|
||||
VENDOR_OPENAI,
|
||||
VENDOR_OPENROUTER,
|
||||
VENDOR_SGLANG,
|
||||
VENDOR_UNKNOWN,
|
||||
VENDOR_VLLM,
|
||||
detect_vendor,
|
||||
stable_model_id_for,
|
||||
)
|
||||
|
||||
|
||||
READER_MODULES = {
|
||||
VENDOR_GENERIC_OPENAI: generic_openai,
|
||||
VENDOR_OPENAI: openai,
|
||||
VENDOR_OPENROUTER: openrouter,
|
||||
VENDOR_GOOGLE: google,
|
||||
VENDOR_LLAMACPP: llamacpp,
|
||||
VENDOR_OLLAMA: ollama,
|
||||
VENDOR_LMSTUDIO: lmstudio,
|
||||
}
|
||||
|
||||
|
||||
PLACEHOLDER_VENDOR_IDS = frozenset(
|
||||
{
|
||||
VENDOR_ANTHROPIC,
|
||||
VENDOR_HUGGINGFACE,
|
||||
VENDOR_SGLANG,
|
||||
VENDOR_VLLM,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def reader_for_vendor(vendor: Any):
|
||||
vendor_id = str(vendor or "").strip().lower().replace("-", "_")
|
||||
return READER_MODULES.get(vendor_id, generic_openai)
|
||||
|
||||
|
||||
def records_from_payload(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
vendor: str | None = None,
|
||||
base_url: str = "",
|
||||
endpoint_kind: str = "",
|
||||
endpoint_id: str = "",
|
||||
) -> tuple[ModelCapabilityRecord, ...]:
|
||||
vendor_id = vendor or detect_vendor(base_url, endpoint_kind)
|
||||
reader = reader_for_vendor(vendor_id)
|
||||
if reader is generic_openai:
|
||||
record_vendor = vendor_id if vendor_id not in {VENDOR_UNKNOWN, ""} else VENDOR_GENERIC_OPENAI
|
||||
return reader.records_from_payload(
|
||||
payload,
|
||||
vendor_id=record_vendor,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
)
|
||||
return reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ModelCapabilityRecord",
|
||||
"PLACEHOLDER_VENDOR_IDS",
|
||||
"READER_MODULES",
|
||||
"VENDOR_ANTHROPIC",
|
||||
"VENDOR_GENERIC_OPENAI",
|
||||
"VENDOR_GOOGLE",
|
||||
"VENDOR_HUGGINGFACE",
|
||||
"VENDOR_LLAMACPP",
|
||||
"VENDOR_LMSTUDIO",
|
||||
"VENDOR_OLLAMA",
|
||||
"VENDOR_OPENAI",
|
||||
"VENDOR_OPENROUTER",
|
||||
"VENDOR_SGLANG",
|
||||
"VENDOR_UNKNOWN",
|
||||
"VENDOR_VLLM",
|
||||
"detect_vendor",
|
||||
"reader_for_vendor",
|
||||
"records_from_payload",
|
||||
"stable_model_id_for",
|
||||
]
|
||||
@@ -0,0 +1,311 @@
|
||||
"""Shared helpers for vendor-specific model capability readers.
|
||||
|
||||
Readers in this package normalize already-fetched provider payload shapes and
|
||||
explicit provider fields. They do not perform network I/O and must not infer
|
||||
authoritative capability from model IDs, names, display names, or ownership
|
||||
labels.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Protocol
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from src import model_capabilities as mc
|
||||
|
||||
|
||||
VENDOR_GENERIC_OPENAI = "generic_openai"
|
||||
VENDOR_OPENAI = "openai"
|
||||
VENDOR_OPENROUTER = "openrouter"
|
||||
VENDOR_GOOGLE = "google"
|
||||
VENDOR_ANTHROPIC = "anthropic"
|
||||
VENDOR_OLLAMA = "ollama"
|
||||
VENDOR_LMSTUDIO = "lmstudio"
|
||||
VENDOR_LLAMACPP = "llamacpp"
|
||||
VENDOR_VLLM = "vllm"
|
||||
VENDOR_SGLANG = "sglang"
|
||||
VENDOR_HUGGINGFACE = "huggingface"
|
||||
VENDOR_UNKNOWN = "unknown"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelCapabilityRecord:
|
||||
vendor: str
|
||||
model_id: str
|
||||
capability: mc.ModelCapability
|
||||
display_name: str = ""
|
||||
stable_model_id: str = ""
|
||||
capability_assertions: tuple[mc.CapabilityAssertion, ...] = ()
|
||||
deterministic_controls: tuple[mc.DeterministicControl, ...] = ()
|
||||
raw: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.stable_model_id:
|
||||
object.__setattr__(self, "stable_model_id", stable_model_id_for(self.vendor, self.model_id))
|
||||
if not self.capability_assertions and self.capability.capabilities:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"capability_assertions",
|
||||
mc.capability_assertions_from_capability(
|
||||
self.capability,
|
||||
status=mc.ASSERTION_CLAIMED,
|
||||
source=self.capability.source,
|
||||
confidence=self.capability.confidence,
|
||||
),
|
||||
)
|
||||
|
||||
def to_dict(self, *, include_raw: bool = False) -> dict[str, Any]:
|
||||
data = {
|
||||
"vendor": self.vendor,
|
||||
"model_id": self.model_id,
|
||||
"stable_model_id": self.stable_model_id,
|
||||
"display_name": self.display_name,
|
||||
"capability": self.capability.to_dict(),
|
||||
"capability_assertions": [assertion.to_dict() for assertion in self.capability_assertions],
|
||||
"deterministic_controls": [control.to_dict() for control in self.deterministic_controls],
|
||||
}
|
||||
if include_raw:
|
||||
data["raw"] = dict(self.raw)
|
||||
return data
|
||||
|
||||
|
||||
class CapabilityReader(Protocol):
|
||||
vendor: str
|
||||
|
||||
def records_from_payload(
|
||||
self,
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> tuple[ModelCapabilityRecord, ...]:
|
||||
"""Normalize a provider model-list payload into capability records."""
|
||||
|
||||
|
||||
def as_mapping(value: Any) -> Mapping[str, Any]:
|
||||
return value if isinstance(value, Mapping) else {}
|
||||
|
||||
|
||||
def as_list(value: Any) -> list[Any]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
if isinstance(value, tuple):
|
||||
return list(value)
|
||||
return [value]
|
||||
|
||||
|
||||
def compact_str(value: Any) -> str:
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
def _identity_part(value: Any) -> str:
|
||||
text = compact_str(value).lower()
|
||||
out = []
|
||||
for char in text:
|
||||
out.append(char if char.isalnum() or char in {"-", "_", ".", "/", ":"} else "_")
|
||||
return "".join(out).strip("_") or "unknown"
|
||||
|
||||
|
||||
def _base_url_scope(base_url: Any) -> str:
|
||||
parsed = urlparse(compact_str(base_url))
|
||||
if not parsed.hostname:
|
||||
return ""
|
||||
port = f":{parsed.port}" if parsed.port else ""
|
||||
path = parsed.path.rstrip("/")
|
||||
normalized = f"{parsed.scheme or 'http'}://{parsed.hostname.lower()}{port}{path}"
|
||||
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:12]
|
||||
return f"url:{digest}"
|
||||
|
||||
|
||||
def stable_model_id_for(vendor: Any, model_id: Any, *, endpoint_id: Any = "", base_url: Any = "") -> str:
|
||||
vendor_part = _identity_part(vendor or VENDOR_UNKNOWN)
|
||||
model_part = _identity_part(model_id)
|
||||
endpoint = compact_str(endpoint_id)
|
||||
if endpoint:
|
||||
scope = f"endpoint:{_identity_part(endpoint)}"
|
||||
else:
|
||||
scope = _base_url_scope(base_url) or "global"
|
||||
return f"{vendor_part}|{scope}|{model_part}"
|
||||
|
||||
|
||||
def model_id_from(raw: Mapping[str, Any], *keys: str) -> str:
|
||||
for key in keys:
|
||||
value = compact_str(raw.get(key))
|
||||
if value:
|
||||
return value.removeprefix("models/")
|
||||
return ""
|
||||
|
||||
|
||||
def int_limit(value: Any) -> int | None:
|
||||
try:
|
||||
limit = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return limit if limit > 0 else None
|
||||
|
||||
|
||||
def merge_unique(*groups: Iterable[str]) -> tuple[str, ...]:
|
||||
out: list[str] = []
|
||||
for group in groups:
|
||||
for value in group:
|
||||
token = compact_str(value)
|
||||
if token and token not in out:
|
||||
out.append(token)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def deterministic_controls_from_supported_parameters(values: Any) -> tuple[mc.DeterministicControl, ...]:
|
||||
return mc.deterministic_controls_from_values(
|
||||
values,
|
||||
status=mc.ASSERTION_CLAIMED,
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
confidence=mc.CONFIDENCE_PROVIDER_REPORTED,
|
||||
)
|
||||
|
||||
|
||||
def openai_model_items(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]:
|
||||
payload = as_mapping(payload)
|
||||
data = payload.get("data")
|
||||
if data is None:
|
||||
data = payload.get("models")
|
||||
return tuple(item for item in as_list(data) if isinstance(item, Mapping))
|
||||
|
||||
|
||||
def normalize_modality_token(value: Any) -> str:
|
||||
token = compact_str(value).lower().replace("-", "_").replace(" ", "_")
|
||||
aliases = {
|
||||
"txt": mc.MODALITY_TEXT,
|
||||
"textual": mc.MODALITY_TEXT,
|
||||
"image_url": mc.MODALITY_IMAGE,
|
||||
"images": mc.MODALITY_IMAGE,
|
||||
"img": mc.MODALITY_IMAGE,
|
||||
"audio_url": mc.MODALITY_AUDIO,
|
||||
"speech": mc.MODALITY_AUDIO,
|
||||
"documents": mc.MODALITY_FILE,
|
||||
"document": mc.MODALITY_FILE,
|
||||
"files": mc.MODALITY_FILE,
|
||||
"file_search": mc.MODALITY_FILE,
|
||||
"pdfs": mc.MODALITY_PDF,
|
||||
"embeddings": mc.MODALITY_EMBEDDING,
|
||||
}
|
||||
token = aliases.get(token, token)
|
||||
return mc.normalize_modality(token)
|
||||
|
||||
|
||||
def modalities_from_value(value: Any) -> tuple[str, ...]:
|
||||
if isinstance(value, str):
|
||||
parts = value.replace(",", "+").replace("/", "+").split("+")
|
||||
else:
|
||||
parts = as_list(value)
|
||||
out: list[str] = []
|
||||
for part in parts:
|
||||
token = normalize_modality_token(part)
|
||||
if token and token not in out:
|
||||
out.append(token)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def split_modality_arrow(value: Any) -> tuple[tuple[str, ...], tuple[str, ...]]:
|
||||
text = compact_str(value).lower()
|
||||
if not text:
|
||||
return (), ()
|
||||
for arrow in ("->", "=>", "to"):
|
||||
if arrow in text:
|
||||
left, right = text.split(arrow, 1)
|
||||
return modalities_from_value(left), modalities_from_value(right)
|
||||
return modalities_from_value(text), ()
|
||||
|
||||
|
||||
def family_from_modalities(input_modalities: Iterable[str], output_modalities: Iterable[str]) -> str:
|
||||
output_set = set(output_modalities)
|
||||
if mc.MODALITY_EMBEDDING in output_set:
|
||||
return mc.FAMILY_EMBEDDING
|
||||
if mc.MODALITY_IMAGE in output_set:
|
||||
return mc.FAMILY_IMAGE
|
||||
if mc.MODALITY_VIDEO in output_set:
|
||||
return mc.FAMILY_VIDEO
|
||||
if mc.MODALITY_AUDIO in output_set:
|
||||
return mc.FAMILY_AUDIO
|
||||
if mc.MODALITY_TEXT in output_set:
|
||||
return mc.FAMILY_CHAT
|
||||
return mc.FAMILY_UNKNOWN
|
||||
|
||||
|
||||
def primary_task_for_family(family: str, capabilities: Iterable[str] = ()) -> str | None:
|
||||
caps = set(capabilities)
|
||||
if family == mc.FAMILY_IMAGE and (mc.CAP_IMAGE_EDITING in caps or mc.CAP_INPAINTING in caps):
|
||||
return mc.TASK_IMAGE_EDIT
|
||||
if family == mc.FAMILY_AUDIO and mc.CAP_TTS in caps:
|
||||
return mc.TASK_AUDIO_SYNTHESIZE
|
||||
if family == mc.FAMILY_AUDIO and mc.CAP_TRANSCRIPTION in caps:
|
||||
return mc.TASK_AUDIO_TRANSCRIBE
|
||||
return None
|
||||
|
||||
|
||||
def build_capability(
|
||||
*,
|
||||
family: str,
|
||||
input_modalities: Iterable[str] = (),
|
||||
output_modalities: Iterable[str] = (),
|
||||
capabilities: Iterable[str] = (),
|
||||
limits: Mapping[str, Any] | None = None,
|
||||
confidence: str = mc.CONFIDENCE_PROVIDER_REPORTED,
|
||||
) -> mc.ModelCapability:
|
||||
return mc.ModelCapability.build(
|
||||
family=family,
|
||||
primary_task=primary_task_for_family(family, capabilities),
|
||||
input_modalities=tuple(input_modalities),
|
||||
output_modalities=tuple(output_modalities),
|
||||
capabilities=tuple(capabilities),
|
||||
limits=limits,
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
confidence=confidence,
|
||||
)
|
||||
|
||||
|
||||
def detect_vendor(base_url: Any = "", endpoint_kind: Any = "") -> str:
|
||||
kind = compact_str(endpoint_kind).lower().replace("-", "_")
|
||||
kind_map = {
|
||||
"openai": VENDOR_OPENAI,
|
||||
"openrouter": VENDOR_OPENROUTER,
|
||||
"google": VENDOR_GOOGLE,
|
||||
"gemini": VENDOR_GOOGLE,
|
||||
"anthropic": VENDOR_ANTHROPIC,
|
||||
"ollama": VENDOR_OLLAMA,
|
||||
"lmstudio": VENDOR_LMSTUDIO,
|
||||
"lm_studio": VENDOR_LMSTUDIO,
|
||||
"llamacpp": VENDOR_LLAMACPP,
|
||||
"llama_cpp": VENDOR_LLAMACPP,
|
||||
"vllm": VENDOR_VLLM,
|
||||
"sglang": VENDOR_SGLANG,
|
||||
"huggingface": VENDOR_HUGGINGFACE,
|
||||
"hf": VENDOR_HUGGINGFACE,
|
||||
}
|
||||
if kind in kind_map:
|
||||
return kind_map[kind]
|
||||
|
||||
parsed = urlparse(compact_str(base_url))
|
||||
host = (parsed.hostname or "").lower()
|
||||
port = parsed.port
|
||||
if host.endswith("openrouter.ai"):
|
||||
return VENDOR_OPENROUTER
|
||||
if host.endswith("openai.com"):
|
||||
return VENDOR_OPENAI
|
||||
if host.endswith("anthropic.com"):
|
||||
return VENDOR_ANTHROPIC
|
||||
if host.endswith("googleapis.com"):
|
||||
return VENDOR_GOOGLE
|
||||
if host.endswith("ollama.com") or port == 11434:
|
||||
return VENDOR_OLLAMA
|
||||
if port == 1234:
|
||||
return VENDOR_LMSTUDIO
|
||||
if port == 8000:
|
||||
return VENDOR_VLLM
|
||||
if port == 30000:
|
||||
return VENDOR_SGLANG
|
||||
return VENDOR_GENERIC_OPENAI if host else VENDOR_UNKNOWN
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Reader for bare OpenAI-compatible model-list payloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from src import model_capabilities as mc
|
||||
from src.model_capability_readers.base import (
|
||||
ModelCapabilityRecord,
|
||||
VENDOR_GENERIC_OPENAI,
|
||||
compact_str,
|
||||
model_id_from,
|
||||
openai_model_items,
|
||||
stable_model_id_for,
|
||||
)
|
||||
|
||||
|
||||
vendor = VENDOR_GENERIC_OPENAI
|
||||
|
||||
|
||||
def record_from_model(
|
||||
raw: Mapping[str, Any],
|
||||
*,
|
||||
vendor_id: str = VENDOR_GENERIC_OPENAI,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> ModelCapabilityRecord | None:
|
||||
model_id = model_id_from(raw, "id", "name", "model")
|
||||
if not model_id:
|
||||
return None
|
||||
capability = mc.unknown_capability(
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
confidence=mc.CONFIDENCE_UNKNOWN,
|
||||
)
|
||||
return ModelCapabilityRecord(
|
||||
vendor=vendor_id,
|
||||
model_id=model_id,
|
||||
stable_model_id=stable_model_id_for(vendor_id, model_id, endpoint_id=endpoint_id, base_url=base_url),
|
||||
display_name=compact_str(raw.get("display_name") or raw.get("name")),
|
||||
capability=capability,
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
|
||||
def records_from_payload(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
vendor_id: str = VENDOR_GENERIC_OPENAI,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> tuple[ModelCapabilityRecord, ...]:
|
||||
records: list[ModelCapabilityRecord] = []
|
||||
for item in openai_model_items(payload):
|
||||
record = record_from_model(item, vendor_id=vendor_id, endpoint_id=endpoint_id, base_url=base_url)
|
||||
if record:
|
||||
records.append(record)
|
||||
return tuple(records)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Google Gemini model metadata reader."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from src.model_capability_readers import google_ai_studio_mapping as ai_studio
|
||||
from src.model_capability_readers.base import (
|
||||
ModelCapabilityRecord,
|
||||
VENDOR_GOOGLE,
|
||||
as_list,
|
||||
compact_str,
|
||||
stable_model_id_for,
|
||||
)
|
||||
|
||||
|
||||
vendor = VENDOR_GOOGLE
|
||||
|
||||
|
||||
def _model_items(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]:
|
||||
models = payload.get("models") if isinstance(payload, Mapping) else None
|
||||
if models is None and isinstance(payload, Mapping) and payload.get("name"):
|
||||
models = [payload]
|
||||
return tuple(item for item in as_list(models) if isinstance(item, Mapping))
|
||||
|
||||
|
||||
def record_from_model(
|
||||
raw: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> ModelCapabilityRecord | None:
|
||||
model_id = ai_studio.google_model_id(raw)
|
||||
if not model_id:
|
||||
return None
|
||||
|
||||
return ModelCapabilityRecord(
|
||||
vendor=VENDOR_GOOGLE,
|
||||
model_id=model_id,
|
||||
stable_model_id=stable_model_id_for(VENDOR_GOOGLE, model_id, endpoint_id=endpoint_id, base_url=base_url),
|
||||
display_name=compact_str(raw.get("displayName")) or model_id,
|
||||
capability=ai_studio.capability_from_model(raw),
|
||||
deterministic_controls=ai_studio.deterministic_controls_from_model(raw),
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
|
||||
def records_from_payload(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> tuple[ModelCapabilityRecord, ...]:
|
||||
records: list[ModelCapabilityRecord] = []
|
||||
for item in _model_items(payload):
|
||||
record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)
|
||||
if record:
|
||||
records.append(record)
|
||||
return tuple(records)
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Google AI Studio / Gemini native Models API capability mapping.
|
||||
|
||||
This module maps already-fetched `models.list` and `models.get` payloads into
|
||||
Odysseus' canonical model capability shape. It performs no network I/O and
|
||||
does not infer model capabilities from model IDs, display names, or product
|
||||
families. Only fields explicitly returned by Google's Model resource are
|
||||
mapped here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from src import model_capabilities as mc
|
||||
from src.model_capability_readers.base import as_list, compact_str, int_limit
|
||||
|
||||
|
||||
METHOD_GENERATE_CONTENT = "generateContent"
|
||||
METHOD_GENERATE_MESSAGE = "generateMessage"
|
||||
METHOD_GENERATE_TEXT = "generateText"
|
||||
METHOD_GENERATE_ANSWER = "generateAnswer"
|
||||
METHOD_EMBED_CONTENT = "embedContent"
|
||||
METHOD_ASYNC_BATCH_EMBED = "asyncBatchEmbedContent"
|
||||
METHOD_PREDICT = "predict"
|
||||
METHOD_PREDICT_LONG_RUNNING = "predictLongRunning"
|
||||
METHOD_BATCH_GENERATE = "batchGenerateContent"
|
||||
METHOD_CREATE_CACHED_CONTENT = "createCachedContent"
|
||||
|
||||
TEXT_GENERATION_METHODS = frozenset(
|
||||
{
|
||||
METHOD_GENERATE_CONTENT,
|
||||
METHOD_GENERATE_MESSAGE,
|
||||
METHOD_GENERATE_TEXT,
|
||||
METHOD_GENERATE_ANSWER,
|
||||
}
|
||||
)
|
||||
EMBEDDING_METHODS = frozenset({METHOD_EMBED_CONTENT, METHOD_ASYNC_BATCH_EMBED})
|
||||
BATCH_METHODS = frozenset({METHOD_BATCH_GENERATE, METHOD_ASYNC_BATCH_EMBED})
|
||||
|
||||
MODEL_FIELD_MAP = {
|
||||
"name": "vendor resource name",
|
||||
"baseModelId": "vendor model id",
|
||||
"displayName": "display name",
|
||||
"description": "display description only",
|
||||
"inputTokenLimit": "limits.input_tokens and limits.context_tokens",
|
||||
"outputTokenLimit": "limits.output_tokens",
|
||||
"supportedGenerationMethods": "provider method support signal",
|
||||
"thinking": "capabilities.reasoning when true",
|
||||
"temperature": "deterministic_controls.temperature when present",
|
||||
"maxTemperature": "deterministic_controls.temperature when present",
|
||||
"topP": "deterministic_controls.top_p when present",
|
||||
"topK": "deterministic_controls.top_k when present",
|
||||
}
|
||||
|
||||
|
||||
def google_model_id(raw: Mapping[str, Any]) -> str:
|
||||
value = compact_str(raw.get("baseModelId")) or compact_str(raw.get("name"))
|
||||
return value.removeprefix("models/")
|
||||
|
||||
|
||||
def supported_methods(raw: Mapping[str, Any]) -> frozenset[str]:
|
||||
return frozenset(compact_str(method) for method in as_list(raw.get("supportedGenerationMethods")) if method)
|
||||
|
||||
|
||||
def limits_from_model(raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||
limits: dict[str, Any] = {}
|
||||
input_limit = int_limit(raw.get("inputTokenLimit"))
|
||||
output_limit = int_limit(raw.get("outputTokenLimit"))
|
||||
if input_limit:
|
||||
limits["input_tokens"] = input_limit
|
||||
limits["context_tokens"] = input_limit
|
||||
if output_limit:
|
||||
limits["output_tokens"] = output_limit
|
||||
return limits
|
||||
|
||||
|
||||
def _capability(
|
||||
*,
|
||||
family: str,
|
||||
input_modalities: tuple[str, ...],
|
||||
output_modalities: tuple[str, ...],
|
||||
capabilities: tuple[str, ...] = (),
|
||||
limits: Mapping[str, Any] | None = None,
|
||||
primary_task: str | None = None,
|
||||
source: str = mc.SOURCE_PROVIDER_READER,
|
||||
confidence: str = mc.CONFIDENCE_PROVIDER_REPORTED,
|
||||
) -> mc.ModelCapability:
|
||||
return mc.ModelCapability.build(
|
||||
family=family,
|
||||
primary_task=primary_task,
|
||||
input_modalities=input_modalities,
|
||||
output_modalities=output_modalities,
|
||||
capabilities=capabilities,
|
||||
limits=limits,
|
||||
source=source,
|
||||
confidence=confidence,
|
||||
)
|
||||
|
||||
|
||||
def capability_from_model(raw: Mapping[str, Any]) -> mc.ModelCapability:
|
||||
methods = supported_methods(raw)
|
||||
capabilities: list[str] = []
|
||||
if raw.get("thinking") is True:
|
||||
capabilities.append(mc.CAP_REASONING)
|
||||
|
||||
if methods & EMBEDDING_METHODS and not methods & TEXT_GENERATION_METHODS:
|
||||
return _capability(
|
||||
family=mc.FAMILY_EMBEDDING,
|
||||
input_modalities=(mc.MODALITY_TEXT,),
|
||||
output_modalities=(mc.MODALITY_EMBEDDING,),
|
||||
capabilities=tuple(capabilities),
|
||||
limits=limits_from_model(raw),
|
||||
)
|
||||
|
||||
# `generateContent` proves the model supports Google's content generation
|
||||
# method, but the Model resource does not expose input/output modalities.
|
||||
# Keep the model unknown instead of guessing chat/image/audio/video from ID.
|
||||
if methods & TEXT_GENERATION_METHODS:
|
||||
return _capability(
|
||||
family=mc.FAMILY_UNKNOWN,
|
||||
input_modalities=(),
|
||||
output_modalities=(),
|
||||
capabilities=tuple(capabilities),
|
||||
limits=limits_from_model(raw),
|
||||
)
|
||||
|
||||
capability = mc.unknown_capability(
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
confidence=mc.CONFIDENCE_UNKNOWN,
|
||||
)
|
||||
limits = limits_from_model(raw)
|
||||
if limits or capabilities:
|
||||
return _capability(
|
||||
family=mc.FAMILY_UNKNOWN,
|
||||
input_modalities=(),
|
||||
output_modalities=(),
|
||||
capabilities=tuple(capabilities),
|
||||
limits=limits,
|
||||
)
|
||||
return capability
|
||||
|
||||
|
||||
def deterministic_controls_from_model(raw: Mapping[str, Any]) -> tuple[mc.DeterministicControl, ...]:
|
||||
methods = supported_methods(raw)
|
||||
controls: list[str] = []
|
||||
if "temperature" in raw or "maxTemperature" in raw:
|
||||
controls.append(mc.CONTROL_TEMPERATURE)
|
||||
if "topP" in raw:
|
||||
controls.append(mc.CONTROL_TOP_P)
|
||||
if raw.get("topK") not in (None, ""):
|
||||
controls.append(mc.CONTROL_TOP_K)
|
||||
if METHOD_CREATE_CACHED_CONTENT in methods:
|
||||
controls.append(mc.CONTROL_PROMPT_CACHING)
|
||||
if methods & BATCH_METHODS:
|
||||
controls.append(mc.CONTROL_BATCH)
|
||||
return mc.deterministic_controls_from_values(
|
||||
controls,
|
||||
status=mc.ASSERTION_CLAIMED,
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
confidence=mc.CONFIDENCE_PROVIDER_REPORTED,
|
||||
)
|
||||
@@ -0,0 +1,428 @@
|
||||
"""llama.cpp server capability reader.
|
||||
|
||||
llama-server exposes OpenAI-compatible model IDs through /v1/models, but its
|
||||
useful runtime metadata lives in native endpoints such as /props and /slots.
|
||||
This reader can normalize each payload independently and can merge the three
|
||||
payloads when the probe script has them all.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from src import model_capabilities as mc
|
||||
from src.model_capability_readers import generic_openai
|
||||
from src.model_capability_readers.base import (
|
||||
ModelCapabilityRecord,
|
||||
VENDOR_LLAMACPP,
|
||||
as_list,
|
||||
as_mapping,
|
||||
build_capability,
|
||||
compact_str,
|
||||
deterministic_controls_from_supported_parameters,
|
||||
int_limit,
|
||||
merge_unique,
|
||||
model_id_from,
|
||||
openai_model_items,
|
||||
stable_model_id_for,
|
||||
)
|
||||
|
||||
|
||||
vendor = VENDOR_LLAMACPP
|
||||
|
||||
|
||||
_SAMPLER_CONTROL_MAP = {
|
||||
"temperature": mc.CONTROL_TEMPERATURE,
|
||||
"top_p": mc.CONTROL_TOP_P,
|
||||
}
|
||||
|
||||
|
||||
def _model_entries(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]:
|
||||
payload = as_mapping(payload)
|
||||
data_items = openai_model_items(payload)
|
||||
if data_items:
|
||||
return data_items
|
||||
return tuple(item for item in as_list(payload.get("models")) if isinstance(item, Mapping))
|
||||
|
||||
|
||||
def _server_model_entries(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]:
|
||||
return tuple(item for item in as_list(as_mapping(payload).get("models")) if isinstance(item, Mapping))
|
||||
|
||||
|
||||
def _model_id_from_props(payload: Mapping[str, Any]) -> str:
|
||||
payload = as_mapping(payload)
|
||||
model_alias = compact_str(payload.get("model_alias"))
|
||||
if model_alias:
|
||||
return model_alias
|
||||
model_path = compact_str(payload.get("model_path"))
|
||||
if model_path:
|
||||
return PurePosixPath(model_path).name
|
||||
return ""
|
||||
|
||||
|
||||
def _capability_tokens_from_server_model(raw: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
out: list[str] = []
|
||||
for value in as_list(raw.get("capabilities")):
|
||||
token = compact_str(value).lower().replace("-", "_")
|
||||
if token in {"embedding", "embeddings"}:
|
||||
continue
|
||||
if token in {"rerank", "reranking"}:
|
||||
continue
|
||||
if token in {"completion", "completions", "chat"}:
|
||||
continue
|
||||
cap = mc.normalize_capability(token)
|
||||
if cap and cap not in out:
|
||||
out.append(cap)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _family_from_server_model(raw: Mapping[str, Any]) -> str:
|
||||
capabilities = {compact_str(value).lower().replace("-", "_") for value in as_list(raw.get("capabilities"))}
|
||||
if "embedding" in capabilities or "embeddings" in capabilities:
|
||||
return mc.FAMILY_EMBEDDING
|
||||
if "rerank" in capabilities or "reranking" in capabilities:
|
||||
return mc.FAMILY_RERANK
|
||||
if "completion" in capabilities or "completions" in capabilities or "chat" in capabilities:
|
||||
return mc.FAMILY_CHAT
|
||||
return mc.FAMILY_UNKNOWN
|
||||
|
||||
|
||||
def _matching_server_model(payload: Mapping[str, Any], model_id: str) -> Mapping[str, Any]:
|
||||
for item in _server_model_entries(payload):
|
||||
if model_id in {
|
||||
model_id_from(item, "id", "name", "model"),
|
||||
compact_str(item.get("name")),
|
||||
compact_str(item.get("model")),
|
||||
}:
|
||||
return item
|
||||
return {}
|
||||
|
||||
|
||||
def _limits_from_model_entry(raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||
meta = as_mapping(raw.get("meta"))
|
||||
limits: dict[str, Any] = {}
|
||||
n_ctx_train = int_limit(raw.get("n_ctx_train") or meta.get("n_ctx_train"))
|
||||
n_params = int_limit(raw.get("n_params") or meta.get("n_params"))
|
||||
size = int_limit(raw.get("size") or meta.get("size"))
|
||||
if n_ctx_train:
|
||||
limits["training_context_tokens"] = n_ctx_train
|
||||
if n_params:
|
||||
limits["parameters"] = n_params
|
||||
if size:
|
||||
limits["model_bytes"] = size
|
||||
return limits
|
||||
|
||||
|
||||
def _props_params(payload: Mapping[str, Any]) -> Mapping[str, Any]:
|
||||
return as_mapping(as_mapping(payload.get("default_generation_settings")).get("params"))
|
||||
|
||||
|
||||
def _limits_from_props(payload: Mapping[str, Any], slots_payload: Any = None) -> dict[str, Any]:
|
||||
default_settings = as_mapping(payload.get("default_generation_settings"))
|
||||
limits: dict[str, Any] = {}
|
||||
n_ctx = int_limit(default_settings.get("n_ctx"))
|
||||
total_slots = int_limit(payload.get("total_slots"))
|
||||
if not n_ctx and isinstance(slots_payload, list):
|
||||
slot_contexts = [int_limit(as_mapping(slot).get("n_ctx")) for slot in slots_payload]
|
||||
slot_contexts = [value for value in slot_contexts if value]
|
||||
if slot_contexts:
|
||||
n_ctx = min(slot_contexts)
|
||||
if n_ctx:
|
||||
limits["context_tokens"] = n_ctx
|
||||
if total_slots:
|
||||
limits["parallel_slots"] = total_slots
|
||||
elif isinstance(slots_payload, list) and slots_payload:
|
||||
limits["parallel_slots"] = len(slots_payload)
|
||||
return limits
|
||||
|
||||
|
||||
def _modalities_from_props(payload: Mapping[str, Any]) -> tuple[tuple[str, ...], tuple[str, ...]]:
|
||||
modalities = as_mapping(payload.get("modalities"))
|
||||
input_modalities = [mc.MODALITY_TEXT]
|
||||
output_modalities = [mc.MODALITY_TEXT]
|
||||
if modalities.get("vision") is True:
|
||||
input_modalities.append(mc.MODALITY_IMAGE)
|
||||
if modalities.get("audio") is True:
|
||||
input_modalities.append(mc.MODALITY_AUDIO)
|
||||
return tuple(input_modalities), tuple(output_modalities)
|
||||
|
||||
|
||||
def _capabilities_from_props(payload: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
caps = as_mapping(payload.get("chat_template_caps"))
|
||||
params = _props_params(payload)
|
||||
out: list[str] = []
|
||||
if caps.get("supports_tools") is True or caps.get("supports_tool_calls") is True:
|
||||
out.append(mc.CAP_TOOL_CALL)
|
||||
if params.get("stream") is not None:
|
||||
out.append(mc.CAP_STREAMING)
|
||||
if as_mapping(payload.get("modalities")).get("vision") is True:
|
||||
out.append(mc.CAP_VISION)
|
||||
if as_mapping(payload.get("modalities")).get("audio") is True:
|
||||
out.append(mc.CAP_AUDIO_INPUT)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _unsupported_assertions_from_props(payload: Mapping[str, Any]) -> tuple[mc.CapabilityAssertion, ...]:
|
||||
modalities = as_mapping(payload.get("modalities"))
|
||||
assertions: list[mc.CapabilityAssertion] = []
|
||||
if modalities.get("vision") is False:
|
||||
assertions.append(
|
||||
mc.CapabilityAssertion.build(
|
||||
capability=mc.CAP_VISION,
|
||||
status=mc.ASSERTION_UNSUPPORTED,
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
confidence=mc.CONFIDENCE_PROVIDER_REPORTED,
|
||||
evidence={"field": "modalities.vision"},
|
||||
)
|
||||
)
|
||||
if modalities.get("audio") is False:
|
||||
assertions.append(
|
||||
mc.CapabilityAssertion.build(
|
||||
capability=mc.CAP_AUDIO_INPUT,
|
||||
status=mc.ASSERTION_UNSUPPORTED,
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
confidence=mc.CONFIDENCE_PROVIDER_REPORTED,
|
||||
evidence={"field": "modalities.audio"},
|
||||
)
|
||||
)
|
||||
return tuple(assertions)
|
||||
|
||||
|
||||
def _deterministic_controls_from_props(payload: Mapping[str, Any]) -> tuple[mc.DeterministicControl, ...]:
|
||||
controls: list[str] = []
|
||||
params = _props_params(payload)
|
||||
for key in ("temperature", "top_p", "seed"):
|
||||
if key in params:
|
||||
controls.append(key)
|
||||
for sampler in as_list(params.get("samplers")):
|
||||
control = _SAMPLER_CONTROL_MAP.get(compact_str(sampler).lower())
|
||||
if control:
|
||||
controls.append(control)
|
||||
template_caps = as_mapping(payload.get("chat_template_caps"))
|
||||
if template_caps.get("supports_system_role") is True:
|
||||
controls.append(mc.CONTROL_SYSTEM_PROMPT)
|
||||
if template_caps.get("supports_tools") is True or template_caps.get("supports_tool_calls") is True:
|
||||
controls.append(mc.CONTROL_TOOL_CHOICE)
|
||||
return deterministic_controls_from_supported_parameters(merge_unique(controls))
|
||||
|
||||
|
||||
def _capability_for_family(
|
||||
family: str,
|
||||
*,
|
||||
capabilities: tuple[str, ...] = (),
|
||||
limits: Mapping[str, Any] | None = None,
|
||||
props_payload: Mapping[str, Any] | None = None,
|
||||
) -> mc.ModelCapability:
|
||||
if family == mc.FAMILY_EMBEDDING:
|
||||
return build_capability(
|
||||
family=mc.FAMILY_EMBEDDING,
|
||||
input_modalities=(mc.MODALITY_TEXT,),
|
||||
output_modalities=(mc.MODALITY_EMBEDDING,),
|
||||
capabilities=capabilities,
|
||||
limits=limits,
|
||||
)
|
||||
if family == mc.FAMILY_RERANK:
|
||||
return build_capability(
|
||||
family=mc.FAMILY_RERANK,
|
||||
input_modalities=(mc.MODALITY_TEXT,),
|
||||
output_modalities=(mc.MODALITY_TEXT,),
|
||||
capabilities=capabilities,
|
||||
limits=limits,
|
||||
)
|
||||
if props_payload:
|
||||
input_modalities, output_modalities = _modalities_from_props(props_payload)
|
||||
else:
|
||||
input_modalities, output_modalities = (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,)
|
||||
return build_capability(
|
||||
family=mc.FAMILY_CHAT,
|
||||
input_modalities=input_modalities,
|
||||
output_modalities=output_modalities,
|
||||
capabilities=capabilities,
|
||||
limits=limits,
|
||||
)
|
||||
|
||||
|
||||
def _record(
|
||||
*,
|
||||
model_id: str,
|
||||
family: str,
|
||||
capabilities: tuple[str, ...] = (),
|
||||
limits: Mapping[str, Any] | None = None,
|
||||
props_payload: Mapping[str, Any] | None = None,
|
||||
deterministic_controls: tuple[mc.DeterministicControl, ...] = (),
|
||||
extra_assertions: tuple[mc.CapabilityAssertion, ...] = (),
|
||||
raw: Mapping[str, Any] | None = None,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> ModelCapabilityRecord:
|
||||
capability = _capability_for_family(
|
||||
family,
|
||||
capabilities=capabilities,
|
||||
limits=limits,
|
||||
props_payload=props_payload,
|
||||
)
|
||||
return ModelCapabilityRecord(
|
||||
vendor=VENDOR_LLAMACPP,
|
||||
model_id=model_id,
|
||||
stable_model_id=stable_model_id_for(VENDOR_LLAMACPP, model_id, endpoint_id=endpoint_id, base_url=base_url),
|
||||
display_name=model_id,
|
||||
capability=capability,
|
||||
capability_assertions=(
|
||||
mc.capability_assertions_from_capability(
|
||||
capability,
|
||||
status=mc.ASSERTION_CLAIMED,
|
||||
source=capability.source,
|
||||
confidence=capability.confidence,
|
||||
)
|
||||
+ extra_assertions
|
||||
),
|
||||
deterministic_controls=deterministic_controls,
|
||||
raw=raw or {},
|
||||
)
|
||||
|
||||
|
||||
def record_from_model_payload(
|
||||
raw: Mapping[str, Any],
|
||||
*,
|
||||
server_model: Mapping[str, Any] | None = None,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> ModelCapabilityRecord | None:
|
||||
model_id = model_id_from(raw, "id", "name", "model")
|
||||
if not model_id:
|
||||
return None
|
||||
server_model = as_mapping(server_model)
|
||||
family = _family_from_server_model(server_model) if server_model else mc.FAMILY_UNKNOWN
|
||||
if family == mc.FAMILY_UNKNOWN:
|
||||
return generic_openai.record_from_model(
|
||||
raw,
|
||||
vendor_id=VENDOR_LLAMACPP,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
)
|
||||
capabilities = _capability_tokens_from_server_model(server_model)
|
||||
return _record(
|
||||
model_id=model_id,
|
||||
family=family,
|
||||
capabilities=capabilities,
|
||||
limits=_limits_from_model_entry(raw),
|
||||
raw=raw,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
)
|
||||
|
||||
|
||||
def record_from_props_payload(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
slots_payload: Any = None,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> ModelCapabilityRecord | None:
|
||||
payload = as_mapping(payload)
|
||||
model_id = _model_id_from_props(payload)
|
||||
if not model_id:
|
||||
return None
|
||||
return _record(
|
||||
model_id=model_id,
|
||||
family=mc.FAMILY_CHAT,
|
||||
capabilities=_capabilities_from_props(payload),
|
||||
limits=_limits_from_props(payload, slots_payload),
|
||||
props_payload=payload,
|
||||
deterministic_controls=_deterministic_controls_from_props(payload),
|
||||
extra_assertions=_unsupported_assertions_from_props(payload),
|
||||
raw=payload,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
)
|
||||
|
||||
|
||||
def records_from_payloads(
|
||||
*,
|
||||
models_payload: Mapping[str, Any] | None = None,
|
||||
props_payload: Mapping[str, Any] | None = None,
|
||||
slots_payload: Any = None,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> tuple[ModelCapabilityRecord, ...]:
|
||||
props_payload = as_mapping(props_payload)
|
||||
models_payload = as_mapping(models_payload)
|
||||
props_record = (
|
||||
record_from_props_payload(props_payload, slots_payload=slots_payload, endpoint_id=endpoint_id, base_url=base_url)
|
||||
if props_payload
|
||||
else None
|
||||
)
|
||||
if not models_payload:
|
||||
return (props_record,) if props_record else ()
|
||||
|
||||
records: list[ModelCapabilityRecord] = []
|
||||
for item in _model_entries(models_payload):
|
||||
model_id = model_id_from(item, "id", "name", "model")
|
||||
if not model_id:
|
||||
continue
|
||||
server_model = _matching_server_model(models_payload, model_id)
|
||||
model_record = record_from_model_payload(
|
||||
item,
|
||||
server_model=server_model,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
)
|
||||
if not model_record:
|
||||
continue
|
||||
if props_record and props_record.model_id == model_id:
|
||||
limits = {**dict(model_record.capability.limits), **dict(props_record.capability.limits)}
|
||||
capability = _capability_for_family(
|
||||
props_record.capability.family,
|
||||
capabilities=merge_unique(model_record.capability.capabilities, props_record.capability.capabilities),
|
||||
limits=limits,
|
||||
props_payload=props_payload,
|
||||
)
|
||||
records.append(
|
||||
ModelCapabilityRecord(
|
||||
vendor=VENDOR_LLAMACPP,
|
||||
model_id=model_id,
|
||||
stable_model_id=stable_model_id_for(
|
||||
VENDOR_LLAMACPP,
|
||||
model_id,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
),
|
||||
display_name=model_id,
|
||||
capability=capability,
|
||||
capability_assertions=(
|
||||
mc.capability_assertions_from_capability(
|
||||
capability,
|
||||
status=mc.ASSERTION_CLAIMED,
|
||||
source=capability.source,
|
||||
confidence=capability.confidence,
|
||||
)
|
||||
+ _unsupported_assertions_from_props(props_payload)
|
||||
),
|
||||
deterministic_controls=props_record.deterministic_controls,
|
||||
raw={"models": item, "props": props_payload, "slots": slots_payload or []},
|
||||
)
|
||||
)
|
||||
else:
|
||||
records.append(model_record)
|
||||
if not records and props_record:
|
||||
records.append(props_record)
|
||||
return tuple(records)
|
||||
|
||||
|
||||
def records_from_payload(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> tuple[ModelCapabilityRecord, ...]:
|
||||
payload = as_mapping(payload)
|
||||
if not payload:
|
||||
return ()
|
||||
if "default_generation_settings" in payload or "chat_template_caps" in payload:
|
||||
record = record_from_props_payload(payload, endpoint_id=endpoint_id, base_url=base_url)
|
||||
return (record,) if record else ()
|
||||
if "models" in payload or "data" in payload:
|
||||
return records_from_payloads(models_payload=payload, endpoint_id=endpoint_id, base_url=base_url)
|
||||
return ()
|
||||
@@ -0,0 +1,186 @@
|
||||
"""LM Studio native model metadata reader."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from src import model_capabilities as mc
|
||||
from src.model_capability_readers import generic_openai
|
||||
from src.model_capability_readers.base import (
|
||||
ModelCapabilityRecord,
|
||||
VENDOR_LMSTUDIO,
|
||||
as_list,
|
||||
as_mapping,
|
||||
build_capability,
|
||||
compact_str,
|
||||
int_limit,
|
||||
merge_unique,
|
||||
model_id_from,
|
||||
openai_model_items,
|
||||
stable_model_id_for,
|
||||
)
|
||||
|
||||
|
||||
vendor = VENDOR_LMSTUDIO
|
||||
|
||||
|
||||
def _loaded_instance_contexts(raw: Mapping[str, Any]) -> tuple[int, ...]:
|
||||
contexts: list[int] = []
|
||||
for instance in as_list(raw.get("loaded_instances")):
|
||||
instance_payload = as_mapping(instance)
|
||||
config = as_mapping(instance_payload.get("config"))
|
||||
value = int_limit(instance_payload.get("context_length")) or int_limit(
|
||||
config.get("context_length")
|
||||
)
|
||||
if value:
|
||||
contexts.append(value)
|
||||
return tuple(contexts)
|
||||
|
||||
|
||||
def _limits_from_model(raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||
limits: dict[str, Any] = {}
|
||||
loaded_contexts = _loaded_instance_contexts(raw)
|
||||
loaded_context = int_limit(raw.get("loaded_context_length")) or (
|
||||
min(loaded_contexts) if loaded_contexts else None
|
||||
)
|
||||
configured_context = int_limit(raw.get("context_length")) or int_limit(raw.get("contextLength"))
|
||||
max_context = int_limit(raw.get("max_context_length")) or int_limit(raw.get("maxContextLength"))
|
||||
context_tokens = loaded_context or configured_context or max_context
|
||||
if context_tokens:
|
||||
limits["context_tokens"] = context_tokens
|
||||
if max_context and max_context != context_tokens:
|
||||
limits["max_context_tokens"] = max_context
|
||||
return limits
|
||||
|
||||
|
||||
def _family_from_type(raw: Mapping[str, Any]) -> str:
|
||||
kind = compact_str(raw.get("type") or raw.get("model_type") or raw.get("task")).lower().replace("-", "_")
|
||||
if kind in {"embedding", "embeddings", "text_embedding", "text_embeddings"}:
|
||||
return mc.FAMILY_EMBEDDING
|
||||
if kind in {"llm", "chat", "vlm", "vision", "text_generation"}:
|
||||
return mc.FAMILY_CHAT
|
||||
return mc.FAMILY_UNKNOWN
|
||||
|
||||
|
||||
def _capabilities_from_native_payload(raw: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
capabilities_payload = as_mapping(raw.get("capabilities"))
|
||||
capabilities: list[str] = []
|
||||
if capabilities_payload.get("vision") is True:
|
||||
capabilities.append(mc.CAP_VISION)
|
||||
if (
|
||||
capabilities_payload.get("trained_for_tool_use") is True
|
||||
or capabilities_payload.get("tools") is True
|
||||
or capabilities_payload.get("tool_use") is True
|
||||
):
|
||||
capabilities.append(mc.CAP_TOOL_CALL)
|
||||
if capabilities_payload.get("reasoning"):
|
||||
capabilities.append(mc.CAP_REASONING)
|
||||
return merge_unique(capabilities)
|
||||
|
||||
|
||||
def _unknown_record(
|
||||
raw: Mapping[str, Any],
|
||||
model_id: str,
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> ModelCapabilityRecord:
|
||||
return ModelCapabilityRecord(
|
||||
vendor=VENDOR_LMSTUDIO,
|
||||
model_id=model_id,
|
||||
stable_model_id=stable_model_id_for(
|
||||
VENDOR_LMSTUDIO,
|
||||
model_id,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
),
|
||||
display_name=compact_str(raw.get("display_name") or raw.get("name")) or model_id,
|
||||
capability=mc.unknown_capability(
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
confidence=mc.CONFIDENCE_UNKNOWN,
|
||||
),
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
|
||||
def record_from_native_model(
|
||||
raw: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> ModelCapabilityRecord | None:
|
||||
model_id = model_id_from(raw, "key", "id", "model", "name")
|
||||
if not model_id:
|
||||
return None
|
||||
|
||||
family = _family_from_type(raw)
|
||||
capabilities = _capabilities_from_native_payload(raw)
|
||||
|
||||
if family == mc.FAMILY_UNKNOWN and capabilities:
|
||||
family = mc.FAMILY_CHAT
|
||||
|
||||
if family == mc.FAMILY_EMBEDDING:
|
||||
input_modalities = (mc.MODALITY_TEXT,)
|
||||
output_modalities = (mc.MODALITY_EMBEDDING,)
|
||||
elif family == mc.FAMILY_CHAT and mc.CAP_VISION in capabilities:
|
||||
input_modalities = (mc.MODALITY_TEXT, mc.MODALITY_IMAGE)
|
||||
output_modalities = (mc.MODALITY_TEXT,)
|
||||
elif family == mc.FAMILY_CHAT:
|
||||
input_modalities = (mc.MODALITY_TEXT,)
|
||||
output_modalities = (mc.MODALITY_TEXT,)
|
||||
else:
|
||||
return generic_openai.record_from_model(
|
||||
raw,
|
||||
vendor_id=VENDOR_LMSTUDIO,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
) or _unknown_record(
|
||||
raw,
|
||||
model_id,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
)
|
||||
|
||||
capability = build_capability(
|
||||
family=family,
|
||||
input_modalities=input_modalities,
|
||||
output_modalities=output_modalities,
|
||||
capabilities=capabilities,
|
||||
limits=_limits_from_model(raw),
|
||||
)
|
||||
return ModelCapabilityRecord(
|
||||
vendor=VENDOR_LMSTUDIO,
|
||||
model_id=model_id,
|
||||
stable_model_id=stable_model_id_for(
|
||||
VENDOR_LMSTUDIO,
|
||||
model_id,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
),
|
||||
display_name=compact_str(raw.get("display_name") or raw.get("name")) or model_id,
|
||||
capability=capability,
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
|
||||
def records_from_payload(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> tuple[ModelCapabilityRecord, ...]:
|
||||
records: list[ModelCapabilityRecord] = []
|
||||
for item in openai_model_items(payload):
|
||||
record = record_from_native_model(item, endpoint_id=endpoint_id, base_url=base_url)
|
||||
if record:
|
||||
records.append(record)
|
||||
if records:
|
||||
return tuple(records)
|
||||
for item in as_list(as_mapping(payload).get("models")):
|
||||
if not isinstance(item, Mapping):
|
||||
continue
|
||||
record = record_from_native_model(item, endpoint_id=endpoint_id, base_url=base_url)
|
||||
if record:
|
||||
records.append(record)
|
||||
return tuple(records)
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Ollama native API capability reader."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from src import model_capabilities as mc
|
||||
from src.model_capability_readers.base import (
|
||||
ModelCapabilityRecord,
|
||||
VENDOR_OLLAMA,
|
||||
as_list,
|
||||
as_mapping,
|
||||
build_capability,
|
||||
compact_str,
|
||||
int_limit,
|
||||
merge_unique,
|
||||
model_id_from,
|
||||
stable_model_id_for,
|
||||
)
|
||||
|
||||
|
||||
vendor = VENDOR_OLLAMA
|
||||
|
||||
|
||||
_CAPABILITY_MAP = {
|
||||
"completion": None,
|
||||
"completions": None,
|
||||
"chat": None,
|
||||
"thinking": mc.CAP_REASONING,
|
||||
"reasoning": mc.CAP_REASONING,
|
||||
"vision": mc.CAP_VISION,
|
||||
"tools": mc.CAP_TOOL_CALL,
|
||||
"tool": mc.CAP_TOOL_CALL,
|
||||
"embedding": None,
|
||||
"embeddings": None,
|
||||
}
|
||||
|
||||
|
||||
def _capability_tokens(values: Any) -> tuple[str, ...]:
|
||||
out: list[str] = []
|
||||
for value in as_list(values):
|
||||
token = compact_str(value).lower().replace("-", "_")
|
||||
cap = _CAPABILITY_MAP.get(token)
|
||||
if cap and cap not in out:
|
||||
out.append(cap)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _family_from_ollama_capabilities(values: Any) -> str:
|
||||
tokens = {compact_str(value).lower().replace("-", "_") for value in as_list(values)}
|
||||
if tokens and tokens.issubset({"embedding", "embeddings"}):
|
||||
return mc.FAMILY_EMBEDDING
|
||||
if "embedding" in tokens or "embeddings" in tokens:
|
||||
return mc.FAMILY_EMBEDDING
|
||||
if tokens.intersection({"completion", "completions", "chat", "thinking", "reasoning", "tools", "tool", "vision"}):
|
||||
return mc.FAMILY_CHAT
|
||||
return mc.FAMILY_UNKNOWN
|
||||
|
||||
|
||||
def _parameters_mapping(value: Any) -> Mapping[str, Any]:
|
||||
if isinstance(value, Mapping):
|
||||
return value
|
||||
text = compact_str(value)
|
||||
if not text:
|
||||
return {}
|
||||
parsed: dict[str, str] = {}
|
||||
for line in text.splitlines():
|
||||
parts = line.strip().split(None, 1)
|
||||
if len(parts) == 2:
|
||||
parsed[parts[0]] = parts[1]
|
||||
return parsed
|
||||
|
||||
|
||||
def _modalities_for_family(family: str, capabilities: tuple[str, ...]) -> tuple[tuple[str, ...], tuple[str, ...]]:
|
||||
if family == mc.FAMILY_EMBEDDING:
|
||||
return (mc.MODALITY_TEXT,), (mc.MODALITY_EMBEDDING,)
|
||||
if family == mc.FAMILY_CHAT and mc.CAP_VISION in capabilities:
|
||||
return (mc.MODALITY_TEXT, mc.MODALITY_IMAGE), (mc.MODALITY_TEXT,)
|
||||
if family == mc.FAMILY_CHAT:
|
||||
return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,)
|
||||
return (), ()
|
||||
|
||||
|
||||
def _first_int_by_key_shape(*mappings: Mapping[str, Any], exact_keys: tuple[str, ...] = ()) -> int | None:
|
||||
for key in exact_keys:
|
||||
for mapping in mappings:
|
||||
value = int_limit(mapping.get(key))
|
||||
if value:
|
||||
return value
|
||||
for mapping in mappings:
|
||||
for key, value in mapping.items():
|
||||
key_text = compact_str(key).lower()
|
||||
if key_text == "context_length" or key_text.endswith(".context_length"):
|
||||
limit = int_limit(value)
|
||||
if limit:
|
||||
return limit
|
||||
return None
|
||||
|
||||
|
||||
def _limits_from_show(raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||
model_info = as_mapping(raw.get("model_info"))
|
||||
parameters = _parameters_mapping(raw.get("parameters"))
|
||||
details = as_mapping(raw.get("details"))
|
||||
limits: dict[str, Any] = {}
|
||||
context_tokens = _first_int_by_key_shape(
|
||||
raw,
|
||||
model_info,
|
||||
parameters,
|
||||
details,
|
||||
exact_keys=("context_length", "num_ctx"),
|
||||
)
|
||||
if context_tokens:
|
||||
limits["context_tokens"] = context_tokens
|
||||
return limits
|
||||
|
||||
|
||||
def record_from_show_payload(
|
||||
model_id: str,
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> ModelCapabilityRecord | None:
|
||||
model_id = compact_str(model_id) or model_id_from(payload, "model", "name")
|
||||
if not model_id:
|
||||
return None
|
||||
capability_values = payload.get("capabilities")
|
||||
capabilities = _capability_tokens(capability_values)
|
||||
family = _family_from_ollama_capabilities(capability_values)
|
||||
if family == mc.FAMILY_UNKNOWN:
|
||||
capability = mc.unknown_capability(
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
confidence=mc.CONFIDENCE_UNKNOWN,
|
||||
)
|
||||
else:
|
||||
input_modalities, output_modalities = _modalities_for_family(family, capabilities)
|
||||
capability = build_capability(
|
||||
family=family,
|
||||
input_modalities=input_modalities,
|
||||
output_modalities=output_modalities,
|
||||
capabilities=merge_unique(capabilities),
|
||||
limits=_limits_from_show(payload),
|
||||
)
|
||||
return ModelCapabilityRecord(
|
||||
vendor=VENDOR_OLLAMA,
|
||||
model_id=model_id,
|
||||
stable_model_id=stable_model_id_for(VENDOR_OLLAMA, model_id, endpoint_id=endpoint_id, base_url=base_url),
|
||||
display_name=model_id,
|
||||
capability=capability,
|
||||
raw=payload,
|
||||
)
|
||||
|
||||
|
||||
def records_from_tags_payload(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> tuple[ModelCapabilityRecord, ...]:
|
||||
records: list[ModelCapabilityRecord] = []
|
||||
for item in as_list(as_mapping(payload).get("models")):
|
||||
if not isinstance(item, Mapping):
|
||||
continue
|
||||
model_id = model_id_from(item, "model", "name")
|
||||
if not model_id:
|
||||
continue
|
||||
records.append(
|
||||
ModelCapabilityRecord(
|
||||
vendor=VENDOR_OLLAMA,
|
||||
model_id=model_id,
|
||||
stable_model_id=stable_model_id_for(
|
||||
VENDOR_OLLAMA,
|
||||
model_id,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
),
|
||||
display_name=model_id,
|
||||
capability=mc.unknown_capability(
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
confidence=mc.CONFIDENCE_UNKNOWN,
|
||||
),
|
||||
raw=item,
|
||||
)
|
||||
)
|
||||
return tuple(records)
|
||||
|
||||
|
||||
def records_from_payload(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> tuple[ModelCapabilityRecord, ...]:
|
||||
payload = as_mapping(payload)
|
||||
if "models" in payload:
|
||||
return records_from_tags_payload(payload, endpoint_id=endpoint_id, base_url=base_url)
|
||||
record = record_from_show_payload(
|
||||
model_id_from(payload, "model", "name"),
|
||||
payload,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
)
|
||||
return (record,) if record else ()
|
||||
@@ -0,0 +1,65 @@
|
||||
"""OpenAI Models API capability reader.
|
||||
|
||||
OpenAI's `/v1/models` list/retrieve shape currently provides model identity
|
||||
metadata only: `id`, `object`, `created`, and `owned_by`. Those fields prove
|
||||
availability, not model capabilities, so this reader keeps capabilities
|
||||
unknown unless OpenAI adds explicit capability fields to the API shape later.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from src import model_capabilities as mc
|
||||
from src.model_capability_readers.base import (
|
||||
ModelCapabilityRecord,
|
||||
VENDOR_OPENAI,
|
||||
compact_str,
|
||||
model_id_from,
|
||||
openai_model_items,
|
||||
stable_model_id_for,
|
||||
)
|
||||
|
||||
|
||||
vendor = VENDOR_OPENAI
|
||||
|
||||
|
||||
OFFICIAL_MODEL_FIELDS = frozenset({"id", "object", "created", "owned_by"})
|
||||
|
||||
|
||||
def record_from_model(
|
||||
raw: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> ModelCapabilityRecord | None:
|
||||
model_id = model_id_from(raw, "id")
|
||||
if not model_id:
|
||||
return None
|
||||
|
||||
return ModelCapabilityRecord(
|
||||
vendor=VENDOR_OPENAI,
|
||||
model_id=model_id,
|
||||
stable_model_id=stable_model_id_for(VENDOR_OPENAI, model_id, endpoint_id=endpoint_id, base_url=base_url),
|
||||
display_name=compact_str(raw.get("name") or raw.get("display_name")),
|
||||
capability=mc.unknown_capability(
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
confidence=mc.CONFIDENCE_UNKNOWN,
|
||||
),
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
|
||||
def records_from_payload(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> tuple[ModelCapabilityRecord, ...]:
|
||||
records: list[ModelCapabilityRecord] = []
|
||||
for item in openai_model_items(payload):
|
||||
record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)
|
||||
if record:
|
||||
records.append(record)
|
||||
return tuple(records)
|
||||
@@ -0,0 +1,200 @@
|
||||
"""OpenRouter model catalog capability reader."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from src import model_capabilities as mc
|
||||
from src.model_capability_readers import generic_openai
|
||||
from src.model_capability_readers.base import (
|
||||
ModelCapabilityRecord,
|
||||
VENDOR_OPENROUTER,
|
||||
as_list,
|
||||
as_mapping,
|
||||
build_capability,
|
||||
compact_str,
|
||||
deterministic_controls_from_supported_parameters,
|
||||
family_from_modalities,
|
||||
int_limit,
|
||||
merge_unique,
|
||||
model_id_from,
|
||||
modalities_from_value,
|
||||
openai_model_items,
|
||||
split_modality_arrow,
|
||||
stable_model_id_for,
|
||||
)
|
||||
|
||||
|
||||
vendor = VENDOR_OPENROUTER
|
||||
|
||||
|
||||
_SUPPORTED_PARAMETER_CAPS = {
|
||||
"tools": mc.CAP_TOOL_CALL,
|
||||
"tool_choice": mc.CAP_TOOL_CALL,
|
||||
"function_calling": mc.CAP_TOOL_CALL,
|
||||
"parallel_tool_calls": mc.CAP_TOOL_CALL,
|
||||
"response_format": mc.CAP_JSON_MODE,
|
||||
"structured_outputs": mc.CAP_STRUCTURED_OUTPUT,
|
||||
"structured_output": mc.CAP_STRUCTURED_OUTPUT,
|
||||
"reasoning": mc.CAP_REASONING,
|
||||
"reasoning_effort": mc.CAP_REASONING,
|
||||
"include_reasoning": mc.CAP_REASONING,
|
||||
"web_search": mc.CAP_WEB_SEARCH,
|
||||
"web_search_options": mc.CAP_WEB_SEARCH,
|
||||
}
|
||||
|
||||
|
||||
def _capabilities_from_supported_parameters(values: Any) -> tuple[str, ...]:
|
||||
iterable = values if isinstance(values, list) else ()
|
||||
out: list[str] = []
|
||||
for value in iterable:
|
||||
cap = _SUPPORTED_PARAMETER_CAPS.get(compact_str(value).lower().replace("-", "_"))
|
||||
if cap and cap not in out:
|
||||
out.append(cap)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _limits_from_model(raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||
architecture = as_mapping(raw.get("architecture"))
|
||||
top_provider = as_mapping(raw.get("top_provider"))
|
||||
per_request_limits = as_mapping(raw.get("per_request_limits"))
|
||||
limits: dict[str, Any] = {}
|
||||
for key, canonical in (
|
||||
("context_length", "context_tokens"),
|
||||
("max_context_length", "context_tokens"),
|
||||
("input_token_limit", "input_tokens"),
|
||||
("output_token_limit", "output_tokens"),
|
||||
("max_completion_tokens", "output_tokens"),
|
||||
):
|
||||
value = int_limit(raw.get(key) or architecture.get(key) or top_provider.get(key))
|
||||
if value:
|
||||
limits[canonical] = value
|
||||
for key, value in per_request_limits.items():
|
||||
limit = int_limit(value)
|
||||
if limit:
|
||||
limits[f"per_request_{key}"] = limit
|
||||
return limits
|
||||
|
||||
|
||||
def _has_supported_voices(value: Any) -> bool:
|
||||
return any(compact_str(item) for item in as_list(value))
|
||||
|
||||
|
||||
def _capabilities_from_modalities(
|
||||
input_modalities: tuple[str, ...],
|
||||
output_modalities: tuple[str, ...],
|
||||
*,
|
||||
supported_voices: Any = None,
|
||||
) -> tuple[str, ...]:
|
||||
input_set = set(input_modalities)
|
||||
output_set = set(output_modalities)
|
||||
capabilities: list[str] = []
|
||||
if mc.MODALITY_IMAGE in input_set and mc.MODALITY_TEXT in output_set:
|
||||
capabilities.append(mc.CAP_VISION)
|
||||
if mc.MODALITY_FILE in input_set:
|
||||
capabilities.append(mc.CAP_FILES)
|
||||
if mc.MODALITY_PDF in input_set:
|
||||
capabilities.append(mc.CAP_PDF)
|
||||
if mc.MODALITY_AUDIO in input_set:
|
||||
capabilities.append(mc.CAP_AUDIO_INPUT)
|
||||
if mc.MODALITY_AUDIO in output_set:
|
||||
capabilities.append(mc.CAP_AUDIO_OUTPUT)
|
||||
if _has_supported_voices(supported_voices):
|
||||
capabilities.append(mc.CAP_TTS)
|
||||
if mc.MODALITY_IMAGE in output_set:
|
||||
capabilities.append(mc.CAP_IMAGE_GENERATION)
|
||||
if mc.MODALITY_IMAGE in input_set:
|
||||
capabilities.append(mc.CAP_IMAGE_EDITING)
|
||||
if mc.MODALITY_VIDEO in output_set:
|
||||
capabilities.append(mc.CAP_VIDEO_GENERATION)
|
||||
return tuple(capabilities)
|
||||
|
||||
|
||||
def _default_parameter_controls(raw: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
defaults = as_mapping(raw.get("default_parameters"))
|
||||
return tuple(key for key, value in defaults.items() if value is not None)
|
||||
|
||||
|
||||
def _deterministic_controls_from_model(raw: Mapping[str, Any]) -> tuple[mc.DeterministicControl, ...]:
|
||||
return deterministic_controls_from_supported_parameters(
|
||||
merge_unique(
|
||||
as_list(raw.get("supported_parameters")),
|
||||
_default_parameter_controls(raw),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def record_from_model(
|
||||
raw: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> ModelCapabilityRecord | None:
|
||||
model_id = model_id_from(raw, "id", "name")
|
||||
if not model_id:
|
||||
return None
|
||||
|
||||
architecture = as_mapping(raw.get("architecture"))
|
||||
input_modalities = modalities_from_value(
|
||||
raw.get("input_modalities") or architecture.get("input_modalities")
|
||||
)
|
||||
output_modalities = modalities_from_value(
|
||||
raw.get("output_modalities") or architecture.get("output_modalities")
|
||||
)
|
||||
if not input_modalities or not output_modalities:
|
||||
arrow_input, arrow_output = split_modality_arrow(
|
||||
raw.get("modality") or architecture.get("modality")
|
||||
)
|
||||
input_modalities = input_modalities or arrow_input
|
||||
output_modalities = output_modalities or arrow_output
|
||||
|
||||
capabilities = list(_capabilities_from_supported_parameters(raw.get("supported_parameters")))
|
||||
capabilities.extend(
|
||||
_capabilities_from_modalities(
|
||||
input_modalities,
|
||||
output_modalities,
|
||||
supported_voices=raw.get("supported_voices"),
|
||||
)
|
||||
)
|
||||
|
||||
family = family_from_modalities(input_modalities, output_modalities)
|
||||
if family == mc.FAMILY_UNKNOWN:
|
||||
fallback = generic_openai.record_from_model(
|
||||
raw,
|
||||
vendor_id=VENDOR_OPENROUTER,
|
||||
endpoint_id=endpoint_id,
|
||||
base_url=base_url,
|
||||
)
|
||||
return fallback
|
||||
|
||||
capability = build_capability(
|
||||
family=family,
|
||||
input_modalities=input_modalities,
|
||||
output_modalities=output_modalities,
|
||||
capabilities=merge_unique(capabilities),
|
||||
limits=_limits_from_model(raw),
|
||||
)
|
||||
return ModelCapabilityRecord(
|
||||
vendor=VENDOR_OPENROUTER,
|
||||
model_id=model_id,
|
||||
stable_model_id=stable_model_id_for(VENDOR_OPENROUTER, model_id, endpoint_id=endpoint_id, base_url=base_url),
|
||||
display_name=compact_str(raw.get("name")) or model_id,
|
||||
capability=capability,
|
||||
deterministic_controls=_deterministic_controls_from_model(raw),
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
|
||||
def records_from_payload(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
endpoint_id: Any = "",
|
||||
base_url: Any = "",
|
||||
) -> tuple[ModelCapabilityRecord, ...]:
|
||||
records: list[ModelCapabilityRecord] = []
|
||||
for item in openai_model_items(payload):
|
||||
record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)
|
||||
if record:
|
||||
records.append(record)
|
||||
return tuple(records)
|
||||
+9
-1
@@ -934,6 +934,13 @@ function initEndpointForm() {
|
||||
function _apiEndpointKind() {
|
||||
return (kindSel && kindSel.value) ? kindSel.value : 'api';
|
||||
}
|
||||
function _modelRefreshModeForApiEndpoint(url, endpointKind) {
|
||||
if (endpointKind === 'proxy') return 'manual';
|
||||
try {
|
||||
if ((new URL(url)).hostname.toLowerCase() === 'generativelanguage.googleapis.com') return '';
|
||||
} catch (_) {}
|
||||
return 'auto';
|
||||
}
|
||||
function _normalizeBaseUrl(raw) {
|
||||
let u = raw.trim();
|
||||
// Fix common protocol typos
|
||||
@@ -1081,7 +1088,8 @@ function initEndpointForm() {
|
||||
fd.append('base_url', url);
|
||||
const endpointKind = _apiEndpointKind();
|
||||
fd.append('endpoint_kind', endpointKind);
|
||||
fd.append('model_refresh_mode', endpointKind === 'proxy' ? 'manual' : 'auto');
|
||||
const refreshMode = _modelRefreshModeForApiEndpoint(url, endpointKind);
|
||||
if (refreshMode) fd.append('model_refresh_mode', refreshMode);
|
||||
fd.append('model_refresh_timeout', '30');
|
||||
if (apiKey) fd.append('api_key', apiKey);
|
||||
if (provider.value && provider.selectedOptions && provider.selectedOptions[0]) {
|
||||
|
||||
@@ -28,6 +28,24 @@ def test_provider_selection_is_inert_and_add_button_starts_device_flow():
|
||||
assert "_startProviderDeviceAuth(deviceAuthProvider" in add_block
|
||||
|
||||
|
||||
def test_google_add_omits_auto_refresh_mode_for_backend_manual_default():
|
||||
refresh_helper = _between(
|
||||
_ADMIN,
|
||||
"function _modelRefreshModeForApiEndpoint",
|
||||
"function _normalizeBaseUrl",
|
||||
)
|
||||
add_block = _between(
|
||||
_ADMIN,
|
||||
"el('adm-epAddBtn').addEventListener('click'",
|
||||
"async function _startProviderDeviceAuth",
|
||||
)
|
||||
|
||||
assert "generativelanguage.googleapis.com" in refresh_helper
|
||||
assert "return '';" in refresh_helper
|
||||
assert "_modelRefreshModeForApiEndpoint(url, endpointKind)" in add_block
|
||||
assert "if (refreshMode) fd.append('model_refresh_mode', refreshMode)" in add_block
|
||||
|
||||
|
||||
def test_device_auth_selection_disables_and_dims_api_test_button():
|
||||
form_block = _between(_ADMIN, "function _setApiFormForProvider()", "function _renderPickerMenu()")
|
||||
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
import src.model_capabilities as mc
|
||||
|
||||
|
||||
def surfaces(capability):
|
||||
return set(mc.display_surfaces_for(capability))
|
||||
|
||||
|
||||
def test_endpoint_type_llm_maps_to_explicit_chat_capability():
|
||||
capability = mc.capability_from_endpoint_type("llm")
|
||||
|
||||
assert capability.family == mc.FAMILY_CHAT
|
||||
assert capability.primary_task == mc.TASK_CHAT_COMPLETIONS
|
||||
assert capability.modalities.input == (mc.MODALITY_TEXT,)
|
||||
assert capability.modalities.output == (mc.MODALITY_TEXT,)
|
||||
assert capability.source == mc.SOURCE_ENDPOINT_CONFIG
|
||||
assert capability.confidence == mc.CONFIDENCE_EXPLICIT
|
||||
assert surfaces(capability) == {"chat"}
|
||||
|
||||
|
||||
def test_endpoint_type_image_maps_to_explicit_image_generation_capability():
|
||||
capability = mc.capability_from_endpoint_type("image")
|
||||
|
||||
assert capability.family == mc.FAMILY_IMAGE
|
||||
assert capability.primary_task == mc.TASK_IMAGE_GENERATE
|
||||
assert capability.modalities.output == (mc.MODALITY_IMAGE,)
|
||||
assert capability.capabilities == (mc.CAP_IMAGE_GENERATION,)
|
||||
assert capability.source == mc.SOURCE_ENDPOINT_CONFIG
|
||||
assert capability.confidence == mc.CONFIDENCE_EXPLICIT
|
||||
assert surfaces(capability) == {"image_generation"}
|
||||
|
||||
|
||||
def test_missing_or_unknown_endpoint_type_does_not_imply_chat():
|
||||
for model_type in (None, "", "openai-compatible", "text"):
|
||||
capability = mc.capability_from_endpoint_type(model_type)
|
||||
|
||||
assert capability.family == mc.FAMILY_UNKNOWN
|
||||
assert capability.primary_task == mc.TASK_UNKNOWN
|
||||
assert capability.source == mc.SOURCE_ENDPOINT_CONFIG
|
||||
assert mc.display_surfaces_for(capability) == ()
|
||||
|
||||
|
||||
def test_provider_record_normalizes_aliases_and_boolean_capability_maps():
|
||||
capability = mc.ModelCapability.from_dict(
|
||||
{
|
||||
"family": "llm",
|
||||
"modalities": {
|
||||
"input": ["text", "images", "docs", "images"],
|
||||
"output": "text",
|
||||
},
|
||||
"capabilities": {
|
||||
"tools": True,
|
||||
"unknown_vendor_flag": True,
|
||||
"vision": True,
|
||||
"tts": False,
|
||||
},
|
||||
"limits": {"max_context_tokens": 32768, "": "ignored"},
|
||||
"source": "provider_reader",
|
||||
"confidence": "provider_reported",
|
||||
}
|
||||
)
|
||||
|
||||
assert capability.family == mc.FAMILY_CHAT
|
||||
assert capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE, mc.MODALITY_FILE)
|
||||
assert capability.modalities.output == (mc.MODALITY_TEXT,)
|
||||
assert capability.capabilities == (mc.CAP_TOOL_CALL, mc.CAP_VISION)
|
||||
assert capability.limits == (("max_context_tokens", 32768),)
|
||||
assert surfaces(capability) == {"chat", "vision_chat", "document_chat"}
|
||||
assert capability.to_dict() == {
|
||||
"family": mc.FAMILY_CHAT,
|
||||
"primary_task": mc.TASK_CHAT_COMPLETIONS,
|
||||
"modalities": {
|
||||
"input": [mc.MODALITY_TEXT, mc.MODALITY_IMAGE, mc.MODALITY_FILE],
|
||||
"output": [mc.MODALITY_TEXT],
|
||||
},
|
||||
"capabilities": [mc.CAP_TOOL_CALL, mc.CAP_VISION],
|
||||
"limits": {"max_context_tokens": 32768},
|
||||
"source": mc.SOURCE_PROVIDER_READER,
|
||||
"confidence": mc.CONFIDENCE_PROVIDER_REPORTED,
|
||||
}
|
||||
|
||||
|
||||
def test_unknown_or_malformed_capability_record_stays_unknown():
|
||||
assert mc.ModelCapability.from_dict(None).to_dict() == mc.unknown_capability().to_dict()
|
||||
|
||||
capability = mc.ModelCapability.build(
|
||||
family="not-real",
|
||||
primary_task=1234,
|
||||
input_modalities=object(),
|
||||
output_modalities=["text", "not-real"],
|
||||
capabilities=["vision", "not-real"],
|
||||
source="not-real",
|
||||
confidence="not-real",
|
||||
)
|
||||
|
||||
assert capability.family == mc.FAMILY_UNKNOWN
|
||||
assert capability.primary_task == "1234"
|
||||
assert capability.modalities.input == ()
|
||||
assert capability.modalities.output == (mc.MODALITY_TEXT,)
|
||||
assert capability.capabilities == (mc.CAP_VISION,)
|
||||
assert capability.source == mc.SOURCE_UNKNOWN
|
||||
assert capability.confidence == mc.CONFIDENCE_UNKNOWN
|
||||
assert mc.display_surfaces_for(capability) == ()
|
||||
|
||||
|
||||
def test_display_surface_queries_cover_core_model_categories():
|
||||
assert surfaces(
|
||||
mc.ModelCapability.build(
|
||||
family=mc.FAMILY_IMAGE,
|
||||
input_modalities=[mc.MODALITY_IMAGE],
|
||||
output_modalities=[mc.MODALITY_IMAGE],
|
||||
capabilities=[mc.CAP_INPAINTING],
|
||||
)
|
||||
) == {"image_editing"}
|
||||
|
||||
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_EMBEDDING)) == {"embeddings"}
|
||||
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_RERANK)) == {"rerank_scoring"}
|
||||
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_MODERATION)) == {"moderation_classification"}
|
||||
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_CLASSIFICATION)) == {"moderation_classification"}
|
||||
|
||||
|
||||
def test_audio_surface_matches_audio_input_or_output_when_capability_is_known():
|
||||
transcription = mc.ModelCapability.build(
|
||||
family=mc.FAMILY_AUDIO,
|
||||
primary_task=mc.TASK_AUDIO_TRANSCRIBE,
|
||||
input_modalities=[mc.MODALITY_AUDIO],
|
||||
output_modalities=[mc.MODALITY_TEXT],
|
||||
capabilities=[mc.CAP_TRANSCRIPTION],
|
||||
)
|
||||
synthesis = mc.ModelCapability.build(
|
||||
family=mc.FAMILY_AUDIO,
|
||||
primary_task=mc.TASK_AUDIO_SYNTHESIZE,
|
||||
input_modalities=[mc.MODALITY_TEXT],
|
||||
output_modalities=[mc.MODALITY_AUDIO],
|
||||
capabilities=[mc.CAP_TTS],
|
||||
)
|
||||
|
||||
assert surfaces(transcription) == {"audio_realtime"}
|
||||
assert surfaces(synthesis) == {"audio_realtime"}
|
||||
|
||||
|
||||
def test_capability_assertion_tracks_claimed_status_separately_from_capability_metadata():
|
||||
assertion = mc.CapabilityAssertion.build(
|
||||
capability="tools",
|
||||
status="claimed",
|
||||
source="provider_reader",
|
||||
confidence="provider_reported",
|
||||
evidence={"field": "supported_parameters"},
|
||||
)
|
||||
|
||||
assert assertion.capability == mc.CAP_TOOL_CALL
|
||||
assert assertion.status == mc.ASSERTION_CLAIMED
|
||||
assert assertion.source == mc.SOURCE_PROVIDER_READER
|
||||
assert assertion.confidence == mc.CONFIDENCE_PROVIDER_REPORTED
|
||||
assert assertion.to_dict() == {
|
||||
"capability": mc.CAP_TOOL_CALL,
|
||||
"status": mc.ASSERTION_CLAIMED,
|
||||
"source": mc.SOURCE_PROVIDER_READER,
|
||||
"confidence": mc.CONFIDENCE_PROVIDER_REPORTED,
|
||||
"evidence": {"field": "supported_parameters"},
|
||||
"tested_at": "",
|
||||
}
|
||||
|
||||
|
||||
def test_capability_probe_result_converts_pass_and_fail_to_assertions():
|
||||
passed = mc.CapabilityProbeResult.build(
|
||||
provider="openrouter",
|
||||
endpoint_id="ep-1",
|
||||
model_id="vendor/model",
|
||||
stable_model_id="openrouter|endpoint:ep-1|vendor/model",
|
||||
capability="tool_calls",
|
||||
status="pass",
|
||||
tested_at="2026-06-04T20:00:00Z",
|
||||
request_hash="abc123",
|
||||
response_fingerprint="fp-test",
|
||||
evidence={"contract": "single_fake_tool"},
|
||||
)
|
||||
failed = mc.CapabilityProbeResult.build(
|
||||
provider="openrouter",
|
||||
model_id="vendor/model",
|
||||
capability="vision",
|
||||
status="fail",
|
||||
)
|
||||
|
||||
pass_assertion = passed.to_assertion()
|
||||
fail_assertion = failed.to_assertion()
|
||||
|
||||
assert pass_assertion.capability == mc.CAP_TOOL_CALL
|
||||
assert pass_assertion.status == mc.ASSERTION_VERIFIED
|
||||
assert pass_assertion.source == mc.SOURCE_CAPABILITY_PROBE
|
||||
assert pass_assertion.confidence == mc.CONFIDENCE_EXPLICIT
|
||||
assert pass_assertion.tested_at == "2026-06-04T20:00:00Z"
|
||||
assert dict(pass_assertion.evidence)["request_hash"] == "abc123"
|
||||
assert dict(pass_assertion.evidence)["contract"] == "single_fake_tool"
|
||||
|
||||
assert fail_assertion.capability == mc.CAP_VISION
|
||||
assert fail_assertion.status == mc.ASSERTION_UNSUPPORTED
|
||||
assert fail_assertion.source == mc.SOURCE_CAPABILITY_PROBE
|
||||
|
||||
|
||||
def test_deterministic_controls_are_normalized_as_claims_not_capabilities():
|
||||
controls = mc.deterministic_controls_from_values(
|
||||
["temp", "top-p", "top-k", "seed", "unknown"],
|
||||
source=mc.SOURCE_PROVIDER_READER,
|
||||
)
|
||||
|
||||
assert [control.control for control in controls] == [
|
||||
mc.CONTROL_TEMPERATURE,
|
||||
mc.CONTROL_TOP_P,
|
||||
mc.CONTROL_TOP_K,
|
||||
mc.CONTROL_SEED,
|
||||
]
|
||||
assert {control.status for control in controls} == {mc.ASSERTION_CLAIMED}
|
||||
assert {control.source for control in controls} == {mc.SOURCE_PROVIDER_READER}
|
||||
|
||||
|
||||
def test_reasoning_control_mechanisms_normalize_known_provider_shapes():
|
||||
values = [
|
||||
"think_directive",
|
||||
"system_prompt_directive",
|
||||
"enable_thinking",
|
||||
"think_bool",
|
||||
"reasoning_object",
|
||||
"thinking_budget",
|
||||
"reasoning_effort",
|
||||
]
|
||||
|
||||
assert [mc.normalize_reasoning_control_mechanism(value) for value in values] == [
|
||||
mc.REASONING_CONTROL_MESSAGE_DIRECTIVE,
|
||||
mc.REASONING_CONTROL_SYSTEM_DIRECTIVE,
|
||||
mc.REASONING_CONTROL_TEMPLATE_KWARG,
|
||||
mc.REASONING_CONTROL_NATIVE_BOOL,
|
||||
mc.REASONING_CONTROL_STRUCTURED_OBJECT,
|
||||
mc.REASONING_CONTROL_BUDGET,
|
||||
mc.REASONING_CONTROL_EFFORT,
|
||||
]
|
||||
|
||||
|
||||
def test_reasoning_control_values_can_describe_provider_supported_auto():
|
||||
values = ["enabled", "disabled", "adaptive", "dynamic", "provider_auto"]
|
||||
|
||||
assert [mc.normalize_reasoning_control_value(value) for value in values] == [
|
||||
mc.REASONING_CONTROL_VALUE_ON,
|
||||
mc.REASONING_CONTROL_VALUE_OFF,
|
||||
mc.REASONING_CONTROL_VALUE_AUTO,
|
||||
mc.REASONING_CONTROL_VALUE_AUTO,
|
||||
mc.REASONING_CONTROL_VALUE_AUTO,
|
||||
]
|
||||
assert mc.normalize_reasoning_control_value("message_directive") == ""
|
||||
@@ -0,0 +1,646 @@
|
||||
import src.model_capabilities as mc
|
||||
import src.model_capability_readers as readers
|
||||
from src.model_capability_readers import generic_openai, google, llamacpp, lmstudio, ollama, openai, openrouter
|
||||
from src.model_capability_readers.base import (
|
||||
VENDOR_GENERIC_OPENAI,
|
||||
VENDOR_GOOGLE,
|
||||
VENDOR_LLAMACPP,
|
||||
VENDOR_LMSTUDIO,
|
||||
VENDOR_OLLAMA,
|
||||
VENDOR_OPENAI,
|
||||
VENDOR_OPENROUTER,
|
||||
detect_vendor,
|
||||
stable_model_id_for,
|
||||
)
|
||||
|
||||
|
||||
def surfaces(record):
|
||||
return set(mc.display_surfaces_for(record.capability))
|
||||
|
||||
|
||||
def test_detect_vendor_uses_endpoint_kind_then_host_and_common_local_ports():
|
||||
assert detect_vendor("https://example.test/v1", endpoint_kind="ollama") == VENDOR_OLLAMA
|
||||
assert detect_vendor("http://127.0.0.1:8080", endpoint_kind="llama_cpp") == VENDOR_LLAMACPP
|
||||
assert detect_vendor("https://openrouter.ai/api/v1") == VENDOR_OPENROUTER
|
||||
assert detect_vendor("https://api.openai.com/v1") == VENDOR_OPENAI
|
||||
assert detect_vendor("https://generativelanguage.googleapis.com/v1beta/openai") == VENDOR_GOOGLE
|
||||
assert detect_vendor("http://127.0.0.1:11434") == VENDOR_OLLAMA
|
||||
assert detect_vendor("http://127.0.0.1:1234") == VENDOR_LMSTUDIO
|
||||
assert detect_vendor("http://127.0.0.1:8080") == VENDOR_GENERIC_OPENAI
|
||||
assert detect_vendor("http://localhost:7000/v1") == VENDOR_GENERIC_OPENAI
|
||||
|
||||
|
||||
def test_generic_openai_reader_keeps_basic_model_payload_unknown():
|
||||
records = generic_openai.records_from_payload(
|
||||
{
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": "gpt-example", "object": "model", "owned_by": "vendor"},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
record = records[0]
|
||||
assert record.vendor == VENDOR_GENERIC_OPENAI
|
||||
assert record.model_id == "gpt-example"
|
||||
assert record.capability.family == mc.FAMILY_UNKNOWN
|
||||
assert record.capability.source == mc.SOURCE_PROVIDER_READER
|
||||
assert record.capability.confidence == mc.CONFIDENCE_UNKNOWN
|
||||
assert record.stable_model_id == "generic_openai|global|gpt-example"
|
||||
assert record.capability_assertions == ()
|
||||
assert record.deterministic_controls == ()
|
||||
assert surfaces(record) == set()
|
||||
|
||||
|
||||
def test_stable_model_id_is_endpoint_scoped_for_local_or_configured_servers():
|
||||
assert stable_model_id_for("ollama", "qwen:latest", endpoint_id="7") == "ollama|endpoint:7|qwen:latest"
|
||||
assert stable_model_id_for("ollama", "qwen:latest", base_url="http://127.0.0.1:11434") != stable_model_id_for(
|
||||
"ollama",
|
||||
"qwen:latest",
|
||||
base_url="http://10.0.0.12:11434",
|
||||
)
|
||||
|
||||
|
||||
def test_registry_uses_openai_reader_for_openai_vendor():
|
||||
records = readers.records_from_payload({"data": [{"id": "shape-only-model"}]}, vendor=VENDOR_OPENAI)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0].vendor == VENDOR_OPENAI
|
||||
assert records[0].stable_model_id == "openai|global|shape-only-model"
|
||||
assert records[0].capability.family == mc.FAMILY_UNKNOWN
|
||||
|
||||
|
||||
def test_openai_reader_keeps_official_model_shape_identity_only():
|
||||
records = openai.records_from_payload(
|
||||
{
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "shape-only-model",
|
||||
"object": "model",
|
||||
"created": 1700000000,
|
||||
"owned_by": "openai",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
record = records[0]
|
||||
assert record.vendor == VENDOR_OPENAI
|
||||
assert record.model_id == "shape-only-model"
|
||||
assert record.capability.family == mc.FAMILY_UNKNOWN
|
||||
assert record.capability.source == mc.SOURCE_PROVIDER_READER
|
||||
assert record.capability.confidence == mc.CONFIDENCE_UNKNOWN
|
||||
assert record.capability_assertions == ()
|
||||
assert record.deterministic_controls == ()
|
||||
assert surfaces(record) == set()
|
||||
|
||||
|
||||
def test_registry_passes_endpoint_context_to_vendor_reader():
|
||||
records = readers.records_from_payload(
|
||||
{"data": [{"id": "local.gguf", "owned_by": "llamacpp"}]},
|
||||
vendor=VENDOR_LLAMACPP,
|
||||
base_url="http://localhost:8000",
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0].stable_model_id == stable_model_id_for(
|
||||
VENDOR_LLAMACPP,
|
||||
"local.gguf",
|
||||
base_url="http://localhost:8000",
|
||||
)
|
||||
|
||||
|
||||
def test_openrouter_reader_maps_rich_architecture_and_supported_parameters():
|
||||
records = openrouter.records_from_payload(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "google/gemini-vision",
|
||||
"name": "Gemini Vision",
|
||||
"architecture": {"modality": "text+image->text"},
|
||||
"supported_parameters": [
|
||||
"tools",
|
||||
"response_format",
|
||||
"reasoning",
|
||||
"include_reasoning",
|
||||
"parallel_tool_calls",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"seed",
|
||||
],
|
||||
"context_length": 1048576,
|
||||
"top_provider": {"max_completion_tokens": 65536},
|
||||
},
|
||||
{
|
||||
"id": "black-forest-labs/flux",
|
||||
"architecture": {"input_modalities": ["text"], "output_modalities": ["image"]},
|
||||
},
|
||||
{
|
||||
"id": "vendor/image-edit-shape",
|
||||
"architecture": {"input_modalities": ["text", "image", "file"], "output_modalities": ["text", "image"]},
|
||||
"supported_parameters": ["structured_outputs", "web_search_options"],
|
||||
},
|
||||
{
|
||||
"id": "vendor/audio-shape",
|
||||
"architecture": {"input_modalities": ["text", "audio"], "output_modalities": ["text", "audio"]},
|
||||
"supported_voices": ["alloy"],
|
||||
"default_parameters": {"temperature": 0.7, "top_p": 0.9, "top_k": None},
|
||||
"per_request_limits": {"prompt_tokens": 12000, "completion_tokens": 4000, "requests": "2"},
|
||||
},
|
||||
{
|
||||
"id": "vendor/embedder",
|
||||
"architecture": {"modality": "text->embedding"},
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert [record.model_id for record in records] == [
|
||||
"google/gemini-vision",
|
||||
"black-forest-labs/flux",
|
||||
"vendor/image-edit-shape",
|
||||
"vendor/audio-shape",
|
||||
"vendor/embedder",
|
||||
]
|
||||
|
||||
vision = records[0]
|
||||
assert vision.capability.family == mc.FAMILY_CHAT
|
||||
assert vision.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE)
|
||||
assert vision.capability.capabilities == (
|
||||
mc.CAP_TOOL_CALL,
|
||||
mc.CAP_JSON_MODE,
|
||||
mc.CAP_REASONING,
|
||||
mc.CAP_VISION,
|
||||
)
|
||||
assert [(assertion.capability, assertion.status) for assertion in vision.capability_assertions] == [
|
||||
(mc.CAP_TOOL_CALL, mc.ASSERTION_CLAIMED),
|
||||
(mc.CAP_JSON_MODE, mc.ASSERTION_CLAIMED),
|
||||
(mc.CAP_REASONING, mc.ASSERTION_CLAIMED),
|
||||
(mc.CAP_VISION, mc.ASSERTION_CLAIMED),
|
||||
]
|
||||
assert [(control.control, control.status) for control in vision.deterministic_controls] == [
|
||||
(mc.CONTROL_TEMPERATURE, mc.ASSERTION_CLAIMED),
|
||||
(mc.CONTROL_TOP_P, mc.ASSERTION_CLAIMED),
|
||||
(mc.CONTROL_SEED, mc.ASSERTION_CLAIMED),
|
||||
]
|
||||
assert dict(vision.capability.limits) == {"context_tokens": 1048576, "output_tokens": 65536}
|
||||
assert surfaces(vision) == {"chat", "vision_chat"}
|
||||
|
||||
assert records[1].capability.family == mc.FAMILY_IMAGE
|
||||
assert records[1].capability.capabilities == (mc.CAP_IMAGE_GENERATION,)
|
||||
assert surfaces(records[1]) == {"image_generation"}
|
||||
|
||||
image_edit = records[2]
|
||||
assert image_edit.capability.family == mc.FAMILY_IMAGE
|
||||
assert image_edit.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE, mc.MODALITY_FILE)
|
||||
assert image_edit.capability.modalities.output == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE)
|
||||
assert image_edit.capability.capabilities == (
|
||||
mc.CAP_STRUCTURED_OUTPUT,
|
||||
mc.CAP_WEB_SEARCH,
|
||||
mc.CAP_VISION,
|
||||
mc.CAP_FILES,
|
||||
mc.CAP_IMAGE_GENERATION,
|
||||
mc.CAP_IMAGE_EDITING,
|
||||
)
|
||||
assert surfaces(image_edit) == {"image_generation", "image_editing"}
|
||||
|
||||
audio = records[3]
|
||||
assert audio.capability.family == mc.FAMILY_AUDIO
|
||||
assert audio.capability.capabilities == (mc.CAP_AUDIO_INPUT, mc.CAP_AUDIO_OUTPUT, mc.CAP_TTS)
|
||||
assert dict(audio.capability.limits) == {
|
||||
"per_request_completion_tokens": 4000,
|
||||
"per_request_prompt_tokens": 12000,
|
||||
"per_request_requests": 2,
|
||||
}
|
||||
assert [control.control for control in audio.deterministic_controls] == [
|
||||
mc.CONTROL_TEMPERATURE,
|
||||
mc.CONTROL_TOP_P,
|
||||
]
|
||||
assert surfaces(audio) == {"audio_realtime"}
|
||||
|
||||
assert records[4].capability.family == mc.FAMILY_EMBEDDING
|
||||
assert surfaces(records[4]) == {"embeddings"}
|
||||
|
||||
|
||||
def test_google_reader_maps_provider_fields_without_claiming_unreported_modalities():
|
||||
records = google.records_from_payload(
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
"name": "models/gemini-3.1-flash-image",
|
||||
"displayName": "Gemini 3.1 Flash Image",
|
||||
"supportedGenerationMethods": ["generateContent"],
|
||||
"inputTokenLimit": 1000000,
|
||||
"outputTokenLimit": 8192,
|
||||
"thinking": True,
|
||||
"temperature": 1.0,
|
||||
"topP": 0.95,
|
||||
"topK": 40,
|
||||
},
|
||||
{
|
||||
"name": "models/text-embedding-example",
|
||||
"supportedGenerationMethods": ["embedContent"],
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert len(records) == 2
|
||||
content = records[0]
|
||||
assert content.vendor == VENDOR_GOOGLE
|
||||
assert content.model_id == "gemini-3.1-flash-image"
|
||||
assert content.capability.family == mc.FAMILY_UNKNOWN
|
||||
assert content.capability.modalities.input == ()
|
||||
assert content.capability.modalities.output == ()
|
||||
assert content.capability.capabilities == (mc.CAP_REASONING,)
|
||||
assert dict(content.capability.limits) == {
|
||||
"context_tokens": 1000000,
|
||||
"input_tokens": 1000000,
|
||||
"output_tokens": 8192,
|
||||
}
|
||||
assert [control.control for control in content.deterministic_controls] == [
|
||||
mc.CONTROL_TEMPERATURE,
|
||||
mc.CONTROL_TOP_P,
|
||||
mc.CONTROL_TOP_K,
|
||||
]
|
||||
assert surfaces(content) == set()
|
||||
|
||||
embedding = records[1]
|
||||
assert embedding.capability.family == mc.FAMILY_EMBEDDING
|
||||
assert surfaces(embedding) == {"embeddings"}
|
||||
|
||||
|
||||
def test_google_ai_studio_mapping_does_not_infer_media_from_model_names():
|
||||
records = google.records_from_payload(
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
"name": "models/imagen-4.0-generate-001",
|
||||
"displayName": "Imagen 4",
|
||||
"supportedGenerationMethods": ["predict"],
|
||||
},
|
||||
{
|
||||
"name": "models/veo-3.1-generate-preview",
|
||||
"displayName": "Veo 3.1",
|
||||
"supportedGenerationMethods": ["predictLongRunning"],
|
||||
},
|
||||
{
|
||||
"name": "models/gemini-3.1-flash-tts-preview",
|
||||
"supportedGenerationMethods": ["generateContent", "countTokens", "createCachedContent", "batchGenerateContent"],
|
||||
},
|
||||
{
|
||||
"name": "models/lyria-3-pro-preview",
|
||||
"displayName": "Lyria 3 Pro Preview",
|
||||
"supportedGenerationMethods": ["generateContent", "countTokens"],
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert len(records) == 4
|
||||
assert [record.capability.family for record in records] == [
|
||||
mc.FAMILY_UNKNOWN,
|
||||
mc.FAMILY_UNKNOWN,
|
||||
mc.FAMILY_UNKNOWN,
|
||||
mc.FAMILY_UNKNOWN,
|
||||
]
|
||||
assert all(record.capability.modalities.input == () for record in records)
|
||||
assert all(record.capability.modalities.output == () for record in records)
|
||||
assert all(surfaces(record) == set() for record in records)
|
||||
assert [control.control for control in records[2].deterministic_controls] == [
|
||||
mc.CONTROL_PROMPT_CACHING,
|
||||
mc.CONTROL_BATCH,
|
||||
]
|
||||
|
||||
|
||||
def test_google_ai_studio_mapping_keeps_unrecognized_predict_models_unknown():
|
||||
records = google.records_from_payload(
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
"name": "models/vendor-future-media-001",
|
||||
"supportedGenerationMethods": ["predict"],
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0].capability.family == mc.FAMILY_UNKNOWN
|
||||
assert surfaces(records[0]) == set()
|
||||
|
||||
|
||||
def test_ollama_reader_maps_show_capabilities_and_tags_are_unknown():
|
||||
vision = ollama.record_from_show_payload(
|
||||
"llava:latest",
|
||||
{
|
||||
"capabilities": ["completion", "vision", "tools"],
|
||||
"model_info": {"llama.context_length": 4096},
|
||||
},
|
||||
)
|
||||
embedding = ollama.record_from_show_payload(
|
||||
"nomic-embed-text:latest",
|
||||
{"capabilities": ["embedding"]},
|
||||
)
|
||||
tags = ollama.records_from_tags_payload({"models": [{"name": "qwen3:latest"}]})
|
||||
|
||||
assert vision is not None
|
||||
assert vision.capability.family == mc.FAMILY_CHAT
|
||||
assert vision.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE)
|
||||
assert vision.capability.capabilities == (mc.CAP_VISION, mc.CAP_TOOL_CALL)
|
||||
assert dict(vision.capability.limits) == {"context_tokens": 4096}
|
||||
assert surfaces(vision) == {"chat", "vision_chat"}
|
||||
|
||||
assert embedding is not None
|
||||
assert embedding.capability.family == mc.FAMILY_EMBEDDING
|
||||
assert surfaces(embedding) == {"embeddings"}
|
||||
|
||||
assert len(tags) == 1
|
||||
assert tags[0].capability.family == mc.FAMILY_UNKNOWN
|
||||
assert surfaces(tags[0]) == set()
|
||||
|
||||
|
||||
def test_ollama_reader_uses_show_shape_without_architecture_name_matching():
|
||||
record = ollama.record_from_show_payload(
|
||||
"local:latest",
|
||||
{
|
||||
"capabilities": ["completion", "thinking", "tools"],
|
||||
"parameters": "temperature 0.7\nnum_ctx 8192",
|
||||
"model_info": {
|
||||
"future_architecture.context_length": 32768,
|
||||
"future_architecture.embedding_length": 4096,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert record is not None
|
||||
assert record.capability.family == mc.FAMILY_CHAT
|
||||
assert record.capability.modalities.input == (mc.MODALITY_TEXT,)
|
||||
assert record.capability.modalities.output == (mc.MODALITY_TEXT,)
|
||||
assert record.capability.capabilities == (mc.CAP_REASONING, mc.CAP_TOOL_CALL)
|
||||
assert dict(record.capability.limits) == {"context_tokens": 8192}
|
||||
assert surfaces(record) == {"chat"}
|
||||
|
||||
|
||||
def test_ollama_reader_uses_generic_model_info_context_length_when_no_num_ctx():
|
||||
record = ollama.record_from_show_payload(
|
||||
"local:latest",
|
||||
{
|
||||
"capabilities": ["completion"],
|
||||
"model_info": {"future_architecture.context_length": 32768},
|
||||
},
|
||||
)
|
||||
|
||||
assert record is not None
|
||||
assert record.capability.family == mc.FAMILY_CHAT
|
||||
assert dict(record.capability.limits) == {"context_tokens": 32768}
|
||||
|
||||
|
||||
def test_lmstudio_reader_uses_native_v1_capabilities_when_present():
|
||||
records = lmstudio.records_from_payload(
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
"type": "llm",
|
||||
"key": "google/gemma-vl",
|
||||
"display_name": "Gemma VL",
|
||||
"capabilities": {
|
||||
"vision": True,
|
||||
"trained_for_tool_use": True,
|
||||
"reasoning": {"allowed_options": ["off", "on"], "default": "on"},
|
||||
},
|
||||
"loaded_instances": [
|
||||
{"config": {"context_length": 8192}},
|
||||
{"config": {"context_length": 4096}},
|
||||
],
|
||||
"max_context_length": 262144,
|
||||
},
|
||||
{
|
||||
"type": "embedding",
|
||||
"key": "nomic/embed",
|
||||
},
|
||||
{"key": "shape-without-type"},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert len(records) == 3
|
||||
vision = records[0]
|
||||
assert vision.vendor == VENDOR_LMSTUDIO
|
||||
assert vision.model_id == "google/gemma-vl"
|
||||
assert vision.display_name == "Gemma VL"
|
||||
assert vision.capability.family == mc.FAMILY_CHAT
|
||||
assert vision.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE)
|
||||
assert vision.capability.capabilities == (mc.CAP_VISION, mc.CAP_TOOL_CALL, mc.CAP_REASONING)
|
||||
assert dict(vision.capability.limits) == {"context_tokens": 4096, "max_context_tokens": 262144}
|
||||
assert surfaces(vision) == {"chat", "vision_chat"}
|
||||
|
||||
assert records[1].capability.family == mc.FAMILY_EMBEDDING
|
||||
assert surfaces(records[1]) == {"embeddings"}
|
||||
|
||||
assert records[2].capability.family == mc.FAMILY_UNKNOWN
|
||||
assert surfaces(records[2]) == set()
|
||||
|
||||
|
||||
def test_lmstudio_reader_uses_legacy_native_v0_shape_for_family_and_limits():
|
||||
records = lmstudio.records_from_payload(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "local-gemma",
|
||||
"type": "llm",
|
||||
"arch": "gemma3",
|
||||
"loaded_context_length": 16384,
|
||||
"max_context_length": 32768,
|
||||
},
|
||||
{
|
||||
"id": "text-embedding-local",
|
||||
"type": "embeddings",
|
||||
"max_context_length": 2048,
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert len(records) == 2
|
||||
chat = records[0]
|
||||
assert chat.vendor == VENDOR_LMSTUDIO
|
||||
assert chat.capability.family == mc.FAMILY_CHAT
|
||||
assert chat.capability.modalities.input == (mc.MODALITY_TEXT,)
|
||||
assert chat.capability.capabilities == ()
|
||||
assert dict(chat.capability.limits) == {"context_tokens": 16384, "max_context_tokens": 32768}
|
||||
assert surfaces(chat) == {"chat"}
|
||||
|
||||
assert records[1].capability.family == mc.FAMILY_EMBEDDING
|
||||
assert dict(records[1].capability.limits) == {"context_tokens": 2048}
|
||||
assert surfaces(records[1]) == {"embeddings"}
|
||||
|
||||
|
||||
def test_lmstudio_openai_compatible_model_list_remains_identity_only():
|
||||
records = lmstudio.records_from_payload(
|
||||
{
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": "local-gemma-3-270m-it-qat-q4_k_m", "object": "model", "owned_by": "organization_owner"},
|
||||
{"id": "text-embedding-nomic-embed-text-v1.5", "object": "model", "owned_by": "organization_owner"},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
assert len(records) == 2
|
||||
for record in records:
|
||||
assert record.vendor == VENDOR_LMSTUDIO
|
||||
assert record.capability.family == mc.FAMILY_UNKNOWN
|
||||
assert record.capability_assertions == ()
|
||||
assert surfaces(record) == set()
|
||||
|
||||
|
||||
def test_lmstudio_unexpected_native_endpoint_error_yields_no_records():
|
||||
assert lmstudio.records_from_payload({"error": "Unexpected endpoint or method. (GET /api/v1/models)"}) == ()
|
||||
|
||||
|
||||
def test_llamacpp_reader_merges_models_props_and_slots_payloads():
|
||||
models_payload = {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "gemma-4-E2B-it-Q8_0.gguf",
|
||||
"owned_by": "llamacpp",
|
||||
"meta": {
|
||||
"n_ctx_train": 131072,
|
||||
"n_params": 4647450147,
|
||||
"size": 5032532108,
|
||||
},
|
||||
}
|
||||
],
|
||||
"models": [
|
||||
{
|
||||
"name": "gemma-4-E2B-it-Q8_0.gguf",
|
||||
"model": "gemma-4-E2B-it-Q8_0.gguf",
|
||||
"capabilities": ["completion"],
|
||||
"details": {"format": "gguf"},
|
||||
}
|
||||
],
|
||||
}
|
||||
props_payload = {
|
||||
"model_alias": "gemma-4-E2B-it-Q8_0.gguf",
|
||||
"model_path": "/models/gemma-4-E2B-it-Q8_0.gguf",
|
||||
"build_info": "b1-c8ac02f",
|
||||
"total_slots": 4,
|
||||
"modalities": {"vision": False, "audio": False},
|
||||
"chat_template_caps": {
|
||||
"supports_object_arguments": True,
|
||||
"supports_parallel_tool_calls": True,
|
||||
"supports_preserve_reasoning": False,
|
||||
"supports_string_content": True,
|
||||
"supports_system_role": True,
|
||||
"supports_tool_calls": True,
|
||||
"supports_tools": True,
|
||||
"supports_typed_content": False,
|
||||
},
|
||||
"default_generation_settings": {
|
||||
"n_ctx": 16384,
|
||||
"params": {
|
||||
"temperature": 1.0,
|
||||
"top_p": 0.95,
|
||||
"seed": 4294967295,
|
||||
"stream": True,
|
||||
"samplers": ["top_p", "temperature"],
|
||||
},
|
||||
},
|
||||
}
|
||||
slots_payload = [
|
||||
{"id": 0, "n_ctx": 16384, "speculative": False, "is_processing": False},
|
||||
{"id": 1, "n_ctx": 16384, "speculative": False, "is_processing": False},
|
||||
{"id": 2, "n_ctx": 16384, "speculative": False, "is_processing": False},
|
||||
{"id": 3, "n_ctx": 16384, "speculative": False, "is_processing": False},
|
||||
]
|
||||
|
||||
records = llamacpp.records_from_payloads(
|
||||
models_payload=models_payload,
|
||||
props_payload=props_payload,
|
||||
slots_payload=slots_payload,
|
||||
base_url="http://localhost:8000",
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
record = records[0]
|
||||
assert record.vendor == VENDOR_LLAMACPP
|
||||
assert record.model_id == "gemma-4-E2B-it-Q8_0.gguf"
|
||||
assert record.stable_model_id == stable_model_id_for(
|
||||
VENDOR_LLAMACPP,
|
||||
"gemma-4-E2B-it-Q8_0.gguf",
|
||||
base_url="http://localhost:8000",
|
||||
)
|
||||
assert record.capability.family == mc.FAMILY_CHAT
|
||||
assert record.capability.modalities.input == (mc.MODALITY_TEXT,)
|
||||
assert record.capability.modalities.output == (mc.MODALITY_TEXT,)
|
||||
assert record.capability.capabilities == (mc.CAP_TOOL_CALL, mc.CAP_STREAMING)
|
||||
assert dict(record.capability.limits) == {
|
||||
"context_tokens": 16384,
|
||||
"model_bytes": 5032532108,
|
||||
"parallel_slots": 4,
|
||||
"parameters": 4647450147,
|
||||
"training_context_tokens": 131072,
|
||||
}
|
||||
assert surfaces(record) == {"chat"}
|
||||
|
||||
assertion_status = {assertion.capability: assertion.status for assertion in record.capability_assertions}
|
||||
assert assertion_status[mc.CAP_TOOL_CALL] == mc.ASSERTION_CLAIMED
|
||||
assert assertion_status[mc.CAP_STREAMING] == mc.ASSERTION_CLAIMED
|
||||
assert assertion_status[mc.CAP_VISION] == mc.ASSERTION_UNSUPPORTED
|
||||
assert assertion_status[mc.CAP_AUDIO_INPUT] == mc.ASSERTION_UNSUPPORTED
|
||||
assert mc.CAP_REASONING not in assertion_status
|
||||
|
||||
controls = {control.control: control.status for control in record.deterministic_controls}
|
||||
assert controls == {
|
||||
mc.CONTROL_TEMPERATURE: mc.ASSERTION_CLAIMED,
|
||||
mc.CONTROL_TOP_P: mc.ASSERTION_CLAIMED,
|
||||
mc.CONTROL_SEED: mc.ASSERTION_CLAIMED,
|
||||
mc.CONTROL_SYSTEM_PROMPT: mc.ASSERTION_CLAIMED,
|
||||
mc.CONTROL_TOOL_CHOICE: mc.ASSERTION_CLAIMED,
|
||||
}
|
||||
|
||||
|
||||
def test_llamacpp_openai_model_list_without_native_capability_shape_stays_unknown():
|
||||
records = llamacpp.records_from_payload(
|
||||
{
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "local-chat.gguf",
|
||||
"owned_by": "llamacpp",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0].capability.family == mc.FAMILY_UNKNOWN
|
||||
assert records[0].capability.capabilities == ()
|
||||
assert records[0].capability_assertions == ()
|
||||
assert surfaces(records[0]) == set()
|
||||
|
||||
|
||||
def test_llamacpp_props_payload_reports_unsupported_modalities_without_model_list():
|
||||
records = llamacpp.records_from_payload(
|
||||
{
|
||||
"model_alias": "local.gguf",
|
||||
"modalities": {"vision": False, "audio": False},
|
||||
"chat_template_caps": {"supports_tools": False, "supports_preserve_reasoning": False},
|
||||
"default_generation_settings": {"n_ctx": 4096, "params": {"stream": True}},
|
||||
}
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
record = records[0]
|
||||
assert record.capability.family == mc.FAMILY_CHAT
|
||||
assert record.capability.capabilities == (mc.CAP_STREAMING,)
|
||||
assert {a.capability: a.status for a in record.capability_assertions} == {
|
||||
mc.CAP_STREAMING: mc.ASSERTION_CLAIMED,
|
||||
mc.CAP_VISION: mc.ASSERTION_UNSUPPORTED,
|
||||
mc.CAP_AUDIO_INPUT: mc.ASSERTION_UNSUPPORTED,
|
||||
}
|
||||
@@ -48,6 +48,9 @@ with preserve_import_state("core.database", "src.database", "core.session_manage
|
||||
_ping_endpoint,
|
||||
_parse_model_list,
|
||||
_normalize_refresh_mode,
|
||||
_normalize_endpoint_refresh_mode,
|
||||
_endpoint_refresh_mode,
|
||||
_is_google_api_base,
|
||||
_truthy,
|
||||
_speech_settings_using_endpoint,
|
||||
_clear_speech_settings_for_endpoint,
|
||||
@@ -464,6 +467,28 @@ class TestClassifyEndpoint:
|
||||
assert _normalize_refresh_mode("manual", "proxy") == "manual"
|
||||
assert _normalize_refresh_mode("auto", "api") == "auto"
|
||||
|
||||
def test_google_refresh_mode_defaults_manual_unless_explicit(self):
|
||||
base = "https://generativelanguage.googleapis.com/v1beta/openai"
|
||||
assert _normalize_endpoint_refresh_mode("", "api", base) == "manual"
|
||||
assert _normalize_endpoint_refresh_mode(None, "auto", base) == "manual"
|
||||
assert _normalize_endpoint_refresh_mode("auto", "api", base) == "auto"
|
||||
|
||||
def test_only_gemini_native_host_uses_google_models_api(self):
|
||||
assert _is_google_api_base("https://generativelanguage.googleapis.com/v1beta/openai") is True
|
||||
assert _is_google_api_base(
|
||||
"https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/us-central1/endpoints/openapi"
|
||||
) is False
|
||||
|
||||
def test_existing_google_endpoint_refresh_mode_defaults_manual(self):
|
||||
ep = SimpleNamespace(
|
||||
model_refresh_mode=None,
|
||||
endpoint_kind="api",
|
||||
base_url="https://generativelanguage.googleapis.com/v1beta/openai",
|
||||
)
|
||||
assert _endpoint_refresh_mode(ep, "api") == "manual"
|
||||
ep.model_refresh_mode = "auto"
|
||||
assert _endpoint_refresh_mode(ep, "api") == "auto"
|
||||
|
||||
def test_parse_model_list_accepts_json_and_text(self):
|
||||
assert _parse_model_list('["a", "b", "a"]') == ["a", "b"]
|
||||
assert _parse_model_list("a, b\nc") == ["a", "b", "c"]
|
||||
@@ -568,6 +593,83 @@ class TestSetupProbeSafety:
|
||||
|
||||
assert _probe_endpoint("https://api.groq.com/openai/v1") == _PROVIDER_CURATED["groq"]
|
||||
|
||||
def test_google_probe_uses_native_paginated_models_api(self, monkeypatch):
|
||||
monkeypatch.setattr(endpoint_resolver, "resolve_url", lambda url: url, raising=False)
|
||||
monkeypatch.setattr(model_routes, "_normalize_base", lambda url: url.rstrip("/"))
|
||||
seen = []
|
||||
|
||||
def fake_get(url, headers=None, params=None, timeout=None, verify=None, **kwargs):
|
||||
seen.append((url, headers, params, timeout, verify))
|
||||
request = httpx.Request("GET", url)
|
||||
page_token = (params or {}).get("pageToken")
|
||||
if page_token:
|
||||
return httpx.Response(
|
||||
200,
|
||||
request=request,
|
||||
json={
|
||||
"models": [{
|
||||
"name": "models/gemini-page-two",
|
||||
"supportedGenerationMethods": ["generateContent"],
|
||||
}]
|
||||
},
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
request=request,
|
||||
json={
|
||||
"models": [
|
||||
{
|
||||
"name": "models/gemini-page-one",
|
||||
"supportedGenerationMethods": ["generateContent"],
|
||||
},
|
||||
{
|
||||
"baseModelId": "gemini-base-id",
|
||||
"name": "models/ignored-version",
|
||||
"supportedGenerationMethods": ["generateText"],
|
||||
},
|
||||
{
|
||||
"name": "models/imagen-4.0-generate-001",
|
||||
"supportedGenerationMethods": ["predict"],
|
||||
},
|
||||
{
|
||||
"name": "models/text-embedding-example",
|
||||
"supportedGenerationMethods": ["embedContent"],
|
||||
},
|
||||
{"name": "models/missing-method-metadata"},
|
||||
],
|
||||
"nextPageToken": "next-page",
|
||||
},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(model_routes.httpx, "get", fake_get)
|
||||
|
||||
assert _probe_endpoint("https://generativelanguage.googleapis.com/v1beta/openai", "google-key") == [
|
||||
"gemini-page-one",
|
||||
"gemini-base-id",
|
||||
"gemini-page-two",
|
||||
]
|
||||
assert [call[0] for call in seen] == [
|
||||
"https://generativelanguage.googleapis.com/v1beta/models",
|
||||
"https://generativelanguage.googleapis.com/v1beta/models",
|
||||
]
|
||||
assert seen[0][1] == {"Accept": "application/json", "x-goog-api-key": "google-key"}
|
||||
assert seen[0][2] == {"pageSize": 1000}
|
||||
assert seen[1][1] == {"Accept": "application/json", "x-goog-api-key": "google-key"}
|
||||
assert seen[1][2] == {"pageSize": 1000, "pageToken": "next-page"}
|
||||
|
||||
def test_google_probe_does_not_use_curated_fallback_on_failure(self, monkeypatch):
|
||||
monkeypatch.setattr(endpoint_resolver, "resolve_url", lambda url: url, raising=False)
|
||||
monkeypatch.setattr(model_routes, "_normalize_base", lambda url: url.rstrip("/"))
|
||||
|
||||
def fake_get(url, headers=None, params=None, timeout=None, verify=None, **kwargs):
|
||||
request = httpx.Request("GET", url)
|
||||
response = httpx.Response(401, request=request)
|
||||
raise httpx.HTTPStatusError("unauthorized", request=request, response=response)
|
||||
|
||||
monkeypatch.setattr(model_routes.httpx, "get", fake_get)
|
||||
|
||||
assert _probe_endpoint("https://generativelanguage.googleapis.com/v1beta/openai", "bad-key") == []
|
||||
|
||||
def test_keyed_anthropic_probe_does_not_fallback_on_failure(self, monkeypatch):
|
||||
monkeypatch.setattr(endpoint_resolver, "resolve_url", lambda url: url, raising=False)
|
||||
monkeypatch.setattr(model_routes, "_normalize_base", lambda url: url.rstrip("/"))
|
||||
@@ -1101,6 +1203,26 @@ def test_post_creates_endpoint_with_pinned_models(monkeypatch):
|
||||
assert json.loads(db.added[0].pinned_models) == ["deploy-1", "deploy-2"]
|
||||
|
||||
|
||||
def test_post_google_endpoint_defaults_to_manual_refresh_when_mode_omitted(monkeypatch):
|
||||
db = _PinnedFakeDb([])
|
||||
_patch_create_deps(monkeypatch, db)
|
||||
monkeypatch.setattr(model_routes, "_probe_endpoint", lambda *args, **kwargs: ["gemini-test"])
|
||||
create = _get_route("/api/model-endpoints", "POST")
|
||||
|
||||
create(
|
||||
_PinnedFakeRequest(),
|
||||
base_url="https://generativelanguage.googleapis.com/v1beta/openai",
|
||||
**_create_form_kwargs(
|
||||
api_key="google-key",
|
||||
endpoint_kind="api",
|
||||
model_refresh_mode="",
|
||||
),
|
||||
)
|
||||
|
||||
assert len(db.added) == 1
|
||||
assert db.added[0].model_refresh_mode == "manual"
|
||||
|
||||
|
||||
def test_post_dedupe_existing_merges_and_returns_pinned(monkeypatch):
|
||||
existing = _make_endpoint(
|
||||
base_url="http://host:1234/v1",
|
||||
|
||||
Reference in New Issue
Block a user