mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-07-19 13:38:04 +00:00
RaresKeY
4f04c347cc
* feat(models): define capability schema and readers * fix(models): harden Google catalog probing Restrict native catalog probing to the Gemini host, keep provider keys out of request URLs, filter non-chat model resources, and preserve the manual refresh default in the built-in Google add flow.
249 lines
9.3 KiB
Python
249 lines
9.3 KiB
Python
import src.model_capabilities as mc
|
|
|
|
|
|
def surfaces(capability):
|
|
return set(mc.display_surfaces_for(capability))
|
|
|
|
|
|
def test_endpoint_type_llm_maps_to_explicit_chat_capability():
|
|
capability = mc.capability_from_endpoint_type("llm")
|
|
|
|
assert capability.family == mc.FAMILY_CHAT
|
|
assert capability.primary_task == mc.TASK_CHAT_COMPLETIONS
|
|
assert capability.modalities.input == (mc.MODALITY_TEXT,)
|
|
assert capability.modalities.output == (mc.MODALITY_TEXT,)
|
|
assert capability.source == mc.SOURCE_ENDPOINT_CONFIG
|
|
assert capability.confidence == mc.CONFIDENCE_EXPLICIT
|
|
assert surfaces(capability) == {"chat"}
|
|
|
|
|
|
def test_endpoint_type_image_maps_to_explicit_image_generation_capability():
|
|
capability = mc.capability_from_endpoint_type("image")
|
|
|
|
assert capability.family == mc.FAMILY_IMAGE
|
|
assert capability.primary_task == mc.TASK_IMAGE_GENERATE
|
|
assert capability.modalities.output == (mc.MODALITY_IMAGE,)
|
|
assert capability.capabilities == (mc.CAP_IMAGE_GENERATION,)
|
|
assert capability.source == mc.SOURCE_ENDPOINT_CONFIG
|
|
assert capability.confidence == mc.CONFIDENCE_EXPLICIT
|
|
assert surfaces(capability) == {"image_generation"}
|
|
|
|
|
|
def test_missing_or_unknown_endpoint_type_does_not_imply_chat():
|
|
for model_type in (None, "", "openai-compatible", "text"):
|
|
capability = mc.capability_from_endpoint_type(model_type)
|
|
|
|
assert capability.family == mc.FAMILY_UNKNOWN
|
|
assert capability.primary_task == mc.TASK_UNKNOWN
|
|
assert capability.source == mc.SOURCE_ENDPOINT_CONFIG
|
|
assert mc.display_surfaces_for(capability) == ()
|
|
|
|
|
|
def test_provider_record_normalizes_aliases_and_boolean_capability_maps():
|
|
capability = mc.ModelCapability.from_dict(
|
|
{
|
|
"family": "llm",
|
|
"modalities": {
|
|
"input": ["text", "images", "docs", "images"],
|
|
"output": "text",
|
|
},
|
|
"capabilities": {
|
|
"tools": True,
|
|
"unknown_vendor_flag": True,
|
|
"vision": True,
|
|
"tts": False,
|
|
},
|
|
"limits": {"max_context_tokens": 32768, "": "ignored"},
|
|
"source": "provider_reader",
|
|
"confidence": "provider_reported",
|
|
}
|
|
)
|
|
|
|
assert capability.family == mc.FAMILY_CHAT
|
|
assert capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE, mc.MODALITY_FILE)
|
|
assert capability.modalities.output == (mc.MODALITY_TEXT,)
|
|
assert capability.capabilities == (mc.CAP_TOOL_CALL, mc.CAP_VISION)
|
|
assert capability.limits == (("max_context_tokens", 32768),)
|
|
assert surfaces(capability) == {"chat", "vision_chat", "document_chat"}
|
|
assert capability.to_dict() == {
|
|
"family": mc.FAMILY_CHAT,
|
|
"primary_task": mc.TASK_CHAT_COMPLETIONS,
|
|
"modalities": {
|
|
"input": [mc.MODALITY_TEXT, mc.MODALITY_IMAGE, mc.MODALITY_FILE],
|
|
"output": [mc.MODALITY_TEXT],
|
|
},
|
|
"capabilities": [mc.CAP_TOOL_CALL, mc.CAP_VISION],
|
|
"limits": {"max_context_tokens": 32768},
|
|
"source": mc.SOURCE_PROVIDER_READER,
|
|
"confidence": mc.CONFIDENCE_PROVIDER_REPORTED,
|
|
}
|
|
|
|
|
|
def test_unknown_or_malformed_capability_record_stays_unknown():
|
|
assert mc.ModelCapability.from_dict(None).to_dict() == mc.unknown_capability().to_dict()
|
|
|
|
capability = mc.ModelCapability.build(
|
|
family="not-real",
|
|
primary_task=1234,
|
|
input_modalities=object(),
|
|
output_modalities=["text", "not-real"],
|
|
capabilities=["vision", "not-real"],
|
|
source="not-real",
|
|
confidence="not-real",
|
|
)
|
|
|
|
assert capability.family == mc.FAMILY_UNKNOWN
|
|
assert capability.primary_task == "1234"
|
|
assert capability.modalities.input == ()
|
|
assert capability.modalities.output == (mc.MODALITY_TEXT,)
|
|
assert capability.capabilities == (mc.CAP_VISION,)
|
|
assert capability.source == mc.SOURCE_UNKNOWN
|
|
assert capability.confidence == mc.CONFIDENCE_UNKNOWN
|
|
assert mc.display_surfaces_for(capability) == ()
|
|
|
|
|
|
def test_display_surface_queries_cover_core_model_categories():
|
|
assert surfaces(
|
|
mc.ModelCapability.build(
|
|
family=mc.FAMILY_IMAGE,
|
|
input_modalities=[mc.MODALITY_IMAGE],
|
|
output_modalities=[mc.MODALITY_IMAGE],
|
|
capabilities=[mc.CAP_INPAINTING],
|
|
)
|
|
) == {"image_editing"}
|
|
|
|
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_EMBEDDING)) == {"embeddings"}
|
|
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_RERANK)) == {"rerank_scoring"}
|
|
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_MODERATION)) == {"moderation_classification"}
|
|
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_CLASSIFICATION)) == {"moderation_classification"}
|
|
|
|
|
|
def test_audio_surface_matches_audio_input_or_output_when_capability_is_known():
|
|
transcription = mc.ModelCapability.build(
|
|
family=mc.FAMILY_AUDIO,
|
|
primary_task=mc.TASK_AUDIO_TRANSCRIBE,
|
|
input_modalities=[mc.MODALITY_AUDIO],
|
|
output_modalities=[mc.MODALITY_TEXT],
|
|
capabilities=[mc.CAP_TRANSCRIPTION],
|
|
)
|
|
synthesis = mc.ModelCapability.build(
|
|
family=mc.FAMILY_AUDIO,
|
|
primary_task=mc.TASK_AUDIO_SYNTHESIZE,
|
|
input_modalities=[mc.MODALITY_TEXT],
|
|
output_modalities=[mc.MODALITY_AUDIO],
|
|
capabilities=[mc.CAP_TTS],
|
|
)
|
|
|
|
assert surfaces(transcription) == {"audio_realtime"}
|
|
assert surfaces(synthesis) == {"audio_realtime"}
|
|
|
|
|
|
def test_capability_assertion_tracks_claimed_status_separately_from_capability_metadata():
|
|
assertion = mc.CapabilityAssertion.build(
|
|
capability="tools",
|
|
status="claimed",
|
|
source="provider_reader",
|
|
confidence="provider_reported",
|
|
evidence={"field": "supported_parameters"},
|
|
)
|
|
|
|
assert assertion.capability == mc.CAP_TOOL_CALL
|
|
assert assertion.status == mc.ASSERTION_CLAIMED
|
|
assert assertion.source == mc.SOURCE_PROVIDER_READER
|
|
assert assertion.confidence == mc.CONFIDENCE_PROVIDER_REPORTED
|
|
assert assertion.to_dict() == {
|
|
"capability": mc.CAP_TOOL_CALL,
|
|
"status": mc.ASSERTION_CLAIMED,
|
|
"source": mc.SOURCE_PROVIDER_READER,
|
|
"confidence": mc.CONFIDENCE_PROVIDER_REPORTED,
|
|
"evidence": {"field": "supported_parameters"},
|
|
"tested_at": "",
|
|
}
|
|
|
|
|
|
def test_capability_probe_result_converts_pass_and_fail_to_assertions():
|
|
passed = mc.CapabilityProbeResult.build(
|
|
provider="openrouter",
|
|
endpoint_id="ep-1",
|
|
model_id="vendor/model",
|
|
stable_model_id="openrouter|endpoint:ep-1|vendor/model",
|
|
capability="tool_calls",
|
|
status="pass",
|
|
tested_at="2026-06-04T20:00:00Z",
|
|
request_hash="abc123",
|
|
response_fingerprint="fp-test",
|
|
evidence={"contract": "single_fake_tool"},
|
|
)
|
|
failed = mc.CapabilityProbeResult.build(
|
|
provider="openrouter",
|
|
model_id="vendor/model",
|
|
capability="vision",
|
|
status="fail",
|
|
)
|
|
|
|
pass_assertion = passed.to_assertion()
|
|
fail_assertion = failed.to_assertion()
|
|
|
|
assert pass_assertion.capability == mc.CAP_TOOL_CALL
|
|
assert pass_assertion.status == mc.ASSERTION_VERIFIED
|
|
assert pass_assertion.source == mc.SOURCE_CAPABILITY_PROBE
|
|
assert pass_assertion.confidence == mc.CONFIDENCE_EXPLICIT
|
|
assert pass_assertion.tested_at == "2026-06-04T20:00:00Z"
|
|
assert dict(pass_assertion.evidence)["request_hash"] == "abc123"
|
|
assert dict(pass_assertion.evidence)["contract"] == "single_fake_tool"
|
|
|
|
assert fail_assertion.capability == mc.CAP_VISION
|
|
assert fail_assertion.status == mc.ASSERTION_UNSUPPORTED
|
|
assert fail_assertion.source == mc.SOURCE_CAPABILITY_PROBE
|
|
|
|
|
|
def test_deterministic_controls_are_normalized_as_claims_not_capabilities():
|
|
controls = mc.deterministic_controls_from_values(
|
|
["temp", "top-p", "top-k", "seed", "unknown"],
|
|
source=mc.SOURCE_PROVIDER_READER,
|
|
)
|
|
|
|
assert [control.control for control in controls] == [
|
|
mc.CONTROL_TEMPERATURE,
|
|
mc.CONTROL_TOP_P,
|
|
mc.CONTROL_TOP_K,
|
|
mc.CONTROL_SEED,
|
|
]
|
|
assert {control.status for control in controls} == {mc.ASSERTION_CLAIMED}
|
|
assert {control.source for control in controls} == {mc.SOURCE_PROVIDER_READER}
|
|
|
|
|
|
def test_reasoning_control_mechanisms_normalize_known_provider_shapes():
|
|
values = [
|
|
"think_directive",
|
|
"system_prompt_directive",
|
|
"enable_thinking",
|
|
"think_bool",
|
|
"reasoning_object",
|
|
"thinking_budget",
|
|
"reasoning_effort",
|
|
]
|
|
|
|
assert [mc.normalize_reasoning_control_mechanism(value) for value in values] == [
|
|
mc.REASONING_CONTROL_MESSAGE_DIRECTIVE,
|
|
mc.REASONING_CONTROL_SYSTEM_DIRECTIVE,
|
|
mc.REASONING_CONTROL_TEMPLATE_KWARG,
|
|
mc.REASONING_CONTROL_NATIVE_BOOL,
|
|
mc.REASONING_CONTROL_STRUCTURED_OBJECT,
|
|
mc.REASONING_CONTROL_BUDGET,
|
|
mc.REASONING_CONTROL_EFFORT,
|
|
]
|
|
|
|
|
|
def test_reasoning_control_values_can_describe_provider_supported_auto():
|
|
values = ["enabled", "disabled", "adaptive", "dynamic", "provider_auto"]
|
|
|
|
assert [mc.normalize_reasoning_control_value(value) for value in values] == [
|
|
mc.REASONING_CONTROL_VALUE_ON,
|
|
mc.REASONING_CONTROL_VALUE_OFF,
|
|
mc.REASONING_CONTROL_VALUE_AUTO,
|
|
mc.REASONING_CONTROL_VALUE_AUTO,
|
|
mc.REASONING_CONTROL_VALUE_AUTO,
|
|
]
|
|
assert mc.normalize_reasoning_control_value("message_directive") == ""
|