mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-07-08 11:56:59 +00:00
Checkpoint Odysseus local update
This commit is contained in:
pewdiepie-archdaemon
File diff suppressed because it is too large
Load Diff
+117
-19
@@ -168,6 +168,19 @@ def _canonical_cpu_backend(system):
|
||||
return "cpu_x86"
|
||||
|
||||
|
||||
def _is_mlx_model(model, native_q=None):
|
||||
name = (model.get("name") or "").lower()
|
||||
provider = (model.get("provider") or "").lower()
|
||||
fmt = (model.get("format") or "").lower()
|
||||
q = (native_q if native_q is not None else _native_quant(model)).lower()
|
||||
return (
|
||||
q.startswith("mlx-")
|
||||
or provider == "mlx-community"
|
||||
or fmt == "mlx"
|
||||
or name.startswith("mlx-community/")
|
||||
)
|
||||
|
||||
|
||||
def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0):
|
||||
"""Estimate tok/s. Uses active params for MoE (only active experts run per token).
|
||||
|
||||
@@ -313,6 +326,22 @@ def _fit_score(required, available):
|
||||
return 50
|
||||
|
||||
|
||||
def _is_unified_memory_system(system):
|
||||
backend = (system.get("backend") or "").lower()
|
||||
return bool(system.get("unified_memory")) or backend in ("metal", "mps", "apple")
|
||||
|
||||
|
||||
def _fit_level_for_budget(required_gb, budget_gb):
|
||||
if not required_gb or not budget_gb or required_gb > budget_gb:
|
||||
return "too_tight"
|
||||
ratio = required_gb / budget_gb
|
||||
if ratio <= 0.50:
|
||||
return "perfect"
|
||||
if ratio <= 0.78:
|
||||
return "good"
|
||||
return "marginal"
|
||||
|
||||
|
||||
def _context_score(ctx, use_case):
|
||||
target = CONTEXT_TARGET.get(use_case, 4096)
|
||||
if ctx >= target:
|
||||
@@ -516,21 +545,42 @@ def analyze_model(model, system, target_quant=None, scoring_use_case=None, targe
|
||||
run_mode, quant, fit_ctx, required_gb = result
|
||||
|
||||
# Determine fit level
|
||||
budget = effective_vram if run_mode == "gpu" else available_ram
|
||||
unified_memory = _is_unified_memory_system(system)
|
||||
total_ram = system.get("total_ram_gb") or available_ram
|
||||
unified_budget = max(total_ram or 0, available_ram or 0, effective_vram or 0)
|
||||
budget = unified_budget if unified_memory else (effective_vram if run_mode == "gpu" else available_ram)
|
||||
if required_gb > budget:
|
||||
return None
|
||||
if run_mode == "gpu":
|
||||
rec = model.get("recommended_ram_gb") or required_gb
|
||||
if rec <= gpu_vram:
|
||||
fit_level = "perfect"
|
||||
elif gpu_vram >= required_gb * 1.2:
|
||||
fit_level = "good"
|
||||
if unified_memory:
|
||||
fit_level = _fit_level_for_budget(required_gb, budget)
|
||||
else:
|
||||
fit_level = "marginal"
|
||||
# GPU-only fit must leave real allocator/KV/runtime headroom. The
|
||||
# old check used recommended_ram_gb (or required_gb as a fallback),
|
||||
# which made any model that barely fit VRAM read as "perfect".
|
||||
# On CUDA/vLLM/SGLang that is misleading: 141 GB on a 160 GB box is
|
||||
# runnable, but not a comfortable perfect fit.
|
||||
if gpu_vram >= required_gb * 1.50:
|
||||
fit_level = "perfect"
|
||||
elif gpu_vram >= required_gb * 1.2:
|
||||
fit_level = "good"
|
||||
else:
|
||||
fit_level = "marginal"
|
||||
elif run_mode == "cpu_offload":
|
||||
fit_level = "good" if available_ram >= required_gb * 1.2 else "marginal"
|
||||
fit_level = _fit_level_for_budget(required_gb, budget)
|
||||
if fit_level == "perfect":
|
||||
fit_level = "good"
|
||||
else:
|
||||
fit_level = "marginal"
|
||||
fit_level = _fit_level_for_budget(required_gb, budget)
|
||||
if fit_level == "too_tight":
|
||||
fit_level = "marginal"
|
||||
|
||||
# Rows that comfortably fit in a huge RAM/unified-memory pool should not all
|
||||
# look "marginal"; that made 1B-70B CPU/Ollama rows orange on 256 GB systems.
|
||||
if fit_level == "marginal" and budget and required_gb <= budget * 0.78:
|
||||
fit_level = "good"
|
||||
if fit_level == "good" and budget and required_gb <= budget * 0.50 and run_mode != "cpu_offload":
|
||||
fit_level = "perfect"
|
||||
|
||||
# Fraction of the model that spills to CPU RAM (drives the offload speed
|
||||
# model). When offloading, anything beyond the GPU's VRAM lives in system RAM.
|
||||
@@ -621,6 +671,40 @@ SORT_KEYS = {
|
||||
}
|
||||
|
||||
|
||||
def _search_blob(*parts):
|
||||
text = " ".join(str(p or "") for p in parts).lower()
|
||||
compact = re.sub(r"[^a-z0-9]+", "", text)
|
||||
spaced = re.sub(r"[^a-z0-9]+", " ", text).strip()
|
||||
return f"{text} {spaced} {compact}"
|
||||
|
||||
|
||||
def _matches_search(model, search):
|
||||
terms = [t for t in re.split(r"\s+", (search or "").strip().lower()) if t]
|
||||
if not terms:
|
||||
return True
|
||||
blob = _search_blob(
|
||||
model.get("name"),
|
||||
model.get("provider"),
|
||||
model.get("architecture"),
|
||||
model.get("quantization"),
|
||||
model.get("format"),
|
||||
model.get("parameter_count"),
|
||||
)
|
||||
for term in terms:
|
||||
norm = re.sub(r"[^a-z0-9]+", "", term)
|
||||
if term not in blob and (not norm or norm not in blob):
|
||||
if re.fullmatch(r"\d+(?:\.\d+)?b?", term):
|
||||
try:
|
||||
wanted = float(term.rstrip("b"))
|
||||
actual = params_b(model)
|
||||
except (TypeError, ValueError):
|
||||
actual = 0
|
||||
if wanted > 0 and actual > 0 and abs(actual - wanted) <= max(5.0, wanted * 0.08):
|
||||
continue
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def rank_models(system, use_case=None, limit=50, search=None, sort="score", quant=None, target_context=None, fit_only=False):
|
||||
"""Rank all models against detected hardware. Returns sorted list of fit results.
|
||||
|
||||
@@ -693,10 +777,11 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
|
||||
|
||||
for m in models:
|
||||
native_q = _native_quant(m)
|
||||
is_mlx = _is_mlx_model(m, native_q)
|
||||
|
||||
# MLX needs the mlx_lm runtime, which Odysseus does not generate serve
|
||||
# commands for. Hide it on every backend, including Metal.
|
||||
if native_q.startswith("mlx-") or "mlx" in (m.get("name") or "").lower():
|
||||
# MLX is Apple Silicon-only. It should never appear on CUDA/ROCm/CPU,
|
||||
# but it is first-class on Metal where mlx_lm.server can serve it.
|
||||
if is_mlx and not apple_silicon:
|
||||
continue
|
||||
|
||||
# ROCm support for vLLM/SGLang quantized safetensors is too brittle to
|
||||
@@ -723,7 +808,7 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
|
||||
# Windows is the same: Odysseus only supports llama.cpp on Windows,
|
||||
# which requires GGUF. vLLM/SGLang are explicitly blocked, so AWQ/GPTQ
|
||||
# models without a GGUF source are unservable there.
|
||||
if (apple_silicon or consumer_amd or is_windows) and not (m.get("is_gguf") or m.get("gguf_sources")):
|
||||
if (apple_silicon or consumer_amd or is_windows) and not is_mlx and not (m.get("is_gguf") or m.get("gguf_sources")):
|
||||
continue
|
||||
|
||||
# Format filter: AWQ tab -> only AWQ models, FP4 tab -> FP4-family models, etc.
|
||||
@@ -741,13 +826,26 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
|
||||
if quant in ("INT4", "INT8", "W4A16", "W8A8", "W8A16") and native_q != quant:
|
||||
continue
|
||||
|
||||
if search:
|
||||
name = m.get("name", "").lower()
|
||||
provider = m.get("provider", "").lower()
|
||||
if search.lower() not in name and search.lower() not in provider:
|
||||
continue
|
||||
if search and not _matches_search(m, search):
|
||||
continue
|
||||
|
||||
result = analyze_model(m, system, target_quant=quant, scoring_use_case=(use_case or "general"), target_context=target_context)
|
||||
model_quant = quant
|
||||
# UI "Q4" means the user's looking for a 4-bit fit. On multi-GPU
|
||||
# CUDA/vLLM/SGLang boxes, many practical 4-bit models are native AWQ
|
||||
# safetensors, not GGUF Q4_K_M. If we pass Q4_K_M into a prequantized
|
||||
# AWQ row, analyze_model correctly rejects it as the wrong serving
|
||||
# format, but the result is confusing: highlighting Quant/Q4 hides the
|
||||
# exact AWQ rows the machine is built to run. Treat Q4 as AWQ-4bit for
|
||||
# native AWQ rows only on accelerator servers that can serve them.
|
||||
if (
|
||||
quant == "Q4_K_M"
|
||||
and system.get("gpu_count", 1) >= 2
|
||||
and not (apple_silicon or consumer_amd or is_windows)
|
||||
and native_q == "AWQ-4bit"
|
||||
):
|
||||
model_quant = native_q
|
||||
|
||||
result = analyze_model(m, system, target_quant=model_quant, scoring_use_case=(use_case or "general"), target_context=target_context)
|
||||
if result is None:
|
||||
continue
|
||||
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
|
||||
from src.constants import DATA_DIR
|
||||
|
||||
|
||||
HF_COLLECTIONS_URL = "https://huggingface.co/api/collections"
|
||||
HW_FIT_CACHE_DIR = Path(DATA_DIR) / "hwfit"
|
||||
MLX_COMMUNITY_CACHE = HW_FIT_CACHE_DIR / "mlx_community_models.json"
|
||||
HF_COLLECTION_MODELS_CACHE = HW_FIT_CACHE_DIR / "hf_collection_models.json"
|
||||
HF_COLLECTION_TTL_SECONDS = 24 * 3600
|
||||
|
||||
|
||||
HF_COLLECTION_SOURCES = (
|
||||
{
|
||||
"key": "mlx_community",
|
||||
"owner": "mlx-community",
|
||||
"provider": "mlx-community",
|
||||
"repo_prefix": "mlx-community/",
|
||||
"mlx_only": True,
|
||||
},
|
||||
{
|
||||
"key": "zai_org",
|
||||
"owner": "zai-org",
|
||||
"provider": "zai-org",
|
||||
},
|
||||
{
|
||||
"key": "deepseek_ai",
|
||||
"owner": "deepseek-ai",
|
||||
"provider": "deepseek-ai",
|
||||
},
|
||||
{
|
||||
"key": "minimax_ai",
|
||||
"owner": "MiniMaxAI",
|
||||
"provider": "MiniMaxAI",
|
||||
},
|
||||
{
|
||||
"key": "qwen",
|
||||
"owner": "Qwen",
|
||||
"provider": "Qwen",
|
||||
},
|
||||
{
|
||||
"key": "stepfun_ai",
|
||||
"owner": "stepfun-ai",
|
||||
"provider": "stepfun-ai",
|
||||
},
|
||||
{
|
||||
"key": "google",
|
||||
"owner": "google",
|
||||
"provider": "google",
|
||||
},
|
||||
{
|
||||
"key": "openai",
|
||||
"owner": "openai",
|
||||
"provider": "openai",
|
||||
},
|
||||
{
|
||||
"key": "mistralai",
|
||||
"owner": "mistralai",
|
||||
"provider": "mistralai",
|
||||
},
|
||||
{
|
||||
"key": "meta_llama",
|
||||
"owner": "meta-llama",
|
||||
"provider": "meta-llama",
|
||||
},
|
||||
{
|
||||
"key": "nousresearch",
|
||||
"owner": "NousResearch",
|
||||
"provider": "NousResearch",
|
||||
},
|
||||
{
|
||||
"key": "moonshotai",
|
||||
"owner": "moonshotai",
|
||||
"provider": "moonshotai",
|
||||
},
|
||||
{
|
||||
"key": "mllama",
|
||||
"owner": "mllama",
|
||||
"provider": "mllama",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _format_params(raw):
|
||||
try:
|
||||
n = int(raw or 0)
|
||||
except (TypeError, ValueError):
|
||||
n = 0
|
||||
if n <= 0:
|
||||
return "", 0
|
||||
if n >= 1_000_000_000_000:
|
||||
return f"{n / 1_000_000_000_000:.3g}T", n
|
||||
if n >= 1_000_000_000:
|
||||
return f"{n / 1_000_000_000:.4g}B", n
|
||||
if n >= 1_000_000:
|
||||
return f"{n / 1_000_000:.4g}M", n
|
||||
if n >= 1_000:
|
||||
return f"{n / 1_000:.4g}K", n
|
||||
return str(n), n
|
||||
|
||||
|
||||
def _parse_params_from_name(repo_id):
|
||||
name = (repo_id or "").rsplit("/", 1)[-1]
|
||||
active = None
|
||||
m_active = re.search(r"[-_][Aa](\d+(?:\.\d+)?)[Bb](?![a-zA-Z])", name)
|
||||
if m_active:
|
||||
active = int(float(m_active.group(1)) * 1_000_000_000)
|
||||
name = name[: m_active.start()] + name[m_active.end() :]
|
||||
total = None
|
||||
for m in re.finditer(r"(\d+(?:\.\d+)?)[Bb](?![a-zA-Z])", name):
|
||||
total = int(float(m.group(1)) * 1_000_000_000)
|
||||
break
|
||||
if total is None:
|
||||
for m in re.finditer(r"(\d+(?:\.\d+)?)[Mm](?![a-zA-Z])", name):
|
||||
total = int(float(m.group(1)) * 1_000_000)
|
||||
break
|
||||
return total or 0, active
|
||||
|
||||
|
||||
def _infer_quant(repo_id, source):
|
||||
name = (repo_id or "").rsplit("/", 1)[-1].lower()
|
||||
if source.get("mlx_only"):
|
||||
if "8bit" in name or "8-bit" in name:
|
||||
return "mlx-8bit"
|
||||
if "6bit" in name or "6-bit" in name:
|
||||
return "mlx-6bit"
|
||||
if "5bit" in name or "5-bit" in name:
|
||||
return "mlx-5bit"
|
||||
if "3bit" in name or "3-bit" in name:
|
||||
return "mlx-3bit"
|
||||
if re.search(r"(^|[-_/])bf16($|[-_/])", name):
|
||||
return "BF16"
|
||||
return "mlx-4bit"
|
||||
if "awq" in name and ("8bit" in name or "8-bit" in name or "int8" in name):
|
||||
return "AWQ-8bit"
|
||||
if "awq" in name or "4bit" in name or "4-bit" in name:
|
||||
return "AWQ-4bit"
|
||||
if "gptq" in name and ("8bit" in name or "8-bit" in name or "int8" in name):
|
||||
return "GPTQ-Int8"
|
||||
if "gptq" in name:
|
||||
return "GPTQ-Int4"
|
||||
if "mxfp4" in name or "nvfp4" in name or re.search(r"(^|[-_/])fp4($|[-_/])", name):
|
||||
return "FP4-MoE-Mixed"
|
||||
if "mxfp8" in name or re.search(r"(^|[-_/])fp8($|[-_/])", name):
|
||||
return "FP8-Mixed"
|
||||
if "gguf" in name or "q4_k" in name or "q4-k" in name:
|
||||
return "Q4_K_M"
|
||||
if re.search(r"(^|[-_/])bf16($|[-_/])", name):
|
||||
return "BF16"
|
||||
return "BF16"
|
||||
|
||||
|
||||
def _quant_bytes_per_param(quant):
|
||||
return {
|
||||
"BF16": 2.2,
|
||||
"FP8": 1.15,
|
||||
"FP8-Mixed": 1.15,
|
||||
"FP4-MoE-Mixed": 0.62,
|
||||
"AWQ-4bit": 0.62,
|
||||
"AWQ-8bit": 1.15,
|
||||
"GPTQ-Int4": 0.62,
|
||||
"GPTQ-Int8": 1.15,
|
||||
"Q4_K_M": 0.62,
|
||||
"mlx-8bit": 1.25,
|
||||
"mlx-6bit": 0.95,
|
||||
"mlx-5bit": 0.82,
|
||||
"mlx-4bit": 0.70,
|
||||
"mlx-3bit": 0.55,
|
||||
}.get(quant, 2.2)
|
||||
|
||||
|
||||
def _infer_context(repo_id, pipeline_tag):
|
||||
text = f"{repo_id or ''} {pipeline_tag or ''}".lower()
|
||||
if any(k in text for k in ("whisper", "asr", "speech-recognition", "tts", "audio", "image", "video", "diffusion")):
|
||||
return 4096
|
||||
if any(k in text for k in ("glm-5.2", "deepseek-v4", "minimax-m3")):
|
||||
return 1_000_000
|
||||
if any(k in text for k in ("qwen3", "glm", "deepseek", "minimax")):
|
||||
return 32768
|
||||
return 32768
|
||||
|
||||
|
||||
def _infer_use_case(repo_id, pipeline_tag):
|
||||
text = f"{repo_id or ''} {pipeline_tag or ''}".lower()
|
||||
if any(k in text for k in ("whisper", "asr", "speech-recognition", "transcrib")):
|
||||
return "stt"
|
||||
if any(k in text for k in ("tts", "text-to-speech", "kokoro", "audio")):
|
||||
return "tts"
|
||||
if any(k in text for k in ("image-text", "vision", "vlm", "vl-", "ocr", "multimodal")):
|
||||
return "multimodal"
|
||||
if any(k in text for k in ("code", "coder")):
|
||||
return "coding"
|
||||
if any(k in text for k in ("reason", "thinking", "thinker", "r1")):
|
||||
return "reasoning"
|
||||
return "general"
|
||||
|
||||
|
||||
def _entry_from_collection_item(collection, item, source):
|
||||
repo_id = item.get("id") or ""
|
||||
if item.get("type") != "model" or not repo_id:
|
||||
return None
|
||||
repo_prefix = source.get("repo_prefix")
|
||||
if repo_prefix and not repo_id.startswith(repo_prefix):
|
||||
return None
|
||||
raw_params = item.get("numParameters") or 0
|
||||
active = None
|
||||
if not raw_params:
|
||||
raw_params, active = _parse_params_from_name(repo_id)
|
||||
param_label, raw_params = _format_params(raw_params)
|
||||
if not raw_params:
|
||||
return None
|
||||
|
||||
quant = _infer_quant(repo_id, source)
|
||||
pipeline_tag = item.get("pipeline_tag") or ""
|
||||
min_ram = round((raw_params / 1_000_000_000) * _quant_bytes_per_param(quant) + 0.8, 1)
|
||||
last_modified = item.get("lastModified") or collection.get("lastUpdated") or ""
|
||||
release_date = ""
|
||||
if last_modified:
|
||||
try:
|
||||
release_date = parsedate_to_datetime(last_modified).date().isoformat()
|
||||
except Exception:
|
||||
release_date = str(last_modified)[:10]
|
||||
|
||||
entry = {
|
||||
"name": repo_id,
|
||||
"provider": source.get("provider") or repo_id.split("/", 1)[0],
|
||||
"parameter_count": param_label,
|
||||
"parameters_raw": raw_params,
|
||||
"min_ram_gb": min_ram,
|
||||
"recommended_ram_gb": round(min_ram * 1.3 + 0.5, 1),
|
||||
"min_vram_gb": 0.0 if source.get("mlx_only") else min_ram,
|
||||
"quantization": quant,
|
||||
"context_length": _infer_context(repo_id, pipeline_tag),
|
||||
"use_case": _infer_use_case(repo_id, pipeline_tag),
|
||||
"capabilities": ["mlx"] if source.get("mlx_only") else ["vllm", "sglang"],
|
||||
"pipeline_tag": pipeline_tag,
|
||||
"architecture": "",
|
||||
"hf_downloads": int(item.get("downloads") or 0),
|
||||
"hf_likes": int(item.get("likes") or 0),
|
||||
"release_date": release_date,
|
||||
"format": "mlx" if source.get("mlx_only") else "safetensors",
|
||||
"collection": collection.get("title") or "",
|
||||
"description": collection.get("description") or "",
|
||||
"_discovered": True,
|
||||
"_source": "hf_collections",
|
||||
"_source_owner": source.get("owner") or "",
|
||||
}
|
||||
if source.get("mlx_only"):
|
||||
entry["mlx_only"] = True
|
||||
if quant == "Q4_K_M":
|
||||
entry["is_gguf"] = True
|
||||
entry["format"] = "gguf"
|
||||
entry["capabilities"] = ["llama.cpp"]
|
||||
if active:
|
||||
entry["is_moe"] = True
|
||||
entry["active_parameters"] = active
|
||||
return entry
|
||||
|
||||
|
||||
def _next_link(header):
|
||||
if not header:
|
||||
return None
|
||||
m = re.search(r'<([^>]+)>;\s*rel="next"', header)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def fetch_collection_models(source, timeout=20, max_pages=20):
|
||||
params = urllib.parse.urlencode({
|
||||
"owner": source["owner"],
|
||||
"limit": "100",
|
||||
"expand": "true",
|
||||
})
|
||||
url = f"{HF_COLLECTIONS_URL}?{params}"
|
||||
models = {}
|
||||
pages = 0
|
||||
while url and pages < max_pages:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "odysseus-hwfit/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
payload = json.load(resp)
|
||||
url = _next_link(resp.headers.get("Link"))
|
||||
pages += 1
|
||||
if not isinstance(payload, list):
|
||||
break
|
||||
for collection in payload:
|
||||
if not isinstance(collection, dict):
|
||||
continue
|
||||
for item in collection.get("items") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
entry = _entry_from_collection_item(collection, item, source)
|
||||
if entry and entry["name"] not in models:
|
||||
models[entry["name"]] = entry
|
||||
rows = list(models.values())
|
||||
rows.sort(key=lambda x: (x.get("hf_downloads") or 0, x.get("release_date") or ""), reverse=True)
|
||||
return rows
|
||||
|
||||
|
||||
def _load_cache(path):
|
||||
try:
|
||||
with path.open(encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
rows = data.get("models") if isinstance(data, dict) else data
|
||||
return rows if isinstance(rows, list) else []
|
||||
except (OSError, ValueError):
|
||||
return []
|
||||
|
||||
|
||||
def _write_cache(path, source, rows):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"source": source,
|
||||
"fetched_at": int(time.time()),
|
||||
"count": len(rows),
|
||||
"models": rows,
|
||||
}
|
||||
tmp = path.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def load_cached_mlx_community_models():
|
||||
return _load_cache(MLX_COMMUNITY_CACHE)
|
||||
|
||||
|
||||
def load_cached_hf_collection_models():
|
||||
return _load_cache(HF_COLLECTION_MODELS_CACHE)
|
||||
|
||||
|
||||
def _cache_fresh(path):
|
||||
try:
|
||||
return (time.time() - path.stat().st_mtime) < HF_COLLECTION_TTL_SECONDS
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def refresh_mlx_community_cache(force=False):
|
||||
if not force and _cache_fresh(MLX_COMMUNITY_CACHE):
|
||||
return load_cached_mlx_community_models()
|
||||
source = next(s for s in HF_COLLECTION_SOURCES if s["key"] == "mlx_community")
|
||||
rows = fetch_collection_models(source)
|
||||
_write_cache(MLX_COMMUNITY_CACHE, "https://huggingface.co/mlx-community/collections", rows)
|
||||
return rows
|
||||
|
||||
|
||||
def refresh_hf_collection_models_cache(force=False):
|
||||
if not force and _cache_fresh(HF_COLLECTION_MODELS_CACHE):
|
||||
return load_cached_hf_collection_models()
|
||||
rows_by_name = {}
|
||||
for source in HF_COLLECTION_SOURCES:
|
||||
if source["key"] == "mlx_community":
|
||||
continue
|
||||
try:
|
||||
for row in fetch_collection_models(source):
|
||||
rows_by_name.setdefault(row["name"], row)
|
||||
except Exception:
|
||||
# Keep partial refreshes useful. A temporary DNS/provider issue for
|
||||
# one brand should not invalidate the other cached collection rows.
|
||||
continue
|
||||
rows = sorted(
|
||||
rows_by_name.values(),
|
||||
key=lambda x: (x.get("hf_downloads") or 0, x.get("release_date") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
if rows:
|
||||
_write_cache(HF_COLLECTION_MODELS_CACHE, "https://huggingface.co/collections", rows)
|
||||
return rows
|
||||
return load_cached_hf_collection_models()
|
||||
+77
-10
@@ -13,7 +13,7 @@ QUANT_BPP = {
|
||||
"AWQ-4bit": 0.50, "AWQ-8bit": 1.0,
|
||||
"GPTQ-Int4": 0.50, "GPTQ-Int8": 1.0,
|
||||
"QAT-INT4": 0.50, "QAT-INT8": 1.0,
|
||||
"mlx-4bit": 0.55, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
|
||||
"mlx-3bit": 0.42, "mlx-4bit": 0.55, "mlx-5bit": 0.65, "mlx-6bit": 0.75, "mlx-8bit": 1.0,
|
||||
# DeepSeek-V4-style mixed: MoE experts in FP4 (bulk), attention + non-
|
||||
# expert dense in FP8, embeddings/LM head in BF16. By weight count the
|
||||
# experts dominate so the effective BPP sits closer to FP4 than FP8.
|
||||
@@ -32,7 +32,7 @@ QUANT_SPEED_MULT = {
|
||||
"AWQ-4bit": 1.2, "AWQ-8bit": 0.85,
|
||||
"GPTQ-Int4": 1.2, "GPTQ-Int8": 0.85,
|
||||
"QAT-INT4": 1.15, "QAT-INT8": 0.85,
|
||||
"mlx-4bit": 1.15, "mlx-8bit": 0.85, "mlx-6bit": 1.0,
|
||||
"mlx-3bit": 1.25, "mlx-4bit": 1.15, "mlx-5bit": 1.05, "mlx-6bit": 1.0, "mlx-8bit": 0.85,
|
||||
"FP4-MoE-Mixed": 1.10, # slightly slower than pure FP4 because of mixed-dtype dispatch
|
||||
"FP8-Mixed": 0.85,
|
||||
}
|
||||
@@ -53,7 +53,7 @@ QUANT_QUALITY_PENALTY = {
|
||||
# QAT-INT4 build lands far closer to bf16 than a post-training Q4/INT4
|
||||
# (Google reports near-bf16 quality). Penalize it lightly, not like Q4_K_M.
|
||||
"QAT-INT4": -1.0, "QAT-INT8": 0.0,
|
||||
"mlx-4bit": -4.0, "mlx-8bit": -0.5, "mlx-6bit": -1.5,
|
||||
"mlx-3bit": -8.0, "mlx-4bit": -4.0, "mlx-5bit": -2.5, "mlx-6bit": -1.5, "mlx-8bit": -0.5,
|
||||
# DeepSeek-V4 mixed: only MoE experts at FP4 (the rest is FP8/BF16),
|
||||
# so the realized quality is much closer to FP8 than to pure FP4 —
|
||||
# the activation-sensitive layers stay high-precision. ~0 penalty.
|
||||
@@ -70,7 +70,7 @@ QUANT_BYTES_PER_PARAM = {
|
||||
"AWQ-4bit": 0.5, "AWQ-8bit": 1.0,
|
||||
"GPTQ-Int4": 0.5, "GPTQ-Int8": 1.0,
|
||||
"QAT-INT4": 0.5, "QAT-INT8": 1.0,
|
||||
"mlx-4bit": 0.5, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
|
||||
"mlx-3bit": 0.375, "mlx-4bit": 0.5, "mlx-5bit": 0.625, "mlx-6bit": 0.75, "mlx-8bit": 1.0,
|
||||
"FP4-MoE-Mixed": 0.55,
|
||||
"FP8-Mixed": 1.0,
|
||||
}
|
||||
@@ -87,8 +87,11 @@ PREQUANTIZED_PREFIXES = (
|
||||
|
||||
def infer_quantization_from_name(name):
|
||||
n = (name or "").lower()
|
||||
model_name = n.rsplit("/", 1)[-1]
|
||||
if "nvfp4" in n:
|
||||
return "NVFP4"
|
||||
if re.search(r"(^|[-_/])bf16($|[-_/])", model_name):
|
||||
return "BF16"
|
||||
if "mxfp4" in n:
|
||||
return "MXFP4"
|
||||
if re.search(r"(^|[-_/])nf4($|[-_/])", n):
|
||||
@@ -106,8 +109,12 @@ def infer_quantization_from_name(name):
|
||||
return "AWQ-8bit" if is8 else "AWQ-4bit"
|
||||
if "gptq" in n:
|
||||
return "GPTQ-Int8" if is8 else "GPTQ-Int4"
|
||||
if "mlx" in n:
|
||||
if "6bit" in n:
|
||||
if n.startswith("mlx-community/") or "mlx" in model_name:
|
||||
if "3bit" in model_name:
|
||||
return "mlx-3bit"
|
||||
if "5bit" in model_name:
|
||||
return "mlx-5bit"
|
||||
if "6bit" in model_name:
|
||||
return "mlx-6bit"
|
||||
return "mlx-8bit" if is8 else "mlx-4bit"
|
||||
if "fp8" in n:
|
||||
@@ -260,15 +267,75 @@ def infer_use_case(model):
|
||||
|
||||
_models_cache = None
|
||||
|
||||
def _load_model_file(path):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
loaded = json.load(f)
|
||||
return loaded if isinstance(loaded, list) else []
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
return []
|
||||
|
||||
def reset_model_cache():
|
||||
global _models_cache
|
||||
_models_cache = None
|
||||
|
||||
def refresh_dynamic_catalogs(force=False):
|
||||
"""Refresh API-backed model catalogs and invalidate the merged cache.
|
||||
|
||||
The bundled JSON files remain the offline fallback. Dynamic catalogs live
|
||||
under DATA_DIR so runtime refreshes do not dirty the source tree.
|
||||
"""
|
||||
from services.hwfit.hf_discovery import (
|
||||
refresh_hf_collection_models_cache,
|
||||
refresh_mlx_community_cache,
|
||||
)
|
||||
|
||||
refreshed = {
|
||||
"mlx_community": len(refresh_mlx_community_cache(force=force)),
|
||||
"hf_collections": len(refresh_hf_collection_models_cache(force=force)),
|
||||
}
|
||||
reset_model_cache()
|
||||
return refreshed
|
||||
|
||||
def get_models():
|
||||
global _models_cache
|
||||
if _models_cache is None:
|
||||
data_path = os.path.join(os.path.dirname(__file__), "data", "hf_models.json")
|
||||
static_mlx_path = os.path.join(os.path.dirname(__file__), "data", "mlx_community_models.json")
|
||||
try:
|
||||
with open(data_path, encoding="utf-8") as f:
|
||||
_models_cache = [_normalize_model_entry(m) for m in json.load(f)]
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
_models_cache = []
|
||||
from services.hwfit.hf_discovery import (
|
||||
load_cached_hf_collection_models,
|
||||
load_cached_mlx_community_models,
|
||||
)
|
||||
dynamic_mlx_models = load_cached_mlx_community_models()
|
||||
dynamic_hf_models = load_cached_hf_collection_models()
|
||||
except Exception:
|
||||
dynamic_mlx_models = []
|
||||
dynamic_hf_models = []
|
||||
seen = set()
|
||||
rows = []
|
||||
def _append_models(models):
|
||||
for model in models:
|
||||
if not isinstance(model, dict):
|
||||
continue
|
||||
name = model.get("name")
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
rows.append(_normalize_model_entry(model))
|
||||
|
||||
for model in _load_model_file(data_path):
|
||||
if not isinstance(model, dict):
|
||||
continue
|
||||
name = model.get("name")
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
rows.append(_normalize_model_entry(model))
|
||||
_append_models(dynamic_hf_models)
|
||||
_append_models(dynamic_mlx_models)
|
||||
_append_models(_load_model_file(static_mlx_path))
|
||||
_models_cache = rows
|
||||
return _models_cache
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user