Jake/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-07-15 12:58:04 +00:00
Code Packages Projects Releases Activity

Raise unified llama context estimate

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-06-27 20:56:57 +00:00
parent a40371d532
commit da7c6a667b
1 changed files with 14 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
static/js/cookbookServe.js
+14 -4
View File
@@ -516,9 +516,19 @@ function _estimateLlamaContextFit(model, fields, modelCtxMax, modelWeightsGb = 0
let ctx = Math.min(modelMax, rounded); let ctx = Math.min(modelMax, rounded);
let reasonSuffix = ''; let reasonSuffix = '';
if (isUnifiedMode) { if (isUnifiedMode) {
// Unified memory should not advertise *less* context than plain GPU mode // Unified memory is not just "GPU math with a slightly bigger VRAM number".
// for the same llama.cpp launch. It may be slower when pages spill into // llama.cpp can spill into system RAM, so a conservative pure-VRAM KV
// system RAM, but the memory pool is at least the GPU path plus overflow. // formula makes confusing recommendations like "58G free unified" but the
// same context as GPU. Use a system-memory-style cap when there is real
// unified headroom, while keeping the GPU estimate as the minimum.
const unifiedCap = freeForKv >= 16
? 131072
: (freeForKv >= 8 ? 65536 : 32768);
const unifiedCtx = Math.min(modelMax, unifiedCap);
if (unifiedCtx > ctx) {
ctx = unifiedCtx;
reasonSuffix = '; unified can spill into system RAM, slower than pure GPU';
}
const gpuUsableGb = Math.max(1, totalVramGb - Math.max(1.0, selectedCount * 0.6)); const gpuUsableGb = Math.max(1, totalVramGb - Math.max(1.0, selectedCount * 0.6));
const gpuFreeForKv = gpuUsableGb - modelGb; const gpuFreeForKv = gpuUsableGb - modelGb;
if (gpuFreeForKv > 0) { if (gpuFreeForKv > 0) {
@@ -527,7 +537,7 @@ function _estimateLlamaContextFit(model, fields, modelCtxMax, modelWeightsGb = 0
const gpuCtx = Math.min(modelMax, gpuRounded); const gpuCtx = Math.min(modelMax, gpuRounded);
if (gpuCtx > ctx) { if (gpuCtx > ctx) {
ctx = gpuCtx; ctx = gpuCtx;
reasonSuffix = `; floored to GPU estimate`; reasonSuffix = '; at least the GPU estimate';
} }
} }
} }