Jake/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-07-08 11:56:59 +00:00
Code Packages Projects Releases Activity

Clamp unified llama context estimate

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-06-27 20:52:31 +00:00
parent be4c4f2926
commit a40371d532
1 changed files with 19 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
static/js/cookbookServe.js
+19 -2
View File
@@ -513,12 +513,29 @@ function _estimateLlamaContextFit(model, fields, modelCtxMax, modelWeightsGb = 0
}
const raw = Math.floor(freeForKv / kvGbPerToken);
const rounded = Math.max(1024, Math.floor(raw / 1024) * 1024);
const ctx = Math.min(modelMax, rounded);
let ctx = Math.min(modelMax, rounded);
let reasonSuffix = '';
if (isUnifiedMode) {
// Unified memory should not advertise *less* context than plain GPU mode
// for the same llama.cpp launch. It may be slower when pages spill into
// system RAM, but the memory pool is at least the GPU path plus overflow.
const gpuUsableGb = Math.max(1, totalVramGb - Math.max(1.0, selectedCount * 0.6));
const gpuFreeForKv = gpuUsableGb - modelGb;
if (gpuFreeForKv > 0) {
const gpuRaw = Math.floor(gpuFreeForKv / kvGbPerToken);
const gpuRounded = Math.max(1024, Math.floor(gpuRaw / 1024) * 1024);
const gpuCtx = Math.min(modelMax, gpuRounded);
if (gpuCtx > ctx) {
ctx = gpuCtx;
reasonSuffix = `; floored to GPU estimate`;
}
}
}
return {
ctx,
modelGb,
kvGbPerToken,
reason: `~${ctx.toLocaleString()} tokens fits llama.cpp KV (${freeForKv.toFixed(1)}G free ${isUnifiedMode ? 'unified' : 'VRAM'})`,
reason: `~${ctx.toLocaleString()} tokens fits llama.cpp KV (${freeForKv.toFixed(1)}G free ${isUnifiedMode ? 'unified' : 'VRAM'}${reasonSuffix})`,
};
}