fix(compaction): budget retained tail with media
This commit is contained in:
@@ -5,18 +5,22 @@ import type { MessageV2 } from "./message-v2"
|
||||
|
||||
const COMPACTION_BUFFER = 20_000
|
||||
|
||||
export function isOverflow(input: { cfg: Config.Info; tokens: MessageV2.Assistant["tokens"]; model: Provider.Model }) {
|
||||
if (input.cfg.compaction?.auto === false) return false
|
||||
export function usable(input: { cfg: Config.Info; model: Provider.Model }) {
|
||||
const context = input.model.limit.context
|
||||
if (context === 0) return false
|
||||
|
||||
const count =
|
||||
input.tokens.total || input.tokens.input + input.tokens.output + input.tokens.cache.read + input.tokens.cache.write
|
||||
if (context === 0) return 0
|
||||
|
||||
const reserved =
|
||||
input.cfg.compaction?.reserved ?? Math.min(COMPACTION_BUFFER, ProviderTransform.maxOutputTokens(input.model))
|
||||
const usable = input.model.limit.input
|
||||
? input.model.limit.input - reserved
|
||||
: context - ProviderTransform.maxOutputTokens(input.model)
|
||||
return count >= usable
|
||||
return input.model.limit.input
|
||||
? Math.max(0, input.model.limit.input - reserved)
|
||||
: Math.max(0, context - ProviderTransform.maxOutputTokens(input.model))
|
||||
}
|
||||
|
||||
export function isOverflow(input: { cfg: Config.Info; tokens: MessageV2.Assistant["tokens"]; model: Provider.Model }) {
|
||||
if (input.cfg.compaction?.auto === false) return false
|
||||
if (input.model.limit.context === 0) return false
|
||||
|
||||
const count =
|
||||
input.tokens.total || input.tokens.input + input.tokens.output + input.tokens.cache.read + input.tokens.cache.write
|
||||
return count >= usable(input)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user