feat: add headerTimeout cfg option, default it on only for openai w/ default of 10s (#29484)
This commit is contained in:
@@ -92,11 +92,19 @@ export const Info = Schema.Struct({
|
||||
timeout: Schema.optional(
|
||||
Schema.Union([PositiveInt, Schema.Literal(false)]).annotate({
|
||||
description:
|
||||
"Timeout in milliseconds for requests to this provider. Default is 300000 (5 minutes). Set to false to disable timeout.",
|
||||
"Timeout in milliseconds for full requests to this provider. Set to false to disable timeout.",
|
||||
}),
|
||||
).annotate({
|
||||
description: "Timeout in milliseconds for full requests to this provider. Set to false to disable timeout.",
|
||||
}),
|
||||
headerTimeout: Schema.optional(
|
||||
Schema.Union([PositiveInt, Schema.Literal(false)]).annotate({
|
||||
description:
|
||||
"Timeout in milliseconds to wait for response headers. Provider integrations may set defaults. Set to false to disable timeout.",
|
||||
}),
|
||||
).annotate({
|
||||
description:
|
||||
"Timeout in milliseconds for requests to this provider. Default is 300000 (5 minutes). Set to false to disable timeout.",
|
||||
"Timeout in milliseconds to wait for response headers. Provider integrations may set defaults. Set to false to disable timeout.",
|
||||
}),
|
||||
chunkTimeout: Schema.optional(PositiveInt).annotate({
|
||||
description:
|
||||
|
||||
@@ -3,6 +3,14 @@ import { STATUS_CODES } from "http"
|
||||
import { iife } from "@/util/iife"
|
||||
import type { ProviderID } from "./schema"
|
||||
|
||||
export class HeaderTimeoutError extends Error {
|
||||
public override readonly name = "ProviderHeaderTimeoutError"
|
||||
|
||||
constructor(public readonly ms: number) {
|
||||
super(`Provider response headers timed out after ${ms}ms`)
|
||||
}
|
||||
}
|
||||
|
||||
// Adapted from overflow detection patterns in:
|
||||
// https://github.com/badlogic/pi-mono/blob/main/packages/ai/src/utils/overflow.ts
|
||||
const OVERFLOW_PATTERNS = [
|
||||
|
||||
@@ -28,9 +28,10 @@ import * as ProviderTransform from "./transform"
|
||||
import { ModelID, ProviderID } from "./schema"
|
||||
import { ModelStatus } from "./model-status"
|
||||
import { RuntimeFlags } from "@/effect/runtime-flags"
|
||||
import { ProviderError } from "./error"
|
||||
|
||||
const log = Log.create({ service: "provider" })
|
||||
|
||||
const OPENAI_HEADER_TIMEOUT_DEFAULT = 10_000
|
||||
function shouldUseCopilotResponsesApi(modelID: string): boolean {
|
||||
const match = /^gpt-(\d+)/.exec(modelID)
|
||||
if (!match) return false
|
||||
@@ -85,6 +86,15 @@ function wrapSSE(res: Response, ms: number, ctl: AbortController) {
|
||||
})
|
||||
}
|
||||
|
||||
function timeoutController(ms: number) {
|
||||
const ctl = new AbortController()
|
||||
const id = setTimeout(() => ctl.abort(new ProviderError.HeaderTimeoutError(ms)), ms)
|
||||
return {
|
||||
signal: ctl.signal,
|
||||
clear: () => clearTimeout(id),
|
||||
}
|
||||
}
|
||||
|
||||
function googleVertexAnthropicBaseURL(project: string | undefined, location: string | undefined) {
|
||||
if (!project) return
|
||||
if (location !== "eu" && location !== "us") return
|
||||
@@ -194,7 +204,7 @@ function custom(dep: CustomDep): Record<string, CustomLoader> {
|
||||
async getModel(sdk: any, modelID: string, _options?: Record<string, any>) {
|
||||
return sdk.responses(modelID)
|
||||
},
|
||||
options: {},
|
||||
options: { headerTimeout: OPENAI_HEADER_TIMEOUT_DEFAULT },
|
||||
}),
|
||||
xai: () =>
|
||||
Effect.succeed({
|
||||
@@ -1601,16 +1611,21 @@ export const layer = Layer.effect(
|
||||
|
||||
const customFetch = options["fetch"]
|
||||
const chunkTimeout = options["chunkTimeout"]
|
||||
const headerTimeout = options["headerTimeout"]
|
||||
delete options["chunkTimeout"]
|
||||
delete options["headerTimeout"]
|
||||
|
||||
options["fetch"] = async (input: any, init?: BunFetchRequestInit) => {
|
||||
const fetchFn = customFetch ?? fetch
|
||||
const opts = init ?? {}
|
||||
const chunkAbortCtl = typeof chunkTimeout === "number" && chunkTimeout > 0 ? new AbortController() : undefined
|
||||
const headerTimeoutMs = headerTimeout === false ? undefined : headerTimeout
|
||||
const headerTimeoutCtl = typeof headerTimeoutMs === "number" ? timeoutController(headerTimeoutMs) : undefined
|
||||
const signals: AbortSignal[] = []
|
||||
|
||||
if (opts.signal) signals.push(opts.signal)
|
||||
if (chunkAbortCtl) signals.push(chunkAbortCtl.signal)
|
||||
if (headerTimeoutCtl) signals.push(headerTimeoutCtl.signal)
|
||||
if (options["timeout"] !== undefined && options["timeout"] !== null && options["timeout"] !== false)
|
||||
signals.push(AbortSignal.timeout(options["timeout"]))
|
||||
|
||||
@@ -1639,7 +1654,7 @@ export const layer = Layer.effect(
|
||||
...opts,
|
||||
// @ts-ignore see here: https://github.com/oven-sh/bun/issues/16682
|
||||
timeout: false,
|
||||
})
|
||||
}).finally(() => headerTimeoutCtl?.clear())
|
||||
|
||||
if (!chunkAbortCtl) return res
|
||||
return wrapSSE(res, chunkTimeout, chunkAbortCtl)
|
||||
|
||||
@@ -1143,6 +1143,18 @@ export function fromError(
|
||||
},
|
||||
{ cause: e },
|
||||
).toObject()
|
||||
case e instanceof ProviderError.HeaderTimeoutError:
|
||||
return new APIError(
|
||||
{
|
||||
message: e.message,
|
||||
isRetryable: true,
|
||||
metadata: {
|
||||
code: e.name,
|
||||
timeoutMs: String(e.ms),
|
||||
},
|
||||
},
|
||||
{ cause: e },
|
||||
).toObject()
|
||||
case APICallError.isInstance(e):
|
||||
const parsed = ProviderError.parseAPICallError({
|
||||
providerID: ctx.providerID,
|
||||
|
||||
Reference in New Issue
Block a user