feat: add headerTimeout cfg option, default it on only for openai w/ default of 10s (#29484)

This commit is contained in:
Aiden Cline
2026-05-26 21:22:24 -05:00
committed by GitHub
parent 519d344470
commit f965db9e13
8 changed files with 257 additions and 8 deletions

View File

@@ -92,11 +92,19 @@ export const Info = Schema.Struct({
timeout: Schema.optional(
Schema.Union([PositiveInt, Schema.Literal(false)]).annotate({
description:
"Timeout in milliseconds for requests to this provider. Default is 300000 (5 minutes). Set to false to disable timeout.",
"Timeout in milliseconds for full requests to this provider. Set to false to disable timeout.",
}),
).annotate({
description: "Timeout in milliseconds for full requests to this provider. Set to false to disable timeout.",
}),
headerTimeout: Schema.optional(
Schema.Union([PositiveInt, Schema.Literal(false)]).annotate({
description:
"Timeout in milliseconds to wait for response headers. Provider integrations may set defaults. Set to false to disable timeout.",
}),
).annotate({
description:
"Timeout in milliseconds for requests to this provider. Default is 300000 (5 minutes). Set to false to disable timeout.",
"Timeout in milliseconds to wait for response headers. Provider integrations may set defaults. Set to false to disable timeout.",
}),
chunkTimeout: Schema.optional(PositiveInt).annotate({
description:

View File

@@ -3,6 +3,14 @@ import { STATUS_CODES } from "http"
import { iife } from "@/util/iife"
import type { ProviderID } from "./schema"
export class HeaderTimeoutError extends Error {
public override readonly name = "ProviderHeaderTimeoutError"
constructor(public readonly ms: number) {
super(`Provider response headers timed out after ${ms}ms`)
}
}
// Adapted from overflow detection patterns in:
// https://github.com/badlogic/pi-mono/blob/main/packages/ai/src/utils/overflow.ts
const OVERFLOW_PATTERNS = [

View File

@@ -28,9 +28,10 @@ import * as ProviderTransform from "./transform"
import { ModelID, ProviderID } from "./schema"
import { ModelStatus } from "./model-status"
import { RuntimeFlags } from "@/effect/runtime-flags"
import { ProviderError } from "./error"
const log = Log.create({ service: "provider" })
const OPENAI_HEADER_TIMEOUT_DEFAULT = 10_000
function shouldUseCopilotResponsesApi(modelID: string): boolean {
const match = /^gpt-(\d+)/.exec(modelID)
if (!match) return false
@@ -85,6 +86,15 @@ function wrapSSE(res: Response, ms: number, ctl: AbortController) {
})
}
function timeoutController(ms: number) {
const ctl = new AbortController()
const id = setTimeout(() => ctl.abort(new ProviderError.HeaderTimeoutError(ms)), ms)
return {
signal: ctl.signal,
clear: () => clearTimeout(id),
}
}
function googleVertexAnthropicBaseURL(project: string | undefined, location: string | undefined) {
if (!project) return
if (location !== "eu" && location !== "us") return
@@ -194,7 +204,7 @@ function custom(dep: CustomDep): Record<string, CustomLoader> {
async getModel(sdk: any, modelID: string, _options?: Record<string, any>) {
return sdk.responses(modelID)
},
options: {},
options: { headerTimeout: OPENAI_HEADER_TIMEOUT_DEFAULT },
}),
xai: () =>
Effect.succeed({
@@ -1601,16 +1611,21 @@ export const layer = Layer.effect(
const customFetch = options["fetch"]
const chunkTimeout = options["chunkTimeout"]
const headerTimeout = options["headerTimeout"]
delete options["chunkTimeout"]
delete options["headerTimeout"]
options["fetch"] = async (input: any, init?: BunFetchRequestInit) => {
const fetchFn = customFetch ?? fetch
const opts = init ?? {}
const chunkAbortCtl = typeof chunkTimeout === "number" && chunkTimeout > 0 ? new AbortController() : undefined
const headerTimeoutMs = headerTimeout === false ? undefined : headerTimeout
const headerTimeoutCtl = typeof headerTimeoutMs === "number" ? timeoutController(headerTimeoutMs) : undefined
const signals: AbortSignal[] = []
if (opts.signal) signals.push(opts.signal)
if (chunkAbortCtl) signals.push(chunkAbortCtl.signal)
if (headerTimeoutCtl) signals.push(headerTimeoutCtl.signal)
if (options["timeout"] !== undefined && options["timeout"] !== null && options["timeout"] !== false)
signals.push(AbortSignal.timeout(options["timeout"]))
@@ -1639,7 +1654,7 @@ export const layer = Layer.effect(
...opts,
// @ts-ignore see here: https://github.com/oven-sh/bun/issues/16682
timeout: false,
})
}).finally(() => headerTimeoutCtl?.clear())
if (!chunkAbortCtl) return res
return wrapSSE(res, chunkTimeout, chunkAbortCtl)

View File

@@ -1143,6 +1143,18 @@ export function fromError(
},
{ cause: e },
).toObject()
case e instanceof ProviderError.HeaderTimeoutError:
return new APIError(
{
message: e.message,
isRetryable: true,
metadata: {
code: e.name,
timeoutMs: String(e.ms),
},
},
{ cause: e },
).toObject()
case APICallError.isInstance(e):
const parsed = ProviderError.parseAPICallError({
providerID: ctx.providerID,