From 9d40159cad3b55c753ed02e7a4dee8cc7b6d15eb Mon Sep 17 00:00:00 2001 From: dev Date: Wed, 21 Oct 2026 18:37:27 +0000 Subject: Add request timeout and retry support to aichat HTTP clients - Add timeout, retries, retry_delay_ms, and retry_max_delay_ms options to client extra config to bound long-running requests and retry failed attempts with exponential backoff. - Apply a total request timeout via reqwest .timeout(). - Implement retry with exponential backoff for non-streaming and streaming chat completions. - Disable reqwest-eventsource internal auto-reconnect so it does not loop forever; outer retry logic is now the single owner of retry behavior. - Document the new options in config.example.yaml. --- config.example.yaml | 12 ++++++ src/client/common.rs | 117 ++++++++++++++++++++++++++++++++++++++++++++++++--- src/client/stream.rs | 6 +++ 3 files changed, 128 insertions(+), 7 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 4cac6c0..24e4cf9 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -116,6 +116,18 @@ clients: # extra: # proxy: socks5://127.0.0.1:1080 # Set proxy # connect_timeout: 10 # Set timeout in seconds for connect to api + # timeout: 600 # Set total request timeout in seconds (0 or omitted disables) + # retries: 3 # Number of retries after the first failed attempt (0 disables) + # retry_delay_ms: 1000 # Base delay between retries in milliseconds (default 1000) + # retry_max_delay_ms: 60000 # Cap exponential backoff delay in milliseconds (default uncapped) + + # A note on long-running providers (e.g. background/queued inference): + # A non-streaming request may hold one HTTP connection for minutes. If the + # connection is dropped by an intermediary, the client otherwise waits + # forever. Set `timeout` to bound each attempt, and `retries` to re-submit + # when an attempt fails. Be aware that retrying a non-idempotent request + # may resubmit the same work; prefer streaming (stream: true) for + # long-running requests so data flows continuously over the wire. # See https://platform.openai.com/docs/quickstart - type: openai diff --git a/src/client/common.rs b/src/client/common.rs index 3fbfd6e..a561ed4 100644 --- a/src/client/common.rs +++ b/src/client/common.rs @@ -51,15 +51,21 @@ pub trait Client: Sync + Send { fn build_client(&self) -> Result { let mut builder = ReqwestClient::builder(); let extra = self.extra_config(); - let timeout = extra.and_then(|v| v.connect_timeout).unwrap_or(10); + let connect_timeout = extra + .map(ExtraConfig::connect_timeout_duration) + .unwrap_or_else(|| Duration::from_secs(10)); + let request_timeout = extra.and_then(ExtraConfig::request_timeout); if let Some(proxy) = extra.and_then(|v| v.proxy.as_deref()) { builder = set_proxy(builder, proxy)?; } if let Some(user_agent) = self.global_config().read().user_agent.as_ref() { builder = builder.user_agent(user_agent); } + if let Some(timeout) = request_timeout { + builder = builder.timeout(timeout); + } let client = builder - .connect_timeout(Duration::from_secs(timeout)) + .connect_timeout(connect_timeout) .build() .with_context(|| "Failed to build client")?; Ok(client) @@ -72,9 +78,42 @@ pub trait Client: Sync + Send { } let client = self.build_client()?; let data = input.prepare_completion_data(self.model(), false)?; - self.chat_completions_inner(&client, data) - .await - .with_context(|| "Failed to call chat-completions api") + let retries = self.extra_config().map(|v| v.retries()).unwrap_or(0); + if retries == 0 { + return self + .chat_completions_inner(&client, data) + .await + .with_context(|| "Failed to call chat-completions api"); + } + + let retry_delay = self + .extra_config() + .map(|v| v.retry_delay()) + .unwrap_or_else(|| Duration::from_millis(1000)); + let retry_max_delay = self.extra_config().and_then(|v| v.retry_max_delay()); + for attempt in 0..=retries { + match self.chat_completions_inner(&client, data.clone()).await { + Ok(output) => return Ok(output), + Err(err) => { + if attempt == retries { + return Err(err).context("Failed to call chat-completions api"); + } + let mut delay = retry_delay + .saturating_mul(2u32.saturating_pow(attempt as u32)); + if let Some(max) = retry_max_delay { + delay = delay.min(max); + } + debug!( + "chat-completions attempt {} failed, retrying in {:?}: {:#}", + attempt + 1, + delay, + err + ); + tokio::time::sleep(delay).await; + } + } + } + unreachable!() } async fn chat_completions_streaming( @@ -83,6 +122,7 @@ pub trait Client: Sync + Send { handler: &mut SseHandler, ) -> Result<()> { let abort_signal = handler.abort(); + let retries = self.extra_config().map(|v| v.retries()).unwrap_or(0); let input = input.clone(); tokio::select! { ret = async { @@ -93,7 +133,37 @@ pub trait Client: Sync + Send { } let client = self.build_client()?; let data = input.prepare_completion_data(self.model(), true)?; - self.chat_completions_streaming_inner(&client, handler, data).await + let retry_delay = self + .extra_config() + .map(|v| v.retry_delay()) + .unwrap_or_else(|| Duration::from_millis(1000)); + let retry_max_delay = self.extra_config().and_then(|v| v.retry_max_delay()); + for attempt in 0..=retries { + match self + .chat_completions_streaming_inner(&client, handler, data.clone()) + .await + { + Ok(()) => return Ok(()), + Err(err) => { + if attempt == retries { + return Err(err); + } + let mut delay = retry_delay + .saturating_mul(2u32.saturating_pow(attempt as u32)); + if let Some(max) = retry_max_delay { + delay = delay.min(max); + } + debug!( + "chat-completions-streaming attempt {} failed, retrying in {:?}: {:#}", + attempt + 1, + delay, + err + ); + tokio::time::sleep(delay).await; + } + } + } + unreachable!() } => { handler.done(); ret.with_context(|| "Failed to call chat-completions api") @@ -201,6 +271,39 @@ impl Default for ClientConfig { pub struct ExtraConfig { pub proxy: Option, pub connect_timeout: Option, + pub timeout: Option, + pub retries: Option, + pub retry_delay_ms: Option, + pub retry_max_delay_ms: Option, +} + +impl ExtraConfig { + /// Total request timeout in seconds (0 or None disables). Defaults to 0 + /// (disabled) so streaming requests that legitimately run for a long time + /// are not cut off. + pub fn request_timeout(&self) -> Option { + self.timeout.filter(|v| *v > 0).map(Duration::from_secs) + } + + /// Connect timeout in seconds. Defaults to 10. + pub fn connect_timeout_duration(&self) -> Duration { + Duration::from_secs(self.connect_timeout.unwrap_or(10)) + } + + /// Number of additional attempts after the first one. Defaults to 0. + pub fn retries(&self) -> u64 { + self.retries.unwrap_or(0) + } + + /// Base delay between retries in milliseconds. Defaults to 1000. + pub fn retry_delay(&self) -> Duration { + Duration::from_millis(self.retry_delay_ms.unwrap_or(1000)) + } + + /// Maximum delay between retries in milliseconds. Defaults to no cap. + pub fn retry_max_delay(&self) -> Option { + self.retry_max_delay_ms.map(Duration::from_millis) + } } #[derive(Debug, Clone, Deserialize, Default)] @@ -277,7 +380,7 @@ impl RequestData { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ChatCompletionsData { pub messages: Vec, pub temperature: Option, diff --git a/src/client/stream.rs b/src/client/stream.rs index fd82c94..94802a8 100644 --- a/src/client/stream.rs +++ b/src/client/stream.rs @@ -4,6 +4,7 @@ use crate::utils::AbortSignal; use anyhow::{anyhow, bail, Context, Result}; use futures_util::{Stream, StreamExt}; use reqwest::RequestBuilder; +use reqwest_eventsource::retry::Never; use reqwest_eventsource::{Error as EventSourceError, Event, RequestBuilderExt}; use serde_json::Value; use tokio::sync::mpsc::UnboundedSender; @@ -95,6 +96,11 @@ where F: FnMut(SseMmessage) -> Result, { let mut es = builder.eventsource()?; + // Disable the eventsource client's automatic reconnect so the outer retry + // logic (driven by ExtraConfig.retries) is the single owner of retry + // behavior. Without this, a dropped connection is re-opened internally in a + // loop that never surfaces the failure to the caller. + es.set_retry_policy(Box::new(Never)); while let Some(event) = es.next().await { match event { Ok(Event::Open) => {} -- cgit v1.2.3