From 9d40159cad3b55c753ed02e7a4dee8cc7b6d15eb Mon Sep 17 00:00:00 2001 From: dev Date: Wed, 21 Oct 2026 18:37:27 +0000 Subject: Add request timeout and retry support to aichat HTTP clients - Add timeout, retries, retry_delay_ms, and retry_max_delay_ms options to client extra config to bound long-running requests and retry failed attempts with exponential backoff. - Apply a total request timeout via reqwest .timeout(). - Implement retry with exponential backoff for non-streaming and streaming chat completions. - Disable reqwest-eventsource internal auto-reconnect so it does not loop forever; outer retry logic is now the single owner of retry behavior. - Document the new options in config.example.yaml. --- src/client/common.rs | 117 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 110 insertions(+), 7 deletions(-) (limited to 'src/client/common.rs') diff --git a/src/client/common.rs b/src/client/common.rs index 3fbfd6e..a561ed4 100644 --- a/src/client/common.rs +++ b/src/client/common.rs @@ -51,15 +51,21 @@ pub trait Client: Sync + Send { fn build_client(&self) -> Result { let mut builder = ReqwestClient::builder(); let extra = self.extra_config(); - let timeout = extra.and_then(|v| v.connect_timeout).unwrap_or(10); + let connect_timeout = extra + .map(ExtraConfig::connect_timeout_duration) + .unwrap_or_else(|| Duration::from_secs(10)); + let request_timeout = extra.and_then(ExtraConfig::request_timeout); if let Some(proxy) = extra.and_then(|v| v.proxy.as_deref()) { builder = set_proxy(builder, proxy)?; } if let Some(user_agent) = self.global_config().read().user_agent.as_ref() { builder = builder.user_agent(user_agent); } + if let Some(timeout) = request_timeout { + builder = builder.timeout(timeout); + } let client = builder - .connect_timeout(Duration::from_secs(timeout)) + .connect_timeout(connect_timeout) .build() .with_context(|| "Failed to build client")?; Ok(client) @@ -72,9 +78,42 @@ pub trait Client: Sync + Send { } let client = self.build_client()?; let data = input.prepare_completion_data(self.model(), false)?; - self.chat_completions_inner(&client, data) - .await - .with_context(|| "Failed to call chat-completions api") + let retries = self.extra_config().map(|v| v.retries()).unwrap_or(0); + if retries == 0 { + return self + .chat_completions_inner(&client, data) + .await + .with_context(|| "Failed to call chat-completions api"); + } + + let retry_delay = self + .extra_config() + .map(|v| v.retry_delay()) + .unwrap_or_else(|| Duration::from_millis(1000)); + let retry_max_delay = self.extra_config().and_then(|v| v.retry_max_delay()); + for attempt in 0..=retries { + match self.chat_completions_inner(&client, data.clone()).await { + Ok(output) => return Ok(output), + Err(err) => { + if attempt == retries { + return Err(err).context("Failed to call chat-completions api"); + } + let mut delay = retry_delay + .saturating_mul(2u32.saturating_pow(attempt as u32)); + if let Some(max) = retry_max_delay { + delay = delay.min(max); + } + debug!( + "chat-completions attempt {} failed, retrying in {:?}: {:#}", + attempt + 1, + delay, + err + ); + tokio::time::sleep(delay).await; + } + } + } + unreachable!() } async fn chat_completions_streaming( @@ -83,6 +122,7 @@ pub trait Client: Sync + Send { handler: &mut SseHandler, ) -> Result<()> { let abort_signal = handler.abort(); + let retries = self.extra_config().map(|v| v.retries()).unwrap_or(0); let input = input.clone(); tokio::select! { ret = async { @@ -93,7 +133,37 @@ pub trait Client: Sync + Send { } let client = self.build_client()?; let data = input.prepare_completion_data(self.model(), true)?; - self.chat_completions_streaming_inner(&client, handler, data).await + let retry_delay = self + .extra_config() + .map(|v| v.retry_delay()) + .unwrap_or_else(|| Duration::from_millis(1000)); + let retry_max_delay = self.extra_config().and_then(|v| v.retry_max_delay()); + for attempt in 0..=retries { + match self + .chat_completions_streaming_inner(&client, handler, data.clone()) + .await + { + Ok(()) => return Ok(()), + Err(err) => { + if attempt == retries { + return Err(err); + } + let mut delay = retry_delay + .saturating_mul(2u32.saturating_pow(attempt as u32)); + if let Some(max) = retry_max_delay { + delay = delay.min(max); + } + debug!( + "chat-completions-streaming attempt {} failed, retrying in {:?}: {:#}", + attempt + 1, + delay, + err + ); + tokio::time::sleep(delay).await; + } + } + } + unreachable!() } => { handler.done(); ret.with_context(|| "Failed to call chat-completions api") @@ -201,6 +271,39 @@ impl Default for ClientConfig { pub struct ExtraConfig { pub proxy: Option, pub connect_timeout: Option, + pub timeout: Option, + pub retries: Option, + pub retry_delay_ms: Option, + pub retry_max_delay_ms: Option, +} + +impl ExtraConfig { + /// Total request timeout in seconds (0 or None disables). Defaults to 0 + /// (disabled) so streaming requests that legitimately run for a long time + /// are not cut off. + pub fn request_timeout(&self) -> Option { + self.timeout.filter(|v| *v > 0).map(Duration::from_secs) + } + + /// Connect timeout in seconds. Defaults to 10. + pub fn connect_timeout_duration(&self) -> Duration { + Duration::from_secs(self.connect_timeout.unwrap_or(10)) + } + + /// Number of additional attempts after the first one. Defaults to 0. + pub fn retries(&self) -> u64 { + self.retries.unwrap_or(0) + } + + /// Base delay between retries in milliseconds. Defaults to 1000. + pub fn retry_delay(&self) -> Duration { + Duration::from_millis(self.retry_delay_ms.unwrap_or(1000)) + } + + /// Maximum delay between retries in milliseconds. Defaults to no cap. + pub fn retry_max_delay(&self) -> Option { + self.retry_max_delay_ms.map(Duration::from_millis) + } } #[derive(Debug, Clone, Deserialize, Default)] @@ -277,7 +380,7 @@ impl RequestData { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ChatCompletionsData { pub messages: Vec, pub temperature: Option, -- cgit v1.2.3