diff options
| author | dev <dev@localhost> | 2026-10-21 18:37:27 +0000 |
|---|---|---|
| committer | dev <dev@localhost> | 2026-10-21 18:37:27 +0000 |
| commit | 9d40159cad3b55c753ed02e7a4dee8cc7b6d15eb (patch) | |
| tree | 05572dd06f27e5817272f80e04bb7609f19b3f61 /src/client/common.rs | |
| parent | 3705a6141c42a828e714853d089c48e88a232de5 (diff) | |
| download | aichat-9d40159cad3b55c753ed02e7a4dee8cc7b6d15eb.tar.gz | |
Add request timeout and retry support to aichat HTTP clients
- Add timeout, retries, retry_delay_ms, and retry_max_delay_ms options to
client extra config to bound long-running requests and retry failed
attempts with exponential backoff.
- Apply a total request timeout via reqwest .timeout().
- Implement retry with exponential backoff for non-streaming and streaming
chat completions.
- Disable reqwest-eventsource internal auto-reconnect so it does not loop
forever; outer retry logic is now the single owner of retry behavior.
- Document the new options in config.example.yaml.
Diffstat (limited to 'src/client/common.rs')
| -rw-r--r-- | src/client/common.rs | 117 |
1 files changed, 110 insertions, 7 deletions
diff --git a/src/client/common.rs b/src/client/common.rs index 3fbfd6e..a561ed4 100644 --- a/src/client/common.rs +++ b/src/client/common.rs @@ -51,15 +51,21 @@ pub trait Client: Sync + Send { fn build_client(&self) -> Result<ReqwestClient> { let mut builder = ReqwestClient::builder(); let extra = self.extra_config(); - let timeout = extra.and_then(|v| v.connect_timeout).unwrap_or(10); + let connect_timeout = extra + .map(ExtraConfig::connect_timeout_duration) + .unwrap_or_else(|| Duration::from_secs(10)); + let request_timeout = extra.and_then(ExtraConfig::request_timeout); if let Some(proxy) = extra.and_then(|v| v.proxy.as_deref()) { builder = set_proxy(builder, proxy)?; } if let Some(user_agent) = self.global_config().read().user_agent.as_ref() { builder = builder.user_agent(user_agent); } + if let Some(timeout) = request_timeout { + builder = builder.timeout(timeout); + } let client = builder - .connect_timeout(Duration::from_secs(timeout)) + .connect_timeout(connect_timeout) .build() .with_context(|| "Failed to build client")?; Ok(client) @@ -72,9 +78,42 @@ pub trait Client: Sync + Send { } let client = self.build_client()?; let data = input.prepare_completion_data(self.model(), false)?; - self.chat_completions_inner(&client, data) - .await - .with_context(|| "Failed to call chat-completions api") + let retries = self.extra_config().map(|v| v.retries()).unwrap_or(0); + if retries == 0 { + return self + .chat_completions_inner(&client, data) + .await + .with_context(|| "Failed to call chat-completions api"); + } + + let retry_delay = self + .extra_config() + .map(|v| v.retry_delay()) + .unwrap_or_else(|| Duration::from_millis(1000)); + let retry_max_delay = self.extra_config().and_then(|v| v.retry_max_delay()); + for attempt in 0..=retries { + match self.chat_completions_inner(&client, data.clone()).await { + Ok(output) => return Ok(output), + Err(err) => { + if attempt == retries { + return Err(err).context("Failed to call chat-completions api"); + } + let mut delay = retry_delay + .saturating_mul(2u32.saturating_pow(attempt as u32)); + if let Some(max) = retry_max_delay { + delay = delay.min(max); + } + debug!( + "chat-completions attempt {} failed, retrying in {:?}: {:#}", + attempt + 1, + delay, + err + ); + tokio::time::sleep(delay).await; + } + } + } + unreachable!() } async fn chat_completions_streaming( @@ -83,6 +122,7 @@ pub trait Client: Sync + Send { handler: &mut SseHandler, ) -> Result<()> { let abort_signal = handler.abort(); + let retries = self.extra_config().map(|v| v.retries()).unwrap_or(0); let input = input.clone(); tokio::select! { ret = async { @@ -93,7 +133,37 @@ pub trait Client: Sync + Send { } let client = self.build_client()?; let data = input.prepare_completion_data(self.model(), true)?; - self.chat_completions_streaming_inner(&client, handler, data).await + let retry_delay = self + .extra_config() + .map(|v| v.retry_delay()) + .unwrap_or_else(|| Duration::from_millis(1000)); + let retry_max_delay = self.extra_config().and_then(|v| v.retry_max_delay()); + for attempt in 0..=retries { + match self + .chat_completions_streaming_inner(&client, handler, data.clone()) + .await + { + Ok(()) => return Ok(()), + Err(err) => { + if attempt == retries { + return Err(err); + } + let mut delay = retry_delay + .saturating_mul(2u32.saturating_pow(attempt as u32)); + if let Some(max) = retry_max_delay { + delay = delay.min(max); + } + debug!( + "chat-completions-streaming attempt {} failed, retrying in {:?}: {:#}", + attempt + 1, + delay, + err + ); + tokio::time::sleep(delay).await; + } + } + } + unreachable!() } => { handler.done(); ret.with_context(|| "Failed to call chat-completions api") @@ -201,6 +271,39 @@ impl Default for ClientConfig { pub struct ExtraConfig { pub proxy: Option<String>, pub connect_timeout: Option<u64>, + pub timeout: Option<u64>, + pub retries: Option<u64>, + pub retry_delay_ms: Option<u64>, + pub retry_max_delay_ms: Option<u64>, +} + +impl ExtraConfig { + /// Total request timeout in seconds (0 or None disables). Defaults to 0 + /// (disabled) so streaming requests that legitimately run for a long time + /// are not cut off. + pub fn request_timeout(&self) -> Option<Duration> { + self.timeout.filter(|v| *v > 0).map(Duration::from_secs) + } + + /// Connect timeout in seconds. Defaults to 10. + pub fn connect_timeout_duration(&self) -> Duration { + Duration::from_secs(self.connect_timeout.unwrap_or(10)) + } + + /// Number of additional attempts after the first one. Defaults to 0. + pub fn retries(&self) -> u64 { + self.retries.unwrap_or(0) + } + + /// Base delay between retries in milliseconds. Defaults to 1000. + pub fn retry_delay(&self) -> Duration { + Duration::from_millis(self.retry_delay_ms.unwrap_or(1000)) + } + + /// Maximum delay between retries in milliseconds. Defaults to no cap. + pub fn retry_max_delay(&self) -> Option<Duration> { + self.retry_max_delay_ms.map(Duration::from_millis) + } } #[derive(Debug, Clone, Deserialize, Default)] @@ -277,7 +380,7 @@ impl RequestData { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ChatCompletionsData { pub messages: Vec<Message>, pub temperature: Option<f64>, |
