summaryrefslogtreecommitdiffstats
path: root/src/client
diff options
context:
space:
mode:
authordev <dev@localhost>2026-10-21 18:37:27 +0000
committerdev <dev@localhost>2026-10-21 18:37:27 +0000
commit9d40159cad3b55c753ed02e7a4dee8cc7b6d15eb (patch)
tree05572dd06f27e5817272f80e04bb7609f19b3f61 /src/client
parent3705a6141c42a828e714853d089c48e88a232de5 (diff)
downloadaichat-9d40159cad3b55c753ed02e7a4dee8cc7b6d15eb.tar.gz
Add request timeout and retry support to aichat HTTP clients
- Add timeout, retries, retry_delay_ms, and retry_max_delay_ms options to client extra config to bound long-running requests and retry failed attempts with exponential backoff. - Apply a total request timeout via reqwest .timeout(). - Implement retry with exponential backoff for non-streaming and streaming chat completions. - Disable reqwest-eventsource internal auto-reconnect so it does not loop forever; outer retry logic is now the single owner of retry behavior. - Document the new options in config.example.yaml.
Diffstat (limited to 'src/client')
-rw-r--r--src/client/common.rs117
-rw-r--r--src/client/stream.rs6
2 files changed, 116 insertions, 7 deletions
diff --git a/src/client/common.rs b/src/client/common.rs
index 3fbfd6e..a561ed4 100644
--- a/src/client/common.rs
+++ b/src/client/common.rs
@@ -51,15 +51,21 @@ pub trait Client: Sync + Send {
fn build_client(&self) -> Result<ReqwestClient> {
let mut builder = ReqwestClient::builder();
let extra = self.extra_config();
- let timeout = extra.and_then(|v| v.connect_timeout).unwrap_or(10);
+ let connect_timeout = extra
+ .map(ExtraConfig::connect_timeout_duration)
+ .unwrap_or_else(|| Duration::from_secs(10));
+ let request_timeout = extra.and_then(ExtraConfig::request_timeout);
if let Some(proxy) = extra.and_then(|v| v.proxy.as_deref()) {
builder = set_proxy(builder, proxy)?;
}
if let Some(user_agent) = self.global_config().read().user_agent.as_ref() {
builder = builder.user_agent(user_agent);
}
+ if let Some(timeout) = request_timeout {
+ builder = builder.timeout(timeout);
+ }
let client = builder
- .connect_timeout(Duration::from_secs(timeout))
+ .connect_timeout(connect_timeout)
.build()
.with_context(|| "Failed to build client")?;
Ok(client)
@@ -72,9 +78,42 @@ pub trait Client: Sync + Send {
}
let client = self.build_client()?;
let data = input.prepare_completion_data(self.model(), false)?;
- self.chat_completions_inner(&client, data)
- .await
- .with_context(|| "Failed to call chat-completions api")
+ let retries = self.extra_config().map(|v| v.retries()).unwrap_or(0);
+ if retries == 0 {
+ return self
+ .chat_completions_inner(&client, data)
+ .await
+ .with_context(|| "Failed to call chat-completions api");
+ }
+
+ let retry_delay = self
+ .extra_config()
+ .map(|v| v.retry_delay())
+ .unwrap_or_else(|| Duration::from_millis(1000));
+ let retry_max_delay = self.extra_config().and_then(|v| v.retry_max_delay());
+ for attempt in 0..=retries {
+ match self.chat_completions_inner(&client, data.clone()).await {
+ Ok(output) => return Ok(output),
+ Err(err) => {
+ if attempt == retries {
+ return Err(err).context("Failed to call chat-completions api");
+ }
+ let mut delay = retry_delay
+ .saturating_mul(2u32.saturating_pow(attempt as u32));
+ if let Some(max) = retry_max_delay {
+ delay = delay.min(max);
+ }
+ debug!(
+ "chat-completions attempt {} failed, retrying in {:?}: {:#}",
+ attempt + 1,
+ delay,
+ err
+ );
+ tokio::time::sleep(delay).await;
+ }
+ }
+ }
+ unreachable!()
}
async fn chat_completions_streaming(
@@ -83,6 +122,7 @@ pub trait Client: Sync + Send {
handler: &mut SseHandler,
) -> Result<()> {
let abort_signal = handler.abort();
+ let retries = self.extra_config().map(|v| v.retries()).unwrap_or(0);
let input = input.clone();
tokio::select! {
ret = async {
@@ -93,7 +133,37 @@ pub trait Client: Sync + Send {
}
let client = self.build_client()?;
let data = input.prepare_completion_data(self.model(), true)?;
- self.chat_completions_streaming_inner(&client, handler, data).await
+ let retry_delay = self
+ .extra_config()
+ .map(|v| v.retry_delay())
+ .unwrap_or_else(|| Duration::from_millis(1000));
+ let retry_max_delay = self.extra_config().and_then(|v| v.retry_max_delay());
+ for attempt in 0..=retries {
+ match self
+ .chat_completions_streaming_inner(&client, handler, data.clone())
+ .await
+ {
+ Ok(()) => return Ok(()),
+ Err(err) => {
+ if attempt == retries {
+ return Err(err);
+ }
+ let mut delay = retry_delay
+ .saturating_mul(2u32.saturating_pow(attempt as u32));
+ if let Some(max) = retry_max_delay {
+ delay = delay.min(max);
+ }
+ debug!(
+ "chat-completions-streaming attempt {} failed, retrying in {:?}: {:#}",
+ attempt + 1,
+ delay,
+ err
+ );
+ tokio::time::sleep(delay).await;
+ }
+ }
+ }
+ unreachable!()
} => {
handler.done();
ret.with_context(|| "Failed to call chat-completions api")
@@ -201,6 +271,39 @@ impl Default for ClientConfig {
pub struct ExtraConfig {
pub proxy: Option<String>,
pub connect_timeout: Option<u64>,
+ pub timeout: Option<u64>,
+ pub retries: Option<u64>,
+ pub retry_delay_ms: Option<u64>,
+ pub retry_max_delay_ms: Option<u64>,
+}
+
+impl ExtraConfig {
+ /// Total request timeout in seconds (0 or None disables). Defaults to 0
+ /// (disabled) so streaming requests that legitimately run for a long time
+ /// are not cut off.
+ pub fn request_timeout(&self) -> Option<Duration> {
+ self.timeout.filter(|v| *v > 0).map(Duration::from_secs)
+ }
+
+ /// Connect timeout in seconds. Defaults to 10.
+ pub fn connect_timeout_duration(&self) -> Duration {
+ Duration::from_secs(self.connect_timeout.unwrap_or(10))
+ }
+
+ /// Number of additional attempts after the first one. Defaults to 0.
+ pub fn retries(&self) -> u64 {
+ self.retries.unwrap_or(0)
+ }
+
+ /// Base delay between retries in milliseconds. Defaults to 1000.
+ pub fn retry_delay(&self) -> Duration {
+ Duration::from_millis(self.retry_delay_ms.unwrap_or(1000))
+ }
+
+ /// Maximum delay between retries in milliseconds. Defaults to no cap.
+ pub fn retry_max_delay(&self) -> Option<Duration> {
+ self.retry_max_delay_ms.map(Duration::from_millis)
+ }
}
#[derive(Debug, Clone, Deserialize, Default)]
@@ -277,7 +380,7 @@ impl RequestData {
}
}
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct ChatCompletionsData {
pub messages: Vec<Message>,
pub temperature: Option<f64>,
diff --git a/src/client/stream.rs b/src/client/stream.rs
index fd82c94..94802a8 100644
--- a/src/client/stream.rs
+++ b/src/client/stream.rs
@@ -4,6 +4,7 @@ use crate::utils::AbortSignal;
use anyhow::{anyhow, bail, Context, Result};
use futures_util::{Stream, StreamExt};
use reqwest::RequestBuilder;
+use reqwest_eventsource::retry::Never;
use reqwest_eventsource::{Error as EventSourceError, Event, RequestBuilderExt};
use serde_json::Value;
use tokio::sync::mpsc::UnboundedSender;
@@ -95,6 +96,11 @@ where
F: FnMut(SseMmessage) -> Result<bool>,
{
let mut es = builder.eventsource()?;
+ // Disable the eventsource client's automatic reconnect so the outer retry
+ // logic (driven by ExtraConfig.retries) is the single owner of retry
+ // behavior. Without this, a dropped connection is re-opened internally in a
+ // loop that never surfaces the failure to the caller.
+ es.set_retry_policy(Box::new(Never));
while let Some(event) = es.next().await {
match event {
Ok(Event::Open) => {}