diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/client/bedrock.rs | 12 | ||||
| -rw-r--r-- | src/client/claude.rs | 1 | ||||
| -rw-r--r-- | src/client/common.rs | 19 | ||||
| -rw-r--r-- | src/client/macros.rs | 10 | ||||
| -rw-r--r-- | src/client/openai.rs | 1 | ||||
| -rw-r--r-- | src/client/vertexai.rs | 9 | ||||
| -rw-r--r-- | src/config/input.rs | 2 | ||||
| -rw-r--r-- | src/serve.rs | 1 |
8 files changed, 40 insertions, 15 deletions
diff --git a/src/client/bedrock.rs b/src/client/bedrock.rs index 3f99713..648771e 100644 --- a/src/client/bedrock.rs +++ b/src/client/bedrock.rs @@ -42,6 +42,7 @@ impl BedrockClient { &self, client: &ReqwestClient, data: ChatCompletionsData, + idempotency_key: Option<&str>, ) -> Result<RequestBuilder> { let access_key_id = self.get_access_key_id()?; let secret_access_key = self.get_secret_access_key()?; @@ -61,8 +62,8 @@ impl BedrockClient { let mut request_data = RequestData::new("", body); self.patch_request_data(&mut request_data); - if self.extra_config().is_some_and(|v| v.idempotency()) { - request_data.header("idempotency-key", uuid::Uuid::new_v4().to_string()); + if let Some(key) = idempotency_key { + request_data.header("idempotency-key", key); } let RequestData { url: _, @@ -155,7 +156,8 @@ impl Client for BedrockClient { client: &ReqwestClient, data: ChatCompletionsData, ) -> Result<ChatCompletionsOutput> { - let builder = self.chat_completions_builder(client, data)?; + let idempotency_key = data.idempotency_key.clone(); + let builder = self.chat_completions_builder(client, data, idempotency_key.as_deref())?; chat_completions(builder).await } @@ -165,7 +167,8 @@ impl Client for BedrockClient { handler: &mut SseHandler, data: ChatCompletionsData, ) -> Result<()> { - let builder = self.chat_completions_builder(client, data)?; + let idempotency_key = data.idempotency_key.clone(); + let builder = self.chat_completions_builder(client, data, idempotency_key.as_deref())?; chat_completions_streaming(builder, handler).await } @@ -328,6 +331,7 @@ fn build_chat_completions_body(data: ChatCompletionsData, model: &Model) -> Resu top_p, functions, stream: _, + idempotency_key: _, } = data; let system_message = extract_system_message(&mut messages); diff --git a/src/client/claude.rs b/src/client/claude.rs index d696b8b..3fa9749 100644 --- a/src/client/claude.rs +++ b/src/client/claude.rs @@ -163,6 +163,7 @@ pub fn claude_build_chat_completions_body( top_p, functions, stream, + idempotency_key: _, } = data; let system_message = extract_system_message(&mut messages); diff --git a/src/client/common.rs b/src/client/common.rs index d0c23e9..58aaf3a 100644 --- a/src/client/common.rs +++ b/src/client/common.rs @@ -77,8 +77,14 @@ pub trait Client: Sync + Send { return Ok(ChatCompletionsOutput::new(&content)); } let client = self.build_client()?; - let data = input.prepare_completion_data(self.model(), false)?; + let mut data = input.prepare_completion_data(self.model(), false)?; let retries = self.extra_config().map(|v| v.retries()).unwrap_or(0); + if self.extra_config().is_some_and(|v| v.idempotency()) { + // A single key is generated once and reused across every retry so + // the provider debounces duplicate submissions for this logical + // request. + data.idempotency_key = Some(uuid::Uuid::new_v4().to_string()); + } if retries == 0 { return self .chat_completions_inner(&client, data) @@ -132,7 +138,10 @@ pub trait Client: Sync + Send { return Ok(()); } let client = self.build_client()?; - let data = input.prepare_completion_data(self.model(), true)?; + let mut data = input.prepare_completion_data(self.model(), true)?; + if self.extra_config().is_some_and(|v| v.idempotency()) { + data.idempotency_key = Some(uuid::Uuid::new_v4().to_string()); + } let retry_delay = self .extra_config() .map(|v| v.retry_delay()) @@ -222,10 +231,11 @@ pub trait Client: Sync + Send { &self, client: &reqwest::Client, mut request_data: RequestData, + idempotency_key: Option<&str>, ) -> RequestBuilder { self.patch_request_data(&mut request_data); - if self.extra_config().is_some_and(|v| v.idempotency()) { - request_data.header("idempotency-key", uuid::Uuid::new_v4().to_string()); + if let Some(key) = idempotency_key { + request_data.header("idempotency-key", key); } request_data.into_builder(client) } @@ -398,6 +408,7 @@ pub struct ChatCompletionsData { pub top_p: Option<f64>, pub functions: Option<Vec<FunctionDeclaration>>, pub stream: bool, + pub idempotency_key: Option<String>, } #[derive(Debug, Clone, Default)] diff --git a/src/client/macros.rs b/src/client/macros.rs index a3730bd..7a9455a 100644 --- a/src/client/macros.rs +++ b/src/client/macros.rs @@ -183,8 +183,9 @@ macro_rules! impl_client_trait { client: &reqwest::Client, data: $crate::client::ChatCompletionsData, ) -> anyhow::Result<$crate::client::ChatCompletionsOutput> { + let idempotency_key = data.idempotency_key.clone(); let request_data = $prepare_chat_completions(self, data)?; - let builder = self.request_builder(client, request_data); + let builder = self.request_builder(client, request_data, idempotency_key.as_deref()); $chat_completions(builder, self.model()).await } @@ -194,8 +195,9 @@ macro_rules! impl_client_trait { handler: &mut $crate::client::SseHandler, data: $crate::client::ChatCompletionsData, ) -> Result<()> { + let idempotency_key = data.idempotency_key.clone(); let request_data = $prepare_chat_completions(self, data)?; - let builder = self.request_builder(client, request_data); + let builder = self.request_builder(client, request_data, idempotency_key.as_deref()); $chat_completions_streaming(builder, handler, self.model()).await } @@ -205,7 +207,7 @@ macro_rules! impl_client_trait { data: &$crate::client::EmbeddingsData, ) -> Result<$crate::client::EmbeddingsOutput> { let request_data = $prepare_embeddings(self, data)?; - let builder = self.request_builder(client, request_data); + let builder = self.request_builder(client, request_data, None); $embeddings(builder, self.model()).await } @@ -215,7 +217,7 @@ macro_rules! impl_client_trait { data: &$crate::client::RerankData, ) -> Result<$crate::client::RerankOutput> { let request_data = $prepare_rerank(self, data)?; - let builder = self.request_builder(client, request_data); + let builder = self.request_builder(client, request_data, None); $rerank(builder, self.model()).await } } diff --git a/src/client/openai.rs b/src/client/openai.rs index 0c46fc3..61293ab 100644 --- a/src/client/openai.rs +++ b/src/client/openai.rs @@ -230,6 +230,7 @@ pub fn openai_build_chat_completions_body(data: ChatCompletionsData, model: &Mod top_p, functions, stream, + idempotency_key: _, } = data; let messages_len = messages.len(); diff --git a/src/client/vertexai.rs b/src/client/vertexai.rs index 16b55d1..5d88280 100644 --- a/src/client/vertexai.rs +++ b/src/client/vertexai.rs @@ -44,8 +44,9 @@ impl Client for VertexAIClient { prepare_gcloud_access_token(client, self.name(), &self.config.adc_file).await?; let model = self.model(); let model_category = ModelCategory::from_str(model.real_name())?; + let idempotency_key = data.idempotency_key.clone(); let request_data = prepare_chat_completions(self, data, &model_category)?; - let builder = self.request_builder(client, request_data); + let builder = self.request_builder(client, request_data, idempotency_key.as_deref()); match model_category { ModelCategory::Gemini => gemini_chat_completions(builder, model).await, ModelCategory::Claude => claude_chat_completions(builder, model).await, @@ -62,8 +63,9 @@ impl Client for VertexAIClient { prepare_gcloud_access_token(client, self.name(), &self.config.adc_file).await?; let model = self.model(); let model_category = ModelCategory::from_str(model.real_name())?; + let idempotency_key = data.idempotency_key.clone(); let request_data = prepare_chat_completions(self, data, &model_category)?; - let builder = self.request_builder(client, request_data); + let builder = self.request_builder(client, request_data, idempotency_key.as_deref()); match model_category { ModelCategory::Gemini => { gemini_chat_completions_streaming(builder, handler, model).await @@ -84,7 +86,7 @@ impl Client for VertexAIClient { ) -> Result<Vec<Vec<f32>>> { prepare_gcloud_access_token(client, self.name(), &self.config.adc_file).await?; let request_data = prepare_embeddings(self, data)?; - let builder = self.request_builder(client, request_data); + let builder = self.request_builder(client, request_data, None); embeddings(builder, self.model()).await } } @@ -316,6 +318,7 @@ pub fn gemini_build_chat_completions_body( top_p, functions, stream: _, + idempotency_key: _, } = data; let system_message = extract_system_message(&mut messages); diff --git a/src/config/input.rs b/src/config/input.rs index 71807f3..facfd62 100644 --- a/src/config/input.rs +++ b/src/config/input.rs @@ -274,6 +274,7 @@ impl Input { top_p: *subquery_top_p, functions: subquery_functions.clone(), stream, + idempotency_key: None, }); } let mut messages = self.build_messages()?; @@ -287,6 +288,7 @@ impl Input { top_p, functions, stream, + idempotency_key: None, }) } diff --git a/src/serve.rs b/src/serve.rs index 9aad17a..0ed1833 100644 --- a/src/serve.rs +++ b/src/serve.rs @@ -316,6 +316,7 @@ impl Server { top_p, functions, stream, + idempotency_key: None, }; if stream { |
