feat: support claude-3-7-sonnet (#1201)

4 files changed, 155 insertions, 38 deletions

diff --git a/models.yaml b/models.yaml
index fdfdfcf..97e26e0 100644
--- a/models.yaml
+++ b/models.yaml
@@ -173,10 +173,33 @@
       max_batch_size: 100
 
 # Links:
-#  - https://docs.anthropic.com/en/docs/about-claude/models
+#  - https://docs.anthropic.com/en/docs/about-claude/models/all-models
 #  - https://docs.anthropic.com/en/api/messages
 - provider: claude
   models:
+    - name: claude-3-7-sonnet-20250219
+      max_input_tokens: 200000
+      max_output_tokens: 8192
+      require_max_tokens: true
+      input_price: 3
+      output_price: 15
+      supports_vision: true
+      supports_function_calling: true
+    - name: claude-3-7-sonnet-20250219:thinking
+      real_name: claude-3-7-sonnet-20250219
+      max_input_tokens: 200000
+      max_output_tokens: 24000
+      require_max_tokens: true
+      input_price: 3
+      output_price: 15
+      supports_vision: true
+      patch:
+        body:
+          temperature: null
+          top_p: null
+          thinking: 
+            type: enabled
+            budget_tokens: 16000
     - name: claude-3-5-sonnet-latest
       max_input_tokens: 200000
       max_output_tokens: 8192
@@ -506,6 +529,29 @@
       output_price: 0.075
       supports_vision: true
       supports_function_calling: true
+    - name: claude-3-7-sonnet@20250219
+      max_input_tokens: 200000
+      max_output_tokens: 8192
+      require_max_tokens: true
+      input_price: 3
+      output_price: 15
+      supports_vision: true
+      supports_function_calling: true
+    - name: claude-3-7-sonnet@20250219:thinking
+      real_name: claude-3-7-sonnet@20250219
+      max_input_tokens: 200000
+      max_output_tokens: 24000
+      require_max_tokens: true
+      input_price: 3
+      output_price: 15
+      supports_vision: true
+      patch:
+        body:
+          temperature: null
+          top_p: null
+          thinking: 
+            type: enabled
+            budget_tokens: 16000
     - name: claude-3-5-sonnet-v2@20241022
       max_input_tokens: 200000
       max_output_tokens: 8192
@@ -586,10 +632,36 @@
 
 # Links:
 #  - https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html#model-ids-arns
+#  - https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-supported-models-features.html
 #  - https://aws.amazon.com/bedrock/pricing/
-#  - https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference-support.html
+#  - https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-call.html
 - provider: bedrock
   models:
+    - name: us.anthropic.claude-3-7-sonnet-20250219-v1:0
+      max_input_tokens: 200000
+      max_output_tokens: 8192
+      require_max_tokens: true
+      input_price: 3
+      output_price: 15
+      supports_vision: true
+      supports_function_calling: true
+    - name: us.anthropic.claude-3-7-sonnet-20250219-v1:0:thinking
+      real_name: us.anthropic.claude-3-7-sonnet-20250219-v1:0
+      max_input_tokens: 200000
+      max_output_tokens: 24000
+      require_max_tokens: true
+      input_price: 3
+      output_price: 15
+      supports_vision: true
+      patch:
+        body:
+          inferenceConfig:
+            temperature: null
+            topP: null
+          additionalModelRequestFields:
+            thinking: 
+              type: enabled
+              budget_tokens: 16000
     - name: anthropic.claude-3-5-sonnet-20241022-v2:0
       max_input_tokens: 200000
       max_output_tokens: 8192
diff --git a/src/client/bedrock.rs b/src/client/bedrock.rs
index ebc0154..089f1c2 100644
--- a/src/client/bedrock.rs
+++ b/src/client/bedrock.rs
@@ -198,6 +198,7 @@ async fn chat_completions_streaming(
     let mut function_name = String::new();
     let mut function_arguments = String::new();
     let mut function_id = String::new();
+    let mut reasoning_state = 0;
 
     let mut stream = res.bytes_stream();
     let mut buffer = BytesMut::new();
@@ -240,11 +241,21 @@ async fn chat_completions_streaming(
                         "contentBlockDelta" => {
                             if let Some(text) = data["delta"]["text"].as_str() {
                                 handler.text(text)?;
+                            } else if let Some(text) = data["delta"]["reasoningContent"]["text"].as_str() {
+                                if reasoning_state == 0 {
+                                    handler.text("<think>\n")?;
+                                    reasoning_state = 1;
+                                }
+                                handler.text(text)?;
                             } else if let Some(input) = data["delta"]["toolUse"]["input"].as_str() {
                                 function_arguments.push_str(input);
                             }
                         }
                         "contentBlockStop" => {
+                            if reasoning_state == 1 {
+                                handler.text("\n</think>\n\n")?;
+                                reasoning_state = 0;
+                            }
                             if !function_name.is_empty() {
                                 let arguments: Value = function_arguments.parse().with_context(|| {
                                     format!("Tool call '{function_name}' have non-JSON arguments '{function_arguments}'")
@@ -448,12 +459,20 @@ fn build_chat_completions_body(data: ChatCompletionsData, model: &Model) -> Resu
 }
 
 fn extract_chat_completions(data: &Value) -> Result<ChatCompletionsOutput> {
-    let mut texts = vec![];
+    let mut text = String::new();
+    let mut reasoning = None;
     let mut tool_calls = vec![];
     if let Some(array) = data["output"]["message"]["content"].as_array() {
         for item in array {
-            if let Some(text) = item["text"].as_str() {
-                texts.push(text);
+            if let Some(v) = item["text"].as_str() {
+                if !text.is_empty() {
+                    text.push_str("\n\n");
+                }
+                text.push_str(v);
+            } else if let Some(reasoning_text) = item["reasoningContent"]["reasoningText"].as_object() {
+                if let Some(text) = json_str_from_map(reasoning_text, "text") {
+                    reasoning = Some(text.to_string());
+                }
             } else if let Some(tool_use) = item["toolUse"].as_object() {
                 if let (Some(id), Some(name), Some(input)) = (
                     json_str_from_map(tool_use, "toolUseId"),
@@ -470,12 +489,16 @@ fn extract_chat_completions(data: &Value) -> Result<ChatCompletionsOutput> {
         }
     }
 
-    if texts.is_empty() && tool_calls.is_empty() {
+    if let Some(reasoning) = reasoning {
+        text = format!("<think>\n{reasoning}\n</think>\n\n{text}")
+    }
+
+    if text.is_empty() && tool_calls.is_empty() {
         bail!("Invalid response data: {data}");
     }
 
     let output = ChatCompletionsOutput {
-        text: texts.join("\n\n"),
+        text,
         tool_calls,
         id: None,
         input_tokens: data["usage"]["inputTokens"].as_u64(),
diff --git a/src/client/claude.rs b/src/client/claude.rs
index 7167e80..202d2f7 100644
--- a/src/client/claude.rs
+++ b/src/client/claude.rs
@@ -80,6 +80,7 @@ pub async fn claude_chat_completions_streaming(
     let mut function_name = String::new();
     let mut function_arguments = String::new();
     let mut function_id = String::new();
+    let mut reasoning_state = 0;
     let handle = |message: SseMmessage| -> Result<bool> {
         let data: Value = serde_json::from_str(&message.data)?;
         debug!("stream-data: {data}");
@@ -110,6 +111,12 @@ pub async fn claude_chat_completions_streaming(
                 "content_block_delta" => {
                     if let Some(text) = data["delta"]["text"].as_str() {
                         handler.text(text)?;
+                    } else if let Some(text) = data["delta"]["thinking"].as_str() {
+                        if reasoning_state == 0 {
+                            handler.text("<think>\n")?;
+                            reasoning_state = 1;
+                        }
+                        handler.text(text)?;
                     } else if let (true, Some(partial_json)) = (
                         !function_name.is_empty(),
                         data["delta"]["partial_json"].as_str(),
@@ -118,6 +125,10 @@ pub async fn claude_chat_completions_streaming(
                     }
                 }
                 "content_block_stop" => {
+                    if reasoning_state == 1 {
+                        handler.text("\n</think>\n\n")?;
+                        reasoning_state = 0;
+                    }
                     if !function_name.is_empty() {
                         let arguments: Value = if function_arguments.is_empty() {
                             json!({})
@@ -282,34 +293,45 @@ pub fn claude_build_chat_completions_body(
 }
 
 pub fn claude_extract_chat_completions(data: &Value) -> Result<ChatCompletionsOutput> {
-    let text = data["content"][0]["text"].as_str().unwrap_or_default();
-
+    let mut text = String::new();
+    let mut reasoning = None;
     let mut tool_calls = vec![];
-    if let Some(calls) = data["content"].as_array().map(|content| {
-        content
-            .iter()
-            .filter(|content| matches!(content["type"].as_str(), Some("tool_use")))
-            .collect::<Vec<&Value>>()
-    }) {
-        tool_calls = calls
-            .into_iter()
-            .filter_map(|call| {
-                if let (Some(name), Some(input), Some(id)) = (
-                    call["name"].as_str(),
-                    call.get("input"),
-                    call["id"].as_str(),
-                ) {
-                    Some(ToolCall::new(
-                        name.to_string(),
-                        input.clone(),
-                        Some(id.to_string()),
-                    ))
-                } else {
-                    None
+    if let Some(list) = data["content"].as_array() {
+        for item in list {
+            match item["type"].as_str() {
+                Some("thinking") => {
+                    if let Some(v) = item["thinking"].as_str() {
+                        reasoning = Some(v.to_string());
+                    }
                 }
-            })
-            .collect();
-    };
+                Some("text") => {
+                    if let Some(v) = item["text"].as_str() {
+                        if !text.is_empty() {
+                            text.push_str("\n\n");
+                        }
+                        text.push_str(v);
+                    }
+                }
+                Some("tool_use") => {
+                    if let (Some(name), Some(input), Some(id)) = (
+                        item["name"].as_str(),
+                        item.get("input"),
+                        item["id"].as_str(),
+                    ) {
+                        tool_calls.push(ToolCall::new(
+                            name.to_string(),
+                            input.clone(),
+                            Some(id.to_string()),
+                        ));
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+    if let Some(reasoning) = reasoning {
+        text = format!("<think>\n{reasoning}\n</think>\n\n{text}")
+    }
 
     if text.is_empty() && tool_calls.is_empty() {
         bail!("Invalid response data: {data}");
diff --git a/src/client/openai.rs b/src/client/openai.rs
index 9be4722..c002f52 100644
--- a/src/client/openai.rs
+++ b/src/client/openai.rs
@@ -106,7 +106,7 @@ pub async fn openai_chat_completions_streaming(
     let mut function_name = String::new();
     let mut function_arguments = String::new();
     let mut function_id = String::new();
-    let mut reason_state = 0;
+    let mut reasoning_state = 0;
     let handle = |message: SseMmessage| -> Result<bool> {
         if message.data == "[DONE]" {
             if !function_name.is_empty() {
@@ -127,9 +127,9 @@ pub async fn openai_chat_completions_streaming(
             .as_str()
             .filter(|v| !v.is_empty())
         {
-            if reason_state == 1 {
+            if reasoning_state == 1 {
                 handler.text("\n</think>\n\n")?;
-                reason_state = 0;
+                reasoning_state = 0;
             }
             handler.text(text)?;
         } else if let Some(text) = data["choices"][0]["delta"]["reasoning_content"]
@@ -137,9 +137,9 @@ pub async fn openai_chat_completions_streaming(
             .or_else(|| data["choices"][0]["delta"]["reasoning"].as_str())
             .filter(|v| !v.is_empty())
         {
-            if reason_state == 0 {
+            if reasoning_state == 0 {
                 handler.text("<think>\n")?;
-                reason_state = 1;
+                reasoning_state = 1;
             }
             handler.text(text)?;
         }