summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--config.example.yaml57
-rw-r--r--models.yaml355
-rw-r--r--src/client/bedrock.rs12
-rw-r--r--src/client/common.rs3
-rw-r--r--src/client/prompt_format.rs4
5 files changed, 348 insertions, 83 deletions
diff --git a/config.example.yaml b/config.example.yaml
index 47f644a..ea9e2fa 100644
--- a/config.example.yaml
+++ b/config.example.yaml
@@ -70,6 +70,7 @@ clients:
# See https://docs.mistral.ai/
- type: openai-compatible
name: mistral
+ api_base: https://api.mistral.ai/v1
api_key: xxx # ENV: {client}_API_KEY
# See https://docs.cohere.com/docs/the-cohere-platform
@@ -77,11 +78,15 @@ clients:
api_key: xxx # ENV: {client}_API_KEY
# See https://docs.perplexity.ai/docs/getting-started
- - type: perplexity
+ - type: openai-compatible
+ name: perplexity
+ api_base: https://api.perplexity.ai
api_key: pplx-xxx # ENV: {client}_API_KEY
# See https://console.groq.com/docs/quickstart
- - type: groq
+ - type: openai-compatible
+ name: groq
+ api_base: https://api.groq.com/openai/v1
api_key: gsk_xxx # ENV: {client}_API_KEY
# See https://github.com/jmorganca/ollama
@@ -137,71 +142,43 @@ clients:
api_key: sk-xxx # ENV: {client}_API_KEY
# See https://platform.moonshot.cn/docs/intro
- - type: moonshot
+ - type: openai-compatible
+ name: moonshot
+ api_base: https://api.moonshot.cn/v1
api_key: sk-xxx # ENV: {client}_API_KEY
# See https://docs.endpoints.anyscale.com/
- type: openai-compatible
name: anyscale
+ api_base: https://api.endpoints.anyscale.com/v1
api_key: xxx # ENV: {client}_API_KEY
- models:
- # https://docs.endpoints.anyscale.com/text-generation/query-a-model#select-a-model
- - name: meta-llama/Meta-Llama-3-70B-Instruct
- max_input_tokens: 8192
- input_price: 1
- output_price: 1
# See https://deepinfra.com/docs
- type: openai-compatible
name: deepinfra
+ api_base: https://api.deepinfra.com/v1/openai
api_key: xxx # ENV: {client}_API_KEY
- models:
- # https://deepinfra.com/models
- - name: meta-llama/Meta-Llama-3-70B-Instruct
- max_input_tokens: 8192
- input_price: 0.59
- output_price: 0.79
# See https://readme.fireworks.ai/docs/quickstart
- type: openai-compatible
name: fireworks
+ api_base: https://api.fireworks.ai/inference/v1
api_key: xxx # ENV: {client}_API_KEY
- models:
- # https://fireworks.ai/models
- - name: accounts/fireworks/models/llama-v3-70b-instruct
- max_input_tokens: 8192
- input_price: 0.9
- output_price: 0.9
# See https://openrouter.ai/docs#quick-start
- type: openai-compatible
name: openrouter
+ api_base: https://openrouter.ai/api/v1
api_key: xxx # ENV: {client}_API_KEY
- models:
- # https://openrouter.ai/docs#models
- - name: meta-llama/llama-3-70b-instruct
- max_input_tokens: 8192
- input_price: 0.81
- output_price: 0.81
# See https://octo.ai/docs/getting-started/quickstart
- type: openai-compatible
name: octoai
+ api_base: https://text.octoai.run/v1
api_key: xxx # ENV: {client}_API_KEY
- models:
- # https://octo.ai/docs/getting-started/inference-models
- - name: meta-llama-3-70b-instruct
- max_input_tokens: 8192
- input_price: 0.86
- output_price: 0.86
# See https://docs.together.ai/docs/quickstart
- type: openai-compatible
name: together
- api_key: xxx # ENV: {client}_API_KEY
- models:
- # https://docs.together.ai/docs/inference-models
- - name: meta-llama/Llama-3-70b-chat-hf
- max_input_tokens: 8192
- input_price: 0.9
- output_price: 0.9 \ No newline at end of file
+ api_base: https://api.together.xyz/v1
+ api_key: xxx # ENV: {client}_API_KEY \ No newline at end of file
diff --git a/models.yaml b/models.yaml
index cca234c..5b57dae 100644
--- a/models.yaml
+++ b/models.yaml
@@ -1,7 +1,3 @@
-# NOTES:
-# - This model list is scheduled to be updated with each new aichat release. Please do not submit PR to add new models.
-# - This model list does not include models officially marked as legacy or beta.
-
- platform: openai
# docs:
# - https://platform.openai.com/docs/models
@@ -129,10 +125,6 @@
max_input_tokens: 32000
input_price: 2
output_price: 6
- - name: mistral-medium-latest
- max_input_tokens: 32000
- input_price: 2.7
- output_price: 8.1
- name: mistral-large-latest
max_input_tokens: 32000
input_price: 8
@@ -310,16 +302,6 @@
input_price: 0.25
output_price: 1.25
supports_vision: true
- - name: meta.llama2-13b-chat-v1
- max_input_tokens: 4096
- max_output_tokens: 2048
- input_price: 0.75
- output_price: 1
- - name: meta.llama2-70b-chat-v1
- max_input_tokens: 4096
- max_output_tokens: 2048
- input_price: 1.95
- output_price: 2.56
- name: meta.llama3-8b-instruct-v1:0
max_input_tokens: 8192
max_output_tokens: 4096
@@ -351,21 +333,23 @@
# - https://developers.cloudflare.com/workers-ai/models/
# - https://developers.cloudflare.com/workers-ai/platform/pricing/
# notes:
- # - get max_output_tokens info from models doc
+ # - unable to get max_output_tokens info
models:
- - name: '@cf/meta/llama-2-7b-chat-fp16'
- max_input_tokens: 3072
- max_output_tokens: 2500
- input_price: 0.56
- output_price: 6.6
- - name: '@cf/meta/llama-2-7b-chat-int8'
- max_input_tokens: 2048
- max_output_tokens: 1800
- input_price: 0.16
- output_price: 0.24
- - name: '@cf/mistral/mistral-7b-instruct-v0.1'
- input_price: 0.11
- output_price: 0.19
+ - name: '@cf/meta/llama-3-8b-instruct'
+ max_input_tokens: 4096
+ max_output_tokens: 4096
+ - name: '@cf/mistral/mistral-7b-instruct-v0.2-lora'
+ max_input_tokens: 4096
+ max_output_tokens: 4096
+ - name: '@cf/google/gemma-7b-it-lora'
+ max_input_tokens: 4096
+ max_output_tokens: 4096
+ - name: '@cf/qwen/qwen1.5-14b-chat-awq'
+ max_input_tokens: 4096
+ max_output_tokens: 4096
+ - name: '@hf/nexusflow/starling-lm-7b-beta'
+ max_input_tokens: 4096
+ max_output_tokens: 4096
- platform: replicate
# docs:
@@ -482,3 +466,310 @@
max_input_tokens: 128000
input_price: 8.4
output_price: 8.4
+
+- platform: anyscale
+ # docs:
+ # - https://docs.endpoints.anyscale.com/text-generation/query-a-model/#select-a-model
+ # - https://docs.endpoints.anyscale.com/pricing
+ models:
+ - name: meta-llama/Meta-Llama-3-8B-Instruct
+ max_input_tokens: 8192
+ input_price: 0.15
+ output_price: 0.15
+ - name: meta-llama/Meta-Llama-3-70B-Instruct
+ max_input_tokens: 8192
+ input_price: 1.0
+ output_price: 1.0
+ - name: codellama/CodeLlama-70b-Instruct-hf
+ max_input_tokens: 4096
+ input_price: 1.0
+ output_price: 1.0
+ - name: mistralai/Mistral-7B-Instruct-v0.1
+ max_input_tokens: 16384
+ input_price: 0.15
+ output_price: 0.15
+ - name: mistralai/Mixtral-8x7B-Instruct-v0.1
+ max_input_tokens: 32768
+ input_price: 0.50
+ output_price: 0.50
+ - name: mistralai/Mixtral-8x22B-Instruct-v0.1
+ max_input_tokens: 65536
+ input_price: 0.90
+ output_price: 0.90
+ - name: google/gemma-7b-it
+ max_input_tokens: 8192
+ input_price: 0.15
+ output_price: 0.15
+
+- platform: deepinfra
+ # docs:
+ # - https://deepinfra.com/models
+ # - https://deepinfra.com/pricing
+ models:
+ - name: meta-llama/Meta-Llama-3-8B-Instruct
+ max_input_tokens: 8192
+ input_price: 0.08
+ output_price: 0.08
+ - name: meta-llama/Meta-Llama-3-70B-Instruct
+ max_input_tokens: 8192
+ input_price: 0.59
+ output_price: 0.79
+ - name: mistralai/Mistral-7B-Instruct-v0.2
+ max_input_tokens: 32768
+ input_price: 0.07
+ output_price: 0.07
+ - name: mistralai/Mixtral-8x7B-Instruct-v0.1
+ max_input_tokens: 32768
+ input_price: 0.24
+ output_price: 0.24
+ - name: mistralai/Mixtral-8x22B-Instruct-v0.1
+ max_input_tokens: 65536
+ input_price: 0.65
+ output_price: 0.65
+ - name: google/gemma-1.1-7b-it
+ max_input_tokens: 8192
+ input_price: 0.07
+ output_price: 0.07
+ - name: databricks/dbrx-instruct
+ max_input_tokens: 32768
+ input_price: 0.6
+ output_price: 0.6
+ - name: 01-ai/Yi-34B-Chat
+ max_input_tokens: 4096
+ input_price: 0.6
+ output_price: 0.6
+
+- platform: fireworks
+ # docs:
+ # - https://fireworks.ai/models
+ # - https://fireworks.ai/pricing
+ models:
+ - name: accounts/fireworks/models/llama-v3-8b-instruct
+ max_input_tokens: 8192
+ input_price: 0.2
+ output_price: 0.2
+ - name: accounts/fireworks/models/llama-v3-70b-instruct
+ max_input_tokens: 8192
+ input_price: 0.9
+ output_price: 0.9
+ - name: accounts/fireworks/models/mistral-7b-instruct-v0p2
+ max_input_tokens: 32768
+ input_price: 0.2
+ output_price: 0.2
+ - name: accounts/fireworks/models/mixtral-8x7b-instruct
+ max_input_tokens: 32768
+ input_price: 0.5
+ output_price: 0.5
+ - name: accounts/fireworks/models/mixtral-8x22b-instruct
+ max_input_tokens: 65536
+ input_price: 0.9
+ output_price: 0.9
+ - name: accounts/fireworks/models/qwen-72b-chat
+ max_input_tokens: 4096
+ input_price: 0.9
+ output_price: 0.9
+ - name: accounts/fireworks/models/gemma-7b-it
+ max_input_tokens: 8192
+ input_price: 0.2
+ output_price: 0.2
+ - name: accounts/fireworks/models/dbrx-instruct
+ max_input_tokens: 32768
+ input_price: 1.6
+ output_price: 1.6
+
+- platform: openrouter
+ # docs:
+ # - https://openrouter.ai/docs#models
+ models:
+ - name: meta-llama/llama-3-8b-instruct
+ max_input_tokens: 8192
+ input_price: 0.1
+ output_price: 0.1
+ - name: meta-llama/llama-3-8b-instruct:nitro
+ max_input_tokens: 8192
+ input_price: 0.2
+ output_price: 0.2
+ - name: meta-llama/llama-3-8b-instruct:extended
+ max_input_tokens: 16384
+ input_price: 0.275
+ output_price: 0.283
+ - name: meta-llama/llama-3-70b-instruct
+ max_input_tokens: 8192
+ input_price: 0.81
+ output_price: 0.81
+ - name: meta-llama/llama-3-70b-instruct:nitro
+ max_input_tokens: 8192
+ input_price: 0.9
+ output_price: 0.9
+ - name: mistralai/mistral-7b-instruct:free
+ max_input_tokens: 32768
+ input_price: 0.0
+ output_price: 0.0
+ - name: codellama/codellama-70b-instruct
+ max_input_tokens: 2048
+ input_price: 0.81
+ output_price: 0.81
+ - name: google/gemma-7b-it:free
+ max_input_tokens: 8192
+ input_price: 0.0
+ output_price: 0.0
+ - name: 01-ai/yi-34b-chat
+ max_input_tokens: 4096
+ input_price: 0.72
+ output_price: 0.72
+ - name: openai/gpt-3.5-turbo
+ max_input_tokens: 16385
+ input_price: 0.5
+ output_price: 1.5
+ - name: openai/gpt-4-turbo
+ max_input_tokens: 128000
+ input_price: 10
+ output_price: 30
+ supports_vision: true
+ - name: openai/gpt-4-turbo-preview
+ max_input_tokens: 128000
+ input_price: 10
+ output_price: 30
+ - name: gpt-4-vision-preview
+ max_input_tokens: 128000
+ max_output_tokens: 4096
+ input_price: 10
+ output_price: 30
+ supports_vision: true
+ - name: openai/gpt-4
+ max_input_tokens: 8192
+ input_price: 30
+ output_price: 60
+ - name: openai/gpt-4-32k
+ max_input_tokens: 32768
+ input_price: 60
+ output_price: 120
+ - name: google/gemini-pro
+ max_input_tokens: 91728
+ input_price: 0.125
+ output_price: 0.375
+ - name: google/gemini-pro-vision
+ max_input_tokens: 45875
+ input_price: 0.125
+ output_price: 0.375
+ supports_vision: true
+ - name: google/gemini-pro-1.5
+ max_input_tokens: 2800000
+ input_price: 2.5
+ output_price: 7.5
+ supports_vision: true
+ - name: anthropic/claude-3-opus
+ max_input_tokens: 200000
+ input_price: 15
+ output_price: 75
+ supports_vision: true
+ - name: anthropic/claude-3-sonnet
+ max_input_tokens: 200000
+ input_price: 3
+ output_price: 15
+ supports_vision: true
+ - name: anthropic/claude-3-haiku
+ max_input_tokens: 200000
+ input_price: 0.25
+ output_price: 1.25
+ supports_vision: true
+ - name: mistralai/mixtral-8x7b-instruct
+ max_input_tokens: 32768
+ input_price: 0.24
+ output_price: 0.24
+ - name: mistralai/mixtral-8x22b-instruct
+ max_input_tokens: 65536
+ input_price: 0.65
+ output_price: 0.65
+ - name: mistralai/mistral-small
+ max_input_tokens: 32000
+ input_price: 2
+ output_price: 6
+ - name: mistralai/mistral-large
+ max_input_tokens: 32000
+ input_price: 8
+ output_price: 24
+ - name: databricks/dbrx-instruct
+ max_input_tokens: 32768
+ input_price: 0.6
+ output_price: 0.6
+ - name: cohere/command-r
+ max_input_tokens: 128000
+ input_price: 0.5
+ output_price: 1.5
+ - name: cohere/command-r-plus
+ max_input_tokens: 128000
+ input_price: 3
+ output_price: 15
+
+- platform: octoai
+ # docs:
+ # - https://octo.ai/docs/getting-started/inference-models
+ # - https://octo.ai/pricing/text-gen-solution/
+ models:
+ - name: meta-llama-3-8b-instruct
+ max_input_tokens: 8192
+ input_price: 0.13
+ output_price: 0.13
+ - name: meta-llama-3-70b-instruct
+ max_input_tokens: 8192
+ input_price: 0.86
+ output_price: 0.86
+ - name: mistral-7b-instruct
+ max_input_tokens: 32768
+ input_price: 0.13
+ output_price: 0.13
+ - name: mixtral-8x7b-instruct
+ max_input_tokens: 32768
+ input_price: 0.34
+ output_price: 0.34
+ - name: mixtral-8x22b-instruct
+ max_input_tokens: 65536
+ input_price: 0.86
+ output_price: 0.86
+
+- platform: together
+ # docs:
+ # - https://docs.together.ai/docs/inference-models
+ # - https://www.together.ai/pricing
+ models:
+ - name: meta-llama/Llama-3-8b-chat-hf
+ max_input_tokens: 8000
+ input_price: 0.2
+ output_price: 0.2
+ - name: meta-llama/Llama-3-70b-chat-hf
+ max_input_tokens: 8000
+ input_price: 0.9
+ output_price: 0.9
+ - name: mistralai/Mistral-7B-Instruct-v0.2
+ max_input_tokens: 32768
+ input_price: 0.2
+ output_price: 0.2
+ - name: mistralai/Mixtral-8x7B-Instruct-v0.1
+ max_input_tokens: 32768
+ input_price: 0.9
+ output_price: 0.9
+ - name: mistralai/Mixtral-8x22B-Instruct-v0.1
+ max_input_tokens: 65536
+ input_price: 1.2
+ output_price: 1.2
+ - name: google/gemma-7b-it
+ max_input_tokens: 8192
+ input_price: 0.2
+ output_price: 0.2
+ - name: Qwen/Qwen1.5-72B-Chat
+ max_input_tokens: 32768
+ input_price: 0.9
+ output_price: 0.9
+ - name: databricks/dbrx-instruct
+ max_input_tokens: 32768
+ input_price: 1.2
+ output_price: 1.2
+ - name: zero-one-ai/Yi-34B-Chat
+ max_input_tokens: 4096
+ input_price: 0.8
+ output_price: 0.8
+ - name: allenai/OLMo-7B-Instruct
+ max_input_tokens: 2048
+ input_price: 0.2
+ output_price: 0.2
diff --git a/src/client/bedrock.rs b/src/client/bedrock.rs
index fb4dad8..d6ab696 100644
--- a/src/client/bedrock.rs
+++ b/src/client/bedrock.rs
@@ -2,7 +2,7 @@ use super::claude::{claude_build_body, claude_extract_completion};
use super::{
catch_error, generate_prompt, BedrockClient, Client, CompletionDetails, ExtraConfig, Model,
ModelConfig, PromptAction, PromptFormat, PromptKind, SendData, SseHandler,
- LLAMA2_PROMPT_FORMAT, LLAMA3_PROMPT_FORMAT,
+ LLAMA3_PROMPT_FORMAT, MISTRAL_PROMPT_FORMAT,
};
use crate::utils::{base64_decode, encode_uri, hex_encode, hmac_sha256, sha256};
@@ -140,7 +140,7 @@ async fn send_message(
match model_category {
ModelCategory::Anthropic => claude_extract_completion(&data),
- ModelCategory::MetaLlama2 | ModelCategory::MetaLlama3 => llama_extract_completion(&data),
+ ModelCategory::MetaLlama3 => llama_extract_completion(&data),
ModelCategory::Mistral => mistral_extrat_completion(&data),
}
}
@@ -183,7 +183,7 @@ async fn send_message_streaming(
}
}
}
- ModelCategory::MetaLlama2 | ModelCategory::MetaLlama3 => {
+ ModelCategory::MetaLlama3 => {
if let Some(text) = data["generation"].as_str() {
handler.text(text)?;
}
@@ -220,7 +220,6 @@ fn build_body(data: SendData, model: &Model, model_category: &ModelCategory) ->
body["anthropic_version"] = "bedrock-2023-05-31".into();
Ok(body)
}
- ModelCategory::MetaLlama2 => meta_llama_build_body(data, model, LLAMA2_PROMPT_FORMAT),
ModelCategory::MetaLlama3 => meta_llama_build_body(data, model, LLAMA3_PROMPT_FORMAT),
ModelCategory::Mistral => mistral_build_body(data, model),
}
@@ -256,7 +255,7 @@ fn mistral_build_body(data: SendData, model: &Model) -> Result<Value> {
top_p,
stream: _,
} = data;
- let prompt = generate_prompt(&messages, LLAMA2_PROMPT_FORMAT)?;
+ let prompt = generate_prompt(&messages, MISTRAL_PROMPT_FORMAT)?;
let mut body = json!({ "prompt": prompt });
if let Some(v) = model.max_output_tokens {
@@ -294,7 +293,6 @@ fn mistral_extrat_completion(data: &Value) -> Result<(String, CompletionDetails)
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ModelCategory {
Anthropic,
- MetaLlama2,
MetaLlama3,
Mistral,
}
@@ -305,8 +303,6 @@ impl FromStr for ModelCategory {
fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
if s.starts_with("anthropic.") {
Ok(ModelCategory::Anthropic)
- } else if s.starts_with("meta.llama2") {
- Ok(ModelCategory::MetaLlama2)
} else if s.starts_with("meta.llama3") {
Ok(ModelCategory::MetaLlama3)
} else if s.starts_with("mistral") {
diff --git a/src/client/common.rs b/src/client/common.rs
index 91d6bca..e889bc8 100644
--- a/src/client/common.rs
+++ b/src/client/common.rs
@@ -390,10 +390,11 @@ pub fn create_openai_compatible_client_config(client: &str) -> Result<Option<(St
.find(|(name, _)| client == *name)
{
None => Ok(None),
- Some((name, _)) => {
+ Some((name, api_base)) => {
let mut config = json!({
"type": "openai-compatible",
"name": name,
+ "api_base": api_base,
});
let prompts = if ALL_CLIENT_MODELS.iter().any(|v| &v.platform == name) {
vec![("api_key", "API Key:", false, PromptKind::String)]
diff --git a/src/client/prompt_format.rs b/src/client/prompt_format.rs
index fca87ac..61647a8 100644
--- a/src/client/prompt_format.rs
+++ b/src/client/prompt_format.rs
@@ -22,7 +22,7 @@ pub const GENERIC_PROMPT_FORMAT: PromptFormat<'static> = PromptFormat {
end: "### Assistant\n",
};
-pub const LLAMA2_PROMPT_FORMAT: PromptFormat<'static> = PromptFormat {
+pub const MISTRAL_PROMPT_FORMAT: PromptFormat<'static> = PromptFormat {
begin: "",
system_pre_message: "[INST] <<SYS>>",
system_post_message: "<</SYS>> [/INST]",
@@ -136,7 +136,7 @@ pub fn smart_prompt_format(model_name: &str) -> PromptFormat<'static> {
|| model_name.contains("mistral")
|| model_name.contains("mixtral")
{
- LLAMA2_PROMPT_FORMAT
+ MISTRAL_PROMPT_FORMAT
} else if model_name.contains("phi3") || model_name.contains("phi-3") {
PHI3_PROMPT_FORMAT
} else if model_name.contains("command-r") {
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-03 00:55:45 +0000