diff options
| author | sigoden <sigoden@gmail.com> | 2024-05-14 15:02:58 +0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-05-14 15:02:58 +0800 |
| commit | a75aef459db3b28e24603617b2584a7f279a3e4d (patch) | |
| tree | 99761c2a86e8f2cffd4cdee53bc93623e36aef3a /src/client/model.rs | |
| parent | 369cf9a36aaf5b76914f8c2fd412124deb51ff92 (diff) | |
| download | aichat-a75aef459db3b28e24603617b2584a7f279a3e4d.tar.gz | |
feat: remove tiktoken (#506)
Diffstat (limited to 'src/client/model.rs')
| -rw-r--r-- | src/client/model.rs | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/src/client/model.rs b/src/client/model.rs index d0d86e4..1af3e59 100644 --- a/src/client/model.rs +++ b/src/client/model.rs @@ -1,6 +1,6 @@ use super::message::{Message, MessageContent}; -use crate::utils::{count_tokens, format_option_value}; +use crate::utils::{estimate_token_length, format_option_value}; use anyhow::{bail, Result}; use serde::Deserialize; @@ -181,7 +181,7 @@ impl Model { .iter() .map(|v| { match &v.content { - MessageContent::Text(text) => count_tokens(text), + MessageContent::Text(text) => estimate_token_length(text), MessageContent::Array(_) => 0, // TODO } }) |