From a75aef459db3b28e24603617b2584a7f279a3e4d Mon Sep 17 00:00:00 2001
From: sigoden <sigoden@gmail.com>
Date: Tue, 14 May 2024 15:02:58 +0800
Subject: feat: remove tiktoken (#506)

---
 src/client/common.rs | 2 +-
 src/client/model.rs  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'src/client')

diff --git a/src/client/common.rs b/src/client/common.rs
index 004a7dc..d42315b 100644
--- a/src/client/common.rs
+++ b/src/client/common.rs
@@ -319,7 +319,7 @@ pub trait Client: Sync + Send {
                     let tokens = tokenize(&content);
                     for token in tokens {
                         tokio::time::sleep(Duration::from_millis(10)).await;
-                        handler.text(&token)?;
+                        handler.text(token)?;
                     }
                     return Ok(());
                 }
diff --git a/src/client/model.rs b/src/client/model.rs
index d0d86e4..1af3e59 100644
--- a/src/client/model.rs
+++ b/src/client/model.rs
@@ -1,6 +1,6 @@
 use super::message::{Message, MessageContent};
 
-use crate::utils::{count_tokens, format_option_value};
+use crate::utils::{estimate_token_length, format_option_value};
 
 use anyhow::{bail, Result};
 use serde::Deserialize;
@@ -181,7 +181,7 @@ impl Model {
             .iter()
             .map(|v| {
                 match &v.content {
-                    MessageContent::Text(text) => count_tokens(text),
+                    MessageContent::Text(text) => estimate_token_length(text),
                     MessageContent::Array(_) => 0, // TODO
                 }
             })
-- 
cgit v1.2.3