From 6286251d320a7f512e49534c30b42452d30d93cc Mon Sep 17 00:00:00 2001
From: sigoden <sigoden@gmail.com>
Date: Tue, 19 Dec 2023 20:37:53 +0800
Subject: feat: support gemini (#273)

---
 src/utils/mod.rs | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'src/utils/mod.rs')

diff --git a/src/utils/mod.rs b/src/utils/mod.rs
index 8a80b23..bd9f381 100644
--- a/src/utils/mod.rs
+++ b/src/utils/mod.rs
@@ -94,4 +94,9 @@ mod tests {
         assert_eq!(tokenize("😊 hello world"), ["😊", " hello", " world"]);
         assert_eq!(tokenize("世界"), ["世", "界"]);
     }
+
+    #[test]
+    fn test_count_tokens() {
+        assert_eq!(count_tokens("😊 hello world"), 4);
+    }
 }
-- 
cgit v1.2.3