feat: support gemini (#273)

1 files changed, 5 insertions, 0 deletions

diff --git a/src/utils/mod.rs b/src/utils/mod.rs
index 8a80b23..bd9f381 100644
--- a/src/utils/mod.rs
+++ b/src/utils/mod.rs
@@ -94,4 +94,9 @@ mod tests {
         assert_eq!(tokenize("😊 hello world"), ["😊", " hello", " world"]);
         assert_eq!(tokenize("世界"), ["世", "界"]);
     }
+
+    #[test]
+    fn test_count_tokens() {
+        assert_eq!(count_tokens("😊 hello world"), 4);
+    }
 }