From 47818159a7ca3910cdc6ed00109a331d2ac2f906 Mon Sep 17 00:00:00 2001
From: sigoden <sigoden@gmail.com>
Date: Fri, 27 Oct 2023 09:33:34 +0800
Subject: refactor: streaming input text in dry-run mode (#154)

---
 src/utils/mod.rs | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'src/utils')

diff --git a/src/utils/mod.rs b/src/utils/mod.rs
index ae75969..3ae1004 100644
--- a/src/utils/mod.rs
+++ b/src/utils/mod.rs
@@ -1,5 +1,6 @@
 mod tiktoken;
 
+use self::tiktoken::cl100k_base;
 pub use self::tiktoken::{cl100k_base_singleton, count_tokens, text_to_tokens, tokens_to_text};
 
 use arboard::Clipboard;
@@ -41,3 +42,10 @@ pub fn copy(src: &str) -> Result<(), arboard::Error> {
     let mut clipboard = Clipboard::new()?;
     clipboard.set_text(src)
 }
+
+pub fn split_text(text: &str) -> Result<Vec<String>, anyhow::Error> {
+    let bpe = cl100k_base()?;
+    let tokens = bpe.encode_with_special_tokens(text);
+    let data: Result<Vec<String>, _> = tokens.into_iter().map(|v| bpe.decode(&[v])).collect();
+    data
+}
-- 
cgit v1.2.3