diff options
| author | sigoden <sigoden@gmail.com> | 2024-05-15 00:31:36 +0000 |
|---|---|---|
| committer | sigoden <sigoden@gmail.com> | 2024-05-15 00:31:36 +0000 |
| commit | 5883a0685ab6c833a974a007f7f5e8a12299988a (patch) | |
| tree | d8d82919afb4e292e96a935ecaa5cc4ddb23bee7 /src/utils/mod.rs | |
| parent | a75aef459db3b28e24603617b2584a7f279a3e4d (diff) | |
| download | aichat-5883a0685ab6c833a974a007f7f5e8a12299988a.tar.gz | |
refactor: improve tokenzie
Diffstat (limited to 'src/utils/mod.rs')
| -rw-r--r-- | src/utils/mod.rs | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 6d9d295..31bb971 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -36,7 +36,8 @@ pub fn get_env_name(key: &str) -> String { pub fn tokenize(text: &str) -> Vec<&str> { if text.is_ascii() { - text.split_whitespace().collect() + text.split_inclusive(|c: char| c.is_ascii_whitespace()) + .collect() } else { unicode_segmentation::UnicodeSegmentation::graphemes(text, true).collect() } |