summaryrefslogtreecommitdiffstats
path: root/src/utils/tiktoken.rs
diff options
context:
space:
mode:
authorsigoden <sigoden@gmail.com>2023-10-30 10:22:00 +0800
committersigoden <sigoden@gmail.com>2023-10-30 10:22:00 +0800
commit44ea384ee59fafb76882e1d6687bc4cbc6d61611 (patch)
tree1ca679e0eb79b75a2e01411cb00509914bec5a23 /src/utils/tiktoken.rs
parent49595053f0a55571f42582ffae4e705c82f2aa00 (diff)
downloadaichat-44ea384ee59fafb76882e1d6687bc4cbc6d61611.tar.gz
chore: remove unnecessary clippy macros
Diffstat (limited to 'src/utils/tiktoken.rs')
-rw-r--r--src/utils/tiktoken.rs2
1 files changed, 0 insertions, 2 deletions
diff --git a/src/utils/tiktoken.rs b/src/utils/tiktoken.rs
index 940d6f9..ce47c17 100644
--- a/src/utils/tiktoken.rs
+++ b/src/utils/tiktoken.rs
@@ -64,7 +64,6 @@ pub fn cl100k_base_singleton() -> Arc<Mutex<CoreBPE>> {
}
fn _byte_pair_merge(piece: &[u8], ranks: &HashMap<Vec<u8>, usize>) -> Vec<std::ops::Range<usize>> {
- #[allow(clippy::range_plus_one)]
let mut parts: Vec<_> = (0..piece.len()).map(|i| i..i + 1).collect();
// If you have n parts and m merges, this does O(mn) work
@@ -340,7 +339,6 @@ impl CoreBPE {
&& self.sorted_token_bytes[point].starts_with(suffix)
{
let possibility = [prefix, self.sorted_token_bytes[point].as_slice()].concat();
- #[allow(clippy::option_if_let_else)]
let encoded = match std::str::from_utf8(&possibility) {
// Morally, this is byte_pair_encode(&possibility, &self.encoder)
// But we might have introduced a regex split which would prevent merges.