From ac89b7ad6716a8af811089b130c2039627d47b48 Mon Sep 17 00:00:00 2001
From: sigoden <sigoden@gmail.com>
Date: Sun, 20 Oct 2024 14:28:18 +0800
Subject: refactor: RAG bm25 (#939)

---
 Cargo.lock     | 7 -------
 Cargo.toml     | 2 +-
 src/rag/mod.rs | 4 ++--
 3 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e670a3b..31723a2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -385,7 +385,6 @@ dependencies = [
  "rayon",
  "rust-stemmers",
  "stop-words",
- "whichlang",
 ]
 
 [[package]]
@@ -3765,12 +3764,6 @@ dependencies = [
  "rustls-pki-types",
 ]
 
-[[package]]
-name = "whichlang"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "213a57fbc76ff74e9dec77cf62e47fa4e4e01dec898dc09cc6873d992eed2ef9"
-
 [[package]]
 name = "widestring"
 version = "1.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 376a3a9..3cadbab 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -64,7 +64,7 @@ sys-locale = "0.3.1"
 html_to_markdown = "0.1.0"
 rust-embed = "8.5.0"
 os_info = { version = "3.8.2", default-features = false }
-bm25 = { version = "0.3.1", features = ["language_detection", "parallelism"] }
+bm25 = { version = "0.3.1", features = ["parallelism"] }
 
 [dependencies.reqwest]
 version = "0.12.0"
diff --git a/src/rag/mod.rs b/src/rag/mod.rs
index 4cee391..87f7a69 100644
--- a/src/rag/mod.rs
+++ b/src/rag/mod.rs
@@ -10,7 +10,7 @@ mod serde_vectors;
 mod splitter;
 
 use anyhow::{anyhow, bail, Context, Result};
-use bm25::{LanguageMode, SearchEngine, SearchEngineBuilder};
+use bm25::{Language, SearchEngine, SearchEngineBuilder};
 use hnsw_rs::prelude::*;
 use indexmap::{IndexMap, IndexSet};
 use inquire::{required, validator::Validation, Confirm, Select, Text};
@@ -724,7 +724,7 @@ impl RagData {
                 documents.push(bm25::Document::new(id, &document.page_content))
             }
         }
-        SearchEngineBuilder::<DocumentId>::with_documents(LanguageMode::Detect, documents)
+        SearchEngineBuilder::<DocumentId>::with_documents(Language::English, documents)
             .k1(1.5)
             .b(0.75)
             .build()
-- 
cgit v1.2.3