diff options
| -rw-r--r-- | Cargo.lock | 7 | ||||
| -rw-r--r-- | Cargo.toml | 2 | ||||
| -rw-r--r-- | src/rag/mod.rs | 4 |
3 files changed, 3 insertions, 10 deletions
@@ -385,7 +385,6 @@ dependencies = [ "rayon", "rust-stemmers", "stop-words", - "whichlang", ] [[package]] @@ -3766,12 +3765,6 @@ dependencies = [ ] [[package]] -name = "whichlang" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213a57fbc76ff74e9dec77cf62e47fa4e4e01dec898dc09cc6873d992eed2ef9" - -[[package]] name = "widestring" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -64,7 +64,7 @@ sys-locale = "0.3.1" html_to_markdown = "0.1.0" rust-embed = "8.5.0" os_info = { version = "3.8.2", default-features = false } -bm25 = { version = "0.3.1", features = ["language_detection", "parallelism"] } +bm25 = { version = "0.3.1", features = ["parallelism"] } [dependencies.reqwest] version = "0.12.0" diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 4cee391..87f7a69 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -10,7 +10,7 @@ mod serde_vectors; mod splitter; use anyhow::{anyhow, bail, Context, Result}; -use bm25::{LanguageMode, SearchEngine, SearchEngineBuilder}; +use bm25::{Language, SearchEngine, SearchEngineBuilder}; use hnsw_rs::prelude::*; use indexmap::{IndexMap, IndexSet}; use inquire::{required, validator::Validation, Confirm, Select, Text}; @@ -724,7 +724,7 @@ impl RagData { documents.push(bm25::Document::new(id, &document.page_content)) } } - SearchEngineBuilder::<DocumentId>::with_documents(LanguageMode::Detect, documents) + SearchEngineBuilder::<DocumentId>::with_documents(Language::English, documents) .k1(1.5) .b(0.75) .build() |