From 2f2b13c891d19e29b18ff09f7f540d7fded15887 Mon Sep 17 00:00:00 2001 From: sigoden Date: Tue, 2 Jul 2024 08:23:16 +0800 Subject: refactor: improve RAG (#675) --- src/config/mod.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'src/config/mod.rs') diff --git a/src/config/mod.rs b/src/config/mod.rs index 25891bd..db58b3d 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1446,12 +1446,16 @@ impl Config { } fn setup_document_loaders(&mut self) { - [("pdf", "pdftotext $1 -"), ("docx", "pandoc --to plain $1")] - .into_iter() - .for_each(|(k, v)| { - let (k, v) = (k.to_string(), v.to_string()); - self.document_loaders.entry(k).or_insert(v); - }); + [ + ("pdf", "pdftotext $1 -"), + ("docx", "pandoc --to plain $1"), + (RECURSIVE_URL_LOADER, "rag-crawler $1 $2"), + ] + .into_iter() + .for_each(|(k, v)| { + let (k, v) = (k.to_string(), v.to_string()); + self.document_loaders.entry(k).or_insert(v); + }); } } -- cgit v1.2.3