refactor: improve RAG (#675)

1 files changed, 10 insertions, 6 deletions

diff --git a/src/config/mod.rs b/src/config/mod.rs
index 25891bd..db58b3d 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -1446,12 +1446,16 @@ impl Config {
     }
 
     fn setup_document_loaders(&mut self) {
-        [("pdf", "pdftotext $1 -"), ("docx", "pandoc --to plain $1")]
-            .into_iter()
-            .for_each(|(k, v)| {
-                let (k, v) = (k.to_string(), v.to_string());
-                self.document_loaders.entry(k).or_insert(v);
-            });
+        [
+            ("pdf", "pdftotext $1 -"),
+            ("docx", "pandoc --to plain $1"),
+            (RECURSIVE_URL_LOADER, "rag-crawler $1 $2"),
+        ]
+        .into_iter()
+        .for_each(|(k, v)| {
+            let (k, v) = (k.to_string(), v.to_string());
+            self.document_loaders.entry(k).or_insert(v);
+        });
     }
 }