feat: support builtin website crawling (recursive_url) (#786)

1 files changed, 6 insertions, 10 deletions

diff --git a/src/config/mod.rs b/src/config/mod.rs
index 1c9ae9d..8db7085 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -1740,16 +1740,12 @@ impl Config {
     }
 
     fn setup_document_loaders(&mut self) {
-        [
-            ("pdf", "pdftotext $1 -"),
-            ("docx", "pandoc --to plain $1"),
-            (RECURSIVE_URL_LOADER, "rag-crawler $1 $2"),
-        ]
-        .into_iter()
-        .for_each(|(k, v)| {
-            let (k, v) = (k.to_string(), v.to_string());
-            self.document_loaders.entry(k).or_insert(v);
-        });
+        [("pdf", "pdftotext $1 -"), ("docx", "pandoc --to plain $1")]
+            .into_iter()
+            .for_each(|(k, v)| {
+                let (k, v) = (k.to_string(), v.to_string());
+                self.document_loaders.entry(k).or_insert(v);
+            });
     }
 }