From f82524fd154a1bf4a7277db868147afdb0cd507a Mon Sep 17 00:00:00 2001 From: sigoden Date: Thu, 27 Jun 2024 12:30:09 +0800 Subject: feat: implement native rag url loader (#660) --- src/rag/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/rag/mod.rs') diff --git a/src/rag/mod.rs b/src/rag/mod.rs index f01fc0c..9e37e6f 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -237,7 +237,7 @@ impl Rag { if let Some(path) = path.strip_suffix("**") { new_paths.push((path.to_string(), RECURSIVE_URL_LOADER.into())); } else { - new_paths.push((path.to_string(), "url".into())) + new_paths.push((path.to_string(), URL_LOADER.into())) } } else { let path = Path::new(path); @@ -275,6 +275,7 @@ impl Rag { for (index, (path, loader_name)) in new_paths.into_iter().enumerate() { println!("Loading {path} [{}/{new_paths_len}]", index + 1); let documents = load(&loaders, &path, &loader_name) + .await .with_context(|| format!("Failed to load '{path}'"))?; let separator = get_separators(&loader_name); let splitter = RecursiveCharacterTextSplitter::new( -- cgit v1.2.3