diff options
Diffstat (limited to 'src/rag/loader.rs')
| -rw-r--r-- | src/rag/loader.rs | 23 |
1 files changed, 11 insertions, 12 deletions
diff --git a/src/rag/loader.rs b/src/rag/loader.rs index c764d78..f6c3b8e 100644 --- a/src/rag/loader.rs +++ b/src/rag/loader.rs @@ -11,15 +11,20 @@ pub async fn load_recursive_url( path: &str, ) -> Result<Vec<(String, RagMetadata)>> { let extension = RECURSIVE_URL_LOADER; - let loader_command = loaders - .get(extension) - .with_context(|| format!("Document loader '{extension}' not configured"))?; - let contents = run_loader_command(path, extension, loader_command)?; - let pages: Vec<WebPage> = serde_json::from_str(&contents).context(r#"The crawler response is invalid. It should follow the JSON format: `[{"path":"...", "text":"..."}]`."#)?; + let pages: Vec<Page> = match loaders.get(extension) { + Some(loader_command) => { + let contents = run_loader_command(path, extension, loader_command)?; + serde_json::from_str(&contents).context(r#"The crawler response is invalid. It should follow the JSON format: `[{"path":"...", "text":"..."}]`."#)? + } + None => { + let options = CrawlOptions::preset(path); + crawl_website(path, options).await? + } + }; let output = pages .into_iter() .map(|v| { - let WebPage { path, text } = v; + let Page { path, text } = v; let mut metadata: RagMetadata = Default::default(); metadata.insert(PATH_METADATA.into(), path); metadata.insert(EXTENSION_METADATA.into(), "md".into()); @@ -29,12 +34,6 @@ pub async fn load_recursive_url( Ok(output) } -#[derive(Debug, Deserialize)] -struct WebPage { - path: String, - text: String, -} - pub async fn load_path( loaders: &HashMap<String, String>, path: &str, |