feat: `--file/.file` accept file formats in document_loaders (#1033)

1 files changed, 0 insertions, 77 deletions

diff --git a/src/rag/loader.rs b/src/rag/loader.rs
deleted file mode 100644
index 658aa2f..0000000
--- a/src/rag/loader.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-use super::*;
-
-use anyhow::{Context, Result};
-use std::collections::HashMap;
-
-pub const EXTENSION_METADATA: &str = "__extension__";
-pub const PATH_METADATA: &str = "__path__";
-
-pub async fn load_recursive_url(
-    loaders: &HashMap<String, String>,
-    path: &str,
-) -> Result<Vec<(String, RagMetadata)>> {
-    let extension = RECURSIVE_URL_LOADER;
-    let pages: Vec<Page> = match loaders.get(extension) {
-        Some(loader_command) => {
-            let contents = run_loader_command(path, extension, loader_command)?;
-            serde_json::from_str(&contents).context(r#"The crawler response is invalid. It should follow the JSON format: `[{"path":"...", "text":"..."}]`."#)?
-        }
-        None => {
-            let options = CrawlOptions::preset(path);
-            crawl_website(path, options).await?
-        }
-    };
-    let output = pages
-        .into_iter()
-        .map(|v| {
-            let Page { path, text } = v;
-            let mut metadata: RagMetadata = Default::default();
-            metadata.insert(PATH_METADATA.into(), path);
-            metadata.insert(EXTENSION_METADATA.into(), "md".into());
-            (text, metadata)
-        })
-        .collect();
-    Ok(output)
-}
-
-pub async fn load_file(
-    loaders: &HashMap<String, String>,
-    path: &str,
-) -> Result<(String, RagMetadata)> {
-    let extension = get_patch_extension(path).unwrap_or_else(|| DEFAULT_EXTENSION.into());
-    match loaders.get(&extension) {
-        Some(loader_command) => load_with_command(path, &extension, loader_command),
-        None => load_plain(path, &extension).await,
-    }
-}
-
-pub async fn load_url(
-    loaders: &HashMap<String, String>,
-    path: &str,
-) -> Result<(String, RagMetadata)> {
-    let (contents, extension) = fetch(loaders, path, false).await?;
-    let mut metadata: RagMetadata = Default::default();
-    metadata.insert(PATH_METADATA.into(), path.into());
-    metadata.insert(EXTENSION_METADATA.into(), extension);
-    Ok((contents, metadata))
-}
-
-async fn load_plain(path: &str, extension: &str) -> Result<(String, RagMetadata)> {
-    let contents = tokio::fs::read_to_string(path).await?;
-    let mut metadata: RagMetadata = Default::default();
-    metadata.insert(PATH_METADATA.into(), path.to_string());
-    metadata.insert(EXTENSION_METADATA.into(), extension.to_string());
-    Ok((contents, metadata))
-}
-
-fn load_with_command(
-    path: &str,
-    extension: &str,
-    loader_command: &str,
-) -> Result<(String, RagMetadata)> {
-    let contents = run_loader_command(path, extension, loader_command)?;
-    let mut metadata: RagMetadata = Default::default();
-    metadata.insert(PATH_METADATA.into(), path.to_string());
-    metadata.insert(EXTENSION_METADATA.into(), DEFAULT_EXTENSION.to_string());
-    Ok((contents, metadata))
-}