feat: enhance `.file` for loading resources from diverse sources (#1155)

3 files changed, 62 insertions, 5 deletions

diff --git a/src/utils/command.rs b/src/utils/command.rs
index 32e470d..3008028 100644
--- a/src/utils/command.rs
+++ b/src/utils/command.rs
@@ -107,9 +107,8 @@ pub fn run_command_with_output<T: AsRef<OsStr>>(
 }
 
 pub fn run_loader_command(path: &str, extension: &str, loader_command: &str) -> Result<String> {
-    let cmd_args = shell_words::split(loader_command).with_context(|| {
-        anyhow!("Invalid rag document loader '{extension}': `{loader_command}`")
-    })?;
+    let cmd_args = shell_words::split(loader_command)
+        .with_context(|| anyhow!("Invalid document loader '{extension}': `{loader_command}`"))?;
     let mut use_stdout = true;
     let outpath = temp_file("-output-", "").display().to_string();
     let cmd_args: Vec<_> = cmd_args
diff --git a/src/utils/loader.rs b/src/utils/loader.rs
index 2ac671d..b36bf4a 100644
--- a/src/utils/loader.rs
+++ b/src/utils/loader.rs
@@ -1,17 +1,19 @@
 use super::*;
 
-use anyhow::{Context, Result};
+use anyhow::{anyhow, Context, Result};
 use indexmap::IndexMap;
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
 pub const EXTENSION_METADATA: &str = "__extension__";
 
 pub type DocumentMetadata = IndexMap<String, String>;
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct LoadedDocument {
     pub path: String,
     pub contents: String,
+    #[serde(default)]
     pub metadata: DocumentMetadata,
 }
 
@@ -80,3 +82,44 @@ fn load_with_command(path: &str, extension: &str, loader_command: &str) -> Resul
     metadata.insert(EXTENSION_METADATA.into(), DEFAULT_EXTENSION.to_string());
     Ok(LoadedDocument::new(path.into(), contents, metadata))
 }
+
+pub fn is_loader_protocol(loaders: &HashMap<String, String>, path: &str) -> bool {
+    match path.split_once(':') {
+        Some((protocol, _)) => loaders.contains_key(protocol),
+        None => false,
+    }
+}
+
+pub fn load_protocol_path(
+    loaders: &HashMap<String, String>,
+    path: &str,
+) -> Result<Vec<LoadedDocument>> {
+    let (protocol, loader_command, new_path) = path
+        .split_once(':')
+        .and_then(|(protocol, path)| {
+            let loader_command = loaders.get(protocol)?;
+            Some((protocol, loader_command, path))
+        })
+        .ok_or_else(|| anyhow!("No document loader for '{}'", path))?;
+    let contents = run_loader_command(new_path, protocol, loader_command)?;
+    let output = if let Ok(list) = serde_json::from_str::<Vec<LoadedDocument>>(&contents) {
+        list.into_iter()
+            .map(|mut v| {
+                if v.path.starts_with(path) {
+                } else if v.path.starts_with(new_path) {
+                    v.path = format!("{}:{}", protocol, v.path);
+                } else {
+                    v.path = format!("{}/{}", path, v.path);
+                }
+                v
+            })
+            .collect()
+    } else {
+        vec![LoadedDocument::new(
+            path.into(),
+            contents,
+            Default::default(),
+        )]
+    };
+    Ok(output)
+}
diff --git a/src/utils/path.rs b/src/utils/path.rs
index bf3d936..20a365e 100644
--- a/src/utils/path.rs
+++ b/src/utils/path.rs
@@ -2,6 +2,7 @@ use std::path::{Component, Path, PathBuf};
 
 use anyhow::{bail, Result};
 use indexmap::IndexSet;
+use path_absolutize::Absolutize;
 
 pub fn safe_join_path<T1: AsRef<Path>, T2: AsRef<Path>>(
     base_path: T1,
@@ -75,6 +76,20 @@ pub fn get_patch_extension(path: &str) -> Option<String> {
         .map(|v| v.to_string_lossy().to_lowercase())
 }
 
+pub fn to_absolute_path(path: &str) -> Result<String> {
+    Ok(Path::new(&path).absolutize()?.display().to_string())
+}
+
+pub fn resolve_home_dir(path: &str) -> String {
+    let mut path = path.to_string();
+    if path.starts_with("~/") || path.starts_with("~\\") {
+        if let Some(home_dir) = dirs::home_dir() {
+            path.replace_range(..1, &home_dir.display().to_string());
+        }
+    }
+    path
+}
+
 fn parse_glob(path_str: &str) -> Result<(String, Vec<String>)> {
     if let Some(start) = path_str.find("/**/*.").or_else(|| path_str.find(r"\**\*.")) {
         let base_path = path_str[..start].to_string();