1 files changed, 4 insertions, 119 deletions

diff --git a/src/rag/loader.rs b/src/rag/loader.rs
index 7fa1fd6..0b0d935 100644
--- a/src/rag/loader.rs
+++ b/src/rag/loader.rs
@@ -1,8 +1,7 @@
 use super::*;
 
-use anyhow::{bail, Context, Result};
-use async_recursion::async_recursion;
-use std::{collections::HashMap, path::Path};
+use anyhow::{Context, Result};
+use std::collections::HashMap;
 
 pub const EXTENSION_METADATA: &str = "__extension__";
 pub const PATH_METADATA: &str = "__path__";
@@ -40,14 +39,7 @@ pub async fn load_path(
     loaders: &HashMap<String, String>,
     path: &str,
 ) -> Result<Vec<(String, RagMetadata)>> {
-    let (path_str, suffixes) = parse_glob(path)?;
-    let suffixes = if suffixes.is_empty() {
-        None
-    } else {
-        Some(&suffixes)
-    };
-    let mut file_paths = vec![];
-    list_files(&mut file_paths, Path::new(&path_str), suffixes).await?;
+    let file_paths = expand_glob_paths(&[path]).await?;
     let mut output = vec![];
     let file_paths_len = file_paths.len();
     match file_paths_len {
@@ -68,7 +60,7 @@ pub async fn load_file(
     loaders: &HashMap<String, String>,
     path: &str,
 ) -> Result<(String, RagMetadata)> {
-    let extension = get_extension(path);
+    let extension = path_extension(path).unwrap_or_else(|| DEFAULT_EXTENSION.into());
     match loaders.get(&extension) {
         Some(loader_command) => load_with_command(path, &extension, loader_command),
         None => load_plain(path, &extension).await,
@@ -105,110 +97,3 @@ fn load_with_command(
     metadata.insert(EXTENSION_METADATA.into(), DEFAULT_EXTENSION.to_string());
     Ok((contents, metadata))
 }
-
-pub fn parse_glob(path_str: &str) -> Result<(String, Vec<String>)> {
-    if let Some(start) = path_str.find("/**/*.").or_else(|| path_str.find(r"\**\*.")) {
-        let base_path = path_str[..start].to_string();
-        if let Some(curly_brace_end) = path_str[start..].find('}') {
-            let end = start + curly_brace_end;
-            let extensions_str = &path_str[start + 6..end + 1];
-            let extensions = if extensions_str.starts_with('{') && extensions_str.ends_with('}') {
-                extensions_str[1..extensions_str.len() - 1]
-                    .split(',')
-                    .map(|s| s.to_string())
-                    .collect::<Vec<String>>()
-            } else {
-                bail!("Invalid path '{path_str}'");
-            };
-            Ok((base_path, extensions))
-        } else {
-            let extensions_str = &path_str[start + 6..];
-            let extensions = vec![extensions_str.to_string()];
-            Ok((base_path, extensions))
-        }
-    } else if path_str.ends_with("/**") || path_str.ends_with(r"\**") {
-        Ok((path_str[0..path_str.len() - 3].to_string(), vec![]))
-    } else {
-        Ok((path_str.to_string(), vec![]))
-    }
-}
-
-#[async_recursion]
-pub async fn list_files(
-    files: &mut Vec<String>,
-    entry_path: &Path,
-    suffixes: Option<&Vec<String>>,
-) -> Result<()> {
-    if !entry_path.exists() {
-        bail!("Not found: {:?}", entry_path);
-    }
-    if entry_path.is_file() {
-        add_file(files, suffixes, entry_path);
-        return Ok(());
-    }
-    if !entry_path.is_dir() {
-        bail!("Not a directory: {:?}", entry_path);
-    }
-    let mut reader = tokio::fs::read_dir(entry_path).await?;
-    while let Some(entry) = reader.next_entry().await? {
-        let path = entry.path();
-        if path.is_file() {
-            add_file(files, suffixes, &path);
-        } else if path.is_dir() {
-            list_files(files, &path, suffixes).await?;
-        }
-    }
-    Ok(())
-}
-
-fn add_file(files: &mut Vec<String>, suffixes: Option<&Vec<String>>, path: &Path) {
-    if is_valid_extension(suffixes, path) {
-        files.push(path.display().to_string());
-    }
-}
-
-fn is_valid_extension(suffixes: Option<&Vec<String>>, path: &Path) -> bool {
-    if let Some(suffixes) = suffixes {
-        if !suffixes.is_empty() {
-            if let Some(extension) = path.extension().map(|v| v.to_string_lossy().to_string()) {
-                return suffixes.contains(&extension);
-            }
-            return false;
-        }
-    }
-    true
-}
-
-fn get_extension(path: &str) -> String {
-    Path::new(&path)
-        .extension()
-        .map(|v| v.to_string_lossy().to_lowercase())
-        .unwrap_or_else(|| DEFAULT_EXTENSION.to_string())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parse_glob() {
-        assert_eq!(parse_glob("dir").unwrap(), ("dir".into(), vec![]));
-        assert_eq!(parse_glob("dir/**").unwrap(), ("dir".into(), vec![]));
-        assert_eq!(
-            parse_glob("dir/file.md").unwrap(),
-            ("dir/file.md".into(), vec![])
-        );
-        assert_eq!(
-            parse_glob("dir/**/*.md").unwrap(),
-            ("dir".into(), vec!["md".into()])
-        );
-        assert_eq!(
-            parse_glob("dir/**/*.{md,txt}").unwrap(),
-            ("dir".into(), vec!["md".into(), "txt".into()])
-        );
-        assert_eq!(
-            parse_glob("C:\\dir\\**\\*.{md,txt}").unwrap(),
-            ("C:\\dir".into(), vec!["md".into(), "txt".into()])
-        );
-    }
-}