diff options
| author | sigoden <sigoden@gmail.com> | 2024-12-04 06:28:43 +0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-12-04 06:28:43 +0800 |
| commit | e86a9f3ee93d6ea0d19e0af04c5a63b41ca72582 (patch) | |
| tree | 9eaffc728d027b23e0b2be63a6b6be51bbea01c7 /src/rag/loader.rs | |
| parent | 46348dfb1d65e6077ebb6441dcfdeeca003c27cb (diff) | |
| download | aichat-e86a9f3ee93d6ea0d19e0af04c5a63b41ca72582.tar.gz | |
refactor: do not refresh documents when `.edit rag-docs` (#1032)
Diffstat (limited to 'src/rag/loader.rs')
| -rw-r--r-- | src/rag/loader.rs | 69 |
1 files changed, 0 insertions, 69 deletions
diff --git a/src/rag/loader.rs b/src/rag/loader.rs index 425187a..658aa2f 100644 --- a/src/rag/loader.rs +++ b/src/rag/loader.rs @@ -1,53 +1,11 @@ use super::*; use anyhow::{Context, Result}; -use path_absolutize::Absolutize; use std::collections::HashMap; pub const EXTENSION_METADATA: &str = "__extension__"; pub const PATH_METADATA: &str = "__path__"; -pub async fn load_document( - loaders: &HashMap<String, String>, - path: &str, - has_error: &mut bool, -) -> (String, Vec<(String, RagMetadata)>) { - let mut path = path.to_string(); - let mut maybe_error = None; - let mut files = vec![]; - if is_url(&path) { - if let Some(path) = path.strip_suffix("**") { - match load_recursive_url(loaders, path).await { - Ok(v) => files.extend(v), - Err(err) => maybe_error = Some(err), - } - } else { - match load_url(loaders, &path).await { - Ok(v) => files.push(v), - Err(err) => maybe_error = Some(err), - } - } - } else { - match Path::new(&path).absolutize() { - Ok(v) => { - path = v.display().to_string(); - match load_path(loaders, &path, has_error).await { - Ok(v) => files.extend(v), - Err(err) => maybe_error = Some(err), - } - } - Err(_) => { - maybe_error = Some(anyhow!("Invalid path")); - } - }; - } - if let Some(err) = maybe_error { - *has_error = true; - println!("{}", warning_text(&format!("⚠️ {err:?}"))); - } - (path, files) -} - pub async fn load_recursive_url( loaders: &HashMap<String, String>, path: &str, @@ -76,33 +34,6 @@ pub async fn load_recursive_url( Ok(output) } -pub async fn load_path( - loaders: &HashMap<String, String>, - path: &str, - has_error: &mut bool, -) -> Result<Vec<(String, RagMetadata)>> { - let file_paths = expand_glob_paths(&[path]).await?; - let mut output = vec![]; - let file_paths_len = file_paths.len(); - match file_paths_len { - 0 => {} - 1 => output.push(load_file(loaders, &file_paths[0]).await?), - _ => { - for path in file_paths { - println!("Load {path}"); - match load_file(loaders, &path).await { - Ok(v) => output.push(v), - Err(err) => { - *has_error = true; - println!("{}", warning_text(&format!("Error: {err:?}"))); - } - } - } - } - } - Ok(output) -} - pub async fn load_file( loaders: &HashMap<String, String>, path: &str, |
