summaryrefslogtreecommitdiffstats
path: root/src/rag/loader.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/rag/loader.rs')
-rw-r--r--src/rag/loader.rs69
1 files changed, 0 insertions, 69 deletions
diff --git a/src/rag/loader.rs b/src/rag/loader.rs
index 425187a..658aa2f 100644
--- a/src/rag/loader.rs
+++ b/src/rag/loader.rs
@@ -1,53 +1,11 @@
use super::*;
use anyhow::{Context, Result};
-use path_absolutize::Absolutize;
use std::collections::HashMap;
pub const EXTENSION_METADATA: &str = "__extension__";
pub const PATH_METADATA: &str = "__path__";
-pub async fn load_document(
- loaders: &HashMap<String, String>,
- path: &str,
- has_error: &mut bool,
-) -> (String, Vec<(String, RagMetadata)>) {
- let mut path = path.to_string();
- let mut maybe_error = None;
- let mut files = vec![];
- if is_url(&path) {
- if let Some(path) = path.strip_suffix("**") {
- match load_recursive_url(loaders, path).await {
- Ok(v) => files.extend(v),
- Err(err) => maybe_error = Some(err),
- }
- } else {
- match load_url(loaders, &path).await {
- Ok(v) => files.push(v),
- Err(err) => maybe_error = Some(err),
- }
- }
- } else {
- match Path::new(&path).absolutize() {
- Ok(v) => {
- path = v.display().to_string();
- match load_path(loaders, &path, has_error).await {
- Ok(v) => files.extend(v),
- Err(err) => maybe_error = Some(err),
- }
- }
- Err(_) => {
- maybe_error = Some(anyhow!("Invalid path"));
- }
- };
- }
- if let Some(err) = maybe_error {
- *has_error = true;
- println!("{}", warning_text(&format!("⚠️ {err:?}")));
- }
- (path, files)
-}
-
pub async fn load_recursive_url(
loaders: &HashMap<String, String>,
path: &str,
@@ -76,33 +34,6 @@ pub async fn load_recursive_url(
Ok(output)
}
-pub async fn load_path(
- loaders: &HashMap<String, String>,
- path: &str,
- has_error: &mut bool,
-) -> Result<Vec<(String, RagMetadata)>> {
- let file_paths = expand_glob_paths(&[path]).await?;
- let mut output = vec![];
- let file_paths_len = file_paths.len();
- match file_paths_len {
- 0 => {}
- 1 => output.push(load_file(loaders, &file_paths[0]).await?),
- _ => {
- for path in file_paths {
- println!("Load {path}");
- match load_file(loaders, &path).await {
- Ok(v) => output.push(v),
- Err(err) => {
- *has_error = true;
- println!("{}", warning_text(&format!("Error: {err:?}")));
- }
- }
- }
- }
- }
- Ok(output)
-}
-
pub async fn load_file(
loaders: &HashMap<String, String>,
path: &str,