summaryrefslogtreecommitdiffstats
path: root/src/rag/loader.rs
diff options
context:
space:
mode:
authorsigoden <sigoden@gmail.com>2024-11-04 19:44:11 +0800
committerGitHub <noreply@github.com>2024-11-04 19:44:11 +0800
commit9fa3d8cd134571b6a16fb1b328bed3c025742576 (patch)
tree4af688326e2ac755c404bcb25ad11fdaaaace3b5 /src/rag/loader.rs
parent6b77890ec400090caa5d9fa3bffee0db91c9adfc (diff)
downloadaichat-9fa3d8cd134571b6a16fb1b328bed3c025742576.tar.gz
refactor: save RAG document path as an absolute path (#965)
Diffstat (limited to 'src/rag/loader.rs')
-rw-r--r--src/rag/loader.rs25
1 files changed, 17 insertions, 8 deletions
diff --git a/src/rag/loader.rs b/src/rag/loader.rs
index 2ea75a9..425187a 100644
--- a/src/rag/loader.rs
+++ b/src/rag/loader.rs
@@ -1,6 +1,7 @@
use super::*;
use anyhow::{Context, Result};
+use path_absolutize::Absolutize;
use std::collections::HashMap;
pub const EXTENSION_METADATA: &str = "__extension__";
@@ -11,31 +12,40 @@ pub async fn load_document(
path: &str,
has_error: &mut bool,
) -> (String, Vec<(String, RagMetadata)>) {
+ let mut path = path.to_string();
let mut maybe_error = None;
let mut files = vec![];
- if is_url(path) {
+ if is_url(&path) {
if let Some(path) = path.strip_suffix("**") {
match load_recursive_url(loaders, path).await {
Ok(v) => files.extend(v),
Err(err) => maybe_error = Some(err),
}
} else {
- match load_url(loaders, path).await {
+ match load_url(loaders, &path).await {
Ok(v) => files.push(v),
Err(err) => maybe_error = Some(err),
}
}
} else {
- match load_path(loaders, path, has_error).await {
- Ok(v) => files.extend(v),
- Err(err) => maybe_error = Some(err),
- }
+ match Path::new(&path).absolutize() {
+ Ok(v) => {
+ path = v.display().to_string();
+ match load_path(loaders, &path, has_error).await {
+ Ok(v) => files.extend(v),
+ Err(err) => maybe_error = Some(err),
+ }
+ }
+ Err(_) => {
+ maybe_error = Some(anyhow!("Invalid path"));
+ }
+ };
}
if let Some(err) = maybe_error {
*has_error = true;
println!("{}", warning_text(&format!("⚠️ {err:?}")));
}
- (path.to_string(), files)
+ (path, files)
}
pub async fn load_recursive_url(
@@ -71,7 +81,6 @@ pub async fn load_path(
path: &str,
has_error: &mut bool,
) -> Result<Vec<(String, RagMetadata)>> {
- let path = Path::new(path).absolutize()?.display().to_string();
let file_paths = expand_glob_paths(&[path]).await?;
let mut output = vec![];
let file_paths_len = file_paths.len();