summaryrefslogtreecommitdiffstats
path: root/src/rag/splitter
diff options
context:
space:
mode:
authorsigoden <sigoden@gmail.com>2024-12-04 06:56:25 +0800
committerGitHub <noreply@github.com>2024-12-04 06:56:25 +0800
commit5efb44c644cf211f64e241ebf961abc36d9a51e1 (patch)
tree530e2e89716206107a75ec6b189dbdb19c57469a /src/rag/splitter
parente86a9f3ee93d6ea0d19e0af04c5a63b41ca72582 (diff)
downloadaichat-5efb44c644cf211f64e241ebf961abc36d9a51e1.tar.gz
feat: `--file/.file` accept file formats in document_loaders (#1033)
Diffstat (limited to 'src/rag/splitter')
-rw-r--r--src/rag/splitter/mod.rs6
1 files changed, 3 insertions, 3 deletions
diff --git a/src/rag/splitter/mod.rs b/src/rag/splitter/mod.rs
index 351c560..a4a9167 100644
--- a/src/rag/splitter/mod.rs
+++ b/src/rag/splitter/mod.rs
@@ -2,7 +2,7 @@ mod language;
pub use self::language::*;
-use super::{RagDocument, RagMetadata};
+use super::{DocumentMetadata, RagDocument};
pub const DEFAULT_SEPARATES: [&str; 4] = ["\n\n", "\n", " ", ""];
@@ -75,7 +75,7 @@ impl RecursiveCharacterTextSplitter {
chunk_header_options: &SplitterChunkHeaderOptions,
) -> Vec<RagDocument> {
let mut texts: Vec<String> = Vec::new();
- let mut metadatas: Vec<RagMetadata> = Vec::new();
+ let mut metadatas: Vec<DocumentMetadata> = Vec::new();
documents.iter().for_each(|d| {
if !d.page_content.is_empty() {
texts.push(d.page_content.clone());
@@ -89,7 +89,7 @@ impl RecursiveCharacterTextSplitter {
pub fn create_documents(
&self,
texts: &[String],
- metadatas: &[RagMetadata],
+ metadatas: &[DocumentMetadata],
chunk_header_options: &SplitterChunkHeaderOptions,
) -> Vec<RagDocument> {
let SplitterChunkHeaderOptions {