diff options
| author | sigoden <sigoden@gmail.com> | 2024-12-04 06:56:25 +0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-12-04 06:56:25 +0800 |
| commit | 5efb44c644cf211f64e241ebf961abc36d9a51e1 (patch) | |
| tree | 530e2e89716206107a75ec6b189dbdb19c57469a /src/rag/splitter | |
| parent | e86a9f3ee93d6ea0d19e0af04c5a63b41ca72582 (diff) | |
| download | aichat-5efb44c644cf211f64e241ebf961abc36d9a51e1.tar.gz | |
feat: `--file/.file` accept file formats in document_loaders (#1033)
Diffstat (limited to 'src/rag/splitter')
| -rw-r--r-- | src/rag/splitter/mod.rs | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/src/rag/splitter/mod.rs b/src/rag/splitter/mod.rs index 351c560..a4a9167 100644 --- a/src/rag/splitter/mod.rs +++ b/src/rag/splitter/mod.rs @@ -2,7 +2,7 @@ mod language; pub use self::language::*; -use super::{RagDocument, RagMetadata}; +use super::{DocumentMetadata, RagDocument}; pub const DEFAULT_SEPARATES: [&str; 4] = ["\n\n", "\n", " ", ""]; @@ -75,7 +75,7 @@ impl RecursiveCharacterTextSplitter { chunk_header_options: &SplitterChunkHeaderOptions, ) -> Vec<RagDocument> { let mut texts: Vec<String> = Vec::new(); - let mut metadatas: Vec<RagMetadata> = Vec::new(); + let mut metadatas: Vec<DocumentMetadata> = Vec::new(); documents.iter().for_each(|d| { if !d.page_content.is_empty() { texts.push(d.page_content.clone()); @@ -89,7 +89,7 @@ impl RecursiveCharacterTextSplitter { pub fn create_documents( &self, texts: &[String], - metadatas: &[RagMetadata], + metadatas: &[DocumentMetadata], chunk_header_options: &SplitterChunkHeaderOptions, ) -> Vec<RagDocument> { let SplitterChunkHeaderOptions { |
