From 9f19108a7157db21e331c0b1c3df3f98611c1a80 Mon Sep 17 00:00:00 2001
From: sigoden <sigoden@gmail.com>
Date: Tue, 25 Jun 2024 12:55:38 +0800
Subject: refactor: rag chunk contains path metadata (#647)

---
 src/rag/mod.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'src/rag/mod.rs')

diff --git a/src/rag/mod.rs b/src/rag/mod.rs
index 3fdad02..ab7c3c7 100644
--- a/src/rag/mod.rs
+++ b/src/rag/mod.rs
@@ -262,8 +262,10 @@ impl Rag {
             );
             let documents = load(&path, &extension)
                 .with_context(|| format!("Failed to load file at '{path}'"))?;
-            let documents =
-                splitter.split_documents(&documents, &SplitterChunkHeaderOptions::default());
+            let split_options = SplitterChunkHeaderOptions::default().with_chunk_header(&format!(
+                "<document_metadata>\npath: {path}\n</document_metadata>\n\n"
+            ));
+            let documents = splitter.split_documents(&documents, &split_options);
             rag_files.push(RagFile { path, documents });
             progress(
                 &progress_tx,
-- 
cgit v1.2.3