feat: rag load websites (#655)

1 files changed, 10 insertions, 22 deletions

diff --git a/src/rag/splitter/mod.rs b/src/rag/splitter/mod.rs
index 88054f7..c3d697e 100644
--- a/src/rag/splitter/mod.rs
+++ b/src/rag/splitter/mod.rs
@@ -8,7 +8,7 @@ use std::cmp::Ordering;
 
 pub const DEFAULT_SEPARATES: [&str; 4] = ["\n\n", "\n", " ", ""];
 
-pub fn detect_separators(extension: &str) -> Vec<&'static str> {
+pub fn get_separators(extension: &str) -> Vec<&'static str> {
     match extension {
         "c" | "cc" | "cpp" => Language::Cpp.separators(),
         "go" => Language::Go.separators(),
@@ -149,16 +149,7 @@ impl RecursiveCharacterTextSplitter {
                 }
 
                 let newlines_count = self.number_of_newlines(&chunk, 0, chunk.len());
-
-                let mut metadata = metadatas[i].clone();
-                metadata.insert(
-                    "loc".into(),
-                    format!(
-                        "{}:{}",
-                        line_counter_index,
-                        line_counter_index + newlines_count
-                    ),
-                );
+                let metadata = metadatas[i].clone();
                 page_content += &chunk;
                 documents.push(RagDocument {
                     page_content,
@@ -348,11 +339,8 @@ mod tests {
     use pretty_assertions::assert_eq;
     use serde_json::{json, Value};
 
-    fn build_metadata(source: &str, loc_from_line: usize, loc_to_line: usize) -> Value {
-        json!({
-            "source": source,
-            "loc": format!("{loc_from_line}:{loc_to_line}"),
-        })
+    fn build_metadata(source: &str) -> Value {
+        json!({ "source": source })
     }
     #[test]
     fn test_split_text() {
@@ -385,15 +373,15 @@ mod tests {
             json!([
                 {
                     "page_content": "foo",
-                    "metadata": build_metadata("1", 1, 1),
+                    "metadata": build_metadata("1"),
                 },
                 {
                     "page_content": "bar",
-                    "metadata": build_metadata("1", 1, 1),
+                    "metadata": build_metadata("1"),
                 },
                 {
                     "page_content": "baz",
-                    "metadata": build_metadata("2", 1, 1),
+                    "metadata": build_metadata("2"),
                 },
             ])
         );
@@ -420,15 +408,15 @@ mod tests {
             json!([
                 {
                     "page_content": "SOURCE NAME: testing\n-----\nfoo",
-                    "metadata": build_metadata("1", 1, 1),
+                    "metadata": build_metadata("1"),
                 },
                 {
                     "page_content": "SOURCE NAME: testing\n-----\n(cont'd) bar",
-                    "metadata": build_metadata("1", 1, 1),
+                    "metadata": build_metadata("1"),
                 },
                 {
                     "page_content": "SOURCE NAME: testing\n-----\nbaz",
-                    "metadata": build_metadata("2", 1, 1),
+                    "metadata": build_metadata("2"),
                 },
             ])
         );