From 64982b4510e38153885bfd0a78c250110b3e03c5 Mon Sep 17 00:00:00 2001 From: sigoden Date: Wed, 12 Jun 2024 19:17:40 +0800 Subject: feat: rag splitter supports languages (#593) --- src/rag/splitter.rs | 564 ---------------------------------------------------- 1 file changed, 564 deletions(-) delete mode 100644 src/rag/splitter.rs (limited to 'src/rag/splitter.rs') diff --git a/src/rag/splitter.rs b/src/rag/splitter.rs deleted file mode 100644 index 5fdacee..0000000 --- a/src/rag/splitter.rs +++ /dev/null @@ -1,564 +0,0 @@ -use super::{RagDocument, RagMetadata}; - -use std::cmp::Ordering; - -pub const DEFAULT_SEPARATES: [&str; 4] = ["\n\n", "\n", " ", ""]; -pub const HTML_SEPARATES: [&str; 28] = [ - // First, try to split along HTML tags - "", "
", "

", "
", "

  • ", "

    ", "

    ", "

    ", "

    ", "

    ", "
    ", - "", "", "", "
    ", "", "
      ", "
        ", "
        ", "