use super::{RagDocument, RagMetadata}; use std::cmp::Ordering; pub const DEFAULT_SEPARATES: [&str; 4] = ["\n\n", "\n", " ", ""]; pub const HTML_SEPARATES: [&str; 28] = [ // First, try to split along HTML tags "", "

", "

", "
", "

", "

", "

", "

", "

", "

", "

", "", "", "", "
", " ", "
", "
", "
", "
", "
", // Head "", "

🦜️🔗 LangChain

⚡ Building applications with LLMs through composability ⚡

As an open source project in a rapidly developing field, we are extremely open to contributions.
"#; let splitter = Splitter::new(175, 20, &HTML_SEPARATES); let output = splitter.split_text(text); let expected_output = vec![ "\n", "\n 🦜️🔗 LangChain", r#" "#, r#"

🦜️🔗 LangChain

⚡ Building applications with LLMs through composability ⚡

"#, r#"
As an open source project in a rapidly developing field, we are extremely open to contributions.
"#, ]; assert_eq!(output, expected_output); } }