From 95bad975f4fb47fe86df1837660a1341009ea12a Mon Sep 17 00:00:00 2001
From: sigoden <sigoden@gmail.com>
Date: Wed, 26 Jun 2024 08:18:58 +0800
Subject: feat: custom rag document loaders (#650)

---
 config.example.yaml | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

(limited to 'config.example.yaml')

diff --git a/config.example.yaml b/config.example.yaml
index 43c3876..286bb83 100644
--- a/config.example.yaml
+++ b/config.example.yaml
@@ -41,14 +41,21 @@ agents:
     dangerously_functions_filter: null
 
 # ---- RAG ----
-rag_embedding_model: null         # Specifies the embedding model to use
-rag_reranker_model: null            # Specifies the rerank model to use
-rag_top_k: 4                      # Specifies the number of documents to retrieve
-rag_chunk_size: null              # Specifies the chunk size
-rag_chunk_overlap: null           # Specifies the chunk overlap
-rag_min_score_vector_search: 0    # Specifies the minimum relevance score for vector-based searching
-rag_min_score_keyword_search: 0   # Specifies the minimum relevance score for keyword-based searching
-rag_min_score_rerank: 0           # Specifies the minimum relevance score for reranking
+rag_embedding_model: null                   # Specifies the embedding model to use
+rag_reranker_model: null                    # Specifies the rerank model to use
+rag_top_k: 4                                # Specifies the number of documents to retrieve
+rag_chunk_size: null                        # Specifies the chunk size
+rag_chunk_overlap: null                     # Specifies the chunk overlap
+rag_min_score_vector_search: 0              # Specifies the minimum relevance score for vector-based searching
+rag_min_score_keyword_search: 0             # Specifies the minimum relevance score for keyword-based searching
+rag_min_score_rerank: 0                     # Specifies the minimum relevance score for reranking
+# Defines document loaders
+rag_document_loaders:
+  # You can add more loaders, here is the syntax:
+  # <file-extension>: <command-to-load-the-file>
+  pdf: 'pdftotext $1 -'                     # Load .pdf file 
+  docx: 'pandoc --to plain $1'              # Load .docx file
+
 # Defines the query structure using variables like __CONTEXT__ and __INPUT__ to tailor searches to specific needs
 rag_template: |
   Use the following context as your learned knowledge, inside <context></context> XML tags.
-- 
cgit v1.2.3