feat: support RAG (#560)

* feat: support RAG

* support more embeddings models and implement concurrent embedding api

* show the progress of addings paths

* ignore embedding context when saving message

* embedding model max_chunk_size => default_chunk_size

* support pdf and pandoc formats (docx, epub, ipynb)

1 files changed, 9 insertions, 0 deletions

diff --git a/src/utils/abort_signal.rs b/src/utils/abort_signal.rs
index af58b35..ac93653 100644
--- a/src/utils/abort_signal.rs
+++ b/src/utils/abort_signal.rs
@@ -53,3 +53,12 @@ impl AbortSignalInner {
         self.ctrld.store(true, Ordering::SeqCst);
     }
 }
+
+pub async fn watch_abort_signal(abort: AbortSignal) {
+    loop {
+        if abort.aborted() {
+            break;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+    }
+}