From 1ec6abfaee2fdc189b348b7e3a8145bd9a84da74 Mon Sep 17 00:00:00 2001
From: sigoden <sigoden@gmail.com>
Date: Wed, 5 Jun 2024 09:02:23 +0800
Subject: feat: support RAG (#560)

* feat: support RAG

* support more embeddings models and implement concurrent embedding api

* show the progress of addings paths

* ignore embedding context when saving message

* embedding model max_chunk_size => default_chunk_size

* support pdf and pandoc formats (docx, epub, ipynb)
---
 models.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

(limited to 'models.yaml')

diff --git a/models.yaml b/models.yaml
index c306f1b..f4c980b 100644
--- a/models.yaml
+++ b/models.yaml
@@ -65,6 +65,16 @@
       max_output_tokens: 4096
       input_price: 60
       output_price: 120
+    - name: text-embedding-3-large
+      mode: embedding
+      max_input_tokens: 8191
+      default_chunk_size: 8000
+      max_concurrent_chunks: 100
+    - name: text-embedding-3-small
+      mode: embedding
+      max_input_tokens: 8191
+      default_chunk_size: 8000
+      max_concurrent_chunks: 100
 
 - platform: gemini 
   # docs:
@@ -100,6 +110,10 @@
       output_price: 10.5
       supports_vision: true
       supports_function_calling: true
+    - name: text-embedding-004
+      mode: embedding
+      max_input_tokens: 2048
+      default_chunk_size: 2000
 
 - platform: claude
   # docs:
@@ -162,6 +176,10 @@
       input_price: 8
       output_price: 24
       supports_function_calling: true
+    - name: mistral-embed
+      mode: embedding
+      max_input_tokens: 8092
+      default_chunk_size: 8000
 
 - platform: cohere
   # docs:
@@ -183,6 +201,16 @@
       input_price: 3
       output_price: 15
       supports_function_calling: true
+    - name: embed-english-v3.0
+      mode: embedding
+      max_input_tokens: 512
+      default_chunk_size: 1000
+      max_concurrent_chunks: 96
+    - name: embed-multilingual-v3.0
+      mode: embedding
+      max_input_tokens: 512
+      default_chunk_size: 1000
+      max_concurrent_chunks: 96
 
 - platform: perplexity
   # docs:
@@ -281,6 +309,16 @@
       input_price: 1.25
       output_price: 3.75
       supports_vision: true
+    - name: text-embedding-004
+      mode: embedding
+      max_input_tokens: 3072
+      default_chunk_size: 3000
+      max_concurrent_chunks: 5
+    - name: text-multilingual-embedding-002
+      mode: embedding
+      max_input_tokens: 3072
+      default_chunk_size: 3000
+      max_concurrent_chunks: 5
 
 - platform: vertexai-claude
   # docs:
@@ -509,6 +547,11 @@
       input_price: 2.8
       output_price: 2.8
       supports_vision: true
+    - name: text-embedding-v2
+      mode: embedding
+      max_input_tokens: 2048
+      default_chunk_size: 2000
+      max_concurrent_chunks: 5
 
 - platform: moonshot
   # docs:
-- 
cgit v1.2.3