From 176ff6f83ecfcf044a7deffad61a33b43480cdc5 Mon Sep 17 00:00:00 2001 From: Kelvie Wong Date: Tue, 30 Jan 2024 03:43:55 -0800 Subject: feat: add `extra_fields` to models of localai/ollama clients (#298) * Add an "extra_fields" config to localai models Because there are so many local AIs out there with a bunch of custom parameters you can set, this allows users to send in extra parameters to a local LLM runner, such as, e.g. `instruction_template: Alpaca`, so that Mixtral can take a system prompt. * support ollama --------- Co-authored-by: sigoden --- src/client/ollama.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/client/ollama.rs') diff --git a/src/client/ollama.rs b/src/client/ollama.rs index 0705f2e..a3e23c9 100644 --- a/src/client/ollama.rs +++ b/src/client/ollama.rs @@ -69,6 +69,7 @@ impl OllamaClient { Model::new(client_name, &v.name) .set_capabilities(v.capabilities) .set_max_tokens(v.max_tokens) + .set_extra_fields(v.extra_fields.clone()) .set_tokens_count_factors(TOKENS_COUNT_FACTORS) }) .collect() @@ -77,7 +78,9 @@ impl OllamaClient { fn request_builder(&self, client: &ReqwestClient, data: SendData) -> Result { let api_key = self.get_api_key().ok(); - let body = build_body(data, self.model.name.clone())?; + let mut body = build_body(data, self.model.name.clone())?; + + self.model.merge_extra_fields(&mut body); let chat_endpoint = self.config.chat_endpoint.as_deref().unwrap_or("/api/chat"); -- cgit v1.2.3