From 6fa697b67842c360613adf99baabcc52dda2a99a Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Mon, 25 Sep 2023 18:34:14 +0300 Subject: [PATCH 01/14] Install llm-core --- requirements/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index a769c8b..3cb78a9 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,4 @@ +# model openai~=0.27 -neon-mq-connector~=0.7 -ovos-utils~=0.0.32 -ovos-config~=0.0.10 \ No newline at end of file +# networking +neon_llm_core==0.0.6 \ No newline at end of file From 9a991169e247745a897b799029c29f9e0ea734d6 Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Mon, 25 Sep 2023 18:36:58 +0300 Subject: [PATCH 02/14] Use core in RMQ --- neon_llm_chatgpt/rmq.py | 76 +++++++++++------------------------------ 1 file changed, 19 insertions(+), 57 deletions(-) diff --git a/neon_llm_chatgpt/rmq.py b/neon_llm_chatgpt/rmq.py index 52fcceb..e73a57f 100644 --- a/neon_llm_chatgpt/rmq.py +++ b/neon_llm_chatgpt/rmq.py @@ -23,71 +23,33 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import pika - -from neon_mq_connector.connector import MQConnector -from neon_mq_connector.utils.network_utils import dict_to_b64 -from neon_mq_connector.utils.rabbit_utils import create_mq_callback -from ovos_utils.log import LOG +from neon_llm_core.rmq import NeonLLMMQConnector from neon_llm_chatgpt.chatgpt import ChatGPT -from neon_llm_chatgpt.config import load_config -class ChatgptMQ(MQConnector): +class ChatgptMQ(NeonLLMMQConnector): + """ + Module for processing MQ requests to ChatGPT """ - Module for processing MQ requests from PyKlatchat to LibreTranslate""" def __init__(self): - config = load_config() - chatgpt_config = config.get("ChatGPT", None) - self.chatGPT = ChatGPT(chatgpt_config) - - self.service_name = 'neon_llm_chatgpt' - - mq_config = config.get("MQ", None) - super().__init__(config=mq_config, service_name=self.service_name) - - self.vhost = "/llm" - self.queue = "chat_gpt_input" - self.register_consumer(name=self.service_name, - vhost=self.vhost, - queue=self.queue, - callback=self.handle_request, - on_error=self.default_error_handler, - auto_ack=False) - - @create_mq_callback(include_callback_props=('channel', 'method', 'body')) - def handle_request(self, - channel: pika.channel.Channel, - method: pika.spec.Basic.Return, - body: dict): - """ - Handles requests from MQ to ChatGPT received on queue - "request_chatgpt" - - :param channel: MQ channel object (pika.channel.Channel) - :param method: MQ return method (pika.spec.Basic.Return) - :param body: request body (dict) - """ - message_id = body["message_id"] - routing_key = body["routing_key"] + super().__init__() + self.warmup() - query = body["query"] - history = body["history"] + @property + def name(self): + return "chatgpt" - response = self.chatGPT.ask(message=query, chat_history=history) + @property + def model(self): + if self._model is None: + self._model = ChatGPT(self.model_config) + return self._model - api_response = { - "message_id": message_id, - "response": response - } + def warmup(self): + self.model - channel.basic_publish(exchange='', - routing_key=routing_key, - body=dict_to_b64(api_response), - properties=pika.BasicProperties( - expiration=str(1000))) - channel.basic_ack(method.delivery_tag) - LOG.info(f"Handled request: {message_id}") + @staticmethod + def compose_opinion_prompt(respondent_nick: str, question: str, answer: str) -> str: + return f'Why Answer "{answer}" to the Question "{question}" generated by Bot named "{respondent_nick}" is good?' From 2a764de60825846cd4e5c7900328979d5a1a97ae Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Mon, 25 Sep 2023 18:40:09 +0300 Subject: [PATCH 03/14] Use config.py as core duplicate --- neon_llm_chatgpt/config.py | 50 -------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 neon_llm_chatgpt/config.py diff --git a/neon_llm_chatgpt/config.py b/neon_llm_chatgpt/config.py deleted file mode 100644 index 2787463..0000000 --- a/neon_llm_chatgpt/config.py +++ /dev/null @@ -1,50 +0,0 @@ -# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# All trademark and other rights reserved by their respective owners -# Copyright 2008-2021 Neongecko.com Inc. -# BSD-3 -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import json - -from os.path import join, dirname, isfile -from ovos_utils.log import LOG -from ovos_config.config import Configuration - - -def load_config() -> dict: - """ - Load and return a configuration object, - """ - legacy_config_path = "/app/app/config.json" - if isfile(legacy_config_path): - LOG.warning(f"Deprecated configuration found at {legacy_config_path}") - with open(legacy_config_path) as f: - config = json.load(f) - return config - config = Configuration() - if not config: - LOG.warning(f"No configuration found! falling back to defaults") - default_config_path = join(dirname(__file__), "default_config.json") - with open(default_config_path) as f: - config = json.load(f) - return config From 64f0dd6a0e74bf722b337ae55aea41cbd772997c Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Mon, 25 Sep 2023 19:39:37 +0300 Subject: [PATCH 04/14] Use core in model --- neon_llm_chatgpt/chatgpt.py | 118 ++++++++++++++++++++++++++++++------ 1 file changed, 98 insertions(+), 20 deletions(-) diff --git a/neon_llm_chatgpt/chatgpt.py b/neon_llm_chatgpt/chatgpt.py index 5751cfa..a596882 100644 --- a/neon_llm_chatgpt/chatgpt.py +++ b/neon_llm_chatgpt/chatgpt.py @@ -25,39 +25,117 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import openai +from openai.embeddings_utils import get_embeddings, distances_from_embeddings +from typing import List, Dict +from neon_llm_core.llm import NeonLLM + + +class ChatGPT(NeonLLM): + + mq_to_llm_role = { + "user": "user", + "llm": "assistant" + } -class ChatGPT: def __init__(self, config): + super().__init__(config) self.model = config["model"] self.role = config["role"] self.context_depth = config["context_depth"] self.max_tokens = config["max_tokens"] - openai.api_key = config["key"] + self.api_key = config["key"] + self.num_parallel_processes = config["num_parallel_processes"] + self.warmup() + + @property + def model(self) -> openai: + if self._model is None: + openai.api_key = self.api_key + self._model = openai + return self._model + + @property + def _system_prompt(self) -> str: + return self.role + + def warmup(self): + self.model - @staticmethod - def convert_role(role): - if role == "user": - role_chatgpt = "user" - elif role == "llm": - role_chatgpt = "assistant" - return role_chatgpt + def get_sorted_answer_indexes(self, question: str, answers: List[str]) -> List[int]: + """ + Creates sorted list of answer indexes with respect to order provided in :param answers based on PPL score + Answers are sorted from best to worst + :param question: incoming question + :param answers: list of answers to rank + :returns list of indexes + """ + if not answers: + return [] + scores = self._score(question=question, answers=answers) + sorted_items = sorted(zip(range(len(answers)), scores), key=lambda x: x[1]) + sorted_items_indexes = [x[0] for x in sorted_items] + return sorted_items_indexes - def ask(self, message, chat_history): + def _call_model(self, prompt: List[Dict[str]]) -> str: + """ + Wrapper for ChatGPT Model generation logic + :param prompt: Input messages sequence + :returns: Output text sequence generated by model + """ + + response = openai.ChatCompletion.create( + model=self.model, + messages=prompt, + temperature=0, + max_tokens=self.max_tokens, + ) + text = response.choices[0].message['content'] + + return text + + def _assemble_prompt(self, message: str, chat_history: List[List[str]]) -> List[Dict[str]]: + """ + Assembles prompt engineering logic + Setup Guidance: + https://platform.openai.com/docs/guides/gpt/chat-completions-api + + :param message: Incoming prompt + :param chat_history: History of preceding conversation + :returns: assembled prompt + """ messages = [ - {"role": "system", "content": self.role}, + {"role": "system", "content": self._system_prompt}, ] # Context N messages for role, content in chat_history[-self.context_depth:]: role_chatgpt = self.convert_role(role) messages.append({"role": role_chatgpt, "content": content}) messages.append({"role": "user", "content": message}) - - response = openai.ChatCompletion.create( - model=self.model, - messages=messages, - temperature=0, - max_tokens=self.max_tokens, - ) - bot_message = response.choices[0].message['content'] - return bot_message + return messages + + def _score(self, prompt: str, targets: List[str]) -> List[float]: + """ + Calculates logarithmic probabilities for the list of provided text sequences + :param prompt: Input text sequence + :param targets: Output text sequences + :returns: List of calculated logarithmic probabilities per output text sequence + """ + + question_embeddings, answers_embeddings = self._embeddings(question=prompt, answers=targets) + scores_list = distances_from_embeddings(question_embeddings, answers_embeddings) + return scores_list + + def _embeddings(self, question: str, answers: List[str]) -> (List[str], List[List[float]]): + """ + Computes embeddings for the list of provided answers + :param question: Question for LLM to response to + :param answers: List of provided answers + :returns ppl values for each answer + """ + response = self.ask(question, []) + texts = [response] + answers + embeddings = get_embeddings(texts, engine="text-embedding-ada-002") + question_embeddings = embeddings[0] + answers_embeddings = embeddings[1:] + return question_embeddings, answers_embeddings \ No newline at end of file From 2bcb5d0dbb0c60b92331e2e8f6f2e664ac4e7c32 Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Mon, 25 Sep 2023 19:41:36 +0300 Subject: [PATCH 05/14] Updated diana config --- docker_overlay/etc/neon/diana.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_overlay/etc/neon/diana.yaml b/docker_overlay/etc/neon/diana.yaml index 78e5933..ec89390 100644 --- a/docker_overlay/etc/neon/diana.yaml +++ b/docker_overlay/etc/neon/diana.yaml @@ -14,7 +14,7 @@ MQ: mq_handler: user: neon_api_utils password: Klatchat2021 -ChatGPT: +LLM_CHATGPT: model: "gpt-3.5-turbo" role: "You are trying to give a short answer in less than 40 words." context_depth: 3 From 5baa61d88e440c0cd66644c62c7ae5891c97e27e Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Mon, 25 Sep 2023 19:41:45 +0300 Subject: [PATCH 06/14] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 353a84c..520c9bc 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ MQ: mq-chatgpt-api: password: user: neon_chatgpt -ChatGPT: +LLM_CHATGPT: key: "" model: "gpt-3.5-turbo" role: "You are trying to give a short answer in less than 40 words." From 434d916185a6d728672db520db33fe8422fc36cf Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Wed, 27 Sep 2023 13:33:30 +0300 Subject: [PATCH 07/14] Fix embeddings output type --- neon_llm_chatgpt/chatgpt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neon_llm_chatgpt/chatgpt.py b/neon_llm_chatgpt/chatgpt.py index a596882..f21b856 100644 --- a/neon_llm_chatgpt/chatgpt.py +++ b/neon_llm_chatgpt/chatgpt.py @@ -126,7 +126,7 @@ def _score(self, prompt: str, targets: List[str]) -> List[float]: scores_list = distances_from_embeddings(question_embeddings, answers_embeddings) return scores_list - def _embeddings(self, question: str, answers: List[str]) -> (List[str], List[List[float]]): + def _embeddings(self, question: str, answers: List[str]) -> (List[float], List[List[float]]): """ Computes embeddings for the list of provided answers :param question: Question for LLM to response to From d56db170112bc7706c9168e367d1259e11c584c4 Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Wed, 27 Sep 2023 13:35:51 +0300 Subject: [PATCH 08/14] Fix model variable duplicate --- neon_llm_chatgpt/chatgpt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neon_llm_chatgpt/chatgpt.py b/neon_llm_chatgpt/chatgpt.py index f21b856..4112717 100644 --- a/neon_llm_chatgpt/chatgpt.py +++ b/neon_llm_chatgpt/chatgpt.py @@ -40,7 +40,7 @@ class ChatGPT(NeonLLM): def __init__(self, config): super().__init__(config) - self.model = config["model"] + self.model_name = config["model"] self.role = config["role"] self.context_depth = config["context_depth"] self.max_tokens = config["max_tokens"] @@ -85,7 +85,7 @@ def _call_model(self, prompt: List[Dict[str]]) -> str: """ response = openai.ChatCompletion.create( - model=self.model, + model=self.model_name, messages=prompt, temperature=0, max_tokens=self.max_tokens, From 09fe083d38a4ca52af70eb7f88db7acb0a7c2361 Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Wed, 27 Sep 2023 13:52:50 +0300 Subject: [PATCH 09/14] Added num_parallel_processes parameter --- README.md | 1 + docker_overlay/etc/neon/diana.yaml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 520c9bc..34f2b5d 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ LLM_CHATGPT: role: "You are trying to give a short answer in less than 40 words." context_depth: 3 max_tokens: 100 + num_parallel_processes: 2 ``` For example, if your configuration resides in `~/.config`: diff --git a/docker_overlay/etc/neon/diana.yaml b/docker_overlay/etc/neon/diana.yaml index ec89390..5fe0c86 100644 --- a/docker_overlay/etc/neon/diana.yaml +++ b/docker_overlay/etc/neon/diana.yaml @@ -18,4 +18,5 @@ LLM_CHATGPT: model: "gpt-3.5-turbo" role: "You are trying to give a short answer in less than 40 words." context_depth: 3 - max_tokens: 100 \ No newline at end of file + max_tokens: 100 + num_parallel_processes: 2 \ No newline at end of file From dfb165a3aa513d33fde1805cc0ec99e8945b2bdb Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Wed, 27 Sep 2023 13:56:07 +0300 Subject: [PATCH 10/14] Fix Dict type def --- neon_llm_chatgpt/chatgpt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neon_llm_chatgpt/chatgpt.py b/neon_llm_chatgpt/chatgpt.py index 4112717..09ce5f4 100644 --- a/neon_llm_chatgpt/chatgpt.py +++ b/neon_llm_chatgpt/chatgpt.py @@ -77,7 +77,7 @@ def get_sorted_answer_indexes(self, question: str, answers: List[str]) -> List[i sorted_items_indexes = [x[0] for x in sorted_items] return sorted_items_indexes - def _call_model(self, prompt: List[Dict[str]]) -> str: + def _call_model(self, prompt: List[Dict[str, str]]) -> str: """ Wrapper for ChatGPT Model generation logic :param prompt: Input messages sequence @@ -94,7 +94,7 @@ def _call_model(self, prompt: List[Dict[str]]) -> str: return text - def _assemble_prompt(self, message: str, chat_history: List[List[str]]) -> List[Dict[str]]: + def _assemble_prompt(self, message: str, chat_history: List[List[str]]) -> List[Dict[str, str]]: """ Assembles prompt engineering logic Setup Guidance: From 6dbcd98cbc0e9a5220edbd8ec4b256820cec3d94 Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Wed, 27 Sep 2023 14:06:45 +0300 Subject: [PATCH 11/14] Init all abstract methods --- neon_llm_chatgpt/chatgpt.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/neon_llm_chatgpt/chatgpt.py b/neon_llm_chatgpt/chatgpt.py index 09ce5f4..d987677 100644 --- a/neon_llm_chatgpt/chatgpt.py +++ b/neon_llm_chatgpt/chatgpt.py @@ -48,6 +48,14 @@ def __init__(self, config): self.num_parallel_processes = config["num_parallel_processes"] self.warmup() + @property + def tokenizer(self) -> None: + return self._tokenizer + + @property + def tokenizer_model_name(self) -> str: + return "" + @property def model(self) -> openai: if self._model is None: @@ -55,6 +63,10 @@ def model(self) -> openai: self._model = openai return self._model + @property + def llm_model_name(self) -> str: + return self.model_name + @property def _system_prompt(self) -> str: return self.role @@ -85,7 +97,7 @@ def _call_model(self, prompt: List[Dict[str, str]]) -> str: """ response = openai.ChatCompletion.create( - model=self.model_name, + model=self.llm_model_name, messages=prompt, temperature=0, max_tokens=self.max_tokens, @@ -126,6 +138,9 @@ def _score(self, prompt: str, targets: List[str]) -> List[float]: scores_list = distances_from_embeddings(question_embeddings, answers_embeddings) return scores_list + def _tokenize(self, prompt: str) -> None: + return None + def _embeddings(self, question: str, answers: List[str]) -> (List[float], List[List[float]]): """ Computes embeddings for the list of provided answers From ebb1c4f3539ce2fad5964d4e6cb22b9300bfe7ce Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Wed, 27 Sep 2023 14:14:50 +0300 Subject: [PATCH 12/14] Fix _score call --- neon_llm_chatgpt/chatgpt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neon_llm_chatgpt/chatgpt.py b/neon_llm_chatgpt/chatgpt.py index d987677..41faefe 100644 --- a/neon_llm_chatgpt/chatgpt.py +++ b/neon_llm_chatgpt/chatgpt.py @@ -84,7 +84,7 @@ def get_sorted_answer_indexes(self, question: str, answers: List[str]) -> List[i """ if not answers: return [] - scores = self._score(question=question, answers=answers) + scores = self._score(prompt=question, targets=answers) sorted_items = sorted(zip(range(len(answers)), scores), key=lambda x: x[1]) sorted_items_indexes = [x[0] for x in sorted_items] return sorted_items_indexes From 4e465fba54281f3719ed4e93df676991e9fee231 Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Thu, 28 Sep 2023 18:24:03 +0300 Subject: [PATCH 13/14] chatgpt -> chat_gpt --- README.md | 4 ++-- docker_overlay/etc/neon/diana.yaml | 2 +- neon_llm_chatgpt/rmq.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 34f2b5d..ce20d97 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,10 @@ MQ: port: server: users: - mq-chatgpt-api: + neon_llm_chat_gpt: password: user: neon_chatgpt -LLM_CHATGPT: +LLM_CHAT_GPT: key: "" model: "gpt-3.5-turbo" role: "You are trying to give a short answer in less than 40 words." diff --git a/docker_overlay/etc/neon/diana.yaml b/docker_overlay/etc/neon/diana.yaml index 5fe0c86..3c0fa01 100644 --- a/docker_overlay/etc/neon/diana.yaml +++ b/docker_overlay/etc/neon/diana.yaml @@ -14,7 +14,7 @@ MQ: mq_handler: user: neon_api_utils password: Klatchat2021 -LLM_CHATGPT: +LLM_CHAT_GPT: model: "gpt-3.5-turbo" role: "You are trying to give a short answer in less than 40 words." context_depth: 3 diff --git a/neon_llm_chatgpt/rmq.py b/neon_llm_chatgpt/rmq.py index e73a57f..3409a95 100644 --- a/neon_llm_chatgpt/rmq.py +++ b/neon_llm_chatgpt/rmq.py @@ -39,7 +39,7 @@ def __init__(self): @property def name(self): - return "chatgpt" + return "chat_gpt" @property def model(self): From 6b899779cd5572f07a83fa404541654e6900e2fd Mon Sep 17 00:00:00 2001 From: NeonBohdan Date: Sun, 15 Oct 2023 16:34:49 +0300 Subject: [PATCH 14/14] Use pass instead of None --- neon_llm_chatgpt/chatgpt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neon_llm_chatgpt/chatgpt.py b/neon_llm_chatgpt/chatgpt.py index 41faefe..0f356c1 100644 --- a/neon_llm_chatgpt/chatgpt.py +++ b/neon_llm_chatgpt/chatgpt.py @@ -139,7 +139,7 @@ def _score(self, prompt: str, targets: List[str]) -> List[float]: return scores_list def _tokenize(self, prompt: str) -> None: - return None + pass def _embeddings(self, question: str, answers: List[str]) -> (List[float], List[List[float]]): """