Port utils from Chatbots project (#183)

* Implement v2 integration tests from `chatbots` repository
Fix v1 ChatBot to define `bot_type`
Add CLI entyrpoint for local discussion session
Implement utils from `chatbots` repository

* Update tests
Fix prompter behavior in local tests

* Troubleshooting skipped test

* Troubleshooting skipped test

* Revert removed test case

* Outline tests for added methods
Continue local test CLI

* Update config handling

* Fix prompter response handling

---------

Co-authored-by: Daniel McKnight <daniel@neon.ai>

chatbot_core/cli.py

@@ -45,14 +45,22 @@ def start_mq_bot(bot_entrypoint):
     run_mq_bot(bot_entrypoint)
 
 
-@chatbot_core_cli.command(help="Start a local debug session")
-@click.option("--bot-dir", default=None, help="Path to legacy chatbots directory")
+@chatbot_core_cli.command(help="Start a local single-bot session")
+@click.option("--bot-dir", default=None,
+              help="Path to legacy chatbots directory")
 def debug_bots(bot_dir):
     from chatbot_core.utils.bot_utils import debug_bots
     bot_dir = expanduser(relpath(bot_dir)) if bot_dir else None
     debug_bots(bot_dir)
 
 
+@chatbot_core_cli.command(help="Start a local CBF-style conversation")
+@click.option("--prompter", "-p", help="ID of Chatbot to submit prompts")
+def run_local_discussion(prompter):
+    from chatbot_core.utils.bot_utils import run_local_discussion
+    run_local_discussion(prompter)
+
+
 # Below are deprecated entrypoints
 def cli_start_mq_bot():
     """

chatbot_core/utils/bot_utils.py

@@ -25,18 +25,18 @@
 import sys
 import yaml
 
-from typing import Optional, Callable, Dict
+from typing import Optional, Callable, Dict, List
 from multiprocessing import Process, Event, synchronize
 from threading import Thread, current_thread
 from ovos_bus_client import Message, MessageBusClient
 from datetime import datetime
 from ovos_utils.xdg_utils import xdg_config_home
-from klat_connector import start_socket
 from ovos_utils.log import LOG, log_deprecation
 from neon_utils.net_utils import get_ip_address
 
 from chatbot_core.chatbot_abc import ChatBotABC
 from chatbot_core.v2 import ChatBot as ChatBotV2
+from chatbot_core.v1 import ChatBot as ChatBotV1
 
 
 ip = get_ip_address()
@@ -65,6 +65,7 @@ def _threaded_start_bot(bot, addr: str, port: int, domain: str, user: str,
     """
     Helper function for _start_bot
     """
+    from klat_connector import start_socket
     # TODO: Deprecate
     if len(inspect.signature(bot).parameters) == 6:
         instance = bot(start_socket(addr, port), domain, user, password, True,
@@ -356,8 +357,11 @@ def debug_bots(bot_dir: str = None):
     #       Automated testing could use pre-built response objects, or run n
     #       other bots and handle their outputs offline
     from klat_connector.mach_server import MachKlatServer
+    from ovos_config.config import Configuration
     server = MachKlatServer()
-
+    Configuration()['socket_io'] = {"server": "0.0.0.0",
+                                    "port": 8888}
+    # TODO: Define alternate `ChatBot` base class with no server dependency
     if bot_dir:
         log_deprecation("Bots should be installed so they may be accessed by "
                         "entrypoint. Specifying a local directory will no "
@@ -375,8 +379,7 @@ def debug_bots(bot_dir: str = None):
                   f'Please choose a bot to talk to')
             bot_name = input('[In]: ')
             if bot_name in subminds:
-                bot = subminds[bot_name](start_socket("0.0.0.0", 8888), None,
-                                         None, None, on_server=False)
+                bot = run_sio_bot(bot_name)
                 while running:
                     utterance = input('[In]: ')
                     response = bot.ask_chatbot('Tester', utterance,
@@ -619,3 +622,96 @@ def run_mq_bot(chatbot_name: str, vhost: str = '/chatbots',
     bot.run(**run_kwargs)
     LOG.info(f"Started {chatbot_name}")
     return bot
+
+
+def run_sio_bot(chatbot_name: str, domain: str = None,
+                is_prompter: bool = False) -> ChatBotV1:
+    """
+    Get an initialized SIO Chatbot instance
+    @param chatbot_name: chatbot entrypoint name and configuration key
+    @param domain: Initial domain to enter
+    @param is_prompter: If true, submit prompts rather than contribute responses
+    @returns: Started ChatBotV2 instance
+    """
+    from ovos_config.config import Configuration
+    from klat_connector import start_socket
+    sio_config = Configuration().get("socket_io", {})
+    os.environ['CHATBOT_VERSION'] = 'v1'
+    domain = domain or "chatbotsforum.org"
+    bots = _find_bot_modules()
+    clazz = bots.get(chatbot_name)
+    if not clazz:
+        raise RuntimeError(f"Requested bot `{chatbot_name}` not found in: "
+                           f"{list(bots.keys())}")
+    sock = start_socket(sio_config.get("server"), sio_config.get("port"))
+    bot = clazz(socket=sock, domain=domain, is_prompter=is_prompter)
+    LOG.info(f"Started {chatbot_name}")
+    return bot
+
+
+def run_all_bots(domain: str = None) -> List[ChatBotABC]:
+    """
+    Run all installed chatbots, connecting to the configured server, considering
+    the value of the `CHATBOT_VERSION` envvar
+    """
+    bots = _find_bot_modules()
+    from chatbot_core.utils.version_utils import get_current_version
+    chatbot_version = get_current_version()
+    chatbots = list()
+    for bot in bots.keys():
+        if chatbot_version == 1:
+            chatbots.append(run_sio_bot(bot, domain=domain))
+        elif chatbot_version == 2:
+            chatbots.append(run_mq_bot(bot))
+        else:
+            from chatbot_core.utils.version_utils import InvalidVersionError
+            raise InvalidVersionError(f"Unable to start chatbot with version: "
+                                      f"{chatbot_version}")
+    return chatbots
+
+
+def run_local_discussion(prompter_bot: str):
+    """
+    Run all installed bots locally with a prompter to submit prompts for
+    discussion.
+    @param prompter_bot: name/entrypoint of bot to be used as a proctor
+    """
+    import click
+    from ovos_config.config import Configuration
+    # Override logging
+    Configuration()['log_level'] = "ERROR"
+
+    # Start local server
+    from klat_connector import start_socket
+    from klat_connector.mach_server import MachKlatServer
+    server = MachKlatServer()
+
+    # Load all installed subminds and facilitators
+    os.environ['CHATBOT_VERSION'] = 'v1'
+    bots = _find_bot_modules()
+    chatbots = list()
+    for name, clazz in bots.items():
+        chatbots.append(clazz(socket=start_socket("0.0.0.0"), domain="local",
+                              username=name, password=name))
+
+    prompter_clazz = bots.get(prompter_bot)
+    prompter = prompter_clazz(socket=start_socket("0.0.0.0"), domain="private",
+                              is_prompter=True, username="Prompter",
+                              password=prompter_bot)
+    chatbots.append(prompter)
+    LOG.info("Local Conversation started")
+    # Make conversation output readable
+    # TODO: prevent log output going to terminal
+
+    def handle_shout(user, shout, cid, dom, timestamp):
+        click.echo(f"{user.rjust(max((len(name) for name in bots)))} : {shout}")
+
+    observer = ChatBotV1(socket=start_socket("0.0.0.0"), domain="local")
+    observer.handle_shout = handle_shout
+
+    prompter.send_shout("@proctor hello")
+    from ovos_utils import wait_for_exit_signal
+    wait_for_exit_signal()
+    for bot in chatbots:
+        bot.exit()
+    server.shutdown_server()

chatbot_core/utils/string_utils.py

@@ -17,6 +17,9 @@
 # US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924
 # China Patent: CN102017585  -  Europe Patent: EU2156652  -  Patents Pending
 
+from copy import copy
+
+
 def remove_prefix(prefixed_string: str, prefix: str):
     """
     Removes the specified prefix from the string
@@ -27,3 +30,21 @@ def remove_prefix(prefixed_string: str, prefix: str):
     if prefixed_string.startswith(prefix):
         return prefixed_string[len(prefix):].lstrip()
     return prefixed_string
+
+
+def enumerate_subminds(subminds: list) -> str:
+    """
+    Enumerates bots in format of type "submind1(,submind2... and submindN)"
+    where N is the number of subminds provided
+
+    :param subminds: list of names to format
+    :returns formatted string reflecting list of subminds provided
+    """
+    if len(subminds) == 0:
+        return 'No one'
+    if len(subminds) == 1:
+        return subminds[0]
+    subminds_copy = copy(subminds)
+    last_submind = subminds_copy.pop()
+    and_str = ", and " if len(subminds_copy) > 1 else " and "
+    return f"{', '.join(subminds_copy)}{and_str}{last_submind}"

chatbot_core/utils/version_utils.py

@@ -33,10 +33,28 @@ def get_class() -> Optional[type(ChatBotABC)]:
     from chatbot_core.v1 import ChatBot as ChatBot_v1
     from chatbot_core.v2 import ChatBot as ChatBot_v2
 
-    version = os.environ.get('CHATBOT_VERSION', 'v1').lower()
+    version = get_current_version()
     LOG.debug(f"version={version}")
     chatbot_versions = {
-        'v1': ChatBot_v1,
-        'v2': ChatBot_v2
+        1: ChatBot_v1,
+        2: ChatBot_v2
     }
-    return chatbot_versions.get(version, None)
+
+    if version not in chatbot_versions:
+        raise InvalidVersionError(f"{version} is not a valid version "
+                                  f"({set(chatbot_versions.keys())}")
+    return chatbot_versions.get(version)
+
+
+def get_current_version() -> int:
+    """
+    Get an int representation of the configured Chatbot version to run
+    """
+    return 2 if os.environ.get('CHATBOT_VERSION',
+                               'v1').lower() in ('v2', '2', 'version2') else 1
+
+
+class InvalidVersionError(Exception):
+    """
+    Exception raised when invalid chatbots version is specified
+    """

chatbot_core/v1/__init__.py

@@ -51,7 +51,9 @@ def __init__(self, *args, **kwargs):
         self.is_prompter = is_prompter
         self.start_domain = domain
         self.enable_responses = False
-        self.bot_type = None
+        self.bot_type = BotTypes.OBSERVER if is_prompter else (
+            BotTypes.PROCTOR) if init_nick.lower() == "proctor" else (
+            BotTypes.SUBMIND)
         self.proposed_responses = dict()
         self.selected_history = list()
 
@@ -63,9 +65,7 @@ def __init__(self, *args, **kwargs):
 
         # Do klat initialization
         klat_timeout = time.time() + 30
-        while not self.ready and time.time() < klat_timeout:
-            time.sleep(1)
-        if not self.ready:
+        if not self.klat_ready.wait(30):
             self.log.error("Klat connection timed out!")
         elif username and password:
             self.login_klat(username, password)
@@ -152,7 +152,7 @@ def handle_shout(self, user: str, shout: str, cid: str, dom: str, timestamp: str
         :param timestamp: formatted timestamp of shout
         """
         if not shout:
-            self.log.error(f"No shout (user={user}")
+            self.log.error(f"No shout (user={user})")
             return
         if not self.nick:
             self.log.error(f"No nick! user is {self.username}")
@@ -209,7 +209,7 @@ def handle_shout(self, user: str, shout: str, cid: str, dom: str, timestamp: str
             if self.is_prompter:
                 self.log.info(f"Prompter bot got reply: {shout}")
                 # private_cid = self.get_private_conversation([user])
-                self.send_shout(resp)
+                self.send_shout(f"@proctor {resp}")
                 return
         # Subminds ignore facilitators
         elif not self._user_is_proctor(user) and user.lower() in self.facilitator_nicks \
@@ -619,7 +619,8 @@ def _handle_next_shout(self):
         next_shout = self.shout_queue.get()
         while next_shout:
             # (user, shout, cid, dom, timestamp)
-            self.handle_shout(next_shout[0], next_shout[1], next_shout[2], next_shout[3], next_shout[4])
+            self.handle_shout(next_shout[0], next_shout[1], next_shout[2],
+                              next_shout[3], next_shout[4])
             next_shout = self.shout_queue.get()
         self.log.warning(f"No next shout to handle! No more shouts will be processed by {self.nick}")
         self.exit()

tests/integration/v1/__init__.py

@@ -0,0 +1,18 @@
+# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System
+#
+# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved
+#
+# Notice of License - Duplicating this Notice of License near the start of any file containing
+# a derivative of this software is a condition of license for this software.
+# Friendly Licensing:
+# No charge, open source royalty free use of the Neon AI software source and object is offered for
+# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and
+# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai
+# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai
+# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied.
+# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM)
+# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds
+#
+# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp.
+# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924
+# China Patent: CN102017585  -  Europe Patent: EU2156652  -  Patents Pending