voice chat with mobs

README.md

@@ -1,13 +1,19 @@
-# AIMobs
-AIMobs is a mod that lets you chat with Minecraft mobs and other entities by creating prompts and using the OpenAI API.
+# AIMobs-Voice
+AIMobs-Voice is a mod that lets you chat or talk (hold R key) with Minecraft mobs and other entities by creating prompts and using the OpenAI API. It's forked from [AIMobs](https://github.com/rebane2001/aimobs) for text only interactions.
 
 ### Requirements
 - Minecraft 1.19.4
 - Fabric
-- Fabric API
+- Fabric API 0.86.1.19.4
+- (Prism Launcher [here](https://prismlauncher.org))
+- (Java 17 [here](https://adoptium.net/en-GB/temurin/releases/?version=17))
+
+### Installation
+1. [YouTube Tutorial] (https://www.youtube.com/watch?v=EKCaTp5a8ZU)
+2. You won't find aimobs-voice via download mods in Prism Launcher yet. Instead please download the .jar here from the release page and add it manually to Prism Launcher -> Mods -> Add file.
 
 ### Usage
-After installing the mod, grab your OpenAI API key from [here](https://beta.openai.com/account/api-keys), and set it with the `/aimobs setkey <key>` command.
+After installing the mod, grab your OpenAI API key from [here](https://beta.openai.com/account/api-keys), and set it with the `/aimobs setkey <key>` command. Also in order to let the mobs talk to you: First enable the Cloud Text-to-Speech API - [here](https://console.cloud.google.com/apis/library/texttospeech.googleapis.com) and then create your API key credential [here](https://console.cloud.google.com/apis/credentials). Once you have it, set it with the `/aimobs setvoicekey <voicekey>` command
 
 You should now be able to **talk to mobs by shift+clicking** on them!
 
@@ -16,16 +22,22 @@ You should now be able to **talk to mobs by shift+clicking** on them!
 - `/aimobs help` - View commands help
 - `/aimobs enable/disable` - Enable/disable the mod
 - `/aimobs setkey <key>` - Set OpenAI API key
+- `/aimobs setvoicekey <voicekey>` - Set Google Text-To-Speech API key
 - `/aimobs setmodel <model>` - Set AI model
 - `/aimobs settemp <temperature>` - Set model temperature
 
 ### Notes
 This project was initially made in 1.12 as a client Forge mod, then ported to 1.19 PaperMC as a server plugin, then ported to Fabric 1.19. Because of this, the code can be a little messy and weird. A couple hardcoded limits are 512 as the max token length and 4096 as the max prompt length (longer prompts will get the beginning cut off), these could be made configurable in the future.
 
-Some plans for the future:  
-- Support for the Forge modloader.
-- Support for other AI APIs.
+## 🛣️ Roadmap:
+- [ ] Let mobs approach player
+- [ ] Stream GPT output to TTS
+- [ ] Replace random voices by mob specific ones
+- [ ] Cut memory within model token limit to prevent overflow
+- [ ] Support for the Forge modloader.
+- [ ] Support for other AI APIs.
+
+Come join the project and add new features or for refined mob prompt engineering. Feel free to contact me on [Twitter/X](https://twitter.com/J_Grenzebach)
 
-An unofficial community-made fork is available with support for Ukranian and Español at [Eianex/aimobs](https://github.com/Eianex/aimobs/releases).
 
 The icon used is the **🧠** emoji from [Twemoji](https://twemoji.twitter.com/) (CC BY 4.0)

build 2/loom-cache/mixin-map-net.fabricmc.yarn.1_19_4.1.19.4+build.2-v2.main.tiny

@@ -0,0 +1 @@
+v1	named	intermediary

build.gradle

@@ -21,6 +21,8 @@ repositories {
 }
 
 dependencies {
+	// needed for whisper ASR in RequestHandler
+	//implementation 'org.apache.httpcomponents:httpmime:4.5.13'
 	// To change the versions see the gradle.properties file
 	minecraft "com.mojang:minecraft:${project.minecraft_version}"
 	mappings "net.fabricmc:yarn:${project.yarn_mappings}:v2"
@@ -33,6 +35,7 @@ dependencies {
 	// These are included in the Fabric API production distribution and allow you to update your mod to the latest modules at a later more convenient time.
 
 	// modImplementation "net.fabricmc.fabric-api:fabric-api-deprecated:${project.fabric_version}"
+
 }
 
 processResources {

src/main/java/com/rebane2001/aimobs/AIMobsCommand.java

@@ -14,49 +14,49 @@
 
 public class AIMobsCommand {
 
+    // Method to setup AIMobs commands
     public static void setupAIMobsCommand(CommandDispatcher<FabricClientCommandSource> dispatcher, CommandRegistryAccess registryAccess) {
+        // Registering the available commands
         dispatcher.register(literal("aimobs")
                 .executes(AIMobsCommand::status)
                 .then(literal("help").executes(AIMobsCommand::help))
-                .then(literal("setkey")
-                        .then(argument("key", StringArgumentType.string())
-                                .executes(AIMobsCommand::setAPIKey)
-                        ))
-                .then(literal("setmodel")
-                        .then(argument("model", StringArgumentType.string())
-                                .executes(AIMobsCommand::setModel)
-                        ))
-                .then(literal("settemp")
-                        .then(argument("temperature", FloatArgumentType.floatArg(0,1))
-                                .executes(AIMobsCommand::setTemp)
-                        ))
+                .then(literal("setkey").then(argument("key", StringArgumentType.string()).executes(AIMobsCommand::setAPIKey)))
+                .then(literal("setvoicekey").then(argument("voicekey", StringArgumentType.string()).executes(AIMobsCommand::setVoiceAPIKey)))
+                .then(literal("setmodel").then(argument("model", StringArgumentType.string()).executes(AIMobsCommand::setModel)))
+                .then(literal("settemp").then(argument("temperature", FloatArgumentType.floatArg(0,1)).executes(AIMobsCommand::setTemp)))
                 .then(literal("enable").executes(context -> setEnabled(context, true)))
                 .then(literal("disable").executes(context -> setEnabled(context, false)))
         );
     }
+
+    // Method to enable or disable the mod
     public static int setEnabled(CommandContext<FabricClientCommandSource> context, boolean enabled) {
         AIMobsConfig.config.enabled = enabled;
         AIMobsConfig.saveConfig();
         context.getSource().sendFeedback(Text.of("AIMobs " + (enabled ? "enabled" : "disabled")));
         return 1;
     }
 
+    // Method to show the status of the mod's configuration
     public static int status(CommandContext<FabricClientCommandSource> context) {
         boolean hasKey = AIMobsConfig.config.apiKey.length() > 0;
+        boolean hasVoiceKey = AIMobsConfig.config.voiceApiKey.length() > 0;
         Text yes = Text.literal("Yes").formatted(Formatting.GREEN);
         Text no = Text.literal("No").formatted(Formatting.RED);
         Text helpText = Text.literal("")
                 .append(Text.literal("AIMobs").formatted(Formatting.UNDERLINE))
                 .append("").formatted(Formatting.RESET)
                 .append("\nEnabled: ").append(AIMobsConfig.config.enabled ? yes : no)
                 .append("\nAPI Key: ").append(hasKey ? yes : no)
+                .append("\nVoice API Key: ").append(hasVoiceKey ? yes : no)
                 .append("\nModel: ").append(AIMobsConfig.config.model)
                 .append("\nTemp: ").append(String.valueOf(AIMobsConfig.config.temperature))
                 .append("\n\nUse ").append(Text.literal("/aimobs help").formatted(Formatting.GRAY)).append(" for help");
         context.getSource().sendFeedback(helpText);
         return 1;
     }
 
+    // Method to show help text
     public static int help(CommandContext<FabricClientCommandSource> context) {
         Text helpText = Text.literal("")
                 .append("AIMobs Commands").formatted(Formatting.UNDERLINE)
@@ -65,12 +65,15 @@ public static int help(CommandContext<FabricClientCommandSource> context) {
                 .append("\n/aimobs help - View commands help")
                 .append("\n/aimobs enable/disable - Enable/disable the mod")
                 .append("\n/aimobs setkey <key> - Set OpenAI API key")
+                .append("\n/aimobs setvoicekey <voicekey> - Set Google Text-To-Speech API key")
                 .append("\n/aimobs setmodel <model> - Set AI model")
                 .append("\n/aimobs settemp <temperature> - Set model temperature")
                 .append("\nYou can talk to mobs by shift-clicking on them!");
         context.getSource().sendFeedback(helpText);
         return 1;
     }
+
+    // Method to set API key
     public static int setAPIKey(CommandContext<FabricClientCommandSource> context) {
         String apiKey = context.getArgument("key", String.class);
         if (apiKey.length() > 0) {
@@ -81,6 +84,20 @@ public static int setAPIKey(CommandContext<FabricClientCommandSource> context) {
         }
         return 0;
     }
+
+    // Method to set Voice API key
+    public static int setVoiceAPIKey(CommandContext<FabricClientCommandSource> context) {
+        String voiceApiKey = context.getArgument("voicekey", String.class);
+        if (voiceApiKey.length() > 0) {
+            AIMobsConfig.config.voiceApiKey = voiceApiKey;
+            AIMobsConfig.saveConfig();
+            context.getSource().sendFeedback(Text.of("Voice API key set"));
+            return 1;
+        }
+        return 0;
+    }
+
+    // Method to set Model
     public static int setModel(CommandContext<FabricClientCommandSource> context) {
         String model = context.getArgument("model", String.class);
         if (model.length() > 0) {
@@ -91,6 +108,8 @@ public static int setModel(CommandContext<FabricClientCommandSource> context) {
         }
         return 0;
     }
+
+    // Method to set Temperature
     public static int setTemp(CommandContext<FabricClientCommandSource> context) {
         AIMobsConfig.config.temperature = context.getArgument("temperature", float.class);
         AIMobsConfig.saveConfig();

src/main/java/com/rebane2001/aimobs/AIMobsConfig.java

@@ -9,30 +9,38 @@
 import java.util.Objects;
 
 public class AIMobsConfig {
+    // Inner class to hold configuration settings
     public static class Config {
-        public boolean enabled = true;
-        public String apiKey = "";
-        public String model = "text-davinci-003";
-        public float temperature = 0.6f;
+        public boolean enabled = true; // Enable or disable the mod
+        public String apiKey = ""; // OpenAI API key
+        public String voiceApiKey = ""; // Voice API key (if applicable)
+        public String model = "gpt-3.5-turbo-16k"; // Model to be used for conversation
+        public float temperature = 0.3f; // Model temperature setting
     }
-    public static Config config;
 
+    public static Config config; // Config instance
+
+    // Get the path to the configuration file
     private static Path getConfigPath() {
         return FabricLoader.getInstance().getConfigDir().resolve("aimobs.json");
     }
 
+    // Load the configuration from the file
     public static void loadConfig() {
         try (FileReader reader = new FileReader(getConfigPath().toFile())) {
             config = new Gson().fromJson(reader, Config.class);
-            Objects.requireNonNull(config);
+            Objects.requireNonNull(config); // Ensure that the config is not null
         } catch (Exception e) {
+            // If an exception occurs, create a new default configuration and save it
             config = new Config();
             saveConfig();
         }
     }
+
+    // Save the configuration to the file
     public static void saveConfig() {
         try (FileWriter writer = new FileWriter(getConfigPath().toFile())) {
-            new Gson().toJson(config, writer);
+            new Gson().toJson(config, writer); // Write the JSON representation of the config to the file
         } catch (Exception e) {
             e.printStackTrace();
         }