Merge pull request #103 from COS301-SE-2023/backend/llm

Backend/llm

blix-plugins/hello-plugin/src/main.js

@@ -30,6 +30,7 @@ const nodes = {
     }
     ,"Jake": (context) => {
         nodeBuilder = context.instantiate("hello-plugin","Jake");
+        nodeBuilder.setTitle("Jake");
         nodeBuilder.setDescription("This is currently a useless node that does nothing.");
 
     }

blix-plugins/input-plugin/src/main.js

@@ -21,7 +21,7 @@ const nodes ={
       nodeBuilder.define((image) => {
           return image;
       });
-     nodeBuilder.addOutput("Image", "res", "Result");
+     nodeBuilder.addOutput("Sharp", "res", "Result");
 
      ui = nodeBuilder.createUIBuilder();
      ui.addImageInput("input image");

blix-plugins/math-plugin/src/main.js

@@ -115,23 +115,23 @@ const nodes ={
         nodeBuilder.addOutput("Number", "res1", "Result");
       },
 
-      // Testing nodes
-    "add": (context) => {
-        nodeBuilder = context.instantiate("math-plugin","add");
-        nodeBuilder.setTitle("Add");
-        nodeBuilder.setDescription("Performs Unary math operations taking one number input and returning one number output");
+    //   // Testing nodes
+    // "add": (context) => {
+    //     nodeBuilder = context.instantiate("math-plugin","add");
+    //     nodeBuilder.setTitle("Add");
+    //     nodeBuilder.setDescription("Performs Unary math operations taking one number input and returning one number output");
 
-        nodeBuilder.define((data) => {
-          return {
-            "res": data.input[0] + data.input[1],
-          }[data.from];
+    //     nodeBuilder.define((data) => {
+    //       return {
+    //         "res": data.input[0] + data.input[1],
+    //       }[data.from];
 
-          });
+    //       });
 
-       nodeBuilder.addInput("Number", "num1","Num");
-       nodeBuilder.addInput("Number", "num2","Num")
-       nodeBuilder.addOutput("Number", "res","Result");
-    },
+    //    nodeBuilder.addInput("Number", "num1","Num");
+    //    nodeBuilder.addInput("Number", "num2","Num")
+    //    nodeBuilder.addOutput("Number", "res","Result");
+    // },
     // Testing ternary
     "ternary": (context) => {
         nodeBuilder = context.instantiate("math-plugin","ternary");

blix-plugins/sharp-plugin/README.md

@@ -0,0 +1,41 @@
+# Math Plugin
+
+The Math Plugin is an essential component of Blix, our AI photo editor. It offers a wide range of basic arithmetic operations, enabling users to perform mathematical manipulations on images within the photo editing graph.
+Features
+
+The Math Plugin provides some of the following key features:
+
+    Addition: Perform addition operations on numerical values.
+    Subtraction: Perform subtraction operations on numerical values.
+    Multiplication: Perform multiplication operations on numerical values.
+    Division: Perform division operations on numerical values.
+    Exponentiation: Apply exponentiation operations to numerical values.
+    Absolute Value: Calculate the absolute value of a numerical value.
+    Minimum: Determine the minimum value between two numerical values.
+    Maximum: Determine the maximum value between two numerical values.
+
+How to Use
+
+To use the Math Plugin within the Blix photo editor, follow these steps:
+
+    Open the Blix photo editor and create a new project.
+    Access the editing graph or workspace.
+    Locate the Math Plugin nodes within the available nodes.
+    Drag and drop the Math Plugin node into the graph.
+    Connect the input and output nodes to the desired locations in the graph.
+    Manipulate the node through the provided components.
+    Observe the effects on the image.
+
+Please note that the Math Plugin can be used in combination with other nodes and plugins within the photo editing graph to achieve complex and customized effects.
+
+Plugin Development
+
+If you are interested in developing additional functionalities for the Math Plugin or creating your own plugins for Blix, please refer to our developer documentation. It provides comprehensive guidelines and resources to help you extend the capabilities of our photo editor.
+Feedback and Support
+
+We highly value your feedback and are dedicated to continuously improving Blix and its plugins. If you encounter any issues, have suggestions for improvement, or need assistance, please don't hesitate to reach out to our support team. We are here to help you make the most out of your photo editing experience with Blix.
+License
+
+The Math Plugin is released under the [GNU GENERAL PUBLIC LICENSE] license. Please review the license file for more information regarding the terms of use and redistribution.
+
+Enjoy using the Math Plugin in Blix, and have fun exploring the possibilities of mathematical transformations in your photo editing projects!

blix-plugins/sharp-plugin/package.json

@@ -0,0 +1,37 @@
+{
+    "name": "Sharp-plugin",
+    "displayName": "Sharp Plugin",
+    "description": "Performs basic image operations using sharp",
+    "version": "0.0.1",
+
+    "author": "Gekota the Frog",
+    "repository": "",
+
+    "contributes": {
+        "commands": [],
+        "nodes": [
+            {
+                "id": "math",
+                "name": "Math",
+                "interface" : {
+                    "inputs": {
+                    "numA": "int"
+                    },
+                    "outputs": {
+                        "numOut": "int"
+                    }
+                }
+            }
+        ]
+    },
+
+    "main": "src/main.js",
+    "renderer": "src/renderer.js",
+
+    "devDependencies": {
+        "@types/node": "^12.0.0",
+        "typescript": "^3.4.5"
+    },
+
+    "comments": [ "This plugin will be expanded to handle more image operations." ]
+}

blix-plugins/sharp-plugin/src/main.js

@@ -0,0 +1,86 @@
+const nodes ={
+    "brightness": (context) => {
+        nodeBuilder = context.instantiate("sharp-plugin","brightness");
+        nodeBuilder.setTitle("Brightness");
+        nodeBuilder.setDescription("Adjusts the brighness of an image taking one image as input and returning one image as output");
+
+        nodeBuilder.define(() => {
+            //TODO: implement
+          });
+
+        nodeBuilder.addInput("Sharp", "img","Img");
+        nodeBuilder.addOutput("Sharp", "res","Result");
+    },
+    "saturation": (context) => {
+        const nodeBuilder = context.instantiate("sharp-plugin", "saturation");
+        nodeBuilder.setTitle("Saturation");
+        nodeBuilder.setDescription("Adjusts the saturation of an image taking one image as input and returning one image as output");
+
+        nodeBuilder.define(() => {
+            //TODO: implement
+        });
+
+       nodeBuilder.addInput("Sharp", "img","Img");
+       nodeBuilder.addOutput("Sharp", "res","Result");
+      },
+    "hue": (context) => {
+        const nodeBuilder = context.instantiate("sharp-plugin", "hue");
+        nodeBuilder.setTitle("Hue");
+        nodeBuilder.setDescription("Adjusts the hue of an image taking one image as input and returning one image as output");
+
+        nodeBuilder.define(() => {
+            //TODO: implement
+        });
+
+        nodeBuilder.addInput("Sharp", "img","Img");
+        nodeBuilder.addOutput("Sharp", "res","Result");
+      },
+    "rotate": (context) => {
+        const nodeBuilder = context.instantiate("sharp-plugin", "rotate");
+        nodeBuilder.setTitle("Rotate");
+        nodeBuilder.setDescription("Rotates an image by an explicit angle taking one image as input and returning one image as output");
+
+        nodeBuilder.define(() => {
+            //TODO: implement
+        });
+
+        nodeBuilder.addInput("Sharp", "img","Img");
+        nodeBuilder.addOutput("Sharp", "res","Result");
+      },
+    "sharpen": (context) => {
+        const nodeBuilder = context.instantiate("sharp-plugin", "sharpen");
+        nodeBuilder.setTitle("sharpen");
+        nodeBuilder.setDescription("Sharpens an image taking one image as input and returning one image as output");
+
+        nodeBuilder.define(() => {
+            //TODO: implement
+        });
+
+        nodeBuilder.addInput("Sharp", "img","Img");
+        nodeBuilder.addOutput("Sharp", "res","Result");
+      },
+    "normalise": (context) => {
+        const nodeBuilder = context.instantiate("sharp-plugin", "normalise");
+        nodeBuilder.setTitle("Normalise");
+        nodeBuilder.setDescription("Enhance image contrast by stretching its luminance to cover a full dynamic range taking one image as input and returning one image as output");
+
+        nodeBuilder.define(() => {
+            //TODO: implement
+        });
+
+        nodeBuilder.addInput("Sharp", "img","Img");
+        nodeBuilder.addOutput("Sharp", "res","Result");
+      }
+}
+
+
+const commands = {}
+
+
+const tiles = {}
+
+module.exports = {
+    nodes,
+    commands,
+    tiles
+};

log.txt

@@ -0,0 +1,12 @@
+[{"status":"success","newGraph":{"nodes":[{"id":"oCQhmd","signature":"sharp-plugin.brightness","inputs":[{"id":"kfFFgc","type":"Sharp"}],"outputs":[{"id":"rgT6YX","type":"Sharp"}]},{"id":"PIK0q2","signature":"sharp-plugin.hue","inputs":[{"id":"q1g9d7","type":"Sharp"}],"outputs":[{"id":"yB4h7P","type":"Sharp"}]}],"edges":[]}}
+ System: When asked for help or to perform a task you will act as an AI assistant
+for node-based AI photo editing application. Your main role is to help the user
+manipulate a node based graph. If a question is asked that does not involve the
+graph or image editing then remind the user of your main role. Don't make
+assumptions about what values to plug into functions. Ask for clarification if a
+user request is ambiguous. Outputs of nodes can only be connected to inputs of
+other nodes. Do not try to connect inputs to inputs or outputs to outputs.
+
+System: Your very final response should be a one sentence summary without any
+JSON.
+]

package.json

@@ -59,7 +59,8 @@
     "lint": "npm-run-all -s lint:electron lint:svelte",
     "prepublishOnly": "npm-run-all -s test lint",
     "preversion": "npm-run-all -s lint format",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "profile:llm": "npm run build:electron:dev && node build/electron/lib/ai/ai.profiler.js"
   },
   "lint-staged": {
     "src/electron/**/*.{js,ts}": "eslint -c eslint.electron.json",

src/electron/lib/Blix.ts

@@ -13,7 +13,7 @@ import { blixCommands } from "./BlixCommands";
 import logger from "../utils/logger";
 import { AiManager } from "./ai/AiManager";
 import { NodeBuilder, NodeUIBuilder } from "./plugins/builders/NodeBuilder";
-
+import { testStuffies } from "./core-graph/CoreGraphTesting";
 // Encapsulates the backend representation for
 // the entire running Blix application
 export class Blix {
@@ -63,7 +63,7 @@ export class Blix {
       logger.info("Result: ", input[0]);
     });
 
-    tempNodeBuilder.addInput("Number", "in", "In");
+    tempNodeBuilder.addInput("", "in", "In");
     tempNodeBuilder.setUI(tempUIBuilder);
     this._toolboxRegistry.addInstance(tempNodeBuilder.build);
 
@@ -76,8 +76,7 @@ export class Blix {
     this._pluginManager = new PluginManager(this);
     await this._pluginManager.loadBasePlugins();
 
-    this._graphManager = new CoreGraphManager(mainWindow, this._toolboxRegistry);
-    // this._aiManager = new AiManager(mainWindow);
+    this._graphManager = new CoreGraphManager();
     this._projectManager = new ProjectManager(mainWindow);
 
     this.initSubscribers();
@@ -96,6 +95,8 @@ export class Blix {
 
     //   this._graphManager.addNode(randId, randomNode);
     // }, 3000);
+
+    this._aiManager = new AiManager(this.toolbox, this._graphManager);
   }
 
   private initSubscribers() {