diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7aad246..d906cdf 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -31,10 +31,15 @@ jobs: - name: Install Task uses: arduino/setup-task@v2 - - name: Install dprint + - name: Install run: | task install + - name: use pnpm + uses: pnpm/action-setup@v4 + with: + version: 10 + - name: Fmt check run: | task fmt:check diff --git a/.gitignore b/.gitignore index 8e6cfa6..7e04415 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ scratch-editor +prg-raise-playground build/ .bin/ -.cache/ \ No newline at end of file +.cache/ diff --git a/README.md b/README.md index 17f42a0..0378155 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,11 @@ NOTE: the `https` is needed by the `getUserMedia()` method for security reason. ## Local development - `task scratch:init` -- `task scratch:local:start` -- `ŧask board:upload` -- change the `const DEFAULT_HOST =``;` in the `scratch-arduino-extensions/packages/scratch-vm/src/extensions/ArduinoUnoQ.js` -- Open local scratch on http://localhost:8601/ +- `task scratch:watch` watch scratch GUI files and reload on save +- Open the `http://localhost:8602?host=BOARD_IP` +- `task watch` watch files changes for both python and sketch, and upload the changes to the board and restart" + +For testing on the board + +- `ŧask app:build` +- `task board:app:upload` diff --git a/Taskfile.yaml b/Taskfile.yaml index 3645624..c26789d 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -1,6 +1,5 @@ version: "3" vars: - SCRATCH_EDITOR_VERSION: v12.0.1 DPRINT_VERSION: 0.48.0 tasks: @@ -8,6 +7,7 @@ tasks: cmds: - curl -fsSL https://dprint.dev/install.sh | sh -s {{ .DPRINT_VERSION }} - mkdir -p .bin && cp $HOME/.dprint/bin/dprint .bin/dprint # workaround for local install + - curl -fsSL https://get.pnpm.io/install.sh | sh - && source ~/.bashrc fmt: desc: Run format @@ -20,21 +20,17 @@ tasks: - ${PWD}/.bin/dprint check scratch:init: - cmds: - - rm -rf scratch-editor - - git clone --depth 1 --branch {{ .SCRATCH_EDITOR_VERSION }} https://github.com/scratchfoundation/scratch-editor.git - - cd scratch-editor && npm install - - cd scratch-editor && npm run build - - task scratch:patch - - scratch:patch: - cmds: - - cd scratch-editor/packages/scratch-gui && node ../../../scratch-arduino-extensions/scripts/patch-gui.js + - git config --global url."https://github.com/mitmedialab/prg-raise-playground-scratch-gui.git".insteadOf "git@github.com:mitmedialab/prg-raise-playground-scratch-gui.git" + - git config --global url."https://github.com/mitmedialab/prg-raise-playground-scratch-vm.git".insteadOf "git@github.com:mitmedialab/prg-raise-playground-scratch-vm.git" + - git clone --recurse-submodules https://github.com/mitmedialab/prg-raise-playground.git + - cd prg-raise-playground && git switch dev && pnpm install + # copy the extension to the rigth place + - ln -s $PWD/scratch-prg-extensions/extensions/src/arduino_basics $PWD/prg-raise-playground/extensions/src/arduino_basics + - cd scratch-prg-extensions/extensions/src/arduino_basics && pnpm install - scratch:local:start: - dir: scratch-editor + scratch:watch: cmds: - - npm start --workspace @scratch/scratch-gui + - cd prg-raise-playground && pnpm dev -i arduino_basics & app:build: desc: "Copy app files (python, assets, app.yaml) to a build directory" @@ -46,17 +42,12 @@ tasks: - cp -r ./python build/scratch-arduino-app/python - cp -r ./certs build/scratch-arduino-app/certs - task scratch:build - - mkdir -p build/scratch-arduino-app/assets - - cp scratch-editor/packages/scratch-gui/build/index.html build/scratch-arduino-app/assets/index.html - - cp scratch-editor/packages/scratch-gui/build/gui.js build/scratch-arduino-app/assets/gui.js - - mkdir -p build/scratch-arduino-app/assets/static - - cp -r scratch-editor/packages/scratch-gui/build/static/blocks-media build/scratch-arduino-app/assets/static/blocks-media + - cp -r prg-raise-playground/build/. build/scratch-arduino-app/assets scratch:build: - desc: "Build Scratch GUI files" - dir: scratch-editor/packages/scratch-gui + dir: prg-raise-playground cmds: - - npm run build:dev --workspace @scratch/scratch-gui + - CI=true pnpm build board:app:upload: desc: "Upload zip file to Arduino board, unzip and deploy to /home/arduino/ArduinoApps" @@ -73,6 +64,9 @@ tasks: adb shell "cd /tmp && unzip -o $ZIP_BASENAME && mkdir -p /home/arduino/ArduinoApps && rm -rf /home/arduino/ArduinoApps/scratch-arduino-app && mv scratch-arduino-app /home/arduino/ArduinoApps/ && rm $ZIP_BASENAME" echo "App deployed to /home/arduino/ArduinoApps/scratch-arduino-app" + app:start: + - adb shell "arduino-app-cli app start user:scratch-arduino-app" + app:zip: desc: "Create a zip file with version (defaults to git commit hash)" vars: @@ -88,7 +82,7 @@ tasks: - cd build && zip -r scratch-arduino-app-{{.APP_VERSION}}.zip scratch-arduino-app && cd .. watch: - desc: "wath cfile changes for both python and sketch, and upload the changes to the board and restart" + desc: "watch files changes for both python and sketch, and upload the changes to the board and restart" deps: - python:watch - sketch:watch diff --git a/app.yaml b/app.yaml index e5fe6e9..d71de2f 100644 --- a/app.yaml +++ b/app.yaml @@ -4,5 +4,4 @@ ports: - 7000 bricks: - arduino:web_ui - - arduino:object_detection icon: 🐱 diff --git a/doc/scratch-unoq.png b/doc/scratch-unoq.png index dc02315..3b5a619 100644 Binary files a/doc/scratch-unoq.png and b/doc/scratch-unoq.png differ diff --git a/python/main.py b/python/main.py index 59513c8..40ffc89 100644 --- a/python/main.py +++ b/python/main.py @@ -1,91 +1,16 @@ from arduino.app_utils import App, Bridge from arduino.app_bricks.web_ui import WebUI -from arduino.app_bricks.object_detection import ObjectDetection -import time -import base64 - -object_detection = ObjectDetection() - - -def on_matrix_draw(_, data): - print(f"Received frame to draw on matrix: {data}") - # from 5x5 to 8x13 matrix - frame_5x5 = data.get("frame") - row0 = "0" * 13 - row1 = "0" * 4 + frame_5x5[0:5] + "0" * 4 - row2 = "0" * 4 + frame_5x5[5:10] + "0" * 4 - row3 = "0" * 4 + frame_5x5[10:15] + "0" * 4 - row4 = "0" * 4 + frame_5x5[15:20] + "0" * 4 - row5 = "0" * 4 + frame_5x5[20:25] + "0" * 4 - row6 = "0" * 13 - row7 = "0" * 13 - frame_8x13 = row0 + row1 + row2 + row3 + row4 + row5 + row6 + row7 - print(f"Transformed frame to draw on 8x13 matrix: {frame_8x13}") - Bridge.call("matrix_draw", frame_8x13) - - -def rgb_to_digital(value, threshold=128) -> bool: - """Convert RGB value (0-255) to digital HIGH(1) or LOW(0)""" - return value >= threshold - - -def on_set_led_rgb(_, data): - led = data.get("led") - r = data.get("r") - g = data.get("g") - b = data.get("b") - - # Convert RGB values (0-255) to digital HIGH/LOW - r_digital = rgb_to_digital(r) - g_digital = rgb_to_digital(g) - b_digital = rgb_to_digital(b) - - print( - f"Setting LED {led} to color: RGB({r},{g},{b}) -> Digital({r_digital},{g_digital},{b_digital})" - ) - Bridge.call("set_led_rgb", led, r_digital, g_digital, b_digital) - - -def on_detect_objects(client_id, data): - """Callback function to handle object detection requests.""" - try: - image_data = data.get("image") - confidence = data.get("confidence", 0.5) - if not image_data: - # TODO: implement the 'detection_error` in the extension - ui.send_message("detection_error", {"error": "No image data"}) - return - - start_time = time.time() * 1000 - results = object_detection.detect(base64.b64decode(image_data), confidence=confidence) - diff = time.time() * 1000 - start_time - - if results is None: - ui.send_message("detection_error", {"error": "No results returned"}) - return - - response = { - "detection": results.get("detection", []), - "detection_count": len(results.get("detection", [])) if results else 0, - "processing_time": f"{diff:.2f} ms", - } - ui.send_message("detection_result", response) - - except Exception as e: - ui.send_message("detection_error", {"error": str(e)}) - ui = WebUI(use_ssl=True) ui.on_connect(lambda sid: (print(f"Client connected: {sid} "),)) -ui.on_message("matrix_draw", on_matrix_draw) -ui.on_message("set_led_rgb", on_set_led_rgb) -ui.on_message("detect_objects", on_detect_objects) -def on_modulino_button_pressed(btn): - ui.send_message("modulino_buttons_pressed", {"btn": btn}) +def on_matrix_draw(_, data): + frame = data.get("frame") + print(f"Frame to draw on 8x13 matrix: {frame}") + Bridge.call("matrix_draw", frame) -Bridge.provide("modulino_button_pressed", on_modulino_button_pressed) +ui.on_message("matrix_draw", on_matrix_draw) App.run() diff --git a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/ArduinoUnoQ.js b/scratch-arduino-extensions/packages/scratch-vm/src/extensions/ArduinoUnoQ.js deleted file mode 100644 index 734401c..0000000 --- a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/ArduinoUnoQ.js +++ /dev/null @@ -1,133 +0,0 @@ -const io = require("./socket.io.min.js"); - -const DEFAULT_HOST = window.location.hostname; - -class ArduinoUnoQ { - constructor() { - this.serverURL = `wss://${DEFAULT_HOST}:7000`; - - this.io = io(this.serverURL, { - path: "/socket.io", - transports: ["polling", "websocket"], - autoConnect: true, - }); - this.isConnected = false; - - this._setupConnectionHandlers(); - } - - on(event, callback) { - if (this.io) { - this.io.on(event, callback); - console.log(`Registered event listener for: ${event}`); - } else { - console.error("Socket.io not initialized"); - } - } - - emit(event, data) { - if (this.io && this.isConnected) { - this.io.emit(event, data); - console.log(`Emitted event: ${event}`, data); - } else { - console.warn(`Cannot emit ${event}: Not connected to Arduino UNO Q`); - } - } - - _setupConnectionHandlers() { - this.io.on("connect", () => { - this.isConnected = true; - console.log(`Connected to Arduino UNO Q at ${this.serverURL}`); - }); - - this.io.on("disconnect", (reason) => { - this.isConnected = false; - console.log(`Disconnected from Arduino UNO Q: ${reason}`); - }); - - this.io.on("connect_error", (error) => { - console.error(`Connection error:`, error.message); - }); - - this.io.on("reconnect", (attemptNumber) => { - console.log(`Reconnected to Arduino UNO Q after ${attemptNumber} attempts`); - }); - } - - connect() { - if (!this.io.connected) { - console.log("Attempting to connect to Arduino UNO Q..."); - this.io.connect(); - } - } - - disconnect() { - if (this.io.connected) { - console.log("Disconnecting from Arduino UNO Q..."); - this.io.disconnect(); - } - } - - // ===== LED CONTROL METHODS ===== - /** - * Set RGB LED color - * @param {string} led - LED identifier ("LED3" or "LED4") - * @param {number} r - Red value (0-255) - * @param {number} g - Green value (0-255) - * @param {number} b - Blue value (0-255) - */ - setLedRGB(led, r, g, b) { - this.io.emit("set_led_rgb", { - led: led, - r: Math.max(0, Math.min(255, r)), - g: Math.max(0, Math.min(255, g)), - b: Math.max(0, Math.min(255, b)), - }); - console.log(`Setting ${led} to RGB(${r}, ${g}, ${b})`); - } - - /** - * Turn off LED - * @param {string} led - LED identifier ("LED3" or "LED4") - */ - turnOffLed(led) { - this.setLedRGB(led, 0, 0, 0); - } - - // ===== MATRIX CONTROL METHODS ===== - - /** - * Draw frame on LED matrix - * @param {string} frame - 25-character string representing 5x5 matrix (0s and 1s) - */ - matrixDraw(frame) { - if (typeof frame !== "string" || frame.length !== 25) { - console.error("Invalid frame format. Expected 25-character string of 0s and 1s"); - return; - } - // Validate frame contains only 0s and 1s - if (!/^[01]+$/.test(frame)) { - console.error("Frame must contain only 0s and 1s"); - return; - } - - this.io.emit("matrix_draw", { frame: frame }); - console.log(`Drawing matrix frame: ${frame}`); - } - - matrixClear() { - const clearFrame = "0".repeat(25); - this.matrixDraw(clearFrame); - } - - // AI object detection - - detectObjects(imageData) { - this.io.emit("detect_objects", { image: imageData }); - console.log("Emitted detect_objects event"); - } - - // ===== EVENT HANDLING METHODS ===== -} - -module.exports = ArduinoUnoQ; diff --git a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_basics/index.js b/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_basics/index.js deleted file mode 100644 index 8f95ab3..0000000 --- a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_basics/index.js +++ /dev/null @@ -1,101 +0,0 @@ -const BlockType = require("../../../../../../scratch-editor/packages/scratch-vm/src/extension-support/block-type"); -const ArgumentType = require( - "../../../../../../scratch-editor/packages/scratch-vm/src/extension-support/argument-type", -); -const ArduinoUnoQ = require("../ArduinoUnoQ"); - -// TODO: add icons -const iconURI = ""; -const menuIconURI = ""; - -class ArduinoBasics { - constructor(runtime) { - this.runtime = runtime; - this.unoq = new ArduinoUnoQ(); - this.unoq.connect(); - } -} - -ArduinoBasics.prototype.getInfo = function() { - return { - id: "arduinobasics", - name: "Arduino Basics", - menuIconURI: menuIconURI, - blockIconURI: iconURI, - blocks: [ - { - opcode: "matrixDraw", - blockType: BlockType.COMMAND, - text: "draw [FRAME] on matrix", - func: "matrixDraw", - arguments: { - FRAME: { - type: ArgumentType.MATRIX, - defaultValue: "0101010101100010101000100", - }, - }, - }, - { - opcode: "matrixClear", - blockType: BlockType.COMMAND, - text: "clear matrix", - func: "matrixClear", - arguments: {}, - }, - { - opcode: "setLed3", - blockType: BlockType.COMMAND, - text: "set LED 3 to [HEX]", - func: "setLed3", - arguments: { - HEX: { - type: ArgumentType.COLOR, - defaultValue: "#ff0000", - }, - }, - }, - { - opcode: "setLed4", - blockType: BlockType.COMMAND, - text: "set LED 4 to [HEX]", - func: "setLed4", - arguments: { - HEX: { - type: ArgumentType.COLOR, - defaultValue: "#0000ff", - }, - }, - }, - ], - }; -}; - -ArduinoBasics.prototype.matrixDraw = function(args) { - this.unoq.matrixDraw(args.FRAME); -}; - -ArduinoBasics.prototype.matrixClear = function() { - this.unoq.matrixClear(); -}; - -ArduinoBasics.prototype.setLed3 = function(args) { - const hexColor = args.HEX; - const rgb = this.hexToRgb(hexColor); - this.unoq.setLedRGB("LED3", rgb.r, rgb.g, rgb.b); -}; - -ArduinoBasics.prototype.setLed4 = function(args) { - const hexColor = args.HEX; - const rgb = this.hexToRgb(hexColor); - this.unoq.setLedRGB("LED4", rgb.r, rgb.g, rgb.b); -}; - -ArduinoBasics.prototype.hexToRgb = function(hex) { - hex = hex.replace("#", ""); - const r = parseInt(hex.substring(0, 2), 16); - const g = parseInt(hex.substring(2, 4), 16); - const b = parseInt(hex.substring(4, 6), 16); - return { r, g, b }; -}; - -module.exports = ArduinoBasics; diff --git a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_modulino/index.js b/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_modulino/index.js deleted file mode 100644 index 572a0c0..0000000 --- a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_modulino/index.js +++ /dev/null @@ -1,61 +0,0 @@ -const BlockType = require("../../../../../../scratch-editor/packages/scratch-vm/src/extension-support/block-type"); -const ArgumentType = require( - "../../../../../../scratch-editor/packages/scratch-vm/src/extension-support/argument-type", -); -const ArduinoUnoQ = require("../ArduinoUnoQ"); - -// TODO: add icons -const iconURI = ""; -const menuIconURI = ""; - -class ArduinoModulino { - constructor(runtime) { - this.runtime = runtime; - this.unoq = new ArduinoUnoQ(); - this.unoq.connect(); - - // TODO: move to ModulinoPeripheral - this._button_pressed = ""; - this.unoq.on("modulino_buttons_pressed", (data) => { - console.log(`Modulino button pressed event received: ${data.btn}`); - this._button_pressed = data.btn.toUpperCase(); - }); - } -} - -ArduinoModulino.prototype.getInfo = function() { - return { - id: "arduinomodulino", - name: "Arduino Modulino", - menuIconURI: menuIconURI, - blockIconURI: iconURI, - blocks: [ - { - opcode: "whenModulinoButtonsPressed", - blockType: BlockType.HAT, - text: "when modulino button [BTN] pressed", - func: "whenModulinoButtonsPressed", - arguments: { - BTN: { - type: ArgumentType.STRING, - menu: "modulinoButtons", - defaultValue: "A", - }, - }, - }, - ], - menus: { - modulinoButtons: ["A", "B", "C"], - }, - }; -}; - -ArduinoModulino.prototype.whenModulinoButtonsPressed = function(args) { - if (args.BTN === this._button_pressed) { - this._button_pressed = ""; - return true; - } - return false; -}; - -module.exports = ArduinoModulino; diff --git a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_object_detection/index.js b/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_object_detection/index.js deleted file mode 100644 index 66d1897..0000000 --- a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_object_detection/index.js +++ /dev/null @@ -1,396 +0,0 @@ -const BlockType = require("../../../../../../scratch-editor/packages/scratch-vm/src/extension-support/block-type"); -const ArgumentType = require( - "../../../../../../scratch-editor/packages/scratch-vm/src/extension-support/argument-type", -); -const Video = require("../../../../../../scratch-editor/packages/scratch-vm/src/io/video"); -const Rectangle = require("../../../../../../scratch-editor/packages/scratch-render/src/Rectangle.js"); -const StageLayering = require("../../../../../../scratch-editor/packages/scratch-vm/src/engine/stage-layering.js"); -const { Detection, MODEL_LABELS } = require("./object_detection"); -const ArduinoUnoQ = require("../ArduinoUnoQ"); - -// TODO add icons -const iconURI = ""; -const menuIconURI = ""; - -/** - * RGB color constants for confidence visualization - */ -const RGB_COLORS = { - RED: { r: 1.0, g: 0.0, b: 0.0 }, - ORANGE: { r: 1.0, g: 0.5, b: 0.0 }, - GREEN: { r: 0.0, g: 1.0, b: 0.0 }, -}; - -class ArduinoObjectDetection { - constructor(runtime) { - this.runtime = runtime; - - this.unoq = new ArduinoUnoQ(); - this.unoq.connect(); - - /** @type {Array} */ - this.detectedObjects = []; - - this._penSkinId = null; - - /** @type {Object} */ - this._detectionStates = this._initializeDetectionStates(); - - /** @type {number|null} */ - this._loopIntervalId = null; - - /** @type {boolean} */ - this._isLoopRunning = false; - - /** @type {boolean} */ - this._enableBoundingBoxes = true; - - this.runtime.on("PROJECT_LOADED", () => { - if (!this.runtime.renderer) { - console.log("Renderer is NOT available in runtime."); - return; - } - if (!this._penSkinId) { - this._penSkinId = this.runtime.renderer.createPenSkin(); - this.penDrawableId = this.runtime.renderer.createDrawable(StageLayering.PEN_LAYER); - this.runtime.renderer.updateDrawableSkinId(this.penDrawableId, this._penSkinId); - } - }); - - this.unoq.on("detection_result", (data) => { - this.detectedObjects = []; - this._clearBoundingBoxes(); - - data.detection.forEach((detection) => { - const [x1, y1, x2, y2] = detection.bounding_box_xyxy; - - const detectionObject = new Detection( - detection.class_name, - this._createRectangleFromBoundingBox(x1, y1, x2, y2), - parseFloat(detection.confidence), - ); - this.detectedObjects.push(detectionObject); - - console.log( - `Detected ${detectionObject.label} with confidence ${ - detectionObject.confidence.toFixed(2) - } took ${data.processing_time}`, - ); - }); - - this._updateDetectionStates(); - - if (this._enableBoundingBoxes) { - this._drawBoundingBoxes(); - } else { - this._clearBoundingBoxes(); - } - }); - } -} - -ArduinoObjectDetection.prototype.getInfo = function() { - return { - id: "ArduinoObjectDetection", - name: "Arduino Object Detection", - menuIconURI: menuIconURI, - blockIconURI: iconURI, - blocks: [ - { - opcode: "whenObjectDetected", - blockType: BlockType.HAT, - text: "when [OBJECT] detected", - func: "whenObjectDetected", - arguments: { - OBJECT: { - type: ArgumentType.STRING, - menu: "modelsLabels", - defaultValue: MODEL_LABELS.PERSON, - }, - }, - }, - { - opcode: "startDetectionLoop", - blockType: BlockType.COMMAND, - text: "start detection", - func: "startDetectionLoop", - arguments: {}, - }, - { - opcode: "stopDetectionLoop", - blockType: BlockType.COMMAND, - text: "stop detection", - func: "stopDetectionLoop", - arguments: {}, - }, - { - opcode: "isObjectDetected", - blockType: BlockType.BOOLEAN, - text: "is [OBJECT] detected", - func: "isObjectDetected", - arguments: { - OBJECT: { - type: ArgumentType.STRING, - menu: "modelsLabels", - defaultValue: MODEL_LABELS.PERSON, - }, - }, - }, - { - opcode: "showBoundingBoxes", - blockType: BlockType.COMMAND, - text: "show bounding boxes", - func: "showBoundingBoxes", - arguments: {}, - }, - { - opcode: "hideBoundingBoxes", - blockType: BlockType.COMMAND, - text: "hide bounding boxes", - func: "hideBoundingBoxes", - arguments: {}, - }, - { - opcode: "getDetectedObjectsCount", - blockType: BlockType.REPORTER, - text: "number", - func: "getDetectedObjectsCount", - arguments: {}, - }, - { - opcode: "getDetectedLabelsAsString", - blockType: BlockType.REPORTER, - text: "labels", - func: "getDetectedLabelsAsString", - arguments: {}, - }, - ], - menus: { - modelsLabels: Object.values(MODEL_LABELS).sort(), - }, - }; -}; - -ArduinoObjectDetection.prototype.startDetectionLoop = function(args) { - if (this._isLoopRunning) { - console.log("Detection loop is already running"); - return; - } - - this._isLoopRunning = true; - this.runtime.ioDevices.video.enableVideo(); - this._loop(); - - this._loopIntervalId = setInterval(() => { - this._loop(); - }, 1000); // 1000ms = 1s -}; - -ArduinoObjectDetection.prototype.stopDetectionLoop = function(args) { - this.runtime.ioDevices.video.disableVideo(); - this.hideBoundingBoxes(); - - if (!this._isLoopRunning) { - console.log("Detection loop is not running"); - return; - } - - console.log("Stopping detection loop"); - this._isLoopRunning = false; - if (this._loopIntervalId) { - clearInterval(this._loopIntervalId); - this._loopIntervalId = null; - } -}; - -ArduinoObjectDetection.prototype._loop = function() { - if (!this._isLoopRunning) { - return; - } - this._detectObjects(); - - // Note: The detection states for all objects will be updated - // automatically when the detection_result event is received -}; - -ArduinoObjectDetection.prototype.whenObjectDetected = function(args) { - const objectLabel = args.OBJECT; - return this.detectedObjects.some(detectionObject => detectionObject.label === objectLabel); -}; - -ArduinoObjectDetection.prototype.isObjectDetected = function(args) { - const objectLabel = args.OBJECT; - return this.detectedObjects.some(detectionObject => detectionObject.label === objectLabel); -}; - -ArduinoObjectDetection.prototype.hideBoundingBoxes = function(args) { - this._enableBoundingBoxes = false; - this._clearBoundingBoxes(); -}; - -ArduinoObjectDetection.prototype.showBoundingBoxes = function(args) { - this._enableBoundingBoxes = true; - this._drawBoundingBoxes(); -}; - -ArduinoObjectDetection.prototype._detectObjects = function(args) { - if (!this.runtime.ioDevices) { - console.log("No ioDevices available."); - return; - } - const canvas = this.runtime.ioDevices.video.getFrame({ - format: Video.FORMAT_CANVAS, - dimensions: [480, 360], // the same as the stage resolution - }); - if (!canvas) { - console.log("No canvas available from video frame."); - return; - } - const dataUrl = canvas.toDataURL("image/png"); - const base64Frame = dataUrl.split(",")[1]; - this.unoq.detectObjects(base64Frame); -}; - -ArduinoObjectDetection.prototype._clearBoundingBoxes = function(args) { - if (!this.runtime.renderer || !this._penSkinId) { - console.log("Renderer or pen skin not available for clearing"); - return; - } - const penSkin = this.runtime.renderer._allSkins[this._penSkinId]; - if (penSkin && penSkin.clear) { - penSkin.clear(); - } else { - console.log("Could not clear pen skin"); - } -}; - -ArduinoObjectDetection.prototype._drawBoundingBoxes = function(args) { - this.detectedObjects.forEach(detectionObject => { - const { r, g, b } = this._getColorByConfidence(detectionObject.confidence); - const penAttributes = { - color4f: [r, g, b, 1.0], - diameter: 3, - }; - this._drawRectangleWithPen(detectionObject.rectangle, penAttributes); - }); -}; - -/** - * Get pen color based on confidence level - * @param {number} confidence - Confidence score (0 to 100) - * @returns {Object} RGB color object {r, g, b} in 0-1 range - */ -ArduinoObjectDetection.prototype._getColorByConfidence = function(confidence) { - if (confidence >= 90) { - return RGB_COLORS.GREEN; - } - if (confidence >= 75 && confidence < 90) { - return RGB_COLORS.ORANGE; - } - return RGB_COLORS.RED; -}; - -/** - * Draw a rectangle using the Rectangle class and pen system - * @param {Rectangle} rectangle - Rectangle object defining the bounds - * @param {Object} penAttributes - Pen drawing attributes (color, thickness) - */ -ArduinoObjectDetection.prototype._drawRectangleWithPen = function(rectangle, penAttributes) { - if (!this.runtime.renderer || !this._penSkinId) { - console.log("Renderer or pen skin not available"); - return; - } - - // TODO: Get the pen skin object in a better way - const penSkin = this.runtime.renderer._allSkins[this._penSkinId]; - if (!penSkin) { - console.log("Pen skin not found"); - return; - } - - const left = rectangle.left; - const right = rectangle.right; - const bottom = rectangle.bottom; - const top = rectangle.top; - - penSkin.drawLine(penAttributes, left, top, right, top); - penSkin.drawLine(penAttributes, right, top, right, bottom); - penSkin.drawLine(penAttributes, right, bottom, left, bottom); - penSkin.drawLine(penAttributes, left, bottom, left, top); -}; - -ArduinoObjectDetection.prototype._createRectangleFromBoundingBox = function(x1, y1, x2, y2) { - x1 = x1 - 240; // 0-480 -> -240 to +240 - y1 = -(y1 - 180); // 0-360 -> -180 to +180 - x2 = x2 - 240; - y2 = -(y2 - 180); - - const left = Math.min(x1, x2); - const right = Math.max(x1, x2); - const bottom = Math.min(y1, y2); - const top = Math.max(y1, y2); - - const rectangle = new Rectangle(); - rectangle.initFromBounds(left, right, bottom, top); - return rectangle; -}; - -/** - * Block function: Get the total number of detected objects - * @returns {number} Number of currently detected objects - */ -ArduinoObjectDetection.prototype.getDetectedObjectsCount = function() { - return this.detectedObjects.length; -}; - -/** - * Block function: Get detected object types as a comma-separated string - * @returns {string} Comma-separated list of detected object types - */ -ArduinoObjectDetection.prototype.getDetectedLabelsAsString = function() { - const detectedLabels = this._getDetectedLabels(); - return detectedLabels.length > 0 ? detectedLabels.join(", ") : "none"; -}; - -/** - * Initialize detection states for all model labels - * @returns {Object} Object with all labels set to false - */ -ArduinoObjectDetection.prototype._initializeDetectionStates = function() { - const states = {}; - Object.values(MODEL_LABELS).forEach(label => { - states[label] = false; - }); - return states; -}; - -/** - * Update detection states based on currently detected objects - */ -ArduinoObjectDetection.prototype._updateDetectionStates = function() { - // Reset all states to false - Object.keys(this._detectionStates).forEach(label => { - this._detectionStates[label] = false; - }); - - // Set to true for currently detected objects - this.detectedObjects.forEach(detectionObject => { - this._detectionStates[detectionObject.label] = true; - }); - - // Log detection updates for debugging - const detectedLabels = Object.keys(this._detectionStates).filter(label => this._detectionStates[label]); - if (detectedLabels.length > 0) { - console.log(`Currently detected: ${detectedLabels.join(", ")}`); - } -}; - -/** - * Get all currently detected object labels - * @returns {Array} Array of currently detected object labels - */ -ArduinoObjectDetection.prototype._getDetectedLabels = function() { - return Object.keys(this._detectionStates).filter(label => this._detectionStates[label]); -}; - -module.exports = ArduinoObjectDetection; diff --git a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_object_detection/object_detection.js b/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_object_detection/object_detection.js deleted file mode 100644 index 5a83b57..0000000 --- a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_object_detection/object_detection.js +++ /dev/null @@ -1,109 +0,0 @@ -const MODEL_LABELS = { - AIRPLANE: "airplane", - APPLE: "apple", - BACKPACK: "backpack", - BANANA: "banana", - BASEBALL_BAT: "baseball bat", - BASEBALL_GLOVE: "baseball glove", - BEAR: "bear", - BED: "bed", - BENCH: "bench", - BICYCLE: "bicycle", - BIRD: "bird", - BOAT: "boat", - BOOK: "book", - BOTTLE: "bottle", - BOWL: "bowl", - BROCCOLI: "broccoli", - BUS: "bus", - CAKE: "cake", - CAR: "car", - CARROT: "carrot", - CAT: "cat", - CELL_PHONE: "cell phone", - CHAIR: "chair", - CLOCK: "clock", - COUCH: "couch", - COW: "cow", - CUP: "cup", - DINING_TABLE: "dining table", - DOG: "dog", - DONUT: "donut", - ELEPHANT: "elephant", - FIRE_HYDRANT: "fire hydrant", - FORK: "fork", - FRISBEE: "frisbee", - GIRAFFE: "giraffe", - HAIR_DRIER: "hair drier", - HANDBAG: "handbag", - HOT_DOG: "hot dog", - HORSE: "horse", - KEYBOARD: "keyboard", - KITE: "kite", - KNIFE: "knife", - LAPTOP: "laptop", - MICROWAVE: "microwave", - MOTORCYCLE: "motorcycle", - MOUSE: "mouse", - ORANGE: "orange", - OVEN: "oven", - PARKING_METER: "parking meter", - PERSON: "person", - PIZZA: "pizza", - POTTED_PLANT: "potted plant", - REFRIGERATOR: "refrigerator", - REMOTE: "remote", - SANDWICH: "sandwich", - SCISSORS: "scissors", - SHEEP: "sheep", - SINK: "sink", - SKATEBOARD: "skateboard", - SKIS: "skis", - SNOWBOARD: "snowboard", - SPOON: "spoon", - SPORTS_BALL: "sports ball", - STOP_SIGN: "stop sign", - SUITCASE: "suitcase", - SURFBOARD: "surfboard", - TEDDY_BEAR: "teddy bear", - TENNIS_RACKET: "tennis racket", - TIE: "tie", - TOASTER: "toaster", - TOILET: "toilet", - TOOTHBRUSH: "toothbrush", - TRAFFIC_LIGHT: "traffic light", - TRAIN: "train", - TRUCK: "truck", - TV: "tv", - UMBRELLA: "umbrella", - VASE: "vase", - WINE_GLASS: "wine glass", - ZEBRA: "zebra", -}; - -class Detection { - /** - * Create a Detection object - * @param {string} label - The object class name (e.g., "person", "car") - * @param {Rectangle} rectangle - The bounding box as a Rectangle object - * @param {number} confidence - The confidence score (0.0 to 1.0) - */ - constructor(label, rectangle, confidence) { - /** @type {string} */ - this.label = label; - - /** @type {Rectangle} */ - this.rectangle = rectangle; - - /** @type {number} */ - this.confidence = confidence; - } - - toString() { - return `Detection: ${this.label} (${(this.confidence * 100).toFixed(1)}%) at [${this.rectangle.left.toFixed(1)}, ${ - this.rectangle.top.toFixed(1) - }, ${this.rectangle.right.toFixed(1)}, ${this.rectangle.bottom.toFixed(1)}]`; - } -} - -export { Detection, MODEL_LABELS }; diff --git a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/socket.io.min.js b/scratch-arduino-extensions/packages/scratch-vm/src/extensions/socket.io.min.js deleted file mode 100644 index 8cd21de..0000000 --- a/scratch-arduino-extensions/packages/scratch-vm/src/extensions/socket.io.min.js +++ /dev/null @@ -1,7 +0,0 @@ -// dprint-ignore-file -/*! - * Socket.IO v4.8.1 - * (c) 2014-2024 Guillermo Rauch - * Released under the MIT License. - */ -!function(t,n){"object"==typeof exports&&"undefined"!=typeof module?module.exports=n():"function"==typeof define&&define.amd?define(n):(t="undefined"!=typeof globalThis?globalThis:t||self).io=n()}(this,(function(){"use strict";function t(t,n){(null==n||n>t.length)&&(n=t.length);for(var i=0,r=Array(n);i=n.length?{done:!0}:{done:!1,value:n[e++]}},e:function(t){throw t},f:o}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var s,u=!0,h=!1;return{s:function(){r=r.call(n)},n:function(){var t=r.next();return u=t.done,t},e:function(t){h=!0,s=t},f:function(){try{u||null==r.return||r.return()}finally{if(h)throw s}}}}function e(){return e=Object.assign?Object.assign.bind():function(t){for(var n=1;n1?{type:l[i],data:t.substring(1)}:{type:l[i]}:d},N=function(t,n){if(B){var i=function(t){var n,i,r,e,o,s=.75*t.length,u=t.length,h=0;"="===t[t.length-1]&&(s--,"="===t[t.length-2]&&s--);var f=new ArrayBuffer(s),c=new Uint8Array(f);for(n=0;n>4,c[h++]=(15&r)<<4|e>>2,c[h++]=(3&e)<<6|63&o;return f}(t);return C(i,n)}return{base64:!0,data:t}},C=function(t,n){return"blob"===n?t instanceof Blob?t:new Blob([t]):t instanceof ArrayBuffer?t:t.buffer},T=String.fromCharCode(30);function U(){return new TransformStream({transform:function(t,n){!function(t,n){y&&t.data instanceof Blob?t.data.arrayBuffer().then(k).then(n):b&&(t.data instanceof ArrayBuffer||w(t.data))?n(k(t.data)):g(t,!1,(function(t){p||(p=new TextEncoder),n(p.encode(t))}))}(t,(function(i){var r,e=i.length;if(e<126)r=new Uint8Array(1),new DataView(r.buffer).setUint8(0,e);else if(e<65536){r=new Uint8Array(3);var o=new DataView(r.buffer);o.setUint8(0,126),o.setUint16(1,e)}else{r=new Uint8Array(9);var s=new DataView(r.buffer);s.setUint8(0,127),s.setBigUint64(1,BigInt(e))}t.data&&"string"!=typeof t.data&&(r[0]|=128),n.enqueue(r),n.enqueue(i)}))}})}function M(t){return t.reduce((function(t,n){return t+n.length}),0)}function x(t,n){if(t[0].length===n)return t.shift();for(var i=new Uint8Array(n),r=0,e=0;e1?n-1:0),r=1;r1&&void 0!==arguments[1]?arguments[1]:{};return t+"://"+this.i()+this.o()+this.opts.path+this.u(n)},i.i=function(){var t=this.opts.hostname;return-1===t.indexOf(":")?t:"["+t+"]"},i.o=function(){return this.opts.port&&(this.opts.secure&&Number(443!==this.opts.port)||!this.opts.secure&&80!==Number(this.opts.port))?":"+this.opts.port:""},i.u=function(t){var n=function(t){var n="";for(var i in t)t.hasOwnProperty(i)&&(n.length&&(n+="&"),n+=encodeURIComponent(i)+"="+encodeURIComponent(t[i]));return n}(t);return n.length?"?"+n:""},n}(I),X=function(t){function n(){var n;return(n=t.apply(this,arguments)||this).h=!1,n}s(n,t);var r=n.prototype;return r.doOpen=function(){this.v()},r.pause=function(t){var n=this;this.readyState="pausing";var i=function(){n.readyState="paused",t()};if(this.h||!this.writable){var r=0;this.h&&(r++,this.once("pollComplete",(function(){--r||i()}))),this.writable||(r++,this.once("drain",(function(){--r||i()})))}else i()},r.v=function(){this.h=!0,this.doPoll(),this.emitReserved("poll")},r.onData=function(t){var n=this;(function(t,n){for(var i=t.split(T),r=[],e=0;e0&&void 0!==arguments[0]?arguments[0]:{};return e(t,{xd:this.xd},this.opts),new Y(tt,this.uri(),t)},n}(K);function tt(t){var n=t.xdomain;try{if("undefined"!=typeof XMLHttpRequest&&(!n||z))return new XMLHttpRequest}catch(t){}if(!n)try{return new(L[["Active"].concat("Object").join("X")])("Microsoft.XMLHTTP")}catch(t){}}var nt="undefined"!=typeof navigator&&"string"==typeof navigator.product&&"reactnative"===navigator.product.toLowerCase(),it=function(t){function n(){return t.apply(this,arguments)||this}s(n,t);var r=n.prototype;return r.doOpen=function(){var t=this.uri(),n=this.opts.protocols,i=nt?{}:_(this.opts,"agent","perMessageDeflate","pfx","key","passphrase","cert","ca","ciphers","rejectUnauthorized","localAddress","protocolVersion","origin","maxPayload","family","checkServerIdentity");this.opts.extraHeaders&&(i.headers=this.opts.extraHeaders);try{this.ws=this.createSocket(t,n,i)}catch(t){return this.emitReserved("error",t)}this.ws.binaryType=this.socket.binaryType,this.addEventListeners()},r.addEventListeners=function(){var t=this;this.ws.onopen=function(){t.opts.autoUnref&&t.ws.C.unref(),t.onOpen()},this.ws.onclose=function(n){return t.onClose({description:"websocket connection closed",context:n})},this.ws.onmessage=function(n){return t.onData(n.data)},this.ws.onerror=function(n){return t.onError("websocket error",n)}},r.write=function(t){var n=this;this.writable=!1;for(var i=function(){var i=t[r],e=r===t.length-1;g(i,n.supportsBinary,(function(t){try{n.doWrite(i,t)}catch(t){}e&&R((function(){n.writable=!0,n.emitReserved("drain")}),n.setTimeoutFn)}))},r=0;rMath.pow(2,21)-1){u.enqueue(d);break}e=v*Math.pow(2,32)+a.getUint32(4),r=3}else{if(M(i)t){u.enqueue(d);break}}}})}(Number.MAX_SAFE_INTEGER,t.socket.binaryType),r=n.readable.pipeThrough(i).getReader(),e=U();e.readable.pipeTo(n.writable),t.U=e.writable.getWriter();!function n(){r.read().then((function(i){var r=i.done,e=i.value;r||(t.onPacket(e),n())})).catch((function(t){}))}();var o={type:"open"};t.query.sid&&(o.data='{"sid":"'.concat(t.query.sid,'"}')),t.U.write(o).then((function(){return t.onOpen()}))}))}))},r.write=function(t){var n=this;this.writable=!1;for(var i=function(){var i=t[r],e=r===t.length-1;n.U.write(i).then((function(){e&&R((function(){n.writable=!0,n.emitReserved("drain")}),n.setTimeoutFn)}))},r=0;r8e3)throw"URI too long";var n=t,i=t.indexOf("["),r=t.indexOf("]");-1!=i&&-1!=r&&(t=t.substring(0,i)+t.substring(i,r).replace(/:/g,";")+t.substring(r,t.length));for(var e,o,s=ut.exec(t||""),u={},h=14;h--;)u[ht[h]]=s[h]||"";return-1!=i&&-1!=r&&(u.source=n,u.host=u.host.substring(1,u.host.length-1).replace(/;/g,":"),u.authority=u.authority.replace("[","").replace("]","").replace(/;/g,":"),u.ipv6uri=!0),u.pathNames=function(t,n){var i=/\/{2,9}/g,r=n.replace(i,"/").split("/");"/"!=n.slice(0,1)&&0!==n.length||r.splice(0,1);"/"==n.slice(-1)&&r.splice(r.length-1,1);return r}(0,u.path),u.queryKey=(e=u.query,o={},e.replace(/(?:^|&)([^&=]*)=?([^&]*)/g,(function(t,n,i){n&&(o[n]=i)})),o),u}var ct="function"==typeof addEventListener&&"function"==typeof removeEventListener,at=[];ct&&addEventListener("offline",(function(){at.forEach((function(t){return t()}))}),!1);var vt=function(t){function n(n,i){var r;if((r=t.call(this)||this).binaryType="arraybuffer",r.writeBuffer=[],r.M=0,r.I=-1,r.R=-1,r.L=-1,r._=1/0,n&&"object"===c(n)&&(i=n,n=null),n){var o=ft(n);i.hostname=o.host,i.secure="https"===o.protocol||"wss"===o.protocol,i.port=o.port,o.query&&(i.query=o.query)}else i.host&&(i.hostname=ft(i.host).host);return $(r,i),r.secure=null!=i.secure?i.secure:"undefined"!=typeof location&&"https:"===location.protocol,i.hostname&&!i.port&&(i.port=r.secure?"443":"80"),r.hostname=i.hostname||("undefined"!=typeof location?location.hostname:"localhost"),r.port=i.port||("undefined"!=typeof location&&location.port?location.port:r.secure?"443":"80"),r.transports=[],r.D={},i.transports.forEach((function(t){var n=t.prototype.name;r.transports.push(n),r.D[n]=t})),r.opts=e({path:"/engine.io",agent:!1,withCredentials:!1,upgrade:!0,timestampParam:"t",rememberUpgrade:!1,addTrailingSlash:!0,rejectUnauthorized:!0,perMessageDeflate:{threshold:1024},transportOptions:{},closeOnBeforeunload:!1},i),r.opts.path=r.opts.path.replace(/\/$/,"")+(r.opts.addTrailingSlash?"/":""),"string"==typeof r.opts.query&&(r.opts.query=function(t){for(var n={},i=t.split("&"),r=0,e=i.length;r1))return this.writeBuffer;for(var t,n=1,i=0;i=57344?i+=3:(r++,i+=4);return i}(t):Math.ceil(1.33*(t.byteLength||t.size))),i>0&&n>this.L)return this.writeBuffer.slice(0,i);n+=2}return this.writeBuffer},i.W=function(){var t=this;if(!this._)return!0;var n=Date.now()>this._;return n&&(this._=0,R((function(){t.F("ping timeout")}),this.setTimeoutFn)),n},i.write=function(t,n,i){return this.J("message",t,n,i),this},i.send=function(t,n,i){return this.J("message",t,n,i),this},i.J=function(t,n,i,r){if("function"==typeof n&&(r=n,n=void 0),"function"==typeof i&&(r=i,i=null),"closing"!==this.readyState&&"closed"!==this.readyState){(i=i||{}).compress=!1!==i.compress;var e={type:t,data:n,options:i};this.emitReserved("packetCreate",e),this.writeBuffer.push(e),r&&this.once("flush",r),this.flush()}},i.close=function(){var t=this,n=function(){t.F("forced close"),t.transport.close()},i=function i(){t.off("upgrade",i),t.off("upgradeError",i),n()},r=function(){t.once("upgrade",i),t.once("upgradeError",i)};return"opening"!==this.readyState&&"open"!==this.readyState||(this.readyState="closing",this.writeBuffer.length?this.once("drain",(function(){t.upgrading?r():n()})):this.upgrading?r():n()),this},i.B=function(t){if(n.priorWebsocketSuccess=!1,this.opts.tryAllTransports&&this.transports.length>1&&"opening"===this.readyState)return this.transports.shift(),this.q();this.emitReserved("error",t),this.F("transport error",t)},i.F=function(t,n){if("opening"===this.readyState||"open"===this.readyState||"closing"===this.readyState){if(this.clearTimeoutFn(this.Y),this.transport.removeAllListeners("close"),this.transport.close(),this.transport.removeAllListeners(),ct&&(this.P&&removeEventListener("beforeunload",this.P,!1),this.$)){var i=at.indexOf(this.$);-1!==i&&at.splice(i,1)}this.readyState="closed",this.id=null,this.emitReserved("close",t,n),this.writeBuffer=[],this.M=0}},n}(I);vt.protocol=4;var lt=function(t){function n(){var n;return(n=t.apply(this,arguments)||this).Z=[],n}s(n,t);var i=n.prototype;return i.onOpen=function(){if(t.prototype.onOpen.call(this),"open"===this.readyState&&this.opts.upgrade)for(var n=0;n1&&void 0!==arguments[1]?arguments[1]:{},r="object"===c(n)?n:i;return(!r.transports||r.transports&&"string"==typeof r.transports[0])&&(r.transports=(r.transports||["polling","websocket","webtransport"]).map((function(t){return st[t]})).filter((function(t){return!!t}))),t.call(this,n,r)||this}return s(n,t),n}(lt);pt.protocol;var dt="function"==typeof ArrayBuffer,yt=function(t){return"function"==typeof ArrayBuffer.isView?ArrayBuffer.isView(t):t.buffer instanceof ArrayBuffer},bt=Object.prototype.toString,wt="function"==typeof Blob||"undefined"!=typeof Blob&&"[object BlobConstructor]"===bt.call(Blob),gt="function"==typeof File||"undefined"!=typeof File&&"[object FileConstructor]"===bt.call(File);function mt(t){return dt&&(t instanceof ArrayBuffer||yt(t))||wt&&t instanceof Blob||gt&&t instanceof File}function kt(t,n){if(!t||"object"!==c(t))return!1;if(Array.isArray(t)){for(var i=0,r=t.length;i=0&&t.num1?e-1:0),s=1;s1?i-1:0),e=1;ei.l.retries&&(i.it.shift(),n&&n(t));else if(i.it.shift(),n){for(var e=arguments.length,o=new Array(e>1?e-1:0),s=1;s0&&void 0!==arguments[0]&&arguments[0];if(this.connected&&0!==this.it.length){var n=this.it[0];n.pending&&!t||(n.pending=!0,n.tryCount++,this.flags=n.flags,this.emit.apply(this,n.args))}},o.packet=function(t){t.nsp=this.nsp,this.io.ct(t)},o.onopen=function(){var t=this;"function"==typeof this.auth?this.auth((function(n){t.vt(n)})):this.vt(this.auth)},o.vt=function(t){this.packet({type:Bt.CONNECT,data:this.lt?e({pid:this.lt,offset:this.dt},t):t})},o.onerror=function(t){this.connected||this.emitReserved("connect_error",t)},o.onclose=function(t,n){this.connected=!1,delete this.id,this.emitReserved("disconnect",t,n),this.yt()},o.yt=function(){var t=this;Object.keys(this.acks).forEach((function(n){if(!t.sendBuffer.some((function(t){return String(t.id)===n}))){var i=t.acks[n];delete t.acks[n],i.withError&&i.call(t,new Error("socket has been disconnected"))}}))},o.onpacket=function(t){if(t.nsp===this.nsp)switch(t.type){case Bt.CONNECT:t.data&&t.data.sid?this.onconnect(t.data.sid,t.data.pid):this.emitReserved("connect_error",new Error("It seems you are trying to reach a Socket.IO server in v2.x with a v3.x client, but they are not compatible (more information here: https://socket.io/docs/v3/migrating-from-2-x-to-3-0/)"));break;case Bt.EVENT:case Bt.BINARY_EVENT:this.onevent(t);break;case Bt.ACK:case Bt.BINARY_ACK:this.onack(t);break;case Bt.DISCONNECT:this.ondisconnect();break;case Bt.CONNECT_ERROR:this.destroy();var n=new Error(t.data.message);n.data=t.data.data,this.emitReserved("connect_error",n)}},o.onevent=function(t){var n=t.data||[];null!=t.id&&n.push(this.ack(t.id)),this.connected?this.emitEvent(n):this.receiveBuffer.push(Object.freeze(n))},o.emitEvent=function(n){if(this.bt&&this.bt.length){var i,e=r(this.bt.slice());try{for(e.s();!(i=e.n()).done;){i.value.apply(this,n)}}catch(t){e.e(t)}finally{e.f()}}t.prototype.emit.apply(this,n),this.lt&&n.length&&"string"==typeof n[n.length-1]&&(this.dt=n[n.length-1])},o.ack=function(t){var n=this,i=!1;return function(){if(!i){i=!0;for(var r=arguments.length,e=new Array(r),o=0;o0&&t.jitter<=1?t.jitter:0,this.attempts=0}_t.prototype.duration=function(){var t=this.ms*Math.pow(this.factor,this.attempts++);if(this.jitter){var n=Math.random(),i=Math.floor(n*this.jitter*t);t=1&Math.floor(10*n)?t+i:t-i}return 0|Math.min(t,this.max)},_t.prototype.reset=function(){this.attempts=0},_t.prototype.setMin=function(t){this.ms=t},_t.prototype.setMax=function(t){this.max=t},_t.prototype.setJitter=function(t){this.jitter=t};var Dt=function(t){function n(n,i){var r,e;(r=t.call(this)||this).nsps={},r.subs=[],n&&"object"===c(n)&&(i=n,n=void 0),(i=i||{}).path=i.path||"/socket.io",r.opts=i,$(r,i),r.reconnection(!1!==i.reconnection),r.reconnectionAttempts(i.reconnectionAttempts||1/0),r.reconnectionDelay(i.reconnectionDelay||1e3),r.reconnectionDelayMax(i.reconnectionDelayMax||5e3),r.randomizationFactor(null!==(e=i.randomizationFactor)&&void 0!==e?e:.5),r.backoff=new _t({min:r.reconnectionDelay(),max:r.reconnectionDelayMax(),jitter:r.randomizationFactor()}),r.timeout(null==i.timeout?2e4:i.timeout),r.st="closed",r.uri=n;var o=i.parser||xt;return r.encoder=new o.Encoder,r.decoder=new o.Decoder,r.et=!1!==i.autoConnect,r.et&&r.open(),r}s(n,t);var i=n.prototype;return i.reconnection=function(t){return arguments.length?(this.kt=!!t,t||(this.skipReconnect=!0),this):this.kt},i.reconnectionAttempts=function(t){return void 0===t?this.At:(this.At=t,this)},i.reconnectionDelay=function(t){var n;return void 0===t?this.jt:(this.jt=t,null===(n=this.backoff)||void 0===n||n.setMin(t),this)},i.randomizationFactor=function(t){var n;return void 0===t?this.Et:(this.Et=t,null===(n=this.backoff)||void 0===n||n.setJitter(t),this)},i.reconnectionDelayMax=function(t){var n;return void 0===t?this.Ot:(this.Ot=t,null===(n=this.backoff)||void 0===n||n.setMax(t),this)},i.timeout=function(t){return arguments.length?(this.Bt=t,this):this.Bt},i.maybeReconnectOnOpen=function(){!this.ot&&this.kt&&0===this.backoff.attempts&&this.reconnect()},i.open=function(t){var n=this;if(~this.st.indexOf("open"))return this;this.engine=new pt(this.uri,this.opts);var i=this.engine,r=this;this.st="opening",this.skipReconnect=!1;var e=It(i,"open",(function(){r.onopen(),t&&t()})),o=function(i){n.cleanup(),n.st="closed",n.emitReserved("error",i),t?t(i):n.maybeReconnectOnOpen()},s=It(i,"error",o);if(!1!==this.Bt){var u=this.Bt,h=this.setTimeoutFn((function(){e(),o(new Error("timeout")),i.close()}),u);this.opts.autoUnref&&h.unref(),this.subs.push((function(){n.clearTimeoutFn(h)}))}return this.subs.push(e),this.subs.push(s),this},i.connect=function(t){return this.open(t)},i.onopen=function(){this.cleanup(),this.st="open",this.emitReserved("open");var t=this.engine;this.subs.push(It(t,"ping",this.onping.bind(this)),It(t,"data",this.ondata.bind(this)),It(t,"error",this.onerror.bind(this)),It(t,"close",this.onclose.bind(this)),It(this.decoder,"decoded",this.ondecoded.bind(this)))},i.onping=function(){this.emitReserved("ping")},i.ondata=function(t){try{this.decoder.add(t)}catch(t){this.onclose("parse error",t)}},i.ondecoded=function(t){var n=this;R((function(){n.emitReserved("packet",t)}),this.setTimeoutFn)},i.onerror=function(t){this.emitReserved("error",t)},i.socket=function(t,n){var i=this.nsps[t];return i?this.et&&!i.active&&i.connect():(i=new Lt(this,t,n),this.nsps[t]=i),i},i.wt=function(t){for(var n=0,i=Object.keys(this.nsps);n=this.At)this.backoff.reset(),this.emitReserved("reconnect_failed"),this.ot=!1;else{var i=this.backoff.duration();this.ot=!0;var r=this.setTimeoutFn((function(){n.skipReconnect||(t.emitReserved("reconnect_attempt",n.backoff.attempts),n.skipReconnect||n.open((function(i){i?(n.ot=!1,n.reconnect(),t.emitReserved("reconnect_error",i)):n.onreconnect()})))}),i);this.opts.autoUnref&&r.unref(),this.subs.push((function(){t.clearTimeoutFn(r)}))}},i.onreconnect=function(){var t=this.backoff.attempts;this.ot=!1,this.backoff.reset(),this.emitReserved("reconnect",t)},n}(I),Pt={};function $t(t,n){"object"===c(t)&&(n=t,t=void 0);var i,r=function(t){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"",i=arguments.length>2?arguments[2]:void 0,r=t;i=i||"undefined"!=typeof location&&location,null==t&&(t=i.protocol+"//"+i.host),"string"==typeof t&&("/"===t.charAt(0)&&(t="/"===t.charAt(1)?i.protocol+t:i.host+t),/^(https?|wss?):\/\//.test(t)||(t=void 0!==i?i.protocol+"//"+t:"https://"+t),r=ft(t)),r.port||(/^(http|ws)$/.test(r.protocol)?r.port="80":/^(http|ws)s$/.test(r.protocol)&&(r.port="443")),r.path=r.path||"/";var e=-1!==r.host.indexOf(":")?"["+r.host+"]":r.host;return r.id=r.protocol+"://"+e+":"+r.port+n,r.href=r.protocol+"://"+e+(i&&i.port===r.port?"":":"+r.port),r}(t,(n=n||{}).path||"/socket.io"),e=r.source,o=r.id,s=r.path,u=Pt[o]&&s in Pt[o].nsps;return n.forceNew||n["force new connection"]||!1===n.multiplex||u?i=new Dt(e,n):(Pt[o]||(Pt[o]=new Dt(e,n)),i=Pt[o]),r.query&&!n.query&&(n.query=r.queryKey),i.socket(r.path,n)}return e($t,{Manager:Dt,Socket:Lt,io:$t,connect:$t}),$t})); diff --git a/scratch-arduino-extensions/scripts/patch-gui.js b/scratch-arduino-extensions/scripts/patch-gui.js deleted file mode 100644 index 5487a7b..0000000 --- a/scratch-arduino-extensions/scripts/patch-gui.js +++ /dev/null @@ -1,59 +0,0 @@ -const path = require("path"); -const fs = require("fs"); - -const extensions = [ - { name: "ArduinoBasics", directory: "arduino_basics" }, - { name: "ArduinoModulino", directory: "arduino_modulino" }, - { name: "ArduinoObjectDetection", directory: "arduino_object_detection" }, -]; - -// base dir is the 'scratch-arduino-extensions' folder -const BaseDir = path.resolve(__dirname, "../"); - -extensions.forEach(extension => { - console.log(`\n${extension.name} (${extension.directory})`); - - process.stdout.write("\t - add symbolic link: "); - const scratchVmExtensionsDir = path.resolve( - BaseDir, - "../scratch-editor/packages/scratch-vm/src/extensions", - extension.directory, - ); - if (!fs.existsSync(scratchVmExtensionsDir)) { - const patchedExtensionDir = path.resolve(BaseDir, "./packages/scratch-vm/src/extensions/", extension.directory); - fs.symlinkSync(patchedExtensionDir, scratchVmExtensionsDir, "dir"); - process.stdout.write("done"); - } else process.stdout.write("skip"); - - process.stdout.write("\n\t - register builtin: "); - const scratchVmExtensionsManagerFile = path.resolve( - BaseDir, - "../scratch-editor/packages/scratch-vm/src/extension-support/extension-manager.js", - ); - let managerCode = fs.readFileSync(scratchVmExtensionsManagerFile, "utf-8"); - if (!managerCode.includes(extension.name)) { - fs.copyFileSync(scratchVmExtensionsManagerFile, `${scratchVmExtensionsManagerFile}.orig`); - managerCode = managerCode.replace( - /builtinExtensions = {[\s\S]*?};/, - `$&\n\nbuiltinExtensions.${extension.name} = () => require('../extensions/${extension.directory}');`, - ); - fs.writeFileSync(scratchVmExtensionsManagerFile, managerCode); - process.stdout.write("done"); - } else process.stdout.write("skip"); - - process.stdout.write("\n\t - register core: "); - const scratchVmVirtualMachineFile = path.resolve( - BaseDir, - "../scratch-editor/packages/scratch-vm/src/virtual-machine.js", - ); - let vmCode = fs.readFileSync(scratchVmVirtualMachineFile, "utf-8"); - if (!vmCode.includes(extension.name)) { - fs.copyFileSync(scratchVmVirtualMachineFile, `${scratchVmVirtualMachineFile}.orig`); - vmCode = vmCode.replace( - /(CORE_EXTENSIONS = \[[\s\S]*?)\];/, - `$1'${extension.name}',\n];`, - ); - fs.writeFileSync(scratchVmVirtualMachineFile, vmCode); - process.stdout.write("done\n"); - } else process.stdout.write("skip"); -}); diff --git a/scratch-prg-extensions/extensions/src/arduino_basics/.gitignore b/scratch-prg-extensions/extensions/src/arduino_basics/.gitignore new file mode 100644 index 0000000..b512c09 --- /dev/null +++ b/scratch-prg-extensions/extensions/src/arduino_basics/.gitignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/scratch-prg-extensions/extensions/src/arduino_basics/ArduinoLogo_Blue.jpg b/scratch-prg-extensions/extensions/src/arduino_basics/ArduinoLogo_Blue.jpg new file mode 100644 index 0000000..79279ae Binary files /dev/null and b/scratch-prg-extensions/extensions/src/arduino_basics/ArduinoLogo_Blue.jpg differ diff --git a/scratch-prg-extensions/extensions/src/arduino_basics/ArduinoLogo_Blue.png b/scratch-prg-extensions/extensions/src/arduino_basics/ArduinoLogo_Blue.png new file mode 100644 index 0000000..cb971c3 Binary files /dev/null and b/scratch-prg-extensions/extensions/src/arduino_basics/ArduinoLogo_Blue.png differ diff --git a/scratch-prg-extensions/extensions/src/arduino_basics/MatrixArgument.svelte b/scratch-prg-extensions/extensions/src/arduino_basics/MatrixArgument.svelte new file mode 100644 index 0000000..768c919 --- /dev/null +++ b/scratch-prg-extensions/extensions/src/arduino_basics/MatrixArgument.svelte @@ -0,0 +1,168 @@ + + + + + +
+
+ {#each matrix as row, rowIndex} +
+ {#each row as ledValue, colIndex} + +
0 ? `rgba(0, 123, 255)` : '#222'} + style:box-shadow={ledValue > 0 ? `0 0 ${ledValue * 2}px rgba(0, 123, 255, 0.8)` : 'none'} + on:mousedown={(e) => handleMouseDown(e, rowIndex, colIndex)} + on:mouseenter={() => handleMouseEnter(rowIndex, colIndex)} + on:contextmenu={handleContextMenu} + tabindex="0" + role="button" + aria-label="LED {rowIndex},{colIndex}: brightness {ledValue}" + >
+ {/each} +
+ {/each} +
+ +
+ + +
+
\ No newline at end of file diff --git a/scratch-prg-extensions/extensions/src/arduino_basics/index.test.ts b/scratch-prg-extensions/extensions/src/arduino_basics/index.test.ts new file mode 100644 index 0000000..fccb213 --- /dev/null +++ b/scratch-prg-extensions/extensions/src/arduino_basics/index.test.ts @@ -0,0 +1,7 @@ +import { createTestSuite } from "$testing"; +import Extension from "."; + +createTestSuite({ Extension, __dirname }, { + unitTests: undefined, + integrationTests: undefined, +}); diff --git a/scratch-prg-extensions/extensions/src/arduino_basics/index.ts b/scratch-prg-extensions/extensions/src/arduino_basics/index.ts new file mode 100644 index 0000000..dbed11f --- /dev/null +++ b/scratch-prg-extensions/extensions/src/arduino_basics/index.ts @@ -0,0 +1,100 @@ +import { type Environment, extension, type ExtensionMenuDisplayDetails, scratch } from "$common"; +import { io, Socket } from "socket.io-client"; +import MatrixArgument from "./MatrixArgument.svelte"; + +const details: ExtensionMenuDisplayDetails = { + name: "Arduino Basics", + description: "Arduino Basics for Uno Q", + iconURL: "ArduinoLogo_Blue.png", + insetIconURL: "ArduinoLogo_Blue.jpg", + // tags: ["Arduino"], + blockColor: "#00878F", + menuColor: "#62AEB2", + menuSelectColor: "#62AEB2", +}; + +// Get Arduino board IP or hostname from URL parameter +const getArduinoBoardHost = () => { + const urlParams = new URLSearchParams(window.location.search); + const boardHost = urlParams.get("host"); + if (boardHost) { + return boardHost; + } + return window.location.hostname; +}; + +// TODO: make the block to support the brightness `0-7' of the leds +const PATTERNS = { + heart: [ + [0, 0, 0, 7, 7, 0, 0, 0, 7, 7, 0, 0, 0], + [0, 0, 7, 0, 0, 7, 0, 7, 0, 0, 7, 0, 0], + [0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7, 0], + [0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0], + [0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0], + [0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0], + [0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0], + ] as number[][], + arduino: [ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 7, 7, 7, 0, 0, 0, 7, 7, 7, 0, 0], + [0, 7, 0, 0, 0, 7, 0, 7, 0, 0, 0, 7, 0], + [7, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 7], + [7, 0, 7, 7, 7, 0, 7, 0, 7, 7, 7, 0, 7], + [7, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 0, 7], + [0, 7, 0, 0, 0, 7, 0, 7, 0, 0, 0, 7, 0], + [0, 0, 7, 7, 7, 0, 0, 0, 7, 7, 7, 0, 0], + ] as number[][], + empty: Array(8).fill(null).map(() => Array(13).fill(0)) as number[][], +} as const; + +export default class ArduinoBasics extends extension(details, "ui", "customArguments") { + private socket: Socket | null = null; + + init(env: Environment) { + const arduinoBoardHost = getArduinoBoardHost(); + var serverURL = `wss://${arduinoBoardHost}:7000`; + + console.log("Connecting to Uno Q", serverURL); + + this.socket = io(serverURL, { + path: "/socket.io", + transports: ["polling", "websocket"], + autoConnect: true, + }); + + this.socket.on("connect", () => { + console.log(`Connected to Arduino UNO Q`); + }); + + this.socket.on("disconnect", (reason) => { + console.log(`Disconnected from Arduino UNO Q: ${reason}`); + }); + } + + @scratch.command(function(_, tag) { + const arg = this.makeCustomArgument({ + component: MatrixArgument, + initial: { + value: PATTERNS.arduino, + text: "arduino", + }, + }); + return tag`draw ${arg} matrix`; + }) + drawMatrix(matrix: number[][]) { + var matrixString = matrix.flat().join(""); + console.log("received matrix update", matrixString); + if (this.socket) { + this.socket.emit("matrix_draw", { frame: matrixString }); + } + } + + @scratch.command`Clear matrix` + clearMatrix(matrix: number[][]) { + var matrixString = PATTERNS.empty.flat().join(""); + if (this.socket) { + this.socket.emit("matrix_draw", { frame: matrixString }); + } + } +} diff --git a/scratch-prg-extensions/extensions/src/arduino_basics/package.json b/scratch-prg-extensions/extensions/src/arduino_basics/package.json new file mode 100644 index 0000000..4a22b55 --- /dev/null +++ b/scratch-prg-extensions/extensions/src/arduino_basics/package.json @@ -0,0 +1,18 @@ +{ + "name": "arduino_basics-extension", + "version": "1.0.0", + "description": "An extension created using the PRG AI Blocks framework", + "main": "index.ts", + "scripts": { + "directory": "echo arduino_basics", + "test": "pnpm --filter prg-extension-root test arduino_basics/index.test.ts", + "dev": "pnpm --filter prg-extension-root dev --include arduino_basics", + "add:ui": "pnpm --filter prg-extension-root add:ui arduino_basics", + "add:arg": "pnpm --filter prg-extension-root add:arg arduino_basics" + }, + "dependencies": { + "socket.io-client": "4.8.1" + }, + "author": "", + "license": "ISC" +} diff --git a/scratch-prg-extensions/extensions/src/arduino_basics/pnpm-lock.yaml b/scratch-prg-extensions/extensions/src/arduino_basics/pnpm-lock.yaml new file mode 100644 index 0000000..81c2b82 --- /dev/null +++ b/scratch-prg-extensions/extensions/src/arduino_basics/pnpm-lock.yaml @@ -0,0 +1,122 @@ +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + .: + dependencies: + socket.io-client: + specifier: 4.8.1 + version: 4.8.1 + +packages: + "@socket.io/component-emitter@3.1.2": + resolution: { + integrity: sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==, + } + + debug@4.3.7: + resolution: { + integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==, + } + engines: { node: ">=6.0" } + peerDependencies: + supports-color: "*" + peerDependenciesMeta: + supports-color: + optional: true + + engine.io-client@6.6.3: + resolution: { + integrity: sha512-T0iLjnyNWahNyv/lcjS2y4oE358tVS/SYQNxYXGAJ9/GLgH4VCvOQ/mhTjqU88mLZCQgiG8RIegFHYCdVC+j5w==, + } + + engine.io-parser@5.2.3: + resolution: { + integrity: sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==, + } + engines: { node: ">=10.0.0" } + + ms@2.1.3: + resolution: { + integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==, + } + + socket.io-client@4.8.1: + resolution: { + integrity: sha512-hJVXfu3E28NmzGk8o1sHhN3om52tRvwYeidbj7xKy2eIIse5IoKX3USlS6Tqt3BHAtflLIkCQBkzVrEEfWUyYQ==, + } + engines: { node: ">=10.0.0" } + + socket.io-parser@4.2.4: + resolution: { + integrity: sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==, + } + engines: { node: ">=10.0.0" } + + ws@8.17.1: + resolution: { + integrity: sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==, + } + engines: { node: ">=10.0.0" } + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ">=5.0.2" + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + xmlhttprequest-ssl@2.1.2: + resolution: { + integrity: sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==, + } + engines: { node: ">=0.4.0" } + +snapshots: + "@socket.io/component-emitter@3.1.2": {} + + debug@4.3.7: + dependencies: + ms: 2.1.3 + + engine.io-client@6.6.3: + dependencies: + "@socket.io/component-emitter": 3.1.2 + debug: 4.3.7 + engine.io-parser: 5.2.3 + ws: 8.17.1 + xmlhttprequest-ssl: 2.1.2 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + engine.io-parser@5.2.3: {} + + ms@2.1.3: {} + + socket.io-client@4.8.1: + dependencies: + "@socket.io/component-emitter": 3.1.2 + debug: 4.3.7 + engine.io-client: 6.6.3 + socket.io-parser: 4.2.4 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + socket.io-parser@4.2.4: + dependencies: + "@socket.io/component-emitter": 3.1.2 + debug: 4.3.7 + transitivePeerDependencies: + - supports-color + + ws@8.17.1: {} + + xmlhttprequest-ssl@2.1.2: {} diff --git a/sketch/sketch.ino b/sketch/sketch.ino index 4ad2f14..0dac15f 100644 --- a/sketch/sketch.ino +++ b/sketch/sketch.ino @@ -53,10 +53,35 @@ void matrix_draw(String frame){ return; } for (int i = 0; i < 104; i++) { - if (frame.charAt(i) == '1') { - shades[i] = 7; - } else{ - shades[i] = 0; + char c = frame.charAt(i); + switch (c) { + case '0': + shades[i] = 0; + break; + case '1': + shades[i] = 1; + break; + case '2': + shades[i] = 2; + break; + case '3': + shades[i] = 3; + break; + case '4': + shades[i] = 4; + break; + case '5': + shades[i] = 5; + break; + case '6': + shades[i] = 6; + break; + case '7': + shades[i] = 7; + break; + default: + shades[i] = 0; // Default to 0 for invalid characters + break; } } matrix.draw(shades);