feat: add server support

package.json

@@ -46,12 +46,14 @@
     "diff-match-patch": "1.0.4",
     "format-message": "6.2.1",
     "htmlparser2": "3.10.0",
+    "jsdom": "^24.1.0",
     "scratch-parser": "github:TurboWarp/scratch-parser#master",
     "scratch-sb1-converter": "0.2.7",
     "scratch-translate-extension-languages": "^1.0.7",
     "text-encoding": "0.7.0",
     "uuid": "8.3.2",
-    "worker-loader": "^1.1.1"
+    "worker-loader": "^1.1.1",
+    "yargs": "^18.0.0"
   },
   "peerDependencies": {
     "@turbowarp/scratch-svg-renderer": "^1.0.0-202312300007-62fe825"

src/engine/runtime.js

@@ -503,6 +503,22 @@ class Runtime extends EventEmitter {
          */
         this.isPackaged = false;
 
+        /**
+         * omni: We support a "privileged" mode. This usually is set when the project is running as a server,
+         * but other privileged clients can use this too.
+         * This is mainly to indicate that system APIs (possibly mocked and/or with a permission system)
+         * can be accessed, as provided by the privileged client.
+         */
+        this.isPrivileged = false;
+
+        /**
+         * omni: Privileged utilities, so that the VM can communicate with a privileged client.
+         * This is usally filled in by the server client, but another client can fill this in too,
+         * as long as they are compatible and set isPrivileged to true.
+         * @type {Object<string, function>}
+         */
+        this.privilegedUtils = Object.create(null);
+
         /**
          * Contains information about the external communication methods that the scripts inside the project
          * can use to send data from inside the project to an external server.
@@ -947,6 +963,20 @@ class Runtime extends EventEmitter {
         return 'PLATFORM_MISMATCH';
     }
 
+    /**
+     * omni: Event name when a web request is forwarded to the VM.
+     */
+    static get SERVER_REQUEST () {
+        return 'SERVER_REQUEST';
+    }
+
+    /**
+     * omni: Event name when a response to a web request is forwarded to the web request handler.
+     */
+    static get SERVER_RESPONSE () {
+        return 'SERVER_RESPONSE';
+    }
+
     /**
      * How rapidly we try to step threads by default, in ms.
      */

src/engine/thread.js

@@ -216,6 +216,27 @@ class Thread {
         this.procedures = null;
         this.executableHat = false;
         this.compatibilityStackFrame = null;
+
+        /**
+         * omni: The object the web server stores a request in.
+         * @type {object}
+         */
+        this.serverRequest = {
+            ip: '',
+            method: '',
+            page: '',
+            headers: '{}',
+            data: ''
+        };
+        /**
+         * omni: The object the web server constructs the response in.
+         * @type {object}
+         */
+        this.serverResponse = {
+            mime: 'text/plain',
+            status: null, // Initialized by the request listener hat.
+            headers: '{}'
+        };
     }
 
     /**

src/extension-support/extension-addon-switchers.js

@@ -1,6 +1,6 @@
 const log = require("../util/log");
 const switches = {};
-const parser = new DOMParser();
+const parser = typeof DOMParser === 'undefined' ? null : new DOMParser();
 
 const define_error_noop = (msg) => {
     log.warn(msg);
@@ -12,6 +12,21 @@ const define_error_noop = (msg) => {
 };
 
 function get_extension_switches(id, blocks) {
+    // I have no idea what this is doing and why it is trying to monkeypatch Blockly via the DOM from
+    // the VM; but, it is blocking server support, so I'm going to mock it for running in Node.js and
+    // hope for the best. Contact @someCatInTheWorld if this mocking breaks something horribly.
+    if (typeof process !== 'undefined' && process.versions?.node) return {
+        opcode: 'un_supported',
+        msg: 'unsupported',
+
+        mapFieldValues: {},
+        remapInputName: {},
+
+        createInputs: {},
+        splitInputs: [],
+        remapShadowType: {},
+    };
+
     let _switches = {};
     for (let block of blocks) {
         var blockswitches = block.info.switches;

src/extension-support/extension-manager.js

@@ -8,10 +8,12 @@ const Cast = require('../util/cast');
 
 const AddonSwitches = require('./extension-addon-switchers');
 
+/* Commenting out for the sake of server support.
+
 const urlParams = new URLSearchParams(location.search);
 
 const IsLocal = String(window.location.href).startsWith(`http://localhost:`);
-const IsLiveTests = urlParams.has('livetests');
+const IsLiveTests = urlParams.has('livetests'); */
 
 // thhank yoh random stack droverflwo person
 async function sha256(source) {
@@ -43,8 +45,9 @@ const defaultBuiltinExtensions = {
     gdxfor: () => require('../extensions/scratch3_gdx_for'),
     // tw: core extension
     tw: () => require('../extensions/tw'),
-    SPjavascriptV2: () => require("../extensions/sp_javascriptV2")
-
+    SPjavascriptV2: () => require("../extensions/sp_javascriptV2"),
+    // omni: Web server blocks.
+    server: () => require('../extensions/omni_server'),
 };
 const CORE_EXTENSIONS = [
     'argument',

src/extension-support/tw-security-manager.js

@@ -171,6 +171,24 @@ class SecurityManager {
     shouldUseLocal(refrenceName) {
         return Promise.resolve(!confirm(`it seems that the extension ${refrenceName} has been updated, use the up-to-date code?`))
     }
+
+    /**
+     * omni: Determine whether a file can be read from a location. Meant for privileged environments.
+     * @param {string} path The file to read
+     * @returns {Promise<boolean>|boolean}
+     */
+    canReadFile (path) {
+        return Promise.resolve(false);
+    }
+
+    /**
+     * omni: Determine whether a file can be written to a location. Meant for privileged environments.
+     * @param {string} path The file to write
+     * @returns {Promise<boolean>|boolean}
+     */
+    canWriteFile (path) {
+        return Promise.resolve(false);
+    }
 }
 
 module.exports = SecurityManager;

src/extension-support/tw-unsandboxed-extension-runner.js

@@ -5,6 +5,7 @@ const createTranslate = require('./tw-l10n');
 const staticFetch = require('../util/tw-static-fetch');
 
 /* eslint-disable require-await */
+/* eslint-env node, browser */
 
 /**
  * Parse a URL object or return null.
@@ -180,12 +181,22 @@ const teardownUnsandboxedExtensionAPI = () => {
 const loadUnsandboxedExtension = (extensionURL, vm) => new Promise((resolve, reject) => {
     setupUnsandboxedExtensionAPI(vm).then(resolve);
 
-    const script = document.createElement('script');
-    script.onerror = () => {
-        reject(new Error(`Error in unsandboxed script ${extensionURL}. Check the console for more information.`));
-    };
-    script.src = extensionURL;
-    document.body.appendChild(script);
+    if (typeof process === 'undefined' || !process.versions?.node) {
+        const script = document.createElement('script');
+        script.onerror = () => {
+            reject(new Error(`Error in unsandboxed script ${extensionURL}. Check the console for more information.`));
+        };
+        script.src = extensionURL;
+        document.body.appendChild(script);
+    } else {
+        fetch(extensionURL).then(res => {
+            res.text().then(data => {
+                const extension = data;
+                const run = Function('Scratch', extension);
+                run(global.Scratch);
+            });
+        });
+    }
 }).then(objects => {
     teardownUnsandboxedExtensionAPI();
     return objects;

src/server/cli.js

@@ -0,0 +1,92 @@
+/* eslint-env node */
+/* eslint-disable no-console */
+
+(async () => {
+    const fs = require('node:fs');
+    const os = require('node:os');
+
+    const Server = require('./server');
+    const setupFileSecurity = require('./setup-file-security');
+
+    const {resolvePath} = require('./resolve-path');
+
+    const {default: yargs} = await import('yargs');
+    const {hideBin} = await import('yargs/helpers');
+
+    const permissions = {
+        fileReadAccess: false,
+        fileWriteAccess: false,
+        fileScope: [os.homedir()],
+        networkAccess: false
+    };
+
+    yargs(hideBin(process.argv))
+        .command(
+            'serve [file] [port]',
+            'Runs the project in server mode',
+            yarg => (
+                yarg
+                    .positional('file', {
+                        type: 'string',
+                        describe: 'The file to run'
+                    })
+                    .positional('port', {
+                        type: 'number',
+                        describe: 'The port to bind on',
+                        default: 8080
+                    })
+                    .option('dev', {
+                        alias: 'D',
+                        type: 'boolean',
+                        description: 'Runs with the ability to hot-swap projects'
+                    })
+                    .option('allow-file-read', {
+                        type: 'boolean',
+                        description: 'Allows the project to read any file in your home folder'
+                    })
+                    .option('allow-file-write', {
+                        type: 'boolean',
+                        description: 'Allows the project to write to any file in your home folder'
+                    })
+                    .option('allow-network-access', {
+                        type: 'boolean',
+                        description: 'Allows the project to access anything on the network'
+                    })
+                    .option('file-scope', {
+                        type: 'array',
+                        description: 'Allows the project to read from the specified folders only'
+                    })
+            ), argv => {
+                if (!argv.file) {
+                    console.log('No project inputted.');
+                    process.exitCode = 1;
+                    return;
+                }
+
+                if (argv.allowFileRead) permissions.fileReadAccess = true;
+                if (argv.allowFileWrite) permissions.fileWriteAccess = true;
+                if (argv.allowNetworkAccess) permissions.networkAccess = true;
+
+                if (argv.fileScope) {
+                    permissions.fileScope = argv.fileScope.map(location => resolvePath(location));
+                }
+
+                const server = new Server(!!argv.dev, argv.port);
+                setupFileSecurity(server.securityManager, permissions);
+
+                const projectLoadError = () => {
+                    console.log('Failed to load the project. :(');
+                    server.halt();
+                    process.exitCode = 2;
+                };
+
+                try {
+                    server.runProject(fs.readFileSync(resolvePath(argv.file))).catch(projectLoadError);
+                } catch {
+                    projectLoadError();
+                    return;
+                }
+            })
+        .demandCommand()
+        .parse();
+})();

src/server/resolve-path.js

@@ -0,0 +1,43 @@
+/* eslint-env node */
+
+const path = require('node:path');
+const os = require('node:os');
+
+/**
+ * omni: A curried function that returns a custom path resolver, based on the inputted values.
+ * @param {() => string} homeDir The home directory.
+ * @param {() => string} workingDir The working directory.
+ * @returns {(location: string) => string} The path resolver.
+ */
+const makePathResolver = (homeDir, workingDir) => {
+    if (typeof homeDir !== 'function') throw new TypeError('"homeDir" must be a function.');
+    if (typeof workingDir !== 'function') throw new TypeError('"workingDir" must be a function.');
+
+    /**
+     * omni: A parser that normalizes a path and converts relative paths to absolute paths,
+     * based on the inputted values.
+     * @param {string} location An absolute or relative path.
+     * @returns {string} An normalized absolute path.
+     */
+    return location => {
+        if (typeof location !== 'string') throw new TypeError('"location" must be a string.');
+        const normalizedPath = path.normalize(location);
+
+        if (location.startsWith('~')) return path.join(homeDir(), normalizedPath.slice(1));
+        if (path.isAbsolute(location)) return normalizedPath;
+        return path.join(workingDir(), normalizedPath);
+    };
+};
+
+/**
+ * omni: A parser that normalizes a path and converts relative paths to absolute paths,
+ * based on the current working directory.
+ * @param {string} location An absolute or relative path.
+ * @returns {string} An normalized absolute path.
+ */
+const resolvePath = makePathResolver(os.homedir, process.cwd);
+
+module.exports = {
+    makePathResolver,
+    resolvePath
+};