diff --git a/package-lock.json b/package-lock.json index b849e92..5bf9fb5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3827,6 +3827,15 @@ "node": ">=10" } }, + "node_modules/istanbul-lib-source-maps/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/istanbul-reports": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.5.tgz", @@ -4468,9 +4477,9 @@ "peer": true }, "node_modules/json5": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", - "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true, "bin": { "json5": "lib/cli.js" @@ -5956,15 +5965,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/source-map-support": { "version": "0.5.13", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", @@ -5975,6 +5975,15 @@ "source-map": "^0.6.0" } }, + "node_modules/source-map-support/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/spdx-correct": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", @@ -9439,6 +9448,14 @@ "debug": "^4.1.1", "istanbul-lib-coverage": "^3.0.0", "source-map": "^0.6.1" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } } }, "istanbul-reports": { @@ -9937,9 +9954,9 @@ "peer": true }, "json5": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", - "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true }, "kleur": { @@ -10994,12 +11011,6 @@ } } }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true - }, "source-map-support": { "version": "0.5.13", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", @@ -11008,6 +11019,14 @@ "requires": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } } }, "spdx-correct": { diff --git a/src/debug.ts b/src/debug.ts new file mode 100644 index 0000000..67d90db --- /dev/null +++ b/src/debug.ts @@ -0,0 +1,519 @@ +export interface DebugConfig { + onStdout?(data: string): void; +} + +export async function debugWASI( + module: WebAssembly.Module | BufferSource, + config: DebugConfig = undefined, +) { + if (!(module instanceof WebAssembly.Module)) { + // Convert `BufferSource` to `Module` + module = await WebAssembly.compile(module); + } + + const wasiPolyfill = createWasiPolyfill(config || {}); + const instance = await runWasmModule(module, wasiPolyfill); + + return { + hexdump(offset?: number, length?: number): string { + return hexdump( + (instance.exports.memory as WebAssembly.Memory).buffer, + offset, + length, + ); + }, + show(offset: number) { + const memory = instance.exports.memory as WebAssembly.Memory; + const view = new DataView(memory.buffer); + return decode(view, offset); + }, + }; +} + +function createWasiPolyfill(config: DebugConfig) { + let moduleInstance: WebAssembly.Instance | undefined; + let memory: WebAssembly.Memory | undefined; + + const WASI_ESUCCESS = 0; + const WASI_EBADF = 8; + const WASI_EINVAL = 28; + const WASI_ENOSYS = 52; + + const WASI_STDOUT_FILENO = 1; + + function setModuleInstance(instance: WebAssembly.Instance) { + moduleInstance = instance; + memory = moduleInstance.exports.memory as WebAssembly.Memory; + } + + function getModuleMemoryDataView() { + // call this any time you'll be reading or writing to a module's memory + // the returned DataView tends to be dissaociated with the module's memory buffer at the will of the WebAssembly engine + // cache the returned DataView at your own peril!! + + return new DataView(memory.buffer); + } + + function fd_prestat_get(fd: number, bufPtr: number) { + return WASI_EBADF; + } + + function fd_prestat_dir_name(fd: number, pathPtr: number, pathLen: number) { + return WASI_EINVAL; + } + + function environ_sizes_get(environCount: number, environBufSize: number) { + const view = getModuleMemoryDataView(); + + view.setUint32(environCount, 0, !0); + view.setUint32(environBufSize, 0, !0); + + return WASI_ESUCCESS; + } + + function environ_get(environ: number, environBuf: any) { + return WASI_ESUCCESS; + } + + function args_sizes_get(argc: number, argvBufSize: number) { + const view = getModuleMemoryDataView(); + + view.setUint32(argc, 0, !0); + view.setUint32(argvBufSize, 0, !0); + + return WASI_ESUCCESS; + } + + function args_get(argv: any, argvBuf: any) { + return WASI_ESUCCESS; + } + + function fd_fdstat_get(fd: number, bufPtr: number) { + const view = getModuleMemoryDataView(); + + view.setUint8(bufPtr, fd); + view.setUint16(bufPtr + 2, 0, !0); + view.setUint16(bufPtr + 4, 0, !0); + + function setBigUint64( + byteOffset: any, + value: number, + littleEndian: boolean, + ) { + const lowWord = value; + const highWord = 0; + + view.setUint32(littleEndian ? 0 : 4, lowWord, littleEndian); + view.setUint32(littleEndian ? 4 : 0, highWord, littleEndian); + } + + setBigUint64(bufPtr + 8, 0, !0); + setBigUint64(bufPtr + 8 + 8, 0, !0); + + return WASI_ESUCCESS; + } + + function fd_write( + fd: number, + iovs: number, + iovsLen: number, + nwritten: number, + ) { + const view = getModuleMemoryDataView(); + + let written = 0; + const bufferBytes: number[] = []; + + function getiovs(iovs: number, iovsLen: number) { + // iovs* -> [iov, iov, ...] + // __wasi_ciovec_t { + // void* buf, + // size_t buf_len, + // } + const buffers = Array.from( + { + length: iovsLen, + }, + function (_, i) { + const ptr = iovs + i * 8; + const buf = view.getUint32(ptr, !0); + const bufLen = view.getUint32(ptr + 4, !0); + + return new Uint8Array(memory.buffer, buf, bufLen); + }, + ); + + return buffers; + } + + const buffers = getiovs(iovs, iovsLen); + + function writev(iov: any) { + let b; + for (b = 0; b < iov.byteLength; b++) { + bufferBytes.push(iov[b]); + } + written += b; + } + + buffers.forEach(writev); + + // if (fd === WASI_STDOUT_FILENO) { + // document.getElementById('output').value += + // String.fromCharCode.apply(null, bufferBytes); + // } + // console.log('[output]', String.fromCharCode(...bufferBytes)); + + const output = String.fromCharCode(...bufferBytes); + config.onStdout?.(output); + + view.setUint32(nwritten, written, true); + + return WASI_ESUCCESS; + } + + function poll_oneoff( + sin: any, + sout: any, + nsubscriptions: any, + nevents: any, + ) { + return WASI_ENOSYS; + } + + function proc_exit(rval: any) { + return WASI_ENOSYS; + } + + function fd_close(fd: number) { + return WASI_ENOSYS; + } + + function fd_seek( + fd: number, + offset: number, + whence: any, + newOffsetPtr: number, + ) {} + + return { + setModuleInstance, + environ_sizes_get, + args_sizes_get, + fd_prestat_get, + fd_fdstat_get, + fd_write, + fd_prestat_dir_name, + environ_get, + args_get, + poll_oneoff, + proc_exit, + fd_close, + fd_seek, + }; +} + +let motokoSections = null; +let motokoHashMap: Record = null; + +async function runWasmModule( + module: WebAssembly.Module, + wasiPolyfill: { + setModuleInstance: (instance: WebAssembly.Instance) => void; + }, +) { + const moduleImports = { + wasi_unstable: wasiPolyfill, + env: {}, + }; + + motokoSections = WebAssembly.Module.customSections(module, 'motoko'); + motokoHashMap = + motokoSections.length > 0 ? decodeMotokoSection(motokoSections) : null; + + const instance = await WebAssembly.instantiate(module, moduleImports); + wasiPolyfill.setModuleInstance(instance); + + (instance.exports._start as () => void)(); + + return instance; +} + +// From https://github.com/bma73/hexdump-js, with fixes +const hexdump = (() => { + const _fillUp = ( + value: string | any[], + count: number, + fillWith: string, + ) => { + let l = count - value.length; + let ret = ''; + while (--l > -1) ret += fillWith; + return ret + value; + }; + return (arrayBuffer: ArrayBufferLike, offset?: number, length?: number) => { + offset = offset || 0; + length = length || arrayBuffer.byteLength; + + const view = new DataView(arrayBuffer); + let out = + _fillUp('Offset', 8, ' ') + + ' 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F\n'; + let row = ''; + for (let i = 0; i < length; i += 16) { + row += _fillUp(offset.toString(16).toUpperCase(), 8, '0') + ' '; + const n = Math.min(16, length - offset); + let string = ''; + for (let j = 0; j < 16; ++j) { + if (j < n) { + const value = view.getUint8(offset); + string += + value >= 32 && value < 0x7f + ? String.fromCharCode(value) + : '.'; + row += + _fillUp(value.toString(16).toUpperCase(), 2, '0') + ' '; + offset++; + } else { + row += ' '; + string += ' '; + } + } + row += ' ' + string + '\n'; + } + out += row; + return out; + }; +})(); + +// function updateHexDump() { +// document.getElementById('memory').value = 'Loading…'; +// if (memory) { +// document.getElementById('memory').value = hexdump(memory.buffer); +// } else { +// document.getElementById('memory').value = 'No memory yet'; +// } +// } + +// Decoding Motoko heap objects + +function getUint32( + view: { getUint32: (arg0: any, arg1: boolean) => any }, + p: any, +) { + return view.getUint32(p, true); +} + +function decodeLabel(hash: string | number) { + return motokoHashMap?.[hash] ?? hash; +} + +function decodeOBJ(view: any, p: number) { + const size = getUint32(view, p + 4); + const m: Record = {}; + let h = getUint32(view, p + 8) + 1; //unskew + let q = p + 12; + for (let i = 0; i < size; i++) { + const hash = getUint32(view, h); + const lab = decodeLabel(hash); + m[lab] = decode(view, getUint32(view, q)); + q += 4; + h += 4; + } + return m; +} + +function decodeVARIANT(view: any, p: number) { + const m: Record = {}; + const hash = getUint32(view, p + 4); + const lab = '#' + decodeLabel(hash); + m[lab] = decode(view, getUint32(view, p + 8)); + return m; +} + +// stolen from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DataView +const bigThirtyTwo = BigInt(32), + bigZero = BigInt(0); +function getUint64BigInt( + dataview: { getUint32: (arg0: number, arg1: boolean) => number }, + byteOffset: number, + littleEndian: boolean = false, +) { + // split 64-bit number into two 32-bit (4-byte) parts + const left = BigInt( + dataview.getUint32(byteOffset | 0, !!littleEndian) >>> 0, + ); + const right = BigInt( + dataview.getUint32(((byteOffset | 0) + 4) | 0, !!littleEndian) >>> 0, + ); + + // combine the two 32-bit values and return + return littleEndian + ? (right << bigThirtyTwo) | left + : (left << bigThirtyTwo) | right; +} + +function decodeBITS64( + view: DataView, + p: number, + littleEndian: boolean = false, +) { + return getUint64BigInt(view, p + 4, littleEndian); +} + +function decodeBITS32(view: DataView, p: number) { + return getUint32(view, p + 4); +} + +function decodeARRAY(view: DataView, p: number) { + const size = getUint32(view, p + 4); + const a = new Array(size); + let q = p + 8; + for (let i = 0; i < size; i++) { + a[i] = decode(view, getUint32(view, q)); + q += 4; + } + return a; +} + +function decodeSOME(view: DataView, p: number): { '?': any } { + return { '?': decode(view, getUint32(view, p + 4)) }; +} + +function decodeNULL(view: DataView, p: any): null { + return null; // Symbol(`null`)? +} + +function decodeMUTBOX(view: DataView, p: number): { mut: any } { + return { mut: decode(view, getUint32(view, p + 4)) }; +} + +function decodeOBJ_IND(view: DataView, p: number): { ind: any } { + return { ind: decode(view, getUint32(view, p + 4)) }; +} + +function decodeCONCAT(view: DataView, p: number): [any, any] { + const q = p + 8; // skip n_bytes + return [ + decode(view, getUint32(view, q)), + decode(view, getUint32(view, q + 4)), + ]; +} + +function decodeBLOB(view: DataView, p: number) { + const size = getUint32(view, p + 4); + const a = new Uint8Array(view.buffer, p + 8, size); + try { + const textDecoder = new TextDecoder('utf-8', { fatal: true }); // hoist and reuse? + return textDecoder.decode(a); + } catch (err) { + return a; + } +} + +const bigInt28 = BigInt(28); +const mask = 2 ** 28 - 1; +function decodeBIGINT(view: DataView, p: number) { + const size = getUint32(view, p + 4); + const sign = getUint32(view, p + 12); + let a = BigInt(0); + const q = p + 20; + for (let r = q + 4 * (size - 1); r >= q; r -= 4) { + a = a << bigInt28; + a += BigInt(getUint32(view, r) & mask); + } + if (sign > 0) { + return -a; + } + return a; +} + +// https://en.wikipedia.org/wiki/LEB128 +function getULEB128(view: DataView, p: number) { + let result = 0; + let shift = 0; + while (true) { + const byte = view.getUint8(p); + p += 1; + result |= (byte & 127) << shift; + if ((byte & 128) === 0) break; + shift += 7; + } + return [result, p]; +} + +function hashLabel(label: string) { + // assumes label is ascii + let s = 0; + for (let i = 0; i < label.length; i++) { + const c = label.charCodeAt(i); + // console.assert('non-ascii label', c < 128); + if (c < 128) { + } + s = s * 223 + label.charCodeAt(i); + } + return (2 ** 31 - 1) & s; +} + +function decodeMotokoSection(customSections: string | any[]) { + const m: Record = {}; + if (customSections.length === 0) return m; + const view = new DataView(customSections[0]); + if (view.byteLength === 0) return m; + const id = view.getUint8(0); + if (!(id === 0)) { + return m; + } + const [_sec_size, p] = getULEB128(view, 1); // always 5 bytes as back patched + let [cnt, p1] = getULEB128(view, 6); + while (cnt > 0) { + const [size, p2] = getULEB128(view, p1); + const a = new Uint8Array(view.buffer, p2, size); + p1 = p2 + size; + const textDecoder = new TextDecoder('utf-8', { fatal: true }); // hoist and reuse? + const id = textDecoder.decode(a); + const hash = hashLabel(id); + m[hash] = id; + cnt -= 1; + } + return m; +} + +function decode(view: DataView, v: number) { + if ((v & 1) === 0) return v >> 1; + const p = v + 1; + const tag = getUint32(view, p); + switch (tag) { + case 1: + return decodeOBJ(view, p); + case 2: + return decodeOBJ_IND(view, p); + case 3: + return decodeARRAY(view, p); + // case 4 : unused? + case 5: + return decodeBITS64(view, p); + case 6: + return decodeMUTBOX(view, p); + case 7: + return ''; + case 8: + return decodeSOME(view, p); + case 9: + return decodeVARIANT(view, p); + case 10: + return decodeBLOB(view, p); + case 11: + return ''; + case 12: + return decodeBITS32(view, p); + case 13: + return decodeBIGINT(view, p); + case 14: + return decodeCONCAT(view, p); + case 15: + return decodeNULL(view, p); + default: + return { address: p, tag: tag }; + } +} diff --git a/src/file.ts b/src/file.ts index e808938..60261e6 100644 --- a/src/file.ts +++ b/src/file.ts @@ -1,4 +1,5 @@ import { Motoko, WasmMode } from '.'; +import { DebugConfig } from './debug'; function getValidPath(path: string): string { if (typeof path !== 'string') { @@ -54,6 +55,9 @@ export const file = (mo: Motoko, path: string) => { run() { return mo.run(path); }, + debug(config: DebugConfig) { + return mo.debug(path, config); + }, candid() { return mo.candid(path); }, diff --git a/src/index.ts b/src/index.ts index 8b2709f..5a084e9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,13 +1,14 @@ -import { Node, simplifyAST, CompilerAST, CompilerNode } from './ast'; +import { CompilerNode, Node, simplifyAST } from './ast'; import { file } from './file'; import { - fetchPackage, - installPackages, Package, PackageInfo, + fetchPackage, + installPackages, validatePackage, } from './package'; -import { resolveMain, resolveLib } from './utils/resolveEntryPoint'; +import { resolveLib, resolveMain } from './utils/resolveEntryPoint'; +import { DebugConfig, debugWASI } from './debug'; export type Motoko = ReturnType; @@ -167,6 +168,10 @@ export default function wrapMotoko(compiler: Compiler) { ): { stdout: string; stderr: string; result: Result } { return invoke('run', false, [libPaths || [], path]); }, + async debug(path: string, config: DebugConfig) { + const { wasm } = mo.wasm(path, 'wasi'); + return debugWASI(wasm, config); + }, candid(path: string): string { return invoke('candid', true, [path]); }, diff --git a/src/utils/wasmSourceMap.ts b/src/utils/wasmSourceMap.ts new file mode 100644 index 0000000..a211476 --- /dev/null +++ b/src/utils/wasmSourceMap.ts @@ -0,0 +1,156 @@ +// Derived from https://github.com/oasislabs/wasm-sourcemap/blob/master/index.js + +import url from 'url'; + +const section = 'sourceMappingURL'; + +// read a variable uint encoding from the buffer stream. +// return the int, and the next position in the stream. +function read_uint(buf: Buffer, pos: number) { + let n = 0; + let shift = 0; + let b = buf[pos]; + let outpos = pos + 1; + while (b >= 128) { + n = n | ((b - 128) << shift); + b = buf[outpos]; + outpos++; + shift += 7; + } + return [n + (b << shift), outpos]; +} + +// Write a buffer with a variable uint encoding of a number. +function encode_uint(n: number) { + const result: number[] = []; + while (n > 127) { + result.push(128 | (n & 127)); + n = n >> 7; + } + result.push(n); + return new Uint8Array(result); +} + +function ab2str(buf: Buffer) { + let str = ''; + let bytes = new Uint8Array(buf); + for (let i = 0; i < bytes.length; i++) { + str += String.fromCharCode(bytes[i]); + } + return str; +} + +function str2ab(str: string) { + let bytes = new Uint8Array(str.length); + for (let i = 0; i < str.length; i++) { + bytes[i] = str[i].charCodeAt(0); + } + return bytes; +} + +/** + * Construct an array buffer representing a WASM 0-id + * sections containing a given name and value pair. + * @param {String} name + * @param {String} value + * @returns {Uint8Array} + */ +function writeSection(name: string, value: string) { + const nameBuf = str2ab(name); + const valBuf = str2ab(value); + const nameLen = encode_uint(nameBuf.length); + const valLen = encode_uint(valBuf.length); + const sectionLen = + nameLen.length + nameBuf.length + valLen.length + valBuf.length; + const headerLen = encode_uint(sectionLen); + let bytes = new Uint8Array(sectionLen + headerLen.length + 1); + let pos = 1; + bytes.set(headerLen, pos); + pos += headerLen.length; + bytes.set(nameLen, pos); + pos += nameLen.length; + bytes.set(nameBuf, pos); + pos += nameBuf.length; + const val_start = pos; + bytes.set(valLen, pos); + pos += valLen.length; + bytes.set(valBuf, pos); + return bytes; +} + +/** + * Search the module sections of a WASM buffer to find + * a section with a given identifier. + * @param {Buffer} buf + * @param {String} id + * @returns {Array.} An array with the index of + * the section, the length of the section, and the index + * of the beginning of the body of the section. + */ +function findSection(buf: Buffer, id: string) { + let pos = 8; + while (pos < buf.byteLength) { + const sec_start = pos; + const [sec_id, pos2] = read_uint(buf, pos); + const [sec_size, body_pos] = read_uint(buf, pos2); + pos = body_pos + sec_size; + if (sec_id == 0) { + const [name_len, name_pos] = read_uint(buf, body_pos); + const name = buf.slice(name_pos, name_pos + name_len); + const nameString = ab2str(name); + if (nameString == id) { + return [ + sec_start, + sec_size + 1 + (body_pos - pos2), + name_pos + name_len, + ]; + } + } + } + return [-1, null, null]; +} + +/** + * GetSourceMapURL extracts the source map from a WASM buffer. + * @param {Buffer} buf The WASM buffer + * @returns {String|null} The linked sourcemap URL if present. + */ +export function getSourceMapURL(buf: Buffer) { + // buf = new Uint8Array(buf); + const [sec_start, _, uri_start] = findSection(buf, section); + if (sec_start == -1) { + return null; + } + const [uri_len, uri_pos] = read_uint(buf, uri_start); + return ab2str(buf.slice(uri_pos, uri_pos + uri_len)); +} + +export function removeSourceMapURL(buf: Buffer) { + // buf = new Uint8Array(buf); + const [sec_start, sec_size, _] = findSection(buf, section); + if (sec_start == -1) { + return buf; + } + let strippedBuf = new Uint8Array(buf.length - sec_size); + strippedBuf.set(buf.slice(0, sec_start)); + strippedBuf.set(buf.slice(sec_start + sec_size), sec_start); + + return strippedBuf; +} + +export function setSourceMapURL(buf: Buffer, url: string) { + const stripped = removeSourceMapURL(buf); + const newSection = writeSection(section, url); + + const outBuf = new Uint8Array(stripped.length + newSection.length); + outBuf.set(stripped); + outBuf.set(newSection, stripped.length); + + return outBuf; +} + +export function setSourceMapURLRelativeTo(buf: Buffer, relativeURL: string) { + const originalURL = getSourceMapURL(buf); + const newURL = url.resolve(relativeURL, originalURL); + return setSourceMapURL(buf, newURL); +} diff --git a/tests/debug.test.ts b/tests/debug.test.ts new file mode 100644 index 0000000..240ae15 --- /dev/null +++ b/tests/debug.test.ts @@ -0,0 +1,41 @@ +import { existsSync, readFileSync } from 'fs'; +import { debugWASI } from '../src/debug'; +import mo from '../src/versions/moc'; +import { join } from 'path'; + +describe('WASI debug', () => { + const expectedOutput = + '(666, true, "hello", "\\FF\\FF\\68\\65\\6C\\6C\\6F", 66, -66, (666, true, "hello", "\\FF\\FF\\68\\65\\6C\\6C\\6F", 66, -66, "abcdefghijklmnopqrstuvwxyz", {fa = 666; fb = "hello"; fc = "state"}, null, ?null, ?(?null), ?666, ?(?666), #fa, #fb("data"), 36_893_488_147_419_103_231, +36_893_488_147_419_103_232, -36_893_488_147_419_103_232))\n'; + + test('debug in memory', async () => { + const wasiFile = mo.file('MemoryWASI.mo'); + wasiFile.write( + readFileSync(join(__dirname, 'wasm/Debug.test.mo'), 'utf8'), + ); + + let stdout = ''; + const result = await wasiFile.debug({ + onStdout(data: string) { + process.stdout.write(data); + stdout += data; + }, + }); + expect(stdout).toStrictEqual(expectedOutput); + + // console.log(result.hexdump(0, 100)); + }); + + // Run additional test when `./wasm/Debug.test.wasm` exists + const wasmFile = join(__dirname, 'wasm/Debug.test.wasm'); + (existsSync(wasmFile) ? test : test.skip)('debug from file', async () => { + const wasm = readFileSync(wasmFile); + let stdout = ''; + const result = await debugWASI(wasm, { + onStdout(data: string) { + process.stdout.write(data); + stdout += data; + }, + }); + expect(stdout).toStrictEqual(expectedOutput); + }); +}); diff --git a/tests/wasm/.gitignore b/tests/wasm/.gitignore new file mode 100644 index 0000000..acdedca --- /dev/null +++ b/tests/wasm/.gitignore @@ -0,0 +1,4 @@ +*.wasm +*.wasm.map +*.wat +*.wast diff --git a/tests/wasm/Debug.test.mo b/tests/wasm/Debug.test.mo new file mode 100644 index 0000000..38ea19e --- /dev/null +++ b/tests/wasm/Debug.test.mo @@ -0,0 +1,31 @@ +import Prim "mo:⛔"; +func id(x : T) : T { x }; // used to suppress const optimization + +func foo(n : Nat8, b : Bool, t : Text) { + // if (n > (0 : Nat8)) { foo(n - (1:Nat8), not b, t # t ) }; + let x = id(666); + let b1 = id(true); + let t2 = id("hello"); + let blob = id("\FF\FFhello" : Blob); + let n2 = id(66 : Nat8); + let i = id(-66 : Int8); + let c = id("abcdefghijklmnop") # id("qrstuvwxyz"); + let o = id({ fa = 666; fb = "hello"; var fc = "state" }); + let z = id(null); + let sz = id(?z); + let ssz = id(??z); + let sn = id(?666); + let ssn = id(??666); + + let v0 = id(#fa); + let v1 = id(#fb "data"); + let ints : [Int] = id([-4294967296, -256, -1, 0, 1, 256, 4294967296]); + let bigNat : Nat = id((2 ** 65) - 1 : Nat); + let bigInt : Int = id(2 ** 65 : Int); + let negBigInt : Int = id(- (2 ** 65) : Int); + let tup = id((x, b1, t2, blob, n2, i, c, o, z, sz, ssz, sn, ssn, v0, v1, bigNat, bigInt, negBigInt)); + + Prim.debugPrint(debug_show (x, b1, t2, blob, n2, i, tup)); +}; + +foo(6, true, "a"); diff --git a/tests/wasm/compile.js b/tests/wasm/compile.js new file mode 100644 index 0000000..153cc4c --- /dev/null +++ b/tests/wasm/compile.js @@ -0,0 +1,35 @@ +const { execSync } = require('child_process'); +const { readFileSync, writeFileSync } = require('fs'); +const { join } = require('path'); +const { setSourceMapURL } = require('../../lib/utils/wasmSourceMap'); + +execSync('$(dfx cache show)/moc -wasi-system-api --map Debug.test.mo', { + cwd: __dirname, + stdio: 'inherit', +}); + +execSync('wasm2wat Debug.test.wasm > Debug.test.wat', { + cwd: __dirname, + stdio: 'inherit', +}); + +const wasmPath = join(__dirname, 'Debug.test.wasm'); + +const buffer = readFileSync(wasmPath); + +const editedBuffer = setSourceMapURL( + buffer, + 'http://localhost:3000/Debug.test.wasm.map', +); + +console.log(buffer.length, editedBuffer.length); + +writeFileSync(wasmPath, editedBuffer); + +// const sourceMap = require('source-map'); +// const rawSourceMap = JSON.parse( +// readFileSync(join(__dirname, 'Debug.test.wasm.map')), +// ); +// sourceMap.SourceMapConsumer.with(rawSourceMap, null, (consumer) => { +// console.log('CONSUMER:', consumer); +// }).catch((err) => console.error(err)); diff --git a/tests/wasm/index.html b/tests/wasm/index.html new file mode 100644 index 0000000..c2fc262 --- /dev/null +++ b/tests/wasm/index.html @@ -0,0 +1,7 @@ + + + + Debug Interface + + + diff --git a/tests/wasm/index.js b/tests/wasm/index.js new file mode 100644 index 0000000..33ca242 --- /dev/null +++ b/tests/wasm/index.js @@ -0,0 +1,615 @@ +// Temporary: browser debugging environment + +'use strict'; +const exports = {}; +var __awaiter = + (this && this.__awaiter) || + function (thisArg, _arguments, P, generator) { + function adopt(value) { + return value instanceof P + ? value + : new P(function (resolve) { + resolve(value); + }); + } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { + try { + step(generator.next(value)); + } catch (e) { + reject(e); + } + } + function rejected(value) { + try { + step(generator['throw'](value)); + } catch (e) { + reject(e); + } + } + function step(result) { + result.done + ? resolve(result.value) + : adopt(result.value).then(fulfilled, rejected); + } + step( + (generator = generator.apply(thisArg, _arguments || [])).next(), + ); + }); + }; +var __generator = + (this && this.__generator) || + function (thisArg, body) { + var _ = { + label: 0, + sent: function () { + if (t[0] & 1) throw t[1]; + return t[1]; + }, + trys: [], + ops: [], + }, + f, + y, + t, + g; + return ( + (g = { + next: verb(0), + throw: verb(1), + return: verb(2), + }), + typeof Symbol === 'function' && + (g[Symbol.iterator] = function () { + return this; + }), + g + ); + function verb(n) { + return function (v) { + return step([n, v]); + }; + } + function step(op) { + if (f) throw new TypeError('Generator is already executing.'); + while (_) + try { + if ( + ((f = 1), + y && + (t = + op[0] & 2 + ? y['return'] + : op[0] + ? y['throw'] || + ((t = y['return']) && t.call(y), 0) + : y.next) && + !(t = t.call(y, op[1])).done) + ) + return t; + if (((y = 0), t)) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: + case 1: + t = op; + break; + case 4: + _.label++; + return { value: op[1], done: false }; + case 5: + _.label++; + y = op[1]; + op = [0]; + continue; + case 7: + op = _.ops.pop(); + _.trys.pop(); + continue; + default: + if ( + !((t = _.trys), + (t = t.length > 0 && t[t.length - 1])) && + (op[0] === 6 || op[0] === 2) + ) { + _ = 0; + continue; + } + if ( + op[0] === 3 && + (!t || (op[1] > t[0] && op[1] < t[3])) + ) { + _.label = op[1]; + break; + } + if (op[0] === 6 && _.label < t[1]) { + _.label = t[1]; + t = op; + break; + } + if (t && _.label < t[2]) { + _.label = t[2]; + _.ops.push(op); + break; + } + if (t[2]) _.ops.pop(); + _.trys.pop(); + continue; + } + op = body.call(thisArg, _); + } catch (e) { + op = [6, e]; + y = 0; + } finally { + f = t = 0; + } + if (op[0] & 5) throw op[1]; + return { value: op[0] ? op[1] : void 0, done: true }; + } + }; +Object.defineProperty(exports, '__esModule', { value: true }); +exports.debugWASI = void 0; +function debugWASI(module, config) { + if (config === void 0) { + config = undefined; + } + return __awaiter(this, void 0, void 0, function () { + var wasiPolyfill, instance; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + if (!!(module instanceof WebAssembly.Module)) + return [3 /*break*/, 2]; + return [4 /*yield*/, WebAssembly.compile(module)]; + case 1: + // Convert `BufferSource` to `Module` + module = _a.sent(); + _a.label = 2; + case 2: + wasiPolyfill = createWasiPolyfill(config || {}); + return [4 /*yield*/, runWasmModule(module, wasiPolyfill)]; + case 3: + instance = _a.sent(); + return [ + 2 /*return*/, + { + hexdump: function (offset, length) { + return hexdump( + instance.exports.memory.buffer, + offset, + length, + ); + }, + show: function (offset) { + var memory = instance.exports.memory; + var view = new DataView(memory.buffer); + return decode(view, offset); + }, + }, + ]; + } + }); + }); +} +exports.debugWASI = debugWASI; +function createWasiPolyfill(config) { + var moduleInstance; + var memory; + var WASI_ESUCCESS = 0; + var WASI_EBADF = 8; + var WASI_EINVAL = 28; + var WASI_ENOSYS = 52; + var WASI_STDOUT_FILENO = 1; + function setModuleInstance(instance) { + moduleInstance = instance; + memory = moduleInstance.exports.memory; + } + function getModuleMemoryDataView() { + // call this any time you'll be reading or writing to a module's memory + // the returned DataView tends to be dissaociated with the module's memory buffer at the will of the WebAssembly engine + // cache the returned DataView at your own peril!! + return new DataView(memory.buffer); + } + function fd_prestat_get(fd, bufPtr) { + return WASI_EBADF; + } + function fd_prestat_dir_name(fd, pathPtr, pathLen) { + return WASI_EINVAL; + } + function environ_sizes_get(environCount, environBufSize) { + var view = getModuleMemoryDataView(); + view.setUint32(environCount, 0, !0); + view.setUint32(environBufSize, 0, !0); + return WASI_ESUCCESS; + } + function environ_get(environ, environBuf) { + return WASI_ESUCCESS; + } + function args_sizes_get(argc, argvBufSize) { + var view = getModuleMemoryDataView(); + view.setUint32(argc, 0, !0); + view.setUint32(argvBufSize, 0, !0); + return WASI_ESUCCESS; + } + function args_get(argv, argvBuf) { + return WASI_ESUCCESS; + } + function fd_fdstat_get(fd, bufPtr) { + var view = getModuleMemoryDataView(); + view.setUint8(bufPtr, fd); + view.setUint16(bufPtr + 2, 0, !0); + view.setUint16(bufPtr + 4, 0, !0); + function setBigUint64(byteOffset, value, littleEndian) { + var lowWord = value; + var highWord = 0; + view.setUint32(littleEndian ? 0 : 4, lowWord, littleEndian); + view.setUint32(littleEndian ? 4 : 0, highWord, littleEndian); + } + setBigUint64(bufPtr + 8, 0, !0); + setBigUint64(bufPtr + 8 + 8, 0, !0); + return WASI_ESUCCESS; + } + function fd_write(fd, iovs, iovsLen, nwritten) { + var _a; + var view = getModuleMemoryDataView(); + var written = 0; + var bufferBytes = []; + function getiovs(iovs, iovsLen) { + // iovs* -> [iov, iov, ...] + // __wasi_ciovec_t { + // void* buf, + // size_t buf_len, + // } + var buffers = Array.from( + { + length: iovsLen, + }, + function (_, i) { + var ptr = iovs + i * 8; + var buf = view.getUint32(ptr, !0); + var bufLen = view.getUint32(ptr + 4, !0); + return new Uint8Array(memory.buffer, buf, bufLen); + }, + ); + return buffers; + } + var buffers = getiovs(iovs, iovsLen); + function writev(iov) { + var b; + for (b = 0; b < iov.byteLength; b++) { + bufferBytes.push(iov[b]); + } + written += b; + } + buffers.forEach(writev); + // if (fd === WASI_STDOUT_FILENO) { + // document.getElementById('output').value += + // String.fromCharCode.apply(null, bufferBytes); + // } + // console.log('[output]', String.fromCharCode(...bufferBytes)); + var output = String.fromCharCode.apply(String, bufferBytes); + (_a = config.onStdout) === null || _a === void 0 + ? void 0 + : _a.call(config, output); + view.setUint32(nwritten, written, true); + return WASI_ESUCCESS; + } + function poll_oneoff(sin, sout, nsubscriptions, nevents) { + return WASI_ENOSYS; + } + function proc_exit(rval) { + return WASI_ENOSYS; + } + function fd_close(fd) { + return WASI_ENOSYS; + } + function fd_seek(fd, offset, whence, newOffsetPtr) {} + return { + setModuleInstance: setModuleInstance, + environ_sizes_get: environ_sizes_get, + args_sizes_get: args_sizes_get, + fd_prestat_get: fd_prestat_get, + fd_fdstat_get: fd_fdstat_get, + fd_write: fd_write, + fd_prestat_dir_name: fd_prestat_dir_name, + environ_get: environ_get, + args_get: args_get, + poll_oneoff: poll_oneoff, + proc_exit: proc_exit, + fd_close: fd_close, + fd_seek: fd_seek, + }; +} +var motokoSections = null; +var motokoHashMap = null; +function runWasmModule(module, wasiPolyfill) { + return __awaiter(this, void 0, void 0, function () { + var moduleImports, instance; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + moduleImports = { + wasi_unstable: wasiPolyfill, + env: {}, + }; + motokoSections = WebAssembly.Module.customSections( + module, + 'motoko', + ); + motokoHashMap = + motokoSections.length > 0 + ? decodeMotokoSection(motokoSections) + : null; + return [ + 4 /*yield*/, + WebAssembly.instantiate(module, moduleImports), + ]; + case 1: + instance = _a.sent(); + wasiPolyfill.setModuleInstance(instance); + instance.exports._start(); + return [2 /*return*/, instance]; + } + }); + }); +} +// From https://github.com/bma73/hexdump-js, with fixes +var hexdump = (function () { + var _fillUp = function (value, count, fillWith) { + var l = count - value.length; + var ret = ''; + while (--l > -1) ret += fillWith; + return ret + value; + }; + return function (arrayBuffer, offset, length) { + offset = offset || 0; + length = length || arrayBuffer.byteLength; + var view = new DataView(arrayBuffer); + var out = + _fillUp('Offset', 8, ' ') + + ' 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F\n'; + var row = ''; + for (var i = 0; i < length; i += 16) { + row += _fillUp(offset.toString(16).toUpperCase(), 8, '0') + ' '; + var n = Math.min(16, length - offset); + var string = ''; + for (var j = 0; j < 16; ++j) { + if (j < n) { + var value = view.getUint8(offset); + string += + value >= 32 && value < 0x7f + ? String.fromCharCode(value) + : '.'; + row += + _fillUp(value.toString(16).toUpperCase(), 2, '0') + ' '; + offset++; + } else { + row += ' '; + string += ' '; + } + } + row += ' ' + string + '\n'; + } + out += row; + return out; + }; +})(); +// function updateHexDump() { +// document.getElementById('memory').value = 'Loading…'; +// if (memory) { +// document.getElementById('memory').value = hexdump(memory.buffer); +// } else { +// document.getElementById('memory').value = 'No memory yet'; +// } +// } +// Decoding Motoko heap objects +function getUint32(view, p) { + return view.getUint32(p, true); +} +function decodeLabel(hash) { + var _a; + return (_a = + motokoHashMap === null || motokoHashMap === void 0 + ? void 0 + : motokoHashMap[hash]) !== null && _a !== void 0 + ? _a + : hash; +} +function decodeOBJ(view, p) { + var size = getUint32(view, p + 4); + var m = {}; + var h = getUint32(view, p + 8) + 1; //unskew + var q = p + 12; + for (var i = 0; i < size; i++) { + var hash = getUint32(view, h); + var lab = decodeLabel(hash); + m[lab] = decode(view, getUint32(view, q)); + q += 4; + h += 4; + } + return m; +} +function decodeVARIANT(view, p) { + var m = {}; + var hash = getUint32(view, p + 4); + var lab = '#' + decodeLabel(hash); + m[lab] = decode(view, getUint32(view, p + 8)); + return m; +} +// stolen from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DataView +var bigThirtyTwo = BigInt(32), + bigZero = BigInt(0); +function getUint64BigInt(dataview, byteOffset, littleEndian) { + if (littleEndian === void 0) { + littleEndian = false; + } + // split 64-bit number into two 32-bit (4-byte) parts + var left = BigInt(dataview.getUint32(byteOffset | 0, !!littleEndian) >>> 0); + var right = BigInt( + dataview.getUint32(((byteOffset | 0) + 4) | 0, !!littleEndian) >>> 0, + ); + // combine the two 32-bit values and return + return littleEndian + ? (right << bigThirtyTwo) | left + : (left << bigThirtyTwo) | right; +} +function decodeBITS64(view, p, littleEndian) { + if (littleEndian === void 0) { + littleEndian = false; + } + return getUint64BigInt(view, p + 4, littleEndian); +} +function decodeBITS32(view, p) { + return getUint32(view, p + 4); +} +function decodeARRAY(view, p) { + var size = getUint32(view, p + 4); + var a = new Array(size); + var q = p + 8; + for (var i = 0; i < size; i++) { + a[i] = decode(view, getUint32(view, q)); + q += 4; + } + return a; +} +function decodeSOME(view, p) { + return { '?': decode(view, getUint32(view, p + 4)) }; +} +function decodeNULL(view, p) { + return null; // Symbol(`null`)? +} +function decodeMUTBOX(view, p) { + return { mut: decode(view, getUint32(view, p + 4)) }; +} +function decodeOBJ_IND(view, p) { + return { ind: decode(view, getUint32(view, p + 4)) }; +} +function decodeCONCAT(view, p) { + var q = p + 8; // skip n_bytes + return [ + decode(view, getUint32(view, q)), + decode(view, getUint32(view, q + 4)), + ]; +} +function decodeBLOB(view, p) { + var size = getUint32(view, p + 4); + var a = new Uint8Array(view.buffer, p + 8, size); + try { + var textDecoder = new TextDecoder('utf-8', { fatal: true }); // hoist and reuse? + return textDecoder.decode(a); + } catch (err) { + return a; + } +} +var bigInt28 = BigInt(28); +var mask = Math.pow(2, 28) - 1; +function decodeBIGINT(view, p) { + var size = getUint32(view, p + 4); + var sign = getUint32(view, p + 12); + var a = BigInt(0); + var q = p + 20; + for (var r = q + 4 * (size - 1); r >= q; r -= 4) { + a = a << bigInt28; + a += BigInt(getUint32(view, r) & mask); + } + if (sign > 0) { + return -a; + } + return a; +} +// https://en.wikipedia.org/wiki/LEB128 +function getULEB128(view, p) { + var result = 0; + var shift = 0; + while (true) { + var byte = view.getUint8(p); + p += 1; + result |= (byte & 127) << shift; + if ((byte & 128) === 0) break; + shift += 7; + } + return [result, p]; +} +function hashLabel(label) { + // assumes label is ascii + var s = 0; + for (var i = 0; i < label.length; i++) { + var c = label.charCodeAt(i); + // console.assert('non-ascii label', c < 128); + if (c < 128) { + } + s = s * 223 + label.charCodeAt(i); + } + return (Math.pow(2, 31) - 1) & s; +} +function decodeMotokoSection(customSections) { + var m = {}; + if (customSections.length === 0) return m; + var view = new DataView(customSections[0]); + if (view.byteLength === 0) return m; + var id = view.getUint8(0); + if (!(id === 0)) { + return m; + } + var _a = getULEB128(view, 1), + _sec_size = _a[0], + p = _a[1]; // always 5 bytes as back patched + var _b = getULEB128(view, 6), + cnt = _b[0], + p1 = _b[1]; + while (cnt > 0) { + var _c = getULEB128(view, p1), + size = _c[0], + p2 = _c[1]; + var a = new Uint8Array(view.buffer, p2, size); + p1 = p2 + size; + var textDecoder = new TextDecoder('utf-8', { fatal: true }); // hoist and reuse? + var id_1 = textDecoder.decode(a); + var hash = hashLabel(id_1); + m[hash] = id_1; + cnt -= 1; + } + return m; +} +function decode(view, v) { + if ((v & 1) === 0) return v >> 1; + var p = v + 1; + var tag = getUint32(view, p); + switch (tag) { + case 1: + return decodeOBJ(view, p); + case 2: + return decodeOBJ_IND(view, p); + case 3: + return decodeARRAY(view, p); + // case 4 : unused? + case 5: + return decodeBITS64(view, p); + case 6: + return decodeMUTBOX(view, p); + case 7: + return ''; + case 8: + return decodeSOME(view, p); + case 9: + return decodeVARIANT(view, p); + case 10: + return decodeBLOB(view, p); + case 11: + return ''; + case 12: + return decodeBITS32(view, p); + case 13: + return decodeBIGINT(view, p); + case 14: + return decodeCONCAT(view, p); + case 15: + return decodeNULL(view, p); + default: + return { address: p, tag: tag }; + } +} + +WebAssembly.compileStreaming(fetch('Debug.test.wasm'), {}) + .then((module) => debugWASI(module, {})) + .then((results) => console.log('DONE', results)) + .catch((err) => console.error(err));