diff --git a/src/_locales/en/messages.json b/src/_locales/en/messages.json index 6c24eab5..5ea24f79 100644 --- a/src/_locales/en/messages.json +++ b/src/_locales/en/messages.json @@ -159,22 +159,6 @@ "message": "Custom (enter server URL)", "description": "Label for custom server." }, - "noHaptics": { - "message": "None", - "description": "Label for no haptics option." - }, - "Haply2diy2gen": { - "message": "Haply 2diy-2Gen", - "description": "Label for Haply 2diy-2Gen option." - }, - "Haply2diy3gen": { - "message": "Haply 2diy-3Gen", - "description": "Label for Haply 2diy-3Gen option." - }, - "Haply2diyNotSupported": { - "message": "Haply 2diy: Unavailable since this browser does not support Web Serial.", - "description": "Label for Haply 2diy option when it is not supported by browser" - }, "preprocessMap": { "message": "Get Preprocessed Map Data", "description": "Fetch preprocessed JSON data for a map without renderings." diff --git a/src/_locales/fr/messages.json b/src/_locales/fr/messages.json index 358ada0a..ae556759 100644 --- a/src/_locales/fr/messages.json +++ b/src/_locales/fr/messages.json @@ -151,22 +151,6 @@ "message": "Personnalisé (entrer l'URL du serveur)", "description": "Étiquette pour serveur personnalisé." }, - "noHaptics": { - "message": "Aucune", - "description": "Étiquette pour pas d'option haptiques." - }, - "Haply2diy2gen": { - "message": "Haply 2diy-2gen", - "description": "Étiquette pour l'option Haply 2diy-2Gen." - }, - "Haply2diy3gen": { - "message": "Haply 2diy-3gen", - "description": "Étiquette pour l'option Haply 2diy-3Gen." - }, - "Haply2diyNotSupported": { - "message": "Haply 2diy: Indisponible car ce navigateur ne supporte pas le Web Serial.", - "description": "Étiquette pour l'option Haply 2diy lorsqu'elle n'est pas supportée par le navigateur" - }, "preprocessMap": { "message": "Obtenir des données cartographiques prétraitées", "description": "Récupérer des données JSON prétraitées pour une carte sans rendu." diff --git a/src/hAPI/README.md b/src/hAPI/README.md deleted file mode 100644 index f70d9fa3..00000000 --- a/src/hAPI/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# What is this? - -This repository contains the necessary components to run the multimodal audio-haptic photo renderer in the IMAGE extension using the Haply 2DIY device. - -## Requirements - -* A [Haply 2DIY](https://2diy.haply.co/) with its firmware installed. -## How To Use - -* Ensure the Docker container hosting the [photo audio haptics handler](https://github.com/Shared-Reality-Lab/IMAGE-server/tree/main/handlers/photo-audio-haptics-handler) is running on the development / main server. -* Connect the Haply 2DIY device through a [serial USB connection](https://developer.mozilla.org/en-US/docs/Web/API/Web_Serial_API). -* In the extension settings under the "Advanced Options", turn on Developer Mode and select the "Haply 2diy" option. Change the server URL if needed. -* Get an IMAGE rendering of a photograph, wait for a response and then and check if the list of available renderers in the pop-up window contains a photo audio-haptic renderer. If not, recheck the first three steps, and/or try obtaining an IMAGE rendering for another photograph. -* Click the "Connect To Haply" button and select the port for the Haply device. Steadily grasp the 2DIY's knob and click the "START" button to initiate the audio-haptic experience. -* The audio-haptic experience is equivalent to that shown in state diagram below, i.e., it plays a TTS narration for each segment, followed by a sonified experience of all subsegments within that segment, and then a haptically guided tour of each subsegment with the 2DIY. Once all segments are covered, the same process is repeated for objects, starting with grouped objects and then ungrouped objects. Subsegments are defined as isolated contours of a segment, as shown in the image below. -* The Previous, Next, and Stop buttons can be used to navigate back and forth in the interaction, which always follows the pattern: -![a state machine diagram explaining how the audio-haptic experience works](./images/haptics_state_machine.png) - -![differentiating an object, segment, and subsegment](./images/object_segments.png) - - For example, the clicking the "Next Button" during the TTS narration of a segment will immediately skip the TTS and move to the sonified experience of that segment. Clicking the "Next" button again will move to the guided tour of that segment, starting with the first subsegment. Clicking "Previous" during a TTS narration of the second segment will run a guided tour of the last subsegment of the first segment. - -## How does it work? - -![the dataflow between the main browser script, the worker, and the 2DIY](./images/browser_arch.png) - -Data from the handler is sent to the ```hapi-utils.ts``` main script file. - -Code to play audio files is contained within the main script, while all code for haptic calculations runs inside a web worker. A web worker allows scripts to be executed in background threads, allowing it to perform tasks without interfering with the main script. A significant portion of the worker code is currently set up to infinitely loop, refreshing at a rate of 1 ms, during which it attempts to read the current position of the end-effector, communicate this position information to the main script, and calculates the required force to transmit to the 2DIY device. However, [stability issues occur](https://github.com/Shared-Reality-Lab/IMAGE-browser/issues/223) when ensuring that this is maintained at all times. - -Since the audio and haptic experiences are intended to be run one after the other in a timed-fashion, there is back and forth communication between the main script and worker to keep track of state variables. For example, the ```doneWithAudio``` variable is set to true once an audio file has finished playing, sent to the worker via a ```postMessage()```, and then acknowledged by the worker to ready the next haptic segment or object to be played. The code that handles the interactivity with the Previous, Next, and Stop buttons is also contained in the ```worker.ts``` file. - - - -## Haptic Loop - -When the worker is instantiated in the ```hapi-utils.ts``` script, a preliminary message containing the data points for the segments and objects contained in the image is sent from the main script. These normalized coordinates returned from the handler are mapped into the 2DIY's frame of reference and then grouped into specific data types (segments, or objects) which are then utilized in functions called from the indefinite "haptic" loop to trace contours or move to certain positions when necessary. - -The ```moveToPos()``` function is responsible for setting the forces based on the next position, calculated using the difference between the current end-effector position and next in the index of the current subsegment or object. The output force calculation utilizes a PID filter, and a moving average filter that uses a weighted average of the previous 5 force vectors. - -For grouped objects, a convex hull is used to generate an array 'connecting' the centroid of each object within the group. Interpolation is then used to generate a much larger array of points using the ```upsample()``` function, which allows the 2DIY to smoothly trace a path connecting the object centroids as if they formed a segment. - -## Visuals - -The visuals are generated in the ```hapi-utils.ts``` file. An HTML canvas is created under the dropdown for the haptic rendering using the ```createCanvas()``` function. This canvas displays the photo along with a visual avatar of the end-effector's instantaneous position and the outline of any segment or object being rendered. - -The [requestAnimationFrame(draw)](https://developer.mozilla.org/en-US/docs/Web/API/window/requestAnimationFrame) function is responsible for drawing graphics on the canvas, including the photograph itself. The ```updateAnimation()``` called by the ```draw()``` callback function takes in the current position of the 2DIY's end-effector and information about the current segment and object, obtained from the web worker's haptic loop. Because the callback function is called 60 times a second, it is advisable to only draw immediately necessary information, such as the 2DIY avatar and outline of the current subsegment or object being drawn only. - -## hAPI TypeScript Library - -The [hAPI library](https://github.com/Shared-Reality-Lab/IMAGE-browser/tree/main/src/hAPI/libraries) used in the project is a TypeScript port of the [Java hAPI library](https://gitlab.com/Haply/hAPI) originally authored by Oliver Anthony, [Haply Robotics](https://haply.co/). ```Vector.ts``` is an additional helper file for vector calculations, modified slightly from [evanw's vector.js](https://evanw.github.io/lightgl.js/docs/vector.html). \ No newline at end of file diff --git a/src/hAPI/convex-hull.ts b/src/hAPI/convex-hull.ts deleted file mode 100644 index 9ef20f3e..00000000 --- a/src/hAPI/convex-hull.ts +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Convex hull algorithm - Library (TypeScript) - * - * Copyright (c) 2021 Project Nayuki - * https://www.nayuki.io/page/convex-hull-algorithm - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program (see COPYING.txt and COPYING.LESSER.txt). - * If not, see . - */ - - -interface Point { - x: number; - y: number; -} - -namespace convexhull { - - // Returns a new array of points representing the convex hull of - // the given set of points. The convex hull excludes collinear points. - // This algorithm runs in O(n log n) time. - export function makeHull

(points: Readonly>): Array

{ - let newPoints: Array

= points.slice(); - newPoints.sort(convexhull.POINT_COMPARATOR); - return convexhull.makeHullPresorted(newPoints); - } - - - // Returns the convex hull, assuming that each points[i] <= points[i + 1]. Runs in O(n) time. - export function makeHullPresorted

(points: Readonly>): Array

{ - if (points.length <= 1) - return points.slice(); - - // Andrew's monotone chain algorithm. Positive y coordinates correspond to "up" - // as per the mathematical convention, instead of "down" as per the computer - // graphics convention. This doesn't affect the correctness of the result. - - let upperHull: Array

= []; - for (let i = 0; i < points.length; i++) { - const p: P = points[i]; - while (upperHull.length >= 2) { - const q: P = upperHull[upperHull.length - 1]; - const r: P = upperHull[upperHull.length - 2]; - if ((q.x - r.x) * (p.y - r.y) >= (q.y - r.y) * (p.x - r.x)) - upperHull.pop(); - else - break; - } - upperHull.push(p); - } - upperHull.pop(); - - let lowerHull: Array

= []; - for (let i = points.length - 1; i >= 0; i--) { - const p: P = points[i]; - while (lowerHull.length >= 2) { - const q: P = lowerHull[lowerHull.length - 1]; - const r: P = lowerHull[lowerHull.length - 2]; - if ((q.x - r.x) * (p.y - r.y) >= (q.y - r.y) * (p.x - r.x)) - lowerHull.pop(); - else - break; - } - lowerHull.push(p); - } - lowerHull.pop(); - - if (upperHull.length == 1 && lowerHull.length == 1 && upperHull[0].x == lowerHull[0].x && upperHull[0].y == lowerHull[0].y) - return upperHull; - else - return upperHull.concat(lowerHull); - } - - - export function POINT_COMPARATOR(a: Point, b: Point): number { - if (a.x < b.x) - return -1; - else if (a.x > b.x) - return +1; - else if (a.y < b.y) - return -1; - else if (a.y > b.y) - return +1; - else - return 0; - } - -} - -export { convexhull } -export { Point } diff --git a/src/hAPI/hapi-utils.ts b/src/hAPI/hapi-utils.ts deleted file mode 100644 index c5216f95..00000000 --- a/src/hAPI/hapi-utils.ts +++ /dev/null @@ -1,417 +0,0 @@ -import { canvasCircle } from "../types/canvas-circle"; -import { canvasRectangle } from "../types/canvas-rectangle"; -import { ImageRendering } from "../types/response.schema"; -import { Vector } from "../types/vector"; - -import * as infoUtils from '../info/info-utils'; -import * as worker from './worker'; - - -import browser from "webextension-polyfill"; -import { BreakKey } from "./worker"; -import { getAllStorageSyncData } from "../utils"; - -// canvas dimensions for haptic rendering -const canvasWidth = 800; -const canvasHeight = 500; - -const pixelsPerMeter = 6000; - -/** - * Updates the canvas at each timeframe. - * @param posEE The 2DIY workspace position. - * @param endEffector Virtual avatar position. - * @param deviceOrigin Starting coordinates of the 2DIY. - * @param border Canvas border. - * @param drawingInfo Info for segment/object to draw. - * @param segments List of segments to draw. - * @param objects List of objects to draw. - * @param ctx Canvas context. - */ -function updateAnimation(posEE: Vector, - endEffector: canvasCircle, - deviceOrigin: Vector, - border: canvasRectangle, - drawingInfo: { haplyType: worker.Type, segIndex: number, subSegIndex: number }, - segments: worker.SubSegment[][], objects: worker.SubSegment[][], - ctx: CanvasRenderingContext2D) { - - // drawing bounding boxes and centroids - drawBoundaries(drawingInfo, segments, objects, ctx); - border.draw(); - - //scaling end effector position to canvas - let xE = pixelsPerMeter * (-posEE.x + 0.014); - let yE = pixelsPerMeter * ((posEE.y / 0.805) - 0.0311); - - // set position of virtual avatar in canvas - endEffector.x = deviceOrigin.x + xE - 100; - endEffector.y = deviceOrigin.y + yE - 167; - endEffector.draw(); -} - -/** - * Draw segment/object boundaries. - * @param drawingInfo Info for segment/object to draw. - * @param segments List of segments to draw. - * @param objects List of objects to draw. - * @param ctx - */ -function drawBoundaries(drawingInfo: { haplyType: worker.Type, segIndex: number, subSegIndex: number }, - segments: worker.SubSegment[][], objects: worker.SubSegment[][], - ctx: CanvasRenderingContext2D) { - if (drawingInfo != undefined) { - - // subsegment and segment index - const [i, j] = [drawingInfo['segIndex'], drawingInfo['subSegIndex']]; - ctx.lineWidth = 4; - - // segments - if (drawingInfo['haplyType'] == 0) { - ctx.strokeStyle = "blue"; - if (segments[i][j] != undefined) { - segments[i][j].coordinates.forEach(coord => { - const pX = coord.x; - const pY = coord.y; - let [pointX, pointY] = imgToWorldFrame(pX, pY); - ctx.strokeRect(pointX, pointY, 1, 1); - }) - } - } - - // objects - else if (drawingInfo['haplyType'] == 1) { - ctx.strokeStyle = "orange"; - if (objects[i][j] != undefined) { - objects[i][j].coordinates.forEach(coord => { - const pX = coord.x; - const pY = coord.y; - let [pointX, pointY] = imgToWorldFrame(pX, pY); - - // bigger size for single point objects - const size = objects[i][j].coordinates.length == 1 ? 20 : 1 - ctx.strokeRect(pointX, pointY, size, size); - }) - } - } - } - -} - -/** - * Converts 2DIY coordinates to Canvas frame of reference coords. - * @param x1 x position in the normalized 0 -> 1 coordinate system - * @param y1 y position in the normalized 0 -> 1 coordinate system - * @returns Tuple containing the [x, y] position for the canvas - */ -function imgToWorldFrame(x1: number, y1: number): [number, number] { - const x = x1 * canvasWidth; - const y = y1 * canvasHeight; - return [x, y] -} - -/** - * Returns a HTML canvas of specified properties. - * @param contentDiv container for canvas. - * @returns Canvas with context. - */ - function createCanvas(contentDiv: HTMLElement, width: number, height: number) { - const canvas: HTMLCanvasElement = document.createElement('canvas'); - canvas.id = "main"; - canvas.width = width; - canvas.height = height; - canvas.style.zIndex = "8"; - canvas.style.position = "relative"; - canvas.style.border = "1px solid"; - contentDiv.append(document.createElement("br")); - contentDiv.append(canvas); - - return canvas; -} - -export async function processHapticsRendering(rendering: ImageRendering, graphic_url: string, container: HTMLElement, contentId : string){ - let endEffector: canvasCircle; - let border: canvasRectangle; - // end effector x/y coordinates - let posEE: Vector; - let deviceOrigin: Vector; - - // virtual end effector avatar offset - let firstCall: boolean = true; - - const data = rendering["data"]["info"] as any; - - const audioCtx = new window.AudioContext(); - - const audioBuffer = await fetch(data["audioFile"] as string).then(resp => { - - return resp.arrayBuffer(); - }).then(buffer => { - return audioCtx.decodeAudioData(buffer); - }).catch(e => { console.error(e); throw e; }); - - - let div = document.createElement("div"); - div.classList.add("row"); - container.append(div); - let contentDiv = document.createElement("div"); - contentDiv.classList.add("collapse"); - contentDiv.classList.add("rendering-content"); - - contentDiv.id = contentId; - div.append(contentDiv); - - // adding buttons - let btn = infoUtils.createButton(contentDiv, "btn", "Connect to Haply"); - let btnStart = infoUtils.createButton(contentDiv, "btnStart", "Start"); - let btnEscape = infoUtils.createButton(contentDiv, "btnEscape", "Stop"); - let btnNext = infoUtils.createButton(contentDiv, "btnNext", "Next"); - let btnPrev = infoUtils.createButton(contentDiv, "btnPrev", "Previous"); - - // creating canvas - const canvas = createCanvas(contentDiv, canvasWidth, canvasHeight); - - if (rendering["metadata"] && rendering["metadata"]["homepage"]) { - infoUtils.addRenderingExplanation(contentDiv, rendering["metadata"]["homepage"]) - } - const res = canvas.getContext('2d'); - if (!res || !(res instanceof CanvasRenderingContext2D)) { - throw new Error('Failed to get 2D context'); - } - const ctx: CanvasRenderingContext2D = res; - - const img = new Image(); - img.src = graphic_url; - - // world resolution properties - const worldPixelWidth = 800; - - posEE = { - x: 0, - y: 0 - }; - - // initial position of end effector avatar - deviceOrigin = { - x: worldPixelWidth / 2, - y: 0 - }; - - border = { - draw: function () { - ctx.strokeRect(0, 0, canvas.width, canvas.height); - } - }; - - // draw end effector - endEffector = { - x: canvas.width / 2, - y: 0, - vx: 5, - vy: 2, - radius: 8, - color: 'brown', - draw: function () { - ctx.beginPath(); - ctx.arc(this.x, this.y, this.radius, 0, Math.PI * 2, true); - ctx.closePath(); - ctx.fillStyle = this.color; - ctx.fill(); - } - }; - - function draw() { - ctx.clearRect(0, 0, canvas.width, canvas.height); - ctx.drawImage(img, 0, 0, canvas.width, canvas.height); - updateAnimation(posEE, endEffector, deviceOrigin, border, drawingInfo, segments, objects, ctx); - window.requestAnimationFrame(draw); - } - - // define segments and objects - let segments: worker.SubSegment[][]; - let objects: worker.SubSegment[][]; - let drawingInfo: { haplyType: worker.Type, segIndex: number, subSegIndex: number }; - // when haply needs to move to a next segment - let waitForInput: boolean = false; - // when user presses a key to break out of the current haply segment - let breakKey: null | BreakKey; - - // Audio Modes - const enum AudioMode { - Play, - Finished, - Idle, - } - - // Keep track of the mode and entity index so we know which file to play - let audioData: { entityIndex: number, mode: null | AudioMode } = { - entityIndex: 0, - mode: null - }; - - // time to wait before audio segment is considered finished - let tAudioBegin: number; - // true when playing an audio segment - let playingAudio = false; - - const worker = new Worker(browser.runtime.getURL("./hAPI/worker.js"), { type: "module"}); - - // Play an audio segment with a given offset and duration. - let sourceNode: AudioBufferSourceNode; - function playAudioSeg(audioBuffer: any, offset: number, duration: number) { - sourceNode = audioCtx.createBufferSource(); - sourceNode.buffer = audioBuffer; - sourceNode.connect(audioCtx.destination); - sourceNode.start(0, offset, duration); - } - - // Start - btnStart.addEventListener("click", async _ => { - let items = await getAllStorageSyncData(); - worker.postMessage({ - start: true, - haply2diy2gen: items["haply2diy2gen"] - }); - }) - - // Stop - btnEscape.addEventListener("click", _ => { - sourceNode.stop(); - breakKey = BreakKey.Escape; - worker.postMessage({ - waitForInput: waitForInput, - breakKey: breakKey, - tKeyPressTime: Date.now() - }); - }) - - // Next - btnNext.addEventListener("click", _ => { - if (audioData.mode == AudioMode.Play) { - stopAudioNode(); - breakKey = BreakKey.NextFromAudio; - } - else { - breakKey = BreakKey.NextHaptic; - } - worker.postMessage({ - waitForInput: waitForInput, - breakKey: breakKey, - tKeyPressTime: Date.now() - }); - - }); - - // Prev - btnPrev.addEventListener("click", _ => { - if (audioData.mode == AudioMode.Play) { - stopAudioNode(); - breakKey = BreakKey.PreviousFromAudio; - } - else { - breakKey = BreakKey.PreviousHaptic; - } - worker.postMessage({ - waitForInput: waitForInput, - breakKey: breakKey, - tKeyPressTime: Date.now() - }); - }); - - // event listener for serial comm button - btn.addEventListener("click", async _ => { - // const worker = new Worker(browser.runtime.getURL("./info/worker.js"), { type: "module" }); - - // only show the Arduino Zero - const filters = [ - { usbVendorId: 0x2341, usbProductId: 0x804D } - ]; - - let hapticPort = await navigator.serial.requestPort({filters}); - - // send all the rendering info - worker.postMessage({ - renderingData: data - }); - - - worker.addEventListener("message", function (msg) { - // we've selected the COM port - btn.style.visibility = 'hidden'; - - const msgdata = msg.data; - - // return end-effector x/y positions and objectData for updating the canvas - - posEE.x = msgdata.positions.x; - posEE.y = msgdata.positions.y; - - waitForInput = msgdata.waitForInput; - - // grab segment data if available - if (msgdata.segmentData != undefined) - segments = msgdata.segmentData; - - // grab object data if available - if (msgdata.objectData != undefined) - objects = msgdata.objectData; - - // grab drawing info if available - if (msgdata.drawingInfo != undefined) - drawingInfo = msgdata.drawingInfo; - - // only request to run draw() once - if (firstCall) { - if (msgdata.segmentData != undefined || - msgdata.objectData != undefined) { - window.requestAnimationFrame(draw); - firstCall = false; - } - } - - // see if the worker wants us to play any audio - if (msgdata.audioInfo != undefined && msgdata.audioInfo.sendAudioSignal) { - audioData.entityIndex = msgdata.audioInfo.entityIndex; - audioData.mode = AudioMode.Play; - worker.postMessage({ - receivedAudioSignal: true - }) - } - - switch (audioData.mode) { - case AudioMode.Play: { - // prevent audio from playing multiple times - if (!playingAudio) { - playingAudio = true; - playAudioSeg(audioBuffer, - data["entities"][audioData.entityIndex]["offset"], - data["entities"][audioData.entityIndex]["duration"]); - tAudioBegin = Date.now(); - } - - // wait for the audio segment to finish - if (Date.now() - tAudioBegin > 1000 * (0.5 + data["entities"][audioData.entityIndex]["duration"])) { - audioData.mode = AudioMode.Finished; - } - break; - } - // we've finished playing the audio segment - case AudioMode.Finished: { - playingAudio = false; - worker.postMessage({ - doneWithAudio: true - }); - audioData.mode = AudioMode.Idle; - } - case AudioMode.Idle: - break; - } - }); - }); - - // Stop the current audio segment from progressing. - function stopAudioNode() { - sourceNode.stop(); - audioData.mode = AudioMode.Finished; - } -} diff --git a/src/hAPI/images/browser_arch.png b/src/hAPI/images/browser_arch.png deleted file mode 100644 index 7ea67ac7..00000000 Binary files a/src/hAPI/images/browser_arch.png and /dev/null differ diff --git a/src/hAPI/images/haptics_state_machine.png b/src/hAPI/images/haptics_state_machine.png deleted file mode 100644 index c489d434..00000000 Binary files a/src/hAPI/images/haptics_state_machine.png and /dev/null differ diff --git a/src/hAPI/images/object_segments.png b/src/hAPI/images/object_segments.png deleted file mode 100644 index 6db35680..00000000 Binary files a/src/hAPI/images/object_segments.png and /dev/null differ diff --git a/src/hAPI/libraries/Actuator.ts b/src/hAPI/libraries/Actuator.ts deleted file mode 100644 index 0d83bada..00000000 --- a/src/hAPI/libraries/Actuator.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2021 IMAGE Project, Shared Reality Lab, McGill University - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * and our Additional Terms along with this program. - * If not, see . - * - * This is based on the hAPI library by Haply Robotics . - */ -class Actuator { - - actuator: number; - direction: number; - actuatorPort: number; - torque: number = 0; - - constructor(actuator?: number, direction?: number, port?: number) { - this.actuator = actuator || 0; - this.direction = direction || 0; - this.actuatorPort = port || 0; - } - - set_actuator(actuator: number) { - this.actuator = actuator; - } - set_direction(direction: number) { - this.direction = direction; - } - set_port(port: number) { - this.actuatorPort = port; - } - set_torque(torque: number) { - this.torque = torque; - } - get_actuator() { - return this.actuator; - } - get_direction() { - return this.direction; - } - get_port() { - return this.actuatorPort; - } - get_torque() { - return this.torque; - } -} - -export { Actuator } diff --git a/src/hAPI/libraries/Board.ts b/src/hAPI/libraries/Board.ts deleted file mode 100644 index f9580ed0..00000000 --- a/src/hAPI/libraries/Board.ts +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright (c) 2021 IMAGE Project, Shared Reality Lab, McGill University - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * and our Additional Terms along with this program. - * If not, see . - * - * This is based on the hAPI library by Haply Robotics . - */ -class Board { - port: any; - reader: any; - writer: any; - encoder: any; - decoder: any; - - // prev_data; - - constructor() { - this.encoder = new TextEncoder(); - this.decoder = new TextDecoder("utf-8"); - } - async init() { - if ('serial' in navigator as any) { - try { - this.port = (await navigator.serial.getPorts())[0]; - await this.port.open({ baudRate: 57600 }); // `baudRate` was `baudrate` in previous versions. - - this.writer = this.port.writable.getWriter(); - this.reader = this.port.readable.getReader(); - - const signals = await this.port.getSignals(); - console.log(signals); - } catch (err) { - console.error('There was an error opening the serial port:', err); - } - } else { - console.error('Web serial doesn\'t seem to be enabled in your browser. Try enabling it by visiting:') - console.error('chrome://flags/#enable-experimental-web-platform-features'); - console.error('opera://flags/#enable-experimental-web-platform-features'); - console.error('edge://flags/#enable-experimental-web-platform-features'); - } - } - - async transmit(communicationType: number, deviceID: number, bData: Uint8Array, fData: Float32Array) { - //bData length is 8, fData length is 4 - let outData: Uint8Array = new Uint8Array(2 + bData.length + 4 * fData.length); - let segments: Uint8Array = new Uint8Array(4); - - outData[0] = communicationType; - outData[1] = deviceID; - - this.arraycopy(bData, 0, outData, 2, bData.length); - - let j = 2 + bData.length; - for (let i = 0; i < fData.length; i++) { - segments = this.FloatToBytes(fData[i]); - this.arraycopy(segments, 0, outData, j, 4); - j = j + 4; - } - - this.writer.write(outData); - return; - } - - async receive(communicationType: number, deviceID: number, expected: number) { - - let segments = new Uint8Array(4); - - let inData = new Uint8Array(1 + 4 * expected); - let data: Float32Array = new Float32Array(expected); - - try { - const readerData = await this.reader.read(); - inData = readerData.value; - - if (inData[0] != deviceID) { - return data; - } - else if (inData.length != 9) { - return data; - } - - let j = 1; - - for (var i = 0; i < expected; i++) { - this.arraycopy(inData, j, segments, 0, 4); - data[i] = this.BytesToFloat(segments); - j = j + 4; - } - - return data; - - } catch (err) { - const errorMessage = `error reading data: ${err}`; - console.error(errorMessage); - return data; - } - } - - data_available() { - let available = false; - - if (this.port.readable) { - available = true; - } - return available; - } - - reset_board() { - let communicationType = 0; - let deviceID = 0; - let bData = new Uint8Array(0); - let fData: Float32Array = new Float32Array(0); - this.transmit(communicationType, deviceID, bData, fData); - } - - FloatToBytes(val: number) { - let segments: Uint8Array = new Uint8Array(4); - let temp = this.floatToRawIntBits(val); - segments[3] = ((temp >> 24) & 0xff); - segments[2] = ((temp >> 16) & 0xff); - segments[1] = ((temp >> 8) & 0xff); - segments[0] = ((temp) & 0xff); - return segments; - } - - - BytesToFloat(segment: Uint8Array) { - - let temp = 0; - - temp = (temp | (segment[3] & 0xff)) << 8; - temp = (temp | (segment[2] & 0xff)) << 8; - temp = (temp | (segment[1] & 0xff)) << 8; - temp = (temp | (segment[0] & 0xff)); - - let val = this.intBitsToFloat(temp); - - return val; - } - - floatToRawIntBits(f: number) { - var buf = new ArrayBuffer(4); - (new Float32Array(buf))[0] = f; - return (new Uint32Array(buf))[0]; - } - - //JS version of intBitsToFloat - intBitsToFloat(f: number) { - var int8 = new Int8Array(4); - var int32 = new Int32Array(int8.buffer, 0, 1); - int32[0] = f; - var float32 = new Float32Array(int8.buffer, 0, 1); - return float32[0]; - } - - arraycopy(src: Uint8Array, srcPos: number, dst: Uint8Array, dstPos: number, length: number) { - while (length--) dst[dstPos++] = src[srcPos++]; return dst; - } - serverConnected() { - console.log("Connected to server"); - } - -} - -export { Board } diff --git a/src/hAPI/libraries/Device.ts b/src/hAPI/libraries/Device.ts deleted file mode 100644 index 24c4b295..00000000 --- a/src/hAPI/libraries/Device.ts +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Copyright (c) 2021 IMAGE Project, Shared Reality Lab, McGill University - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * and our Additional Terms along with this program. - * If not, see . - * - * This is based on the hAPI library by Haply Robotics . - */ -import { Pwm } from "./Pwm"; -import { Actuator } from "./Actuator"; -import { Sensor } from "./Sensor"; -import { Board } from "./Board"; -import { Pantograph } from "./Pantograph"; - -class Device { - - deviceLink: Board; - - deviceID: number; - mechanism: any; - - communicationType: number | undefined; - - actuatorsActive = 0; - motors: Array = []; - - encodersActive = 0; - encoders: Array = []; - - sensorsActive: number = 0; - sensors: Array = []; - - pwmsActive: number = 0; - pwms: Array = []; - - actuatorPositions: Array = [0, 0, 0, 0]; - encoderPositions: Array = [0, 0, 0, 0]; - - constructor(deviceID: number, deviceLink: Board) { - this.deviceID = deviceID; - this.deviceLink = deviceLink; - - } - - add_actuator(actuator: number, rotation: number, port: number) { - let error = false; - - if (port < 1 || port > 4) { - console.log("error: encoder port index out of bounds"); - error = true; - } - - if (actuator < 1 || actuator > 4) { - console.log("error: encoder index out of bound!"); - error = true; - } - - let j = 0; - for (let i = 0; i < this.actuatorsActive; i++) { - if (this.motors[i].get_actuator() < actuator) { - j++; - } - - if (this.motors[i].get_actuator() == actuator) { - console.log("error: actuator " + actuator + " has already been set"); - error = true; - } - } - - if (!error) { - - let temp = []; - for (var i = 0; i < this.actuatorsActive + 1; i++) { - temp[i] = this.motors[i]; - } - - temp[this.actuatorsActive] = new Actuator(actuator, rotation, port); - this.actuator_assignment(actuator, port); - - this.motors = temp; - this.actuatorsActive++; - } - } - - add_encoder(encoder: number, rotation: number, offset: number, resolution: number, port: number) { - let error = false; - - if (port < 1 || port > 4) { - console.log("error: encoder port index out of bounds"); - error = true; - } - - if (encoder < 1 || encoder > 4) { - console.log("error: encoder index out of bound!"); - error = true; - } - - // determine index for copying - let j = 0; - for (let i = 0; i < this.encodersActive; i++) { - if (this.encoders[i].get_encoder() < encoder) { - j++; - } - - if (this.encoders[i].get_encoder() == encoder) { - console.log("error: encoder " + encoder + " has already been set"); - error = true; - } - } - - if (!error) { - - let temp = []; - for (var i = 0; i < this.encodersActive + 1; i++) { - temp[i] = this.encoders[i]; - } - - temp[this.encodersActive] = new Sensor(encoder, rotation, offset, resolution, port); - this.encoder_assignment(encoder, port); - - this.encoders = temp; - this.encodersActive++; - } - } - - add_analog_sensor(pin: string) { - // set sensor to be size zero - let error = false; - - let port = pin.charAt(0); - let number = pin.substring(1); - - let value = parseInt(number); - value = value + 54; - - for (let i = 0; i < this.sensorsActive; i++) { - if (value == this.sensors[i].get_port()) { - console.log("error: Analog pin: A" + (value - 54) + " has already been set"); - error = true; - } - } - - if (port != 'A' || value < 54 || value > 65) { - console.log("error: outside analog pin range"); - error = true; - } - - if (!error) { - let temp = this.sensors; - temp[this.sensorsActive] = new Sensor(); - temp[this.sensorsActive].set_port(value); - this.sensors = temp; - this.sensorsActive++; - } - } - - add_pwm_pin(pin: number) { - - let error = false; - - for (let i = 0; i < this.pwmsActive; i++) { - if (pin == this.pwms[i].get_pin()) { - console.log("error: pwm pin: " + pin + " has already been set"); - error = true; - } - } - - if (pin < 0 || pin > 13) { - console.log("error: outside pwn pin range"); - error = true; - } - - if (pin == 0 || pin == 1) { - console.log("warning: 0 and 1 are not pwm pins on Haply M3 or Haply original"); - } - - - if (!error) { - const temp = this.pwms; - temp[this.pwmsActive] = new Pwm(); - temp[this.pwmsActive].set_pin(pin); - this.pwms = temp; - this.pwmsActive++; - } - } - - set_mechanism(mechanism: any) { - this.mechanism = mechanism; - } - - device_set_parameters() { - - this.communicationType = 1; - - let control; - - let encoderParameters = new Float32Array(); - - let encoderParams: Uint8Array; - let motorParams: Uint8Array = new Uint8Array(); - let sensorParams: Uint8Array; - let pwmParams: Uint8Array; - - if (this.encodersActive > 0) { - encoderParams = new Uint8Array(this.encodersActive + 1); - control = 0; - - for (let i = 0; i < this.encoders.length; i++) { - if (this.encoders[i].get_encoder() != (i + 1)) { - console.log("warning, improper encoder indexing"); - this.encoders[i].set_encoder(i + 1); - this.encoderPositions[this.encoders[i].get_port() - 1] = this.encoders[i].get_encoder(); - } - } - - for (let i = 0; i < this.encoderPositions.length; i++) { - control = control >> 1; - - if (this.encoderPositions[i] > 0) { - control = control | 0x0008; - } - } - - encoderParams[0] = control; - - encoderParameters = new Float32Array(2 * this.encodersActive); - - let j = 0; - for (let i = 0; i < this.encoderPositions.length; i++) { - if (this.encoderPositions[i] > 0) { - encoderParameters[2 * j] = this.encoders[this.encoderPositions[i] - 1].get_offset(); - encoderParameters[2 * j + 1] = this.encoders[this.encoderPositions[i] - 1].get_resolution(); - j++; - encoderParams[j] = this.encoders[this.encoderPositions[i] - 1].get_direction(); - } - } - } - else { - encoderParams = new Uint8Array(1); - encoderParams[0] = 0; - encoderParameters = new Float32Array(0); - } - - - if (this.actuatorsActive > 0) { - motorParams = new Uint8Array(this.actuatorsActive + 1); - control = 0; - - for (let i = 0; i < this.motors.length; i++) { - if (this.motors[i].get_actuator() != (i + 1)) { - console.log("warning, improper actuator indexing"); - this.motors[i].set_actuator(i + 1); - this.actuatorPositions[this.motors[i].get_port() - 1] = this.motors[i].get_actuator(); - } - } - - for (let i = 0; i < this.actuatorPositions.length; i++) { - control = control >> 1; - - if (this.actuatorPositions[i] > 0) { - control = control | 0x0008; - } - } - - motorParams[0] = control; - - let j = 1; - for (let i = 0; i < this.actuatorPositions.length; i++) { - if (this.actuatorPositions[i] > 0) { - motorParams[j] = this.motors[this.actuatorPositions[i] - 1].get_direction(); - j++; - } - } - } else { - const motorParams = new Uint8Array(1); - motorParams[0] = 0; - } - - if (this.sensorsActive > 0) { - sensorParams = new Uint8Array(this.sensorsActive + 1); - sensorParams[0] = this.sensorsActive; - - for (let i = 0; i < this.sensorsActive; i++) { - sensorParams[i + 1] = this.sensors[i].get_port(); - } - - sensorParams = sensorParams.sort(); - - for (let i = 0; i < this.sensorsActive; i++) { - this.sensors[i].set_port(sensorParams[i + 1]); - } - - } else { - sensorParams = new Uint8Array(1); - sensorParams[0] = 0; - } - - if (this.pwmsActive > 0) { - let temp = new Uint8Array(this.pwmsActive); - - pwmParams = new Uint8Array(this.pwmsActive + 1); - pwmParams[0] = this.pwmsActive; - - - for (let i = 0; i < this.pwmsActive; i++) { - temp[i] = this.pwms[i].get_pin(); - } - - temp = temp.sort(); - - for (let i = 0; i < this.pwmsActive; i++) { - this.pwms[i].set_pin(temp[i]); - pwmParams[i + 1] = this.pwms[i].get_pin(); - } - - } else { - pwmParams = new Uint8Array(1);//byte[1]; - pwmParams[0] = 0; - } - - const encMtrSenPwm = new Uint8Array(motorParams.length + encoderParams.length + sensorParams.length + pwmParams.length); - this.arraycopy(motorParams, 0, encMtrSenPwm, 0, motorParams.length); - this.arraycopy(encoderParams, 0, encMtrSenPwm, motorParams.length, encoderParams.length); - this.arraycopy(sensorParams, 0, encMtrSenPwm, motorParams.length + encoderParams.length, sensorParams.length); - this.arraycopy(pwmParams, 0, encMtrSenPwm, motorParams.length + encoderParams.length + sensorParams.length, pwmParams.length); - this.deviceLink.transmit(this.communicationType, this.deviceID, encMtrSenPwm, encoderParameters); - } - - actuator_assignment(actuator: number, port: number) { - if (this.actuatorPositions[port - 1] > 0) { - console.log("warning, double check actuator port usage"); - } - - this.actuatorPositions[port - 1] = actuator; - } - - - arraycopy(src: Uint8Array, srcPos: number, dst: Uint8Array, dstPos: number, length: number) { - while (length--) dst[dstPos++] = src[srcPos++]; return dst; - } - - /** - * assigns encoder positions based on actuator port - */ - encoder_assignment(encoder: number, port: number) { - if (this.encoderPositions[port - 1] > 0) { - console.log("warning, double check encoder port usage"); - } - - this.encoderPositions[port - 1] = encoder; - } - - async device_read_data() { - let communicationType = 2; - let dataCount = 0; - - const device_data = await this.deviceLink.receive(communicationType, this.deviceID, this.sensorsActive + this.encodersActive); - - //do not process garbled data from the serial comms - if (device_data[0] == 0 && device_data[1] == 0) - return; - - for (let i = 0; i < this.sensorsActive; i++) { - this.sensors[i].set_value(device_data[dataCount]); - dataCount++; - } - - for (let i = 0; i < this.encoderPositions.length; i++) { - if (this.encoderPositions[i] > 0) { - this.encoders[this.encoderPositions[i] - 1].set_value(device_data[dataCount]); - dataCount++; - } - } - } - - device_read_request() { - let communicationType = 2; - const pulses = new Uint8Array(this.pwmsActive); - const encoderRequest = new Float32Array(this.actuatorsActive); - - for (let i = 0; i < this.pwms.length; i++) { - pulses[i] = this.pwms[i].get_value(); - } - - let j = 0; - for (let i = 0; i < this.actuatorPositions.length; i++) { - if (this.actuatorPositions[i] > 0) { - encoderRequest[j] = 0; - j++; - } - } - - this.deviceLink.transmit(communicationType, this.deviceID, pulses, encoderRequest); - } - - device_write_torques() { - let communicationType = 2; - const pulses = new Uint8Array(this.pwmsActive); - const deviceTorques = new Float32Array(this.actuatorsActive); - - for (let i = 0; i < this.pwms.length; i++) { - pulses[i] = this.pwms[i].get_value(); - } - - let j = 0; - for (let i = 0; i < this.actuatorPositions.length; i++) { - if (this.actuatorPositions[i] > 0) { - deviceTorques[j] = this.motors[this.actuatorPositions[i] - 1].get_torque(); - j++; - } - } - - this.deviceLink.transmit(communicationType, this.deviceID, pulses, deviceTorques); - } - - set_pwm_pulse(pin: number, pulse: number) { - - for (let i = 0; i < this.pwms.length; i++) { - if (this.pwms[i].get_pin() == pin) { - this.pwms[i].set_pulse(pulse); - } - } - } - - get_pwm_pulse(pin: number) { - - let pulse = 0; - - for (let i = 0; i < this.pwms.length; i++) { - if (this.pwms[i].get_pin() == pin) { - pulse = this.pwms[i].get_pulse(); - } - } - - return pulse; - } - - get_device_angles() { - const angles = new Float32Array(this.encodersActive); - - for (let i = 0; i < this.encodersActive; i++) { - angles[i] = this.encoders[i].get_value(); - } - return angles; - } - - get_sensor_data() { - const data = new Float32Array(this.sensorsActive); - - let j = 0; - for (let i = 0; i < this.sensorsActive; i++) { - data[i] = this.sensors[i].get_value(); - } - - return data; - } - - get_device_position(angles: Float32Array) { - this.mechanism.forwardKinematics(angles); - var endEffectorPosition = this.mechanism.get_coordinate(); - - return endEffectorPosition; - } - - set_device_torques(forces: Float32Array) { - this.mechanism.torqueCalculation(forces); - var torques = this.mechanism.get_torque(); - - for (let i = 0; i < this.actuatorsActive; i++) { - this.motors[i].set_torque(torques[i]); - } - - return torques; - } -} - -export { Device } \ No newline at end of file diff --git a/src/hAPI/libraries/Pantograph.ts b/src/hAPI/libraries/Pantograph.ts deleted file mode 100644 index a035d65b..00000000 --- a/src/hAPI/libraries/Pantograph.ts +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2021 IMAGE Project, Shared Reality Lab, McGill University - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * and our Additional Terms along with this program. - * If not, see . - * - * This is based on the hAPI library by Haply Robotics . - */ - -class Pantograph { - l: number; - L: number; - d: number; - th1: number; - th2: number; - tau1: number; - tau2: number; - f_x: number; - f_y: number; - x_E: number; - y_E: number; - pi: number; - JT11: number; - JT12: number; - JT21: number; - JT22: number; - gain: number; - - constructor() { - this.l = 0.07; - this.L = 0.09; - this.d = 0.00; - - this.th1 = 0; - this.th2 = 0; - this.tau1 = 0; - this.tau2 = 0; - this.f_x = 0; - this.f_y = 0; - this.x_E = 0; - this.y_E = 0; - - this.pi = 3.14159265359; - this.JT11 = 0; - this.JT12 = 0; - this.JT21 = 0; - this.JT22 = 0; - this.gain = 1.0; - } - - torqueCalculation(force: Float32Array) { - this.f_x = force[0]; - this.f_y = force[1]; - - this.tau1 = this.JT11 * this.f_x + this.JT12 * this.f_y; - this.tau2 = this.JT21 * this.f_x + this.JT22 * this.f_y; - - this.tau1 = this.tau1 * this.gain; - this.tau2 = this.tau2 * this.gain; - } - - forwardKinematics(angles: Float32Array) { - - let l1 = this.l; - let l2 = this.l; - let L1 = this.L; - let L2 = this.L; - - this.th1 = (this.pi / 180) * angles[0]; - this.th2 = (this.pi / 180) * angles[1]; - - // Forward Kinematics - let c1 = Math.cos(this.th1); - let c2 = Math.cos(this.th2); - let s1 = Math.sin(this.th1); - let s2 = Math.sin(this.th2); - - let xA = l1 * c1; - let yA = l1 * s1; - let xB = this.d + l2 * c2; - - let yB = l2 * s2; - let hx = xB - xA; - let hy = yB - yA; - - let hh2 = Math.pow(hx, 2) + Math.pow(hy, 2); - let hh = (hx * hx) + (hy * hy); - - let hm = (Math.sqrt(hh)); - - let cB; - let h1x; - let h1y; - - if (hm == 0) { - cB = 0; - h1x = 0; - h1y = 0; - } else { - cB = -1 * (((Math.pow(L2, 2) - (Math.pow(L1, 2)) - hh)) / (2 * L1 * hm)); - h1x = L1 * cB * hx / hm; - h1y = L1 * cB * hy / hm; - } - - - let h1h1 = Math.pow(h1x, 2) + Math.pow(h1y, 2); - let h1m = Math.sqrt(h1h1); - let sB = Math.sqrt(1 - Math.pow(cB, 2)); - - let lx; - let ly; - - if (h1m == 0) { - lx = 0; - ly = 0; - } else { - lx = -L1 * sB * h1y / h1m; - ly = L1 * sB * h1x / h1m; - } - - let x_P = xA + h1x + lx; - let y_P = yA + h1y + ly; - - let phi1 = Math.acos((x_P - l1 * c1) / L1); - let phi2 = Math.acos((x_P - this.d - l2 * c2) / L2); - - let c11 = Math.cos(phi1); - let s11 = Math.sin(phi1); - let c22 = Math.cos(phi2); - let s22 = Math.sin(phi2); - - let dn = L1 * (c11 * s22 - c22 * s11); - - let eta; - let nu; - - if (dn == 0) { - eta = 0; - nu = 0; - } else { - eta = (-L1 * c11 * s22 + L1 * c22 * s11 - c1 * l1 * s22 + c22 * l1 * s1) / dn; - nu = l2 * (c2 * s22 - c22 * s2) / dn; - } - - this.JT11 = -L1 * eta * s11 - L1 * s11 - l1 * s1; - this.JT12 = L1 * c11 * eta + L1 * c11 + c1 * l1; - this.JT21 = -L1 * s11 * nu; - this.JT22 = L1 * c11 * nu; - - this.x_E = x_P; - this.y_E = y_P; - - } - - set_mechanism_parameters(parameters: Float32Array) { - this.l = parameters[0]; - this.L = parameters[1]; - this.d = parameters[2]; - } - - - set_sensor_data(data: Float32Array) { - } - - get_coordinate() { - let temp = [this.x_E, this.y_E]; - return temp; - } - - - get_torque() { - let temp = [this.tau1, this.tau2]; - return temp; - } - - get_angle() { - let temp = [this.th1, this.th2]; - return temp; - } -} - -export { Pantograph } diff --git a/src/hAPI/libraries/PantographV3.ts b/src/hAPI/libraries/PantographV3.ts deleted file mode 100644 index a5cc19e1..00000000 --- a/src/hAPI/libraries/PantographV3.ts +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2021 IMAGE Project, Shared Reality Lab, McGill University - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * and our Additional Terms along with this program. - * If not, see . - * - * This is based on the hAPI library by Haply Robotics . - */ - -class PantographV3 { - l: number; - L: number; - d: number; - th1: number; - th2: number; - tau1: number; - tau2: number; - f_x: number; - f_y: number; - x_E: number; - y_E: number; - pi: number; - JT11: number; - JT12: number; - JT21: number; - JT22: number; - gain: number; - - constructor() { - this.l = 0.07; - this.L = 0.09; - this.d = 0.038; - - this.th1 = 0; - this.th2 = 0; - this.tau1 = 0; - this.tau2 = 0; - this.f_x = 0; - this.f_y = 0; - this.x_E = 0; - this.y_E = 0; - - this.pi = 3.14159265359; - this.JT11 = 0; - this.JT12 = 0; - this.JT21 = 0; - this.JT22 = 0; - this.gain = 1.0; - } - - torqueCalculation(force: Float32Array) { - this.f_x = force[0]; - this.f_y = force[1]; - - this.tau1 = this.JT11 * this.f_x + this.JT12 * this.f_y; - this.tau2 = this.JT21 * this.f_x + this.JT22 * this.f_y; - - this.tau1 = this.tau1 * this.gain; - this.tau2 = this.tau2 * this.gain; - } - - forwardKinematics(angles: Float32Array) { - - let l1 = this.l; - let l2 = this.l; - let L1 = this.L; - let L2 = this.L; - - this.th1 = (this.pi / 180) * angles[0]; - this.th2 = (this.pi / 180) * angles[1]; - - // Forward Kinematics - let c1 = Math.cos(this.th1); - let c2 = Math.cos(this.th2); - let s1 = Math.sin(this.th1); - let s2 = Math.sin(this.th2); - - let xA = l1 * c1; - let yA = l1 * s1; - let xB = this.d + l2 * c2; - - let yB = l2 * s2; - let hx = xB - xA; - let hy = yB - yA; - - let hh2 = Math.pow(hx, 2) + Math.pow(hy, 2); - let hh = (hx * hx) + (hy * hy); - - let hm = (Math.sqrt(hh)); - - let cB; - let h1x; - let h1y; - - if (hm == 0) { - cB = 0; - h1x = 0; - h1y = 0; - } else { - cB = -1 * (((Math.pow(L2, 2) - (Math.pow(L1, 2)) - hh)) / (2 * L1 * hm)); - h1x = L1 * cB * hx / hm; - h1y = L1 * cB * hy / hm; - } - - - let h1h1 = Math.pow(h1x, 2) + Math.pow(h1y, 2); - let h1m = Math.sqrt(h1h1); - let sB = Math.sqrt(1 - Math.pow(cB, 2)); - - let lx; - let ly; - - if (h1m == 0) { - lx = 0; - ly = 0; - } else { - lx = -L1 * sB * h1y / h1m; - ly = L1 * sB * h1x / h1m; - } - - let x_P = xA + h1x + lx; - let y_P = yA + h1y + ly; - - let phi1 = Math.acos((x_P - l1 * c1) / L1); - let phi2 = Math.acos((x_P - this.d - l2 * c2) / L2); - - let c11 = Math.cos(phi1); - let s11 = Math.sin(phi1); - let c22 = Math.cos(phi2); - let s22 = Math.sin(phi2); - - let dn = L1 * (c11 * s22 - c22 * s11); - - let eta; - let nu; - - if (dn == 0) { - eta = 0; - nu = 0; - } else { - eta = (-L1 * c11 * s22 + L1 * c22 * s11 - c1 * l1 * s22 + c22 * l1 * s1) / dn; - nu = l2 * (c2 * s22 - c22 * s2) / dn; - } - - this.JT11 = -L1 * eta * s11 - L1 * s11 - l1 * s1; - this.JT12 = L1 * c11 * eta + L1 * c11 + c1 * l1; - this.JT21 = -L1 * s11 * nu; - this.JT22 = L1 * c11 * nu; - - this.x_E = x_P; - this.y_E = y_P; - - } - - set_mechanism_parameters(parameters: Float32Array) { - this.l = parameters[0]; - this.L = parameters[1]; - this.d = parameters[2]; - } - - - set_sensor_data(data: Float32Array) { - } - - get_coordinate() { - let temp = [this.x_E, this.y_E]; - return temp; - } - - - get_torque() { - let temp = [this.tau1, this.tau2]; - return temp; - } - - get_angle() { - let temp = [this.th1, this.th2]; - return temp; - } -} - -export { PantographV3 } diff --git a/src/hAPI/libraries/Pwm.ts b/src/hAPI/libraries/Pwm.ts deleted file mode 100644 index 26640e21..00000000 --- a/src/hAPI/libraries/Pwm.ts +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2021 IMAGE Project, Shared Reality Lab, McGill University - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * and our Additional Terms along with this program. - * If not, see . - * - * This is based on the hAPI library by Haply Robotics . - */ -class Pwm { - pin: number; - value: number; - - constructor(pin?: number, pulseWidth?: number) { - this.pin = pin || 0; - if (pulseWidth || 0 > 100.0) { - this.value = 255; - } else { - this.value = (pulseWidth || 0 * 255 / 100); - } - } - - set_pin(pin: number) { - this.pin = pin; - } - set_pulse(percent: number) { - if (percent > 100.0) { - this.value = 255; - } else { - if (percent < 0) { - this.value = 0; - } else { - this.value = (percent * 255 / 100); - } - } - } - - get_pin() { - return this.pin; - } - - get_value() { - return this.value; - } - - get_pulse() { - if (this.value != undefined) { - let percent = this.value * 100 / 255; - return percent; - } else - return -1; - } -} - -export { Pwm } diff --git a/src/hAPI/libraries/Sensor.ts b/src/hAPI/libraries/Sensor.ts deleted file mode 100644 index 43a678f6..00000000 --- a/src/hAPI/libraries/Sensor.ts +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2021 IMAGE Project, Shared Reality Lab, McGill University - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * and our Additional Terms along with this program. - * If not, see . - * - * This is based on the hAPI library by Haply Robotics . - */ -class Sensor { - encoder: number; - direction: number; - encoder_resolution: number; - encoder_offset: number; - value: number; - port: number; - - constructor(encoder?: number, direction?: number, offset?: number, resolution?: number, port?: number) { - this.encoder = encoder || 0; - this.direction = direction || 0; - this.encoder_offset = offset || 0; - this.encoder_resolution = resolution || 0; - this.value = 0; - this.port = port || 0; - } - - set_encoder(encoder: number) { - this.encoder = encoder; - } - set_direction(direction: number) { - this.direction = direction; - } - set_offset(offset: number) { - this.encoder_offset = offset; - } - set_resolution(resolution: number) { - this.encoder_resolution = resolution; - } - set_port(port: number) { - this.port = port; - } - set_value(value: number) { - this.value = value; - } - get_encoder() { - return this.encoder; - } - get_direction() { - return this.direction; - } - get_offset() { - return this.encoder_offset; - } - get_resolution() { - return this.encoder_resolution; - } - get_port() { - return this.port; - } - get_value() { - return this.value; - } -} - -export { Sensor } diff --git a/src/hAPI/libraries/Vector.ts b/src/hAPI/libraries/Vector.ts deleted file mode 100644 index f589a9ca..00000000 --- a/src/hAPI/libraries/Vector.ts +++ /dev/null @@ -1,114 +0,0 @@ -class Vector { - x: number = 0; - y: number = 0; - - constructor(x: number, y: number) { - this.x = x; - this.y = y; - - } - negative() { - return new Vector(-this.x, -this.y); - } - add(v: any) { - if (v instanceof Vector) return new Vector(this.x + v.x, this.y + v.y); - else return new Vector(this.x + v, this.y + v); - } - - subtract(v: any) { - if (v instanceof Vector) return new Vector(this.x - v.x, this.y - v.y); - else return new Vector(this.x - v, this.y - v); - } - - multiply(v: any) { - if (v instanceof Vector) return new Vector(this.x * v.x, this.y * v.y); - else return new Vector(this.x * v, this.y * v); - } - - divide(v: any) { - if (v instanceof Vector) return new Vector(this.x / v.x, this.y / v.y); - else return new Vector(this.x / v, this.y / v); - } - equals(v: Vector) { - return this.x == v.x && this.y == v.y; - } - dot(v: Vector) { - return this.x * v.x + this.y * v.y; - } - - // cross(v:Vector) { - // return new Vector( - // this.y * v.z - this.z * v.y, - // this.z * v.x - this.x * v.z, - // this.x * v.y - this.y * v.x - // ); - // } - - length() { - return Math.sqrt(this.dot(this)); - } - unit() { - return this.divide(this.length()); - } - min() { - return Math.min(this.x, this.y); - } - max() { - return Math.max(this.x, this.y); - } - dist(v: Vector) { - return Math.sqrt(Math.pow(this.x - v.x, 2) + Math.pow(this.y - v.y, 2)); - } - normalize() { - let len = this.mag(); - - if (len !== 0) { - this.x = this.x / len; - this.y = this.y / len; - - } - - return this; - } - // toAngles() { - // return { - // // theta: Math.atan2(this.z, this.x), - // phi: Math.asin(this.y / this.length()) - // }; - // } - // angleTo(a:number) { - // return Math.acos(this.dot(a) / (this.length() * a.length())); - // } - toArray(n?: number) { - return [this.x, this.y].slice(0, n || 3); - } - clone() { - return new Vector(this.x, this.y); - } - mag() { - return Math.sqrt(this.x * this.x + this.y * this.y); - } - set(x: any, y?: number) { - if (x instanceof Vector) { - this.x = x.x || 0; - this.y = x.y || 0; - return this; - } - if (x instanceof Array) { - this.x = x[0] || 0; - this.y = x[1] || 0; - return this; - } - this.x = x || 0; - this.y = y || 0; - return this; - } - init(x: number, y: number) { - this.x = x; this.y = y; - return this; - } - -} - -export { Vector } - diff --git a/src/hAPI/worker.ts b/src/hAPI/worker.ts deleted file mode 100644 index ae20def5..00000000 --- a/src/hAPI/worker.ts +++ /dev/null @@ -1,1059 +0,0 @@ -/* - * Copyright (c) 2021 IMAGE Project, Shared Reality Lab, McGill University - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * and our Additional Terms along with this program. - * If not, see . - */ - -import { Vector } from "./libraries/Vector"; -import { Board } from "./libraries/Board"; -import { Device } from "./libraries/Device"; -import { PantographV3 } from './libraries/PantographV3'; -import { Pantograph } from "./libraries/Pantograph"; -import { convexhull } from './convex-hull'; - -// TODO: set object types - -// set to false if using old 2DIY -let usePantographV3 = true; - -// declaration of haply specific variables -const widgetOneID = 5; -let widgetOne: any; -let pantograph; -let haplyBoard; - -// store required handler json -let baseObjectData: Array = []; -let objectData: Array = [] -let segmentData: Array = []; -let baseSegmentData: Array = []; -let audioData: Array = []; - -// store the angles and positions of the end effector, which are sent back to info.ts -let angles = new Vector(0, 0); -let positions = new Vector(0, 0); - - -// end-effector x/y coords -let posEE = new Vector(0, 0); -let prevPosEE = new Vector(0.5, 0); - -// transformed end-effector coordinates -let convPosEE = new Vector(0, 0); - -// get force needed for torques -let force = new Vector(0, 0); - -// the force applied to the end effector -let fEE = new Vector(0, 0); -let fEEPrev5 = new Vector(0, 0); -let fEEPrev4 = new Vector(0, 0); -let fEEPrev3 = new Vector(0, 0); -let fEEPrev2 = new Vector(0, 0); -let fEEPrev = new Vector(0, 0); -// keeps track of many times a message has been received in the worker -let messageCount: number = 0; - -// Index for where objects begin. -let objHeaderIndex: number = 0; - -// Bool to let us know when to start guidance. -let guidance: boolean = false; - -// Home position of end-effector -let xHome = new Vector(-0.00001291268703669304, 0.022095726821870526); - -enum Transition { - GetPoints, - Move, - Rest -} - -// for transition between segments -let transition: Transition = Transition.GetPoints; -let idx: number = 0; -let prevIdx: number = 0; -let finishTransition: boolean = false; -let upSampled: Vector[] = []; -const tWaitTime: number = 8; -let tHoldTimeSegToSeg: number; - -// distance threshold for stopping segment to segment guidance -const threshold: number = 0.008; - -/** - * Defines a subsegment. - * Contains an array of vectors - * But also can contain rectangular bounds (for objects) - */ -export type SubSegment = { - coordinates: Vector[], - bounds?: [number, number, number, number] -} - -let segments: SubSegment[][] = []; -let objects: SubSegment[][] = []; - -export const enum Mode { - InitializeAudio, - StartAudio, - PlayAudio, - DoneAudio, - StartHaply, - WaitHaply, - MoveHaply, - Reset -} - -let mode = Mode.InitializeAudio; - -// Type of segment to trace. -export const enum Type { - SEGMENT, - OBJECT, - IDLE -} - -let haplyType = Type.IDLE; - -function device_to_graphics(deviceFrame: Vector) { - return new Vector(-deviceFrame.x, deviceFrame.y); -} - -function graphics_to_device(graphicsFrame: any) { - return graphicsFrame.set(-graphicsFrame.x, graphicsFrame.y); -} - -self.addEventListener("message", async function (event) { - // get image data from the main script - if (event) { - - if(event.data.haply2diy2gen){ - usePantographV3 = false; - } - - if (event.data.doneWithAudio != undefined) { - doneWithAudio = event.data.doneWithAudio; - } - - // handshake - if (event.data.receivedAudioSignal != undefined - && sendAudioSignal == true) { - sendAudioSignal = !event.data.receivedAudioSignal; - } - - // if the user presses a key to move back and forth - if (event.data.breakKey != undefined) { - breakKey = event.data.breakKey; - } - - if (event.data.start != undefined) { - guidance = true; - haplyType = Type.SEGMENT; - } - - if (curSegmentDone) { - tLastChangeSegment = event.data.tKeyPressTime; - } - - // Read object, segment, and audio data. - if (event.data.renderingData != undefined) { - - let rendering = event.data.renderingData.entities; - // index marking object start location - objHeaderIndex = rendering.findIndex((x: { entityType: string }) => x.entityType == "object") - - for (let i = 0; i < rendering.length; i++) { - - // find objects - // keep the base object data for sending to main script for renderign - // but also keep object data for 2DIY - if (rendering[i].entityType == "object") { - const name = rendering[i].name; - const centroid = rendering[i].centroid.map((x: any) => transformToVector(x)); - const coords = rendering[i].contours; - const tCentroid = rendering[i].centroid.map((x: any) => transformPtToWorkspace(x)) - - let baseObj = { - name: name, - centroid: centroid, - coords: coords - } - - let tObj = { - name: name, - centroid: tCentroid, - coords: coords - } - - baseObjectData.push(baseObj); - objectData.push(tObj); - } - - // find segments and map them to 2DIY workspace - if (rendering[i].entityType == "segment") { - const coords = rendering[i].contours.map((y: any) => y.map((x: any) => mapCoordsToVec(x.coordinates))); - const tCoords = rendering[i].contours.map((y: any) => y.map((x: any) => mapCoords(x.coordinates))); - let baseSeg = { coords: coords } - let tSeg = { coords: tCoords } - baseSegmentData.push(baseSeg); - segmentData.push(tSeg); - } - - // find audio files - let audio = { - name: rendering[i].name, - offset: rendering[i].offset, - duration: rendering[i].duration, - entityType: rendering[i].entityType, - isStaticSegment: rendering[i].entityType.includes("static") ? true : false - } - audioData.push(audio); - } - - objects = createObjs(objectData); - segments = createSegs(segmentData); - - this.self.postMessage({ - positions: { x: positions.x, y: positions.y }, - objectData: createObjs(baseObjectData), - segmentData: createSegs(baseSegmentData), - }); - } - } - - - /** - * Transforms base ML coordinate data for segments into array of segments. - * @param segmentInfo Array containing objects that contain segment coordinate data. - * @returns Array of segments. - */ - function createSegs(segmentInfo: any): SubSegment[][] { - let data: SubSegment[][] = []; - for (const segs of segmentInfo) { - const segment: Array = []; - const segmentCoords = segs.coords[0]; - for (let i = 0; i < segmentCoords.length; i++) { - let coordinates = segmentCoords[i]; - segment[i] = { coordinates }; - } - data.push(segment); - } - return data; - } - - /** - * Returns an array of objects in transformed 2DIY plane. - * @param objectData Array containing objects that contain object coordinate data. - * @returns Array of objects. - */ - function createObjs(objectData: any): SubSegment[][] { - let data: SubSegment[][] = []; - - // loop through each object entity - for (const obj of objectData) { - const object: Array = []; - - // if ungrouped, just add the object directly - if (obj.centroid && obj.centroid.length == 1) { - for (let i = 0; i < obj.centroid.length; i++) { - let coordinates = [obj.centroid[i]]; - let bounds = obj.coords[i]; - object[i] = { coordinates, bounds }; - } - } - // if we have more than 1 point, i.e., grouped object - // then we'll make a convex hull - else { - // make hull from the obj centroids and then upsample - const objCoords: Vector[] = obj.centroid; - const hull: Vector[] = convexhull.makeHull(objCoords); - const coordinates: Vector[] = upsample(hull, 5000); - object[0] = { coordinates }; - } - data.push(object); - } - return data; - } - - /** - * Maps coordinates from anormalized 0 -> 1 coordinate system into the 2DIY frame of reference. - * @param coordinates Array of 2D coordinate data. - * @returns Vector array of {x, y} data. - */ - function mapCoords(coordinates: [number, number][]): Vector[] { - - return coordinates.map(x => transformPtToWorkspace(x)); - } - - function mapCoordsToVec(coordinates: [number, number][]): Vector[] { - return coordinates.map(x => transformToVector(x)); - } - - /** - * Transforms a tuple into a vector. - * @param coords Tuple containing the coordinate data. - * @returns Vector containing the x and y positions. - */ - function transformToVector(coords: [number, number]): Vector { - const x = (coords[0]); - const y = (coords[1]); - return new Vector(x, y); - } - - /************ BEGIN SETUP CODE *****************/ - if (messageCount < 1) { - messageCount++; - haplyBoard = new Board(); - await haplyBoard.init(); - - widgetOne = new Device(widgetOneID, haplyBoard); - if (usePantographV3) { - - pantograph = new PantographV3(); - widgetOne.set_mechanism(pantograph); - - widgetOne.add_actuator(1, 1, 2); //CCW - widgetOne.add_actuator(2, 1, 1); //CCW - - widgetOne.add_encoder(1, 1, 97.23, 2048 * 2.5 * 1.0194 * 1.0154, 2); //right in theory - widgetOne.add_encoder(2, 1, 82.77, 2048 * 2.5 * 1.0194, 1); //left in theory - } else { - - pantograph = new Pantograph(); - widgetOne.set_mechanism(pantograph); - - // Haply v1 config - widgetOne.add_actuator(1, 1, 2); //CCW - widgetOne.add_actuator(2, 0, 1); //CW - - widgetOne.add_encoder(1, 1, 241, 10752, 2); - widgetOne.add_encoder(2, 0, -61, 10752, 1); - } - - widgetOne.device_set_parameters(); - fEE.set(0, 0); - } - - /************************ END SETUP CODE ************************* */ - - /********** BEGIN CONTROL LOOP CODE *********************/ - - while (true) { - - // find position and angle data - widgetOne.device_read_data(); - angles = widgetOne.get_device_angles(); - - positions = transformToVector(widgetOne.get_device_position(angles)); - - posEE.set(device_to_graphics(positions)); - convPosEE = posEE.clone(); - - - if (guidance) { - posEE.set(device_to_graphics(posEE)); - - // depending on the type of entity to trace - switch (haplyType) { - case Type.SEGMENT: { - if (segments.length != 0) { - audioHapticContours(segments, 3000, 3000, 5); // prev: 15 - } - break; - } - case Type.OBJECT: { - if (objects.length != 0) { - audioHapticContours(objects, 2000, 2000, 20); - } - break; - } - case Type.IDLE: - break; - } - } - prevPosEE.set(convPosEE.clone()); - - // send required data back - /** - * positions: x/y position in 2DIY frame of reference - * waitForInput: if we need to request input from the user - * entityIndex: index of the entity needed to play audio - * sendAudioSignal: signal to let main script know we're ready to play audio - * haplyType: segment or object for drawing - * segIndex: index of the current segment or object - * subSegIndex: index of the current subsegment or object in a group - */ - const data = { - positions: - { x: positions.x, y: positions.y }, - waitForInput: waitForInput, - audioInfo: { - entityIndex: entityIndex, - sendAudioSignal: sendAudioSignal - }, - drawingInfo: { - haplyType: haplyType, - segIndex: currentSegmentIndex, - subSegIndex: currentSubSegmentIndex - } - } - - // // sending end effector position back to info.ts to update visuals - this.self.postMessage(data); - - // calculate and set torques - widgetOne.set_device_torques(fEE.toArray()); - widgetOne.device_write_torques(); - - await new Promise(r => setTimeout(r, 1)); - } - - /********** END CONTROL LOOP CODE *********************/ -}); - -// Index for audio information. -let entityIndex: number = 0; - -// Indicates whether we are done tracing the current segment. -let curSegmentDone: boolean = false; - -// Indicates whether we are done tracing the current subsegment. -let curSubSegmentDone: boolean = false; - -// Indicates the current index of segments being traced. -let currentSegmentIndex: number = 0; - -// Indicates the current index of the sub-segment being traced. -let currentSubSegmentIndex: number = 0; - -// Indicates the current index (point) in the current segment -let currentSubSegmentPointIndex: number = 0; - -// Wait for current user input. -let waitForInput: boolean = false; - -// Spring constant. -let springConst = 200; - -// Mode when controlled by the user. -export const enum BreakKey { - None, - PreviousHaptic, - NextHaptic, - PreviousFromAudio, - NextFromAudio, - Escape -} - -let breakKey: BreakKey; -let tLastChangePoint: number = Number.NEGATIVE_INFINITY; -let tLastChangeSegment: number = Number.NEGATIVE_INFINITY; -let tLastChangeSubSegment: number = Number.NEGATIVE_INFINITY; -let tHoldTime: number = Number.NEGATIVE_INFINITY; - -// Let's us know if that the main script is done playing audio. -let doneWithAudio = false; - -// To let main script know it's time to play audio. -let sendAudioSignal = false; - -// TODO: rewrite all of this within a class -function audioHapticContours(segments: SubSegment[][], tSegDuration: number, - tSubSegDuration: number, tSubSegPointDuration: number) { - - const t0 = tSegDuration; - const t1 = tSubSegDuration; - const t2 = tSubSegPointDuration; - - switch (mode) { - - case Mode.InitializeAudio: { - // called when starting a segment or object experience - // depending on the type, the starting audio index may vary - // objHeaderIndex contains the index of the first object returned in the entities list - entityIndex = haplyType == Type.SEGMENT ? 0 : objHeaderIndex - 1; - mode = Mode.StartAudio; - } - - case Mode.StartAudio: { - // make sure we don't go beyond the max index - if (entityIndex > audioData.length) { - mode = Mode.Reset; - } - else { - // tell main script we need audio played - sendAudioSignal = true; - mode = Mode.PlayAudio; - } - break; - } - case Mode.PlayAudio: { - // wait for a response from main script to see if we're done - if (doneWithAudio) { - mode = Mode.DoneAudio; - } - break; - } - case Mode.DoneAudio: { - - // reset flag - doneWithAudio = false; - - // in case we need to play another audio segment right after one is finished - // typically after static segments - // otherwise get ready for the 2DIY - let audioSeg = audioData[entityIndex]; - if (audioSeg.isStaticSegment) { - mode = Mode.StartAudio; - } - else { - mode = Mode.StartHaply; - } - - // since we've finished an audio chunk, move on to the next one - entityIndex++; - break; - } - - // grace 1.5s buffer before we actually begin - case Mode.StartHaply: { - tHoldTime = Date.now(); - mode = Mode.WaitHaply; - break; - } - - case Mode.WaitHaply: { - if (Date.now() - tHoldTime > 1000) { - mode = Mode.MoveHaply; - tLastChangePoint = Date.now(); - } - break; - } - case Mode.MoveHaply: { - activeGuidance(segments, t0, t1, t2); - break; - } - case Mode.Reset: { - mode = Mode.InitializeAudio; - return; - } - } -} - -/** - * Moves the Haply 2DIY in ascending order of segments/objects. - * @param segments list of objects or segments to trace as SubSegments[] - * @param tSegmentDuration Buffer time when moving from one segment to the next. - * @param tSubSegmentDuration Buffer time when moving from one subsegment to the next. - * @param tSubSegmentPointDuration Buffer time when moving from one point to the next in a subsegment. - */ -function activeGuidance(segments: SubSegment[][], tSegmentDuration: number, - tSubSegmentDuration: number, tSubSegmentPointDuration: number) { - - // first check for breakout conditions by user - if (breakKey != BreakKey.None) { - fEE.set(0, 0); // reset forces - - // escape means we want to cancel any tracing - if (breakKey == BreakKey.Escape) { - finishTracing(); - haplyType = Type.IDLE; - mode = Mode.Reset; - } - - // the user skipped forward while on a haptic segment - if (breakKey == BreakKey.NextHaptic) { - finishSubSegment(); - } - - // the user skipped forward while on an audio segment - if (breakKey == BreakKey.NextFromAudio) { - // the only difference with audio is that we don't - // increment the subsegment index - // because this will be a fresh segment - changeSubSegment(); - } - // the user wants to go back - if (breakKey == BreakKey.PreviousHaptic) { - prevSubSegment(); - } - - if (breakKey == BreakKey.PreviousFromAudio) { - - // note our pattern is A.A.H.A.H.A since the first seg is static - // TODO: rewrite - if (currentSegmentIndex == 0 && entityIndex <= 2) { - entityIndex = 0; - mode = Mode.StartAudio; - } - else { - // go back one index - currentSegmentIndex = currentSegmentIndex == 0 ? 0 : currentSegmentIndex - 1; - - // this will always be equivalent to the last subsegment to trace when coming from an audio segment - currentSubSegmentIndex = segments[currentSegmentIndex].length - 1; - entityIndex--; - changeSubSegment(); - } - } - // reset after we've finished - breakKey = BreakKey.None; - } - - else { - - // if we have traced every index, then we are done with the current mode - if (currentSegmentIndex == segments.length) { - finishTracing(); - switchMode(); - return; - } - - let currentSegment: SubSegment[] = segments[currentSegmentIndex]; - let currentSubSegment: SubSegment = currentSegment[currentSubSegmentIndex]; - - //TODO: further abstract some of these into functions - // if we are done with the current segment... - if (curSegmentDone) { - - // check if the buffer time is over, then play audio - if (Date.now() - tLastChangeSegment > tSegmentDuration) { - mode = Mode.StartAudio; - startNewSegment(); - } - - // if we are done with a subsegment ... - } else if (curSubSegmentDone) { - - // check to see if this is the last subsegment in the list - if (currentSubSegmentIndex != currentSegment.length) { - // if not, move on to next subsegment - // but make sure we're not waiting for input - if (waitForInput) { - guidance = false; - } - // we'll only start a new subsegment once the buffer time is over - else { - if (Date.now() - tLastChangeSubSegment > tSubSegmentDuration) { - startNewSubSegment(); - } - } - } - - else { - finishSegment(); - } - } - - // we're not done with the current subsegment - else { - - // get the distance from where we currently are - const coord = currentSubSegment.coordinates[currentSubSegmentPointIndex]; - const setPoint = new Vector(coord.x, coord.y); - const diff = setPoint.subtract(convPosEE.clone()); - - // don't set the time-based guidance until we have reached the first position - // set dist threshold - if (currentSubSegmentPointIndex == 0 && diff.mag() >= threshold) { - tSubSegmentPointDuration = Number.POSITIVE_INFINITY; - } - else { - // set back original point duration and reset variables - tSubSegmentPointDuration = tSubSegmentPointDuration; - idx = 0; - finishTransition = true; - transition = Transition.GetPoints; - } - // check if we have to move to the next point in the subsegment - if (Date.now() - tLastChangePoint > tSubSegmentPointDuration) { - currentSubSegmentPointIndex++; - - // if we are done tracing each point in this subsegment, end it - if (currentSubSegmentPointIndex >= currentSubSegment.coordinates.length) { - finishSubSegment(); - } - tLastChangePoint = Date.now(); - } else { - - // move to the point - // to avoid setting a large force at once - // when moving from the end of one segment to the beginning of another - // we linear interpolate the # of points - // to avoid setting large forces - // except in the case of setting the force from home pos - // which needs a larger force to compensate - if (currentSubSegmentPointIndex == 0 && !atHomePos()) { - - // case where we need to transition from segment to segment - switch (transition) { - case Transition.GetPoints: { - // interpolate point data - const vEndEffector = new Vector(convPosEE.x, convPosEE.y); - upSampled = upsample([vEndEffector, coord], 6500); - tHoldTimeSegToSeg = Date.now(); - transition = Transition.Move; - break; - } - case Transition.Move: { - // if we are done then end - if (idx >= upSampled.length - 1) { - //finishTransition = true; - transition = Transition.Rest; - idx = 0; - break; - } - if (diff.mag() < 0.01) { - finishTransition = true; - } - else { - // force reduction as we approach the threshold - let k = 3; - let coeff = 1; - if (diff.mag() < 0.025) { - let unit = diff.unit(); - const multiplier = 1 / (diff.mag() + 0.9); - coeff = Math.min(1, 1 - ((unit.x * unit.y) * multiplier * multiplier)); - } - // move to new point with the WaitTime refresh rate - if (Date.now() - tHoldTimeSegToSeg > tWaitTime) { - moveToPos(upSampled[idx], k); - prevIdx = idx; - idx++; - tHoldTimeSegToSeg = Date.now(); - } - } - break; - } - case Transition.Rest: { - force.set(0, 0); - fEE.set(graphics_to_device(force)); - break; - } - } - } - // case where we need to transition from point to point in a segment - else { - moveToPos(coord); - } - } - } - } -} - - - -function atHomePos() { - if (haplyType == Type.SEGMENT && - currentSegmentIndex == 0 && - currentSubSegmentIndex == 0) - return true; - return false; -} -/** - * Called when we are done tracing all entities or want to cancel. - */ -function finishTracing() { - currentSegmentIndex = 0; - currentSubSegmentIndex = 0; - curSegmentDone = false; - mode = Mode.Reset; -} - -/** - * Switch between guidance modes. - */ -function switchMode() { - - switch (haplyType) { - case Type.SEGMENT: { - haplyType = Type.OBJECT; - break; - } - case Type.OBJECT: { - haplyType = Type.IDLE; - break; - } - } -} - -/** - * Called when moving to the previous subsegment. - */ -function prevSubSegment() { - // check if this is the first subsegment - // if so we'll have to change back to audio mode - if (currentSubSegmentIndex == 0) { - entityIndex--; - mode = Mode.StartAudio; - } - else { - currentSubSegmentIndex--; - changeSubSegment(); - } -} -/** - * Called when starting a new segment. - */ -function startNewSegment() { - curSegmentDone = false; - waitForInput = false; -} - -/** - * Called when starting a new subsegment. - */ -function startNewSubSegment() { - curSubSegmentDone = false; - waitForInput = false; - // reset the point to point time buffer - tLastChangePoint = Date.now(); -} - -/** - * Called as soon as we are done tracing a full segment. - */ -function finishSegment() { - currentSegmentIndex++; - currentSubSegmentIndex = 0; - currentSubSegmentPointIndex = 0; - curSegmentDone = true; - curSubSegmentDone = false; - waitForInput = true; - fEE.set(0, 0); -} - -/** - * Called when we are done tracing a subsegment. - */ -function finishSubSegment() { - currentSubSegmentIndex++; - changeSubSegment(); -} - -/** - * Called by either prevSubSegment() or nextSubSegment(). - * Change the subsegment index before calling this. - */ -function changeSubSegment() { - currentSubSegmentPointIndex = 0; - curSubSegmentDone = true; - tLastChangeSubSegment = Date.now(); - fEE.set(0, 0); -} - -/** - * Moves the end-effector to the specified vector position. - * @param vector Vector containing {x,y} position of the Haply coordinates. - */ -function moveToPos(vector: Vector, - springConstMultiplier = 2, - ki = 0.5, - kd = 1.2) { - - if (vector == undefined) - return; - - // find the distance between our current position and target - const targetPos = new Vector(vector.x, vector.y); - const xDiff = targetPos.subtract(convPosEE.clone()); - const kx = xDiff.multiply(springConst).multiply(springConstMultiplier); - - - // allow for higher tolerance when moving from the home position - // apparently needs more force to move from there - const constrainedMax = atHomePos() ? 6 : 3 - - // D controller - const dx = (convPosEE.clone()).subtract(prevPosEE); - const dt = 1 / 1000; - const c = 1.8; - const cdxdt = (dx.divide(dt)).multiply(kd); - - // I controller - const cumError = dx.add(dx.multiply(dt)); - - // set forces - let fx = constrain(kx.x + cdxdt.x + ki * cumError.x, -1 * constrainedMax, constrainedMax); - let fy = constrain(kx.y + cdxdt.y + ki * cumError.y, -1 * constrainedMax, constrainedMax); - const forceMag = new Vector(fx, fy).mag(); - const maxMag = new Vector(constrainedMax, constrainedMax).mag(); - - // if outside of the initial movement from the home position the force is too high, ignore it - if (!atHomePos() && forceMag >= maxMag) { - force.set(0, 0); - } - else { - if (!atHomePos()) { - - // this will break if we have less than 10 points in a subsegment - //console.log(finishTransition); - if (finishTransition) { - const w = 21; - const i = 6; - - const x1 = (i / w) * fx; - const x2 = ((i - 1) / w) * fEEPrev.x; - const x3 = ((i - 2) / w) * fEEPrev2.x; - const x4 = ((i - 3) / w) * fEEPrev3.x; - const x5 = ((i - 4) / w) * fEEPrev4.x; - const x6 = ((i - 5) / w) * fEEPrev5.x; - - const y1 = (i / w) * fy; - const y2 = ((i - 1) / w) * fEEPrev.y; - const y3 = ((i - 2) / w) * fEEPrev2.y; - const y4 = ((i - 3) / w) * fEEPrev3.y; - const y5 = ((i - 4) / w) * fEEPrev4.y; - const y6 = ((i - 5) / w) * fEEPrev5.y; - - fx = x1 + x2 + x3 + x4 + x5 + x6; - fy = y1 + y2 + y3 + y4 + y5 + y6; - const stdX = getStd([x1, x2, x3, x4, x5, x6]) - const stdY = getStd([y1, y2, y3, y4, y5, y6]); - - if (stdX < 0.2 && stdY < 0.2) { - finishTransition = false; - } - } - else { - fx = (1 / 2 * fEEPrev.x + fx); - fy = (1 / 2 * fEEPrev.y + fy); - } - - if (!isFinite(fx)) - fx = 0; - if (!isFinite(fy)) - fy = 0; - - fx = constrain(fx, -constrainedMax, constrainedMax); - fy = constrain(fy, -constrainedMax, constrainedMax); - } - } - - force.set(fx, fy); - - fEEPrev5 = fEEPrev4.clone(); - fEEPrev4 = fEEPrev3.clone(); - fEEPrev3 = fEEPrev2.clone(); - fEEPrev2 = fEEPrev.clone(); - fEEPrev = force.clone(); - - console.log(idx, currentSubSegmentPointIndex, force, finishTransition); - fEE.set(graphics_to_device(force)); -} - -/** - * Calculate the standard deviation of an array of numbers. - * @param numArray Array of numbers from which to calculate the standard deviation. - * @returns Standard deviation. - */ -function getStd(numArray: number[]) { - const mean = numArray.reduce((s: number, n: number) => s + n) / numArray.length; - const variance = numArray.reduce((s: number, n: number) => s + (n - mean) ** 2, 0) / (numArray.length - 1); - return Math.sqrt(variance); -} - -/** - * - * @param val Value to constrain. - * @param min Minimum constrained value. - * @param max Maximum constrained value. - * @returns - */ -function constrain(val: number, min: number, max: number) { - return val > max ? max : val < min ? min : val; -} -/** - * Linearly upsamples an array of points. - * @param pointArray Array containing the {x, y} positions in the 2DIY frame of reference. - * @param k Constant that determines the sampling resolution. - * @returns Upsampled array of {x, y} points. - */ -function upsample(pointArray: Vector[], k = 2000) { - let upsampledSeg = []; - - // for each point (except the last one)... - for (let n = 0; n < pointArray.length - 1; n++) { - - let upsampleSubSeg: Array = []; - - // get the location of both points - const currentPoint = new Vector(pointArray[n].x, pointArray[n].y); - const nextPoint = new Vector(pointArray[n + 1].x, pointArray[n + 1].y); - - const x1 = currentPoint.x; - const y1 = currentPoint.y; - const x2 = nextPoint.x; - const y2 = nextPoint.y; - - // find vars for equation - const m = (y2 - y1) / (x2 - x1); - const c = m == Number.POSITIVE_INFINITY ? 0 : y2 - (m * x2); - - // let the # of sample points be a function of the distance - const euclidean1 = currentPoint.dist(nextPoint); - const samplePoints = Math.round(k * euclidean1); - - // get distance between the two points - const sampleDistX = Math.abs(x2 - x1); - const sampleDistY = Math.abs(y2 - y1); - - for (let v = 0; v < samplePoints; v++) { - // find the location of each interpolated point - const distX = (sampleDistX / (samplePoints - 1)) * v; - const distY = (sampleDistY / (samplePoints - 1)) * v; - - let xLocation = 0; - let yLocation = 0; - - // case where the x values are the same - if (x1 == x2) { - xLocation = x1; - yLocation = y2; - } - - // case where y values are the same - else if (y1 == y2) { - xLocation = x2; - yLocation = y1; - } - - // standard case - else { - xLocation = x2 > x1 ? x1 + distX : x1 - distX; - yLocation = m * xLocation + c; - } - - // add new interpolated point to vector array for these two points - const p = new Vector(xLocation, yLocation); - upsampleSubSeg.push(p); - } - upsampledSeg.push(...upsampleSubSeg); - } - return [...upsampledSeg]; -} - -/** - -/** - * - * @param coords 2D array containing normalized x/y positions - * @returns Vector of {x,y} corresponding to position on Haply 2DIY workspace. - */ -export function transformPtToWorkspace(coords: [number, number]): Vector { - const x = (coords[0] * 0.1333) - 0.064; - const y = (coords[1] * 0.0547) + 0.0589; - return new Vector(x, y); -} diff --git a/src/info/info.ts b/src/info/info.ts index c888762e..32bb0a0d 100644 --- a/src/info/info.ts +++ b/src/info/info.ts @@ -25,7 +25,6 @@ import { IMAGEResponse } from "../types/response.schema"; import { IMAGERequest } from "../types/request.schema"; import * as utils from "./info-utils"; -import * as hapiUtils from '../hAPI/hapi-utils'; import { RENDERERS } from '../config'; import { createSVG } from './info-utils'; import { getAllStorageSyncData, getContext } from '../utils'; @@ -197,10 +196,6 @@ port.onMessage.addListener(async (message) => { } } - if (rendering["type_id"] === RENDERERS.photoAudioHaptics) { - hapiUtils.processHapticsRendering(rendering, graphic_url, container, contentId) - } - if (rendering["type_id"] === RENDERERS.svgLayers) { let contentDiv = utils.addRenderingContent(container, contentId); const imgContainer = document.createElement("div"); diff --git a/src/manifest.json b/src/manifest.json index 7124c4db..1702b7be 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -56,7 +56,6 @@ "launchpad/*", "firstLaunch/*", "maps/*", - "hAPI/*.js", "errors/*", "progressBar/progressBar.html", "audio/*", diff --git a/src/utils.ts b/src/utils.ts index e0450b36..bf88f753 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -18,9 +18,6 @@ export function getAllStorageSyncData() { mcgillServer: true, developerMode: false, previousToggleState: false, - noHaptics: true, - haply2diy2gen: false, - haply2diy3gen: false, audio: true, text: true, processItem: "", @@ -49,10 +46,6 @@ export async function getRenderers(): Promise { if (items["text"]) { renderers.push(RENDERERS.text); } - if (items["haply2diy2gen"] || items["haply2diy3gen"]) { - renderers.push(RENDERERS.simpleHaptics); - renderers.push(RENDERERS.photoAudioHaptics); - } if (items['developerMode']) { renderers.push(RENDERERS.svgLayers); } diff --git a/webpack.config.ts b/webpack.config.ts index 8f99745b..8ff5c263 100644 --- a/webpack.config.ts +++ b/webpack.config.ts @@ -25,8 +25,6 @@ const config: webpack.Configuration = { 'feedback/feedback': './src/feedback/feedback.ts', 'launchpad/launchpad': './src/launchpad/launchpad.ts', 'progressBar/progressBar': './src/progressBar/progressBar.ts', - 'hAPI/hapi-utils': './src/hAPI/hapi-utils.ts', - 'hAPI/worker': './src/hAPI/worker.ts', 'monarch/utils': './src/monarch/utils.ts', 'monarch/types': './src/monarch/types.ts' },