default browser face landmark sucks

UTA-CONNECT · Apr 27, 2021 · 3818952 · 3818952
1 parent dc5e2b8
commit 3818952
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 11 deletions.
diff --git a/Samples/TypeScript/Demo2/index.html b/Samples/TypeScript/Demo2/index.html
@@ -21,6 +21,8 @@
 <body>
     <!-- <img id="faceTest" src="https://miro.medium.com/max/1838/1*DW3-mBLhOOAFIFUVYUkgsQ.png" /> -->
     <canvas id="fcanvas"></canvas>
+    <video id="webcam"></video>
+    <canvas id="rcanvas"></canvas>
     <canvas id="canvas"></canvas>
 </body>
 </html>
diff --git a/Samples/TypeScript/Demo2/src/flandmark.ts b/Samples/TypeScript/Demo2/src/flandmark.ts
@@ -16,38 +16,85 @@ export class FLandmark {
 
     canvas: HTMLCanvasElement;
     renderer: CanvasRenderingContext2D;
+    rcanvas: HTMLCanvasElement;
+    rrenderer: CanvasRenderingContext2D;
+    video: HTMLVideoElement;
 
-    constructor(canvas: HTMLCanvasElement) {
+    constructor(canvas: HTMLCanvasElement, video: HTMLVideoElement) {
         this.canvas = canvas;
+        this.video = video;
         console.log('[FLandmark] [constructor] canvas', canvas);
         // @ts-ignore
         // faceapi.env.monkeyPatch({ Canvas, Image, ImageData })
         this.renderer = this.canvas.getContext('2d');
+
+        this.rcanvas = document.getElementById('rcanvas') as HTMLCanvasElement;
+        this.rcanvas.width = 640;
+        this.rcanvas.height = 360;
+        this.rrenderer = this.rcanvas.getContext('2d');
     }
 
     init() {
         return faceapi.nets.faceLandmark68Net.loadFromUri('./models');
     }
 
-    detectFace() {
+    webcam() {
+
+        if (navigator.mediaDevices.getUserMedia) {
+            navigator.mediaDevices.getUserMedia({ video: { width: 640, height: 360 }})
+                .then((stream) => {
+                    this.video.srcObject = stream;
+                    this.video.onloadedmetadata = (e) => {
+                        this.video.play();
+                        this.draw();
+                      };
+                })
+                .catch((err0r) => {
+                    console.log("Something went wrong!", err0r);
+                });
+        }
+
+    }
+
+    draw() {
+        requestAnimationFrame(() => {
+            this.draw();
+        });
+        this.renderer.clearRect(0, 0, this.canvas.width, this.canvas.height);
+        this.canvas.width = 640;
+        this.canvas.height = 360;
+        // console.log(`w: ${this.video.width} h: ${this.video.height}`);
+        this.renderer.drawImage(this.video, 0, 0, this.canvas.width, this.canvas.height);
+        // rrenderer.drawImage(this.video, 0, 0, this.canvas.width, this.canvas.height);
+        this.detectFace(this.canvas.toDataURL(), this.rrenderer, this.video);
+    }
+
+    detectFace(src: string = 'https://miro.medium.com/max/1022/1*osGdB2BNMThhk1rTwo07JA.jpeg', renderer: CanvasRenderingContext2D = null, video: HTMLVideoElement = null) {
         let currentImg = null;
-        faceapi.fetchImage('https://miro.medium.com/max/1022/1*osGdB2BNMThhk1rTwo07JA.jpeg')
+        faceapi.fetchImage(src)
         .then((imgEle) => {
             currentImg = imgEle;
-            console.log('currentImg', currentImg);
+            // console.log('currentImg', currentImg);
             return faceapi.detectFaceLandmarks(imgEle);
         })
         .then((landmarks) => {
-            console.log('landmarks', landmarks);
             // faceapi.draw.drawLa
 
             const tmpCanvas = faceapi.createCanvasFromMedia(currentImg);
-            this.canvas.width = tmpCanvas.width;
-            this.canvas.height = tmpCanvas.height;
-            this.renderer.clearRect(0, 0, this.canvas.width, this.canvas.height);
+            // this.canvas.width = tmpCanvas.width;
+            // this.canvas.height = tmpCanvas.height;
             // document.querySelector('body').appendChild(tmpCanvas);
-            // @ts-ignore
-            new faceapi.draw.DrawFaceLandmarks(landmarks, { drawLines: true }).draw(this.renderer);
+            if (renderer) {
+                console.log('landmarks', landmarks);
+                renderer.clearRect(0, 0, this.rcanvas.width, this.rcanvas.height);
+                // @ts-ignore
+                new faceapi.draw.DrawFaceLandmarks(landmarks, { drawLines: true }).draw(renderer);
+                if (video) {
+                    renderer.globalAlpha = 0.4;
+                    renderer.drawImage(this.video, 0, 0, this.rcanvas.width, this.rcanvas.height);
+                    renderer.globalAlpha = 1;
+                }
+            }
         })
     }
 }
diff --git a/Samples/TypeScript/Demo2/src/main.ts b/Samples/TypeScript/Demo2/src/main.ts
@@ -18,7 +18,8 @@ window.onload = () => {
 
     lcanvas.run();
 
-    let flandmark = new FLandmark(document.getElementById('fcanvas') as HTMLCanvasElement);
+    let flandmark = new FLandmark(document.getElementById('fcanvas') as HTMLCanvasElement, document.getElementById('webcam') as HTMLVideoElement);
+    flandmark.webcam();
     flandmark.init()
     .then(() => {
         flandmark.detectFace();