feat: video will not go alone

wldhg · Jun 28, 2022 · 9c69962 · 9c69962
1 parent 22a366e
commit 9c69962
Show file tree

Hide file tree

Showing 8 changed files with 103 additions and 65 deletions.
diff --git a/release/app/package.json b/release/app/package.json
@@ -1,6 +1,6 @@
 {
   "name": "viseme-labeler",
-  "version": "1.2.5",
+  "version": "1.2.6",
   "description": "Viseme labeling tool",
   "license": "MIT",
   "author": {

diff --git a/src/renderer/AppNotice.tsx b/src/renderer/AppNotice.tsx
@@ -5,6 +5,9 @@ const AppNotice = () => {
   return (
     <Stack spacing={2} style={{ margin: '12px 0' }}>
       <Typography>VisLab version {version}.</Typography>
+      <Typography>
+        Known issue : keyboard shortcut is not working for now.
+      </Typography>
     </Stack>
   );
 };

diff --git a/src/renderer/Editor.tsx b/src/renderer/Editor.tsx
@@ -10,10 +10,10 @@ import {
 } from '@mui/material';
 import Selecto from 'react-selecto';
 import EditorContext from '../context/editor';
-import EditorVideoInfo from './EditorVideoInfo';
-import EditorVideoProgress from './EditorVideoProgress';
-import EditorVideoPlayer from './EditorVideoPlayer';
-import EditorVideoControl from './EditorVideoControl';
+import EditorPlayerInfo from './EditorPlayerInfo';
+import EditorPlayerProgress from './EditorPlayerProgress';
+import EditorPlayer from './EditorPlayer';
+import EditorPlayerControl from './EditorPlayerControl';
 import EditorLabelControl from './EditorLabelControl';
 import EditorLabelVisualizer from './EditorLabelVisualizer';
 import EditorBanner from './EditorBanner';
@@ -23,17 +23,19 @@ const Editor = () => {
   const ed = useContext(EditorContext);
 
   const selecto = useRef<Selecto>(null);
-  const player = useRef<HTMLVideoElement>(null);
-  const [progress, setProgress] = useState<EditorVideoProgressInfo>({
+  const videoPlayer = useRef<HTMLVideoElement>(null);
+  const audioPlayer = useRef<HTMLAudioElement>(null);
+  const [progress, setProgress] = useState<EditorPlayerProgressInfo>({
     currentTime: 0,
     currentFrame: 0,
   });
 
   return (
     <>
       <Stack direction="row" spacing={4}>
-        <EditorVideoPlayer
-          playerRef={player}
+        <EditorPlayer
+          videoPlayerRef={videoPlayer}
+          audioPlayerRef={audioPlayer}
           setProgress={setProgress}
           style={{
             width: 'auto',
@@ -42,11 +44,12 @@ const Editor = () => {
           }}
         />
         <Box>
-          <EditorVideoInfo currentFrame={progress.currentFrame} />
+          <EditorPlayerInfo currentFrame={progress.currentFrame} />
           <Divider light style={{ margin: '12px 0' }} />
-          <EditorVideoProgress player={player.current} />
-          <EditorVideoControl
-            player={player.current}
+          <EditorPlayerProgress player={videoPlayer.current} />
+          <EditorPlayerControl
+            videoPlayer={videoPlayer.current}
+            audioPlayer={audioPlayer.current}
             selecto={selecto.current}
             currentFrame={progress.currentFrame}
             style={{

diff --git a/src/renderer/EditorVideoPlayer.tsx → src/renderer/EditorPlayer.tsx b/src/renderer/EditorVideoPlayer.tsx → src/renderer/EditorPlayer.tsx
@@ -4,29 +4,35 @@ import { useEffect, useState, useContext, useRef } from 'react';
 import GlobalContext from '../context/global';
 import EditorContext, { EditorLabelContent } from '../context/editor';
 
-type EditorVideoPlayerProps = {
-  playerRef: React.RefObject<HTMLVideoElement>;
-  setProgress: React.Dispatch<React.SetStateAction<EditorVideoProgressInfo>>;
+type EditorPlayerProps = {
+  videoPlayerRef: React.RefObject<HTMLVideoElement>;
+  audioPlayerRef: React.RefObject<HTMLAudioElement>;
+  setProgress: React.Dispatch<React.SetStateAction<EditorPlayerProgressInfo>>;
   style: React.CSSProperties;
 };
 
-const EditorVideoPlayerProgressEventInterval = 10;
+const EditorPlayerProgressEventInterval = 10;
 
-const EditorVideoPlayer = (props: EditorVideoPlayerProps) => {
-  const { playerRef, setProgress, style } = props;
+const EditorPlayer = (props: EditorPlayerProps) => {
+  const { videoPlayerRef, audioPlayerRef, setProgress, style } = props;
   const ctx = useContext(GlobalContext);
   const ed = useContext(EditorContext);
   const [beforeItemIndex, setBeforeItemIndex] = useState<number>(-1);
-  const [playerElement, setPlayerElement] = useState<React.ReactNode>(null);
+  const [videoPlayerElement, setVideoPlayerElement] =
+    useState<React.ReactNode>(null);
+  const [audioPlayerElement, setAudioPlayerElement] =
+    useState<React.ReactNode>(null);
   const videoProgressUpdateInterval = useRef<NodeJS.Timeout>();
   const fpsRef = useRef<number>(1);
 
   const updateProgress = (currentTime = -1) => {
     setProgress({
       currentTime:
-        currentTime === -1 ? playerRef.current?.currentTime || 0 : currentTime,
+        currentTime === -1
+          ? videoPlayerRef.current?.currentTime || 0
+          : currentTime,
       currentFrame: Math.round(
-        (playerRef.current?.currentTime || 0) * fpsRef.current
+        (videoPlayerRef.current?.currentTime || 0) * fpsRef.current
       ),
     });
   };
@@ -40,24 +46,36 @@ const EditorVideoPlayer = (props: EditorVideoPlayerProps) => {
         currentFrame: 0,
       });
       setBeforeItemIndex(ctx.currentItemIndex);
-      setPlayerElement(
+      setAudioPlayerElement(
+        <audio ref={audioPlayerRef}>
+          <source
+            src={ctx.items[ctx.currentItemIndex].srcFilePath}
+            type="audio/mpeg"
+          />
+        </audio>
+      );
+      setVideoPlayerElement(
         <video
-          ref={playerRef}
+          ref={videoPlayerRef}
           src={ctx.items[ctx.currentItemIndex].srcFilePath}
           onPlay={() => {
             if (videoProgressUpdateInterval.current) {
               clearInterval(videoProgressUpdateInterval.current);
             }
             videoProgressUpdateInterval.current = setInterval(() => {
               updateProgress();
-            }, EditorVideoPlayerProgressEventInterval);
+            }, EditorPlayerProgressEventInterval);
             updateProgress();
           }}
           onPause={() => {
             if (videoProgressUpdateInterval.current) {
               clearInterval(videoProgressUpdateInterval.current);
             }
             updateProgress();
+            if (audioPlayerRef.current) {
+              audioPlayerRef.current.currentTime =
+                videoPlayerRef.current?.currentTime || 0;
+            }
           }}
           onSeeked={() => {
             updateProgress();
@@ -106,7 +124,12 @@ const EditorVideoPlayer = (props: EditorVideoPlayerProps) => {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [ctx.currentItemIndex]);
 
-  return <>{playerElement}</>;
+  return (
+    <>
+      {videoPlayerElement}
+      {audioPlayerElement}
+    </>
+  );
 };
 
-export default EditorVideoPlayer;
+export default EditorPlayer;
diff --git a/src/renderer/EditorVideoControl.tsx → src/renderer/EditorPlayerControl.tsx b/src/renderer/EditorVideoControl.tsx → src/renderer/EditorPlayerControl.tsx
@@ -5,15 +5,16 @@ import { useHotkeys } from 'react-hotkeys-hook';
 import Selecto from 'react-selecto';
 import EditorContext, { EditorContextDefault } from '../context/editor';
 
-type EditorVideoControlProps = {
-  player: HTMLVideoElement | null;
+type EditorPlayerControlProps = {
+  videoPlayer: HTMLVideoElement | null;
+  audioPlayer: HTMLAudioElement | null;
   currentFrame: number;
   selecto: Selecto | null;
   style?: React.CSSProperties;
 };
 
-const EditorVideoControl = (props: EditorVideoControlProps) => {
-  const { player, currentFrame, style, selecto } = props;
+const EditorPlayerControl = (props: EditorPlayerControlProps) => {
+  const { videoPlayer, audioPlayer, currentFrame, style, selecto } = props;
   const ed = useContext(EditorContext);
 
   const emptySelection = () => {
@@ -22,48 +23,54 @@ const EditorVideoControl = (props: EditorVideoControlProps) => {
   };
 
   const playPause = () => {
-    if (player) {
+    if (videoPlayer) {
       emptySelection();
-      if (player.paused) {
-        player.play();
+      if (videoPlayer.paused) {
+        videoPlayer.play();
       } else {
-        player.pause();
+        videoPlayer.pause();
       }
     }
   };
 
   const back1F = () => {
-    if (player) {
+    if (videoPlayer && audioPlayer) {
       emptySelection();
-      player.currentTime = (currentFrame - 1) / ed.videoInfo.fps;
+      const currentTime = (currentFrame - 1) / ed.videoInfo.fps;
+      videoPlayer.currentTime = currentTime;
+      audioPlayer.currentTime = currentTime;
     }
   };
 
   const go1F = () => {
-    if (player) {
+    if (videoPlayer && audioPlayer) {
+      audioPlayer.currentTime = videoPlayer.currentTime;
       emptySelection();
       const seekTime = (currentFrame + 1) / ed.videoInfo.fps;
       setTimeout(() => {
-        if (player) {
-          player.pause();
-          player.currentTime = seekTime;
+        if (audioPlayer) {
+          audioPlayer.pause();
+          audioPlayer.currentTime = seekTime;
         }
-      }, (1 / ed.videoInfo.fps) * 1000);
-      player.play();
+      }, (1 / ed.videoInfo.fps) * 900);
+      audioPlayer.play();
+      videoPlayer.currentTime = seekTime;
     }
   };
 
   const back5s = () => {
-    if (player) {
+    if (videoPlayer && audioPlayer) {
       emptySelection();
-      player.currentTime -= 5;
+      videoPlayer.currentTime -= 5;
+      audioPlayer.currentTime = videoPlayer.currentTime;
     }
   };
 
   const go5s = () => {
-    if (player) {
+    if (videoPlayer && audioPlayer) {
       emptySelection();
-      player.currentTime += 5;
+      videoPlayer.currentTime += 5;
+      audioPlayer.currentTime = videoPlayer.currentTime;
     }
   };
 
@@ -85,7 +92,7 @@ const EditorVideoControl = (props: EditorVideoControlProps) => {
       </span>
       <Tooltip title="Space">
         <Button size="small" variant="outlined" onClick={playPause}>
-          {player && player.paused ? <PlayArrow /> : <Pause />}
+          {videoPlayer && videoPlayer.paused ? <PlayArrow /> : <Pause />}
         </Button>
       </Tooltip>
       <Tooltip title="< key">
@@ -112,9 +119,10 @@ const EditorVideoControl = (props: EditorVideoControlProps) => {
         size="small"
         variant="text"
         onClick={() => {
-          if (player) {
+          if (videoPlayer && audioPlayer) {
             emptySelection();
-            player.currentTime = 0;
+            videoPlayer.currentTime = 0;
+            audioPlayer.currentTime = 0;
           }
         }}
       >
@@ -124,9 +132,10 @@ const EditorVideoControl = (props: EditorVideoControlProps) => {
         size="small"
         variant="text"
         onClick={() => {
-          if (player) {
+          if (videoPlayer && audioPlayer) {
             emptySelection();
-            player.currentTime = ed.videoInfo.duration;
+            videoPlayer.currentTime = ed.videoInfo.duration;
+            audioPlayer.currentTime = ed.videoInfo.duration;
           }
         }}
       >
@@ -136,8 +145,8 @@ const EditorVideoControl = (props: EditorVideoControlProps) => {
   );
 };
 
-EditorVideoControl.defaultProps = {
+EditorPlayerControl.defaultProps = {
   style: {},
 };
 
-export default EditorVideoControl;
+export default EditorPlayerControl;
diff --git a/src/renderer/EditorVideoInfo.tsx → src/renderer/EditorPlayerInfo.tsx b/src/renderer/EditorVideoInfo.tsx → src/renderer/EditorPlayerInfo.tsx
@@ -2,11 +2,11 @@ import { useContext } from 'react';
 import { Breadcrumbs, Card } from '@mui/material';
 import EditorContext from '../context/editor';
 
-type EditorVideoInfoProps = {
+type EditorPlayerInfoProps = {
   currentFrame: number;
 };
 
-const EditorVideoInfo = (props: EditorVideoInfoProps) => {
+const EditorPlayerInfo = (props: EditorPlayerInfoProps) => {
   const ed = useContext(EditorContext);
   const { currentFrame } = props;
 
@@ -21,4 +21,4 @@ const EditorVideoInfo = (props: EditorVideoInfoProps) => {
     </Card>
   );
 };
-export default EditorVideoInfo;
+export default EditorPlayerInfo;
diff --git a/src/renderer/EditorVideoProgress.tsx → src/renderer/EditorPlayerProgress.tsx b/src/renderer/EditorVideoProgress.tsx → src/renderer/EditorPlayerProgress.tsx
@@ -2,20 +2,20 @@ import { useContext } from 'react';
 import { Stack, LinearProgress } from '@mui/material';
 import EditorContext from '../context/editor';
 
-type EditorVideoProgressProps = {
+type EditorPlayerProgressProps = {
   player: HTMLVideoElement | null;
 };
 
-type EditorVideoProgressTextStyleType = {
+type EditorPlayerProgressTextStyleType = {
   minWidth: string;
   textAlign: 'center';
 };
-const EditorVideoProgressTextStyle: EditorVideoProgressTextStyleType = {
+const EditorPlayerProgressTextStyle: EditorPlayerProgressTextStyleType = {
   minWidth: '11ex',
   textAlign: 'center',
 };
 
-const EditorVideoProgress = (props: EditorVideoProgressProps) => {
+const EditorPlayerProgress = (props: EditorPlayerProgressProps) => {
   const { player } = props;
   const ed = useContext(EditorContext);
 
@@ -27,7 +27,7 @@ const EditorVideoProgress = (props: EditorVideoProgressProps) => {
       direction="row"
       style={{ fontFamily: 'Inconsolata, monospace' }}
     >
-      <span style={EditorVideoProgressTextStyle}>
+      <span style={EditorPlayerProgressTextStyle}>
         {Math.floor(currentTime / 60)}:
         {String(Math.floor(currentTime % 60)).padStart(2, '0')}:
         {String(
@@ -39,7 +39,7 @@ const EditorVideoProgress = (props: EditorVideoProgressProps) => {
         variant="determinate"
         value={(currentTime / ed.videoInfo.duration) * 100}
       />
-      <span style={EditorVideoProgressTextStyle}>
+      <span style={EditorPlayerProgressTextStyle}>
         -{Math.max(0, Math.floor((ed.videoInfo.duration - currentTime) / 60))}:
         {String(
           Math.max(0, Math.floor((ed.videoInfo.duration - currentTime) % 60))
@@ -58,4 +58,4 @@ const EditorVideoProgress = (props: EditorVideoProgressProps) => {
   );
 };
 
-export default EditorVideoProgress;
+export default EditorPlayerProgress;
diff --git a/src/renderer/EditorVideoProgressInfo.d.ts → src/renderer/EditorPlayerProgressInfo.d.ts b/src/renderer/EditorVideoProgressInfo.d.ts → src/renderer/EditorPlayerProgressInfo.d.ts
@@ -1,4 +1,4 @@
-type EditorVideoProgressInfo = {
+type EditorPlayerProgressInfo = {
   currentTime: number;
   currentFrame: number;
 };