From b159cf99b891fb6477de1f58a3c7eadccc8e0083 Mon Sep 17 00:00:00 2001
From: Fausto Morales <fausto@robinbay.com>
Date: Sun, 24 Sep 2023 13:29:09 -0500
Subject: [PATCH] Integrate video segment labeler. Fix targets being passed to
 BatchImageLabeler.

---
 qsl/common.py                                 |  30 +++-
 qsl/version.py                                |   2 +-
 qslwidgets/package.json                       |   2 +-
 .../src/components/BatchImageLabeler.svelte   |   4 +-
 qslwidgets/src/components/VideoLabeler.svelte |  44 +++---
 qslwidgets/src/components/VideoPair.svelte    |  16 +-
 .../src/components/VideoSegmentLabeler.svelte | 142 +++++++++++++++---
 .../src/components/VideoWithPlaybar.svelte    |  17 ++-
 qslwidgets/src/components/Widget.svelte       |  20 ++-
 .../src/stories/VideoSegmentLabeler.svelte    |   1 +
 10 files changed, 223 insertions(+), 55 deletions(-)

diff --git a/qsl/common.py b/qsl/common.py
index f4f9c0a..9d1e17d 100644
--- a/qsl/common.py
+++ b/qsl/common.py
@@ -630,7 +630,12 @@ def update(self, reset: bool):
             (
                 base_item.get("labels")
                 or base_item.get("defaults")
-                or ([] if base_item.get("type", "image") == "video" else {})
+                or (
+                    []
+                    if base_item.get("type", "image")
+                    in ("video", "video-segment-pairs")
+                    else {}
+                )
             ),
         )
         sIdx = self.sortedIdxs.index(self.idx)
@@ -687,6 +692,29 @@ def set_urls_and_type(self):
             )
             if self.type == "time-series":
                 self.urls = [self.targets[0]["target"]]
+            elif self.type == "video-segment-pairs":
+                target = self.targets[0]["target"]
+                self.urls = [
+                    {
+                        "video1": {
+                            **target["video1"],
+                            "target": files.build_url(
+                                target["video1"]["target"],
+                                base=self.base,
+                                get_tempdir=self.get_temporary_directory,
+                            ),
+                        },
+                        "video2": {
+                            **target["video2"],
+                            "target": files.build_url(
+                                target["video2"]["target"],
+                                base=self.base,
+                                get_tempdir=self.get_temporary_directory,
+                            ),
+                        },
+                    }
+                ]
+
             elif self.type in ["image-group", "image-stack"]:
                 target = self.targets[0]["target"]
                 self.urls = [
diff --git a/qsl/version.py b/qsl/version.py
index ad30018..df1ed62 100644
--- a/qsl/version.py
+++ b/qsl/version.py
@@ -1,2 +1,2 @@
-version_info = (0, 2, 30)
+version_info = (0, 2, 33)
 __version__ = ".".join(map(str, version_info))
diff --git a/qslwidgets/package.json b/qslwidgets/package.json
index cb47171..d656130 100644
--- a/qslwidgets/package.json
+++ b/qslwidgets/package.json
@@ -1,6 +1,6 @@
 {
   "name": "qslwidgets",
-  "version": "0.2.32",
+  "version": "0.2.33",
   "description": "Widgets for the QSL media labeling package.",
   "keywords": [
     "jupyter",
diff --git a/qslwidgets/src/components/BatchImageLabeler.svelte b/qslwidgets/src/components/BatchImageLabeler.svelte
index d216447..4cd73be 100644
--- a/qslwidgets/src/components/BatchImageLabeler.svelte
+++ b/qslwidgets/src/components/BatchImageLabeler.svelte
@@ -15,7 +15,7 @@
   export let labels: Labels,
     config: Config,
     states: BatchEntry[] = [],
-    targets: (string | undefined)[] = [],
+    targets: (string | undefined)[] | undefined = [],
     navigation: boolean = false,
     editableConfig: boolean = false,
     transitioning: boolean = false,
@@ -39,7 +39,7 @@
     })));
   $: targets, labels, draft.reset(labels);
   $: items =
-    targets.length === states.length
+    targets && targets.length === states.length
       ? targets
           .map((target, index) => ({ target, state: states[index], index }))
           .filter((entry) => entry.state.visible)
diff --git a/qslwidgets/src/components/VideoLabeler.svelte b/qslwidgets/src/components/VideoLabeler.svelte
index 27572a3..572e0e0 100644
--- a/qslwidgets/src/components/VideoLabeler.svelte
+++ b/qslwidgets/src/components/VideoLabeler.svelte
@@ -39,14 +39,10 @@
     paused: true,
     muted: false,
     playhead: 0,
-    t1: 0,
-    t2: undefined,
   } as {
     paused: boolean;
     muted: boolean;
     playhead: number;
-    t1: number;
-    t2: undefined | number;
   };
   $: ({ callbacks: loadCallbacks, state: loadState } = createContentLoader({
     targets: [target],
@@ -63,15 +59,17 @@
   }));
   const synchronize = () => {
     if (playback.paused) {
-      const existing = labels4timestamp(labels, playback.t1);
+      const existing = labels4timestamp(
+        labels,
+        $draft.timestampInfo?.timestamp || 0
+      );
       frame = existing.label;
-      if (!existing.exists) {
-        playback = {
-          ...playback,
-          t2: $loadState.mediaState?.states[0].duration,
-        };
-      }
-      draft.reset(frame.labels);
+      draft.reset(frame.labels, {
+        timestamp: existing.label.timestamp,
+        end: existing.exists
+          ? existing.label.end
+          : $loadState.mediaState?.states[0].duration,
+      });
     } else {
       console.error(
         "synchronize() was called when unpaused, which should not occur."
@@ -89,13 +87,17 @@
   // If the external inputs change ...
   $: target, labels, $loadState, synchronize();
   // If our current timestamp changes.
-  $: if (frame.timestamp !== playback.t1) synchronize();
+  $: if (frame.timestamp !== $draft.timestampInfo?.timestamp) synchronize();
   const save = () => {
-    if (playback.paused && playback.t1 !== undefined) {
+    if (playback.paused && $draft.timestampInfo?.timestamp !== undefined) {
+      // We can't edit start points of labels because moving the start point
+      // results in a reset/history erasure. Is that okay? If not,
+      // we'll need to add a way to differentiate between "move start time"
+      // and "create new label."
       const current = {
         labels: draft.export($loadState.mediaState?.states[0].size),
-        timestamp: playback.t1!,
-        end: playback.t2,
+        timestamp: $draft.timestampInfo.timestamp,
+        end: $draft.timestampInfo.end,
       };
       labels = insertOrAppendByTimestamp(current, labels || []);
       dispatcher("save");
@@ -134,10 +136,14 @@
     mains={[main]}
     secondaries={[mini]}
     bind:playhead={playback.playhead}
-    t1={playback.t1}
-    t2={playback.t2}
+    t1={$draft.timestampInfo?.timestamp || 0}
+    t2={$draft.timestampInfo?.end}
     on:setMarkers={(event) => {
-      playback = { ...playback, ...event.detail };
+      draft.snapshot();
+      draft.set({
+        ...$draft,
+        timestampInfo: { timestamp: event.detail.t1, end: event.detail.t2 },
+      });
     }}
     bind:paused={playback.paused}
     bind:muted={playback.muted}
diff --git a/qslwidgets/src/components/VideoPair.svelte b/qslwidgets/src/components/VideoPair.svelte
index e947f83..8b0fa9a 100644
--- a/qslwidgets/src/components/VideoPair.svelte
+++ b/qslwidgets/src/components/VideoPair.svelte
@@ -36,8 +36,6 @@
     }
   };
 </script>
-
-{#if timestampInfo}
   <div class="video-pair">
     <div style="--icon-button-size: 2.5rem">
       <IconButton on:click={onClick}>
@@ -53,14 +51,16 @@
     <!-- svelte-ignore a11y-media-has-caption -->
     <VideoWithPlaybar
       target={target.video1.target}
-      t1={timestampInfo.timestamp}
-      t2={timestampInfo.end}
+      t1={timestampInfo?.timestamp}
+      t2={timestampInfo?.end}
       bind:playhead={playbackState.video1.playhead}
       bind:paused={playbackState.video1.paused}
+      on:loaded={(event) => dispatcher("loaded-video1", event.detail)}
       on:setMarkers={(event) => {
         dispatcher("change");
         draft = {
           ...draft,
+          dirty: true,
           timestampInfo: {
             ...draft.timestampInfo,
             timestamp: event.detail.t1,
@@ -72,10 +72,11 @@
     <!-- svelte-ignore a11y-media-has-caption -->
     <VideoWithPlaybar
       target={target.video2.target}
-      t1={timestampInfo.match.timestamp}
-      t2={timestampInfo.match.end}
+      t1={timestampInfo?.match.timestamp}
+      t2={timestampInfo?.match.end}
       bind:playhead={playbackState.video2.playhead}
       bind:paused={playbackState.video2.paused}
+      on:loaded={(event) => dispatcher("loaded-video2", event.detail)}
       on:setMarkers={(event) => {
         if (
           event.detail.t1 !== undefined &&
@@ -86,6 +87,7 @@
           dispatcher("change");
           draft = {
             ...draft,
+            dirty: true,
             timestampInfo: {
               ...draft.timestampInfo,
               match: {
@@ -98,8 +100,6 @@
       }}
     />
   </div>
-{/if}
-
 <style>
   .video-pair {
     display: flex;
diff --git a/qslwidgets/src/components/VideoSegmentLabeler.svelte b/qslwidgets/src/components/VideoSegmentLabeler.svelte
index 2360019..bee1df7 100644
--- a/qslwidgets/src/components/VideoSegmentLabeler.svelte
+++ b/qslwidgets/src/components/VideoSegmentLabeler.svelte
@@ -5,24 +5,41 @@
     VideoSegmentTarget,
     WidgetActions,
     Config,
+    TimestampedLabel,
     TimestampInfoWithMatch,
+    ArbitraryMetadata,
   } from "../library/types";
   import { createEventDispatcher } from "svelte";
   import { createDraftStore } from "../library/common";
+  import Metadata from "./Metadata.svelte";
   import LabelPanelEntry from "./LabelPanelEntry.svelte";
   import ControlMenu from "./ControlMenu.svelte";
   import VideoPair from "./VideoPair.svelte";
+  import ButtonGroup from "./ButtonGroup.svelte";
 
   export let target: VideoSegmentTarget | undefined,
-    labels: TimestampedLabelWithMatch[] | undefined,
+    transitioning: boolean = false,
+    metadata: ArbitraryMetadata | undefined = undefined,
+    editableConfig: boolean = false,
+    labels: (TimestampedLabelWithMatch | TimestampedLabel)[] | undefined,
     config: Config,
     navigation: boolean = false,
     actions: WidgetActions = {};
   const dispatcher = createEventDispatcher();
   let { draft, history } = createDraftStore();
+  const format = (value: number) => (Math.round(value * 1e2) / 1e2).toString();
   const labelToString = (label: TimestampedLabelWithMatch) =>
-    `${label.timestamp} to ${label.end} and ${label.match.timestamp} to ${label.match.end}`;
+    `${format(label.timestamp)} to ${format(label.end)} and ${format(
+      label.match.timestamp
+    )} to ${format(label.match.end)}`;
   let selected = labels && labels.length > 0 ? 0 : null;
+  let mediaStates: {
+    video1: { duration: number } | null;
+    video2: { duration: number } | null;
+  } = {
+    video1: null,
+    video2: null,
+  };
   let playbackState = {
     video1: {
       paused: true,
@@ -31,7 +48,10 @@
     video2: {
       paused: true,
       playhead:
-        selected !== null && labels ? labels[selected].match.timestamp : 0,
+        selected && labels && labels[selected]
+          ? (labels[selected] as TimestampedLabelWithMatch).match?.timestamp ||
+            0
+          : 0,
     },
   };
   const reset = () => {
@@ -40,10 +60,25 @@
       draft.reset(current.labels, {
         timestamp: current.timestamp,
         end: current.end,
-        match: current.match,
+        match: (current as TimestampedLabelWithMatch)?.match || {
+          timestamp: 0,
+          end: mediaStates.video2?.duration,
+        },
       });
     } else {
-      draft.reset({}, undefined);
+      draft.reset(
+        {},
+        mediaStates.video1 && mediaStates.video2
+          ? {
+              timestamp: 0,
+              end: mediaStates.video1.duration,
+              match: {
+                timestamp: 0,
+                end: mediaStates.video2.duration,
+              },
+            }
+          : undefined
+      );
     }
   };
   const changeSelected = (event: { detail: { selected: string[] } }) => {
@@ -70,6 +105,16 @@
     }
   };
   $: labels, reset();
+  $: mediaStates, reset();
+  $: controlConfig = {
+    name: "Segment Pair",
+    multiple: false,
+    freeform: false,
+    options: (labels || []).map((l, i) => ({
+      name: i.toString(),
+      displayName: labelToString(l as TimestampedLabelWithMatch),
+    })),
+  };
 </script>
 
 <div>
@@ -79,25 +124,82 @@
       bind:draft={$draft}
       {playbackState}
       on:change={draft.snapshot}
+      on:loaded-video1={(event) =>
+        (mediaStates = {
+          ...mediaStates,
+          video1: { duration: event.detail.duration },
+        })}
+      on:loaded-video2={(event) =>
+        (mediaStates = {
+          ...mediaStates,
+          video2: { duration: event.detail.duration },
+        })}
     />
+    <Metadata {metadata} />
   {/if}
-  {#if labels}
+  {#if labels && labels.length > 0}
     <LabelPanelEntry
-      config={{
-        name: "Segment Pair",
-        multiple: false,
-        freeform: false,
-        options: labels.map((l, i) => ({
-          name: i.toString(),
-          displayName: labelToString(l),
-        })),
-      }}
+      config={controlConfig}
       disabled={false}
       selected={selected !== null ? [selected.toString()] : []}
       on:change={changeSelected}
       editableConfig={false}
     />
+  {:else}
+    <p>
+      There are no pair labels for this video pair. Please add a new pair using
+      the "Add" button.
+    </p>
   {/if}
+  <div class="add-delete-controls">
+    <ButtonGroup
+      configs={[
+        {
+          text: "Add",
+          event: "add",
+          shortcuts: [],
+          tooltip: "Add a new pair label.",
+          disabled: $draft.dirty,
+        },
+        {
+          text: "Delete",
+          event: "delete",
+          shortcuts: [],
+          tooltip: "Delete a pair label.",
+          disabled: $draft.dirty || !labels || labels.length == 0,
+        },
+      ]}
+      on:click={(event) => {
+        if (
+          event.detail.name == "add" &&
+          mediaStates.video1 &&
+          mediaStates.video2
+        ) {
+          labels = [
+            ...(labels || []),
+            {
+              labels: {},
+              timestamp: 0,
+              end: mediaStates.video1.duration,
+              match: {
+                timestamp: 0,
+                end: mediaStates.video2.duration,
+              },
+            },
+          ];
+          selected = labels.length - 1;
+        } else if (
+          event.detail.name == "delete" &&
+          labels &&
+          selected !== null &&
+          labels.length > selected
+        ) {
+          labels = labels.slice(0, selected).concat(labels.slice(selected + 1));
+          selected = labels.length > 0 ? Math.max(selected - 1, 0) : null;
+        }
+      }}
+    />
+  </div>
   <ControlMenu
     bind:config
     bind:draft={$draft}
@@ -111,9 +213,15 @@
     on:undo={() => history.undo()}
     on:save={save}
     on:reset={reset}
-    disabled={false}
-    editableConfig={false}
+    disabled={transitioning}
+    {editableConfig}
     {navigation}
     actions={{ ...actions, undo: $history > 0 }}
   />
 </div>
+
+<style>
+  .add-delete-controls {
+    margin-top: 5px;
+  }
+</style>
diff --git a/qslwidgets/src/components/VideoWithPlaybar.svelte b/qslwidgets/src/components/VideoWithPlaybar.svelte
index 8a72c3a..ecee235 100644
--- a/qslwidgets/src/components/VideoWithPlaybar.svelte
+++ b/qslwidgets/src/components/VideoWithPlaybar.svelte
@@ -1,28 +1,32 @@
 <script lang="ts">
   import Playbar from "./Playbar.svelte";
+  import {createEventDispatcher } from "svelte"
   import type { RangeSliderMark } from "../library/types";
   import { createContentLoader } from "../library/common";
   export let target: string,
-    t1: number,
-    t2: number,
-    playhead: number = t1,
+    t1: number | undefined,
+    t2: number | undefined,
+    playhead: number = t1 || 0,
     paused: boolean = true,
     muted: boolean = false,
     limitToBounds: boolean = true,
     marks: RangeSliderMark[] = [],
     disabled: boolean = false;
   let video: HTMLVideoElement;
+  let dispatcher = createEventDispatcher()
   $: ({ callbacks, state } = createContentLoader({
     targets: [target],
     load: async (event: { currentTarget: HTMLElement }) => {
       const target = event.currentTarget as HTMLVideoElement;
-      return {
+      const mediaState = {
         size: {
           width: target.videoWidth,
           height: target.videoHeight,
         },
         duration: target.duration,
-      };
+      }
+      dispatcher("loaded", mediaState)
+      return mediaState;
     },
   }));
 </script>
@@ -55,4 +59,7 @@
     display: flex;
     flex-direction: column;
   }
+  .player video {
+    width: 100%;
+  }
 </style>
diff --git a/qslwidgets/src/components/Widget.svelte b/qslwidgets/src/components/Widget.svelte
index baeca07..ec9eaed 100644
--- a/qslwidgets/src/components/Widget.svelte
+++ b/qslwidgets/src/components/Widget.svelte
@@ -6,6 +6,7 @@
   import MediaIndex from "./MediaIndex.svelte";
   import VideoLabeler from "./VideoLabeler.svelte";
   import TimeSeriesLabeler from "./TimeSeriesLabeler.svelte";
+  import VideoSegmentLabeler from "./VideoSegmentLabeler.svelte";
   import ImagePreloader from "./ImagePreloader.svelte";
   import ImageGroupLabeler from "./ImageGroupLabeler.svelte";
   import Labeler from "./Labeler.svelte";
@@ -102,6 +103,23 @@
             on:save={createAction("save")}
             on:showIndex={createAction("index")}
           />
+        {:else if $type == "video-segment-pairs" && Array.isArray($labels)}
+          <VideoSegmentLabeler
+            transitioning={$viewState === "transitioning"}
+            target={urlObjects[0]}
+            bind:config={$config}
+            bind:labels={$labels}
+            metadata={$states[0].metadata}
+            editableConfig={$buttons.config}
+            actions={{ ...$buttons, showIndex: true }}
+            on:next={createAction("next")}
+            on:prev={createAction("prev")}
+            on:delete={createAction("delete")}
+            on:ignore={createAction("ignore")}
+            on:unignore={createAction("unignore")}
+            on:save={createAction("save")}
+            on:showIndex={createAction("index")}
+          />
         {:else if $type == "time-series" && !Array.isArray($labels) && urlObjects.length == 1 && urlObjects[0] && urlObjects[0].plots}
           <TimeSeriesLabeler
             transitioning={$viewState === "transitioning"}
@@ -143,7 +161,7 @@
       {:else if !Array.isArray($labels)}
         <BatchImageLabeler
           transitioning={$viewState === "transitioning"}
-          targets={urlObjects[0]}
+          targets={urlStrings}
           bind:states={$states}
           bind:config={$config}
           bind:labels={$labels}
diff --git a/qslwidgets/src/stories/VideoSegmentLabeler.svelte b/qslwidgets/src/stories/VideoSegmentLabeler.svelte
index 53d7ebe..1802727 100644
--- a/qslwidgets/src/stories/VideoSegmentLabeler.svelte
+++ b/qslwidgets/src/stories/VideoSegmentLabeler.svelte
@@ -7,6 +7,7 @@
 <Labeler>
   <VideoSegmentLabeler
     config={{}}
+    actions={{ save: true }}
     target={{
       video1: {
         target: data.videos[0].url,