refactor: Temporal comparison result data improvements #121

* Refactor using 'close' boolean to 'match' granularity
  * Makes using granularity for future logic easier
  * Easier logging for granularity in summary
* Remove intermediate temporal functions in classes for DRY and so we can use comparison results
* Add Time Detail to match breakdown for more visibility during logging

Author: FoxxMD

src/backend/scrobblers/AbstractScrobbleClient.ts

@@ -19,15 +19,26 @@ import {Logger} from '@foxxmd/winston';
 import { CommonClientConfig } from "../common/infrastructure/config/client/index";
 import { Notifiers } from "../notifier/Notifiers";
 import {FixedSizeList} from 'fixed-size-list';
-import {DeadLetterScrobble, PlayObject, QueuedScrobble, SourceScrobble, TrackStringOptions} from "../../core/Atomic";
+import {
+    DeadLetterScrobble,
+    PlayObject,
+    QueuedScrobble,
+    TA_CLOSE, TA_FUZZY,
+    TrackStringOptions
+} from "../../core/Atomic";
 import {buildTrackString, capitalize, truncateStringToLength} from "../../core/StringUtils";
 import EventEmitter from "events";
 import {compareScrobbleArtists, compareScrobbleTracks, normalizeStr} from "../utils/StringUtils";
 import {hasUpstreamError, UpstreamError} from "../common/errors/UpstreamError";
 import {nanoid} from "nanoid";
 import {ErrorWithCause, messageWithCauses} from "pony-cause";
 import {hasNodeNetworkException} from "../common/errors/NodeErrors";
-import {isPlayTemporallyClose} from "../utils/TimeUtils";
+import {
+    comparePlayTemporally,
+    temporalAccuracyIsAtLeast,
+    temporalAccuracyToString,
+    temporalPlayComparisonSummary
+} from "../utils/TimeUtils";
 
 export default abstract class AbstractScrobbleClient implements Authenticatable {
 
@@ -246,21 +257,13 @@ export default abstract class AbstractScrobbleClient implements Authenticatable
         }
 
         const matchPlayDate = dtInvariantMatches.find((x: ScrobbledPlayObject) => {
-            const [closeTime, fuzzyTime = false] = this.compareExistingScrobbleTime(x.play, playObj);
-            return closeTime;
+            const temporalComparison = comparePlayTemporally(x.play, playObj);
+            return temporalAccuracyIsAtLeast(TA_CLOSE, temporalComparison.match)
         });
 
         return [matchPlayDate, dtInvariantMatches];
     }
 
-    protected compareExistingScrobbleTime = (existing: PlayObject, candidate: PlayObject): [boolean, boolean?] => {
-        let closeTime = isPlayTemporallyClose(existing, candidate);
-        let fuzzyTime = false;
-        if(!closeTime) {
-            fuzzyTime = isPlayTemporallyClose(existing, candidate, {fuzzyDuration: true});
-        }
-        return [closeTime, fuzzyTime];
-    }
     protected compareExistingScrobbleTitle = (existing: PlayObject, candidate: PlayObject): number => {
         return Math.min(compareScrobbleTracks(existing, candidate)/100, 1);
     }
@@ -335,8 +338,13 @@ export default abstract class AbstractScrobbleClient implements Authenticatable
                 //const referenceMatch = referenceApiScrobbleResponse !== undefined && playObjDataMatch(x, referenceApiScrobbleResponse);
 
 
-                const [closeTime, fuzzyTime = false] = this.compareExistingScrobbleTime(x, playObj);
-                const timeMatch = (closeTime ? 1 : (fuzzyTime ? 0.6 : 0));
+                const temporalComparison = comparePlayTemporally(x, playObj);
+                let timeMatch = 0;
+                if(temporalAccuracyIsAtLeast(TA_CLOSE, temporalComparison.match)) {
+                    timeMatch = 1;
+                } else if(temporalComparison.match === TA_FUZZY) {
+                    timeMatch = 0.6;
+                }
 
                 const titleMatch = this.compareExistingScrobbleTitle(x, playObj);
 
@@ -377,7 +385,8 @@ export default abstract class AbstractScrobbleClient implements Authenticatable
                     //`Reference: ${(referenceMatch ? 1 : 0)} * ${REFERENCE_WEIGHT} = ${referenceScore.toFixed(2)}`,
                     artistBreakdown,
                     `Title: ${titleMatch.toFixed(2)} * ${TITLE_WEIGHT} = ${titleScore.toFixed(2)}`,
-                    `Time: ${timeMatch} * ${TIME_WEIGHT} = ${timeScore.toFixed(2)}`,
+                    `Time: (${capitalize(temporalAccuracyToString(temporalComparison.match))}) ${timeMatch} * ${TIME_WEIGHT} = ${timeScore.toFixed(2)}`,
+                    `Time Detail => ${temporalPlayComparisonSummary(temporalComparison, x, playObj)}`,
                     `Score ${score.toFixed(2)} => ${score >= DUP_SCORE_THRESHOLD ? 'Matched!' : 'No Match'}`
                 ];
 

src/backend/scrobblers/MalojaScrobbler.ts

@@ -3,10 +3,7 @@ import request from 'superagent';
 import dayjs from 'dayjs';
 import compareVersions from 'compare-versions';
 import {
-    playObjDataMatch,
-    setIntersection,
     sleep,
-    sortByOldestPlayDate,
     parseRetryAfterSecsFromObj,
 
 } from "../utils";
@@ -25,9 +22,7 @@ import {buildTrackString, capitalize} from "../../core/StringUtils";
 import EventEmitter from "events";
 import normalizeUrl from "normalize-url";
 import {UpstreamError} from "../common/errors/UpstreamError";
-import {ar} from "@faker-js/faker";
 import {ErrorWithCause} from "pony-cause";
-import {isPlayTemporallyClose} from "../utils/TimeUtils";
 
 const feat = ["ft.", "ft", "feat.", "feat", "featuring", "Ft.", "Ft", "Feat.", "Feat", "Featuring"];
 

src/backend/sources/AbstractSource.ts

@@ -29,11 +29,11 @@ import { SourceConfig } from "../common/infrastructure/config/source/sources";
 import {EventEmitter} from "events";
 import {FixedSizeList} from "fixed-size-list";
 import TupleMap from "../common/TupleMap";
-import { PlayObject } from "../../core/Atomic";
+import {PlayObject, TA_CLOSE} from "../../core/Atomic";
 import {buildTrackString, capitalize} from "../../core/StringUtils";
 import {isNodeNetworkException} from "../common/errors/NodeErrors";
 import {ErrorWithCause} from "pony-cause";
-import {isPlayTemporallyClose} from "../utils/TimeUtils";
+import {comparePlayTemporally, temporalAccuracyIsAtLeast} from "../utils/TimeUtils";
 
 export interface RecentlyPlayedOptions {
     limit?: number
@@ -176,7 +176,7 @@ export default abstract class AbstractSource implements Authenticatable {
             });
         }
         for(const list of lists) {
-            const existing = list.find(x => playObjDataMatch(x, play) && isPlayTemporallyClose(x, play));
+            const existing = list.find(x => playObjDataMatch(x, play) && temporalAccuracyIsAtLeast(TA_CLOSE, comparePlayTemporally(x, play).match));
             if(existing) {
                 return existing;
             }

src/backend/sources/JellyfinSource.ts

@@ -13,10 +13,14 @@ import EventEmitter from "events";
 import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState";
 import {Logger} from "@foxxmd/winston";
 import { JellyfinPlayerState } from "./PlayerState/JellyfinPlayerState";
-import { PlayObject } from "../../core/Atomic";
+import {PlayObject, TA_CLOSE} from "../../core/Atomic";
 import {buildTrackString, splitByFirstFound, truncateStringToLength} from "../../core/StringUtils";
 import {source} from "common-tags";
-import {comparePlayTemporally, isPlayTemporallyClose, temporalPlayComparisonSummary} from "../utils/TimeUtils";
+import {
+    comparePlayTemporally,
+    temporalAccuracyIsAtLeast,
+    temporalPlayComparisonSummary
+} from "../utils/TimeUtils";
 
 const shortDeviceId = truncateStringToLength(10, '');
 
@@ -308,7 +312,7 @@ export default class JellyfinSource extends MemorySource {
                         
                         Temporal Comparison => ${temporalPlayComparisonSummary(temporalResult, currPlay, playObj)}`);
                     }
-                    if(temporalResult.close) {
+                    if(temporalAccuracyIsAtLeast(TA_CLOSE,temporalResult.match)) {
                         existingTracked = currPlay;
                     }
                     break;

src/backend/sources/MemorySource.ts

@@ -4,8 +4,6 @@ import {
     sortByOldestPlayDate,
     toProgressAwarePlayObject,
     getProgress,
-    playPassesScrobbleThreshold,
-    timePassesScrobbleThreshold,
     thresholdResultSummary,
     genGroupId,
     genGroupIdStr,
@@ -32,6 +30,7 @@ import {SimpleIntervalJob, Task, ToadScheduler} from "toad-scheduler";
 import {SourceConfig} from "../common/infrastructure/config/source/sources";
 import {EventEmitter} from "events";
 import objectHash from 'object-hash';
+import {timePassesScrobbleThreshold} from "../utils/TimeUtils";
 
 export default class MemorySource extends AbstractSource {
 

src/backend/utils.ts

@@ -7,8 +7,6 @@ import {TimeoutError, WebapiError} from "spotify-web-api-node/src/response-error
 import Ajv, {Schema} from 'ajv';
 import {
     asPlayerStateData,
-    DEFAULT_SCROBBLE_DURATION_THRESHOLD,
-    DEFAULT_SCROBBLE_PERCENT_THRESHOLD,
     NO_DEVICE,
     NO_USER,
     numberFormatOptions,
@@ -23,7 +21,6 @@ import {Request} from "express";
 import pathUtil from "path";
 import {ErrorWithCause, getErrorCause} from "pony-cause";
 import backoffStrategies from '@kenyip/backoff-strategies';
-import {ScrobbleThresholds} from "./common/infrastructure/config/source";
 import {replaceResultTransformer, stripIndentTransformer, TemplateTag, trimResultTransformer} from 'common-tags';
 import {Duration} from "dayjs/plugin/duration.js";
 import {PlayObject} from "../core/Atomic";
@@ -476,41 +473,6 @@ export const getProgress = (initial: ProgressAwarePlayObject, curr: PlayObject):
     return undefined;
 }
 
-export const playPassesScrobbleThreshold = (play: PlayObject, thresholds: ScrobbleThresholds): ScrobbleThresholdResult => {
-    const progressed = Math.round(Math.abs(dayjs().diff(play.data.playDate, 's')));
-    return timePassesScrobbleThreshold(thresholds, progressed, play.data.duration);
-}
-
-export const timePassesScrobbleThreshold = (thresholds: ScrobbleThresholds, secondsTracked: number, playDuration?: number): ScrobbleThresholdResult => {
-    let durationPasses = undefined,
-        durationThreshold: number | null = thresholds.duration ?? DEFAULT_SCROBBLE_DURATION_THRESHOLD,
-        percentPasses = undefined,
-        percentThreshold: number | null = thresholds.percent ?? DEFAULT_SCROBBLE_PERCENT_THRESHOLD,
-        percent: number | undefined;
-
-    if (percentThreshold !== null && playDuration !== undefined && playDuration !== 0) {
-        percent = Math.round(((secondsTracked / playDuration) * 100));
-        percentPasses = percent >= percentThreshold;
-    }
-    if (durationThreshold !== null || percentPasses === undefined) {
-        durationPasses = secondsTracked >= durationThreshold;
-    }
-
-    return {
-        passes: (durationPasses ?? false) || (percentPasses ?? false),
-        duration: {
-            passes: durationPasses,
-            threshold: durationThreshold,
-            value: secondsTracked
-        },
-        percent: {
-            passes: percentPasses,
-            value: percent,
-            threshold: percentThreshold
-        }
-    }
-}
-
 export const thresholdResultSummary = (result: ScrobbleThresholdResult) => {
     const parts: string[] = [];
     if(result.duration.passes !== undefined) {

src/backend/utils/TimeUtils.ts

@@ -1,14 +1,21 @@
-import {PlayObject, TemporalPlayComparison} from "../../core/Atomic";
-import {lowGranularitySources} from "../common/infrastructure/Atomic";
+import {
+    PlayObject,
+    TA_CLOSE,
+    TA_EXACT,
+    TA_FUZZY,
+    TA_NONE,
+    TemporalAccuracy,
+    TemporalPlayComparison
+} from "../../core/Atomic";
+import {
+    DEFAULT_SCROBBLE_DURATION_THRESHOLD,
+    DEFAULT_SCROBBLE_PERCENT_THRESHOLD,
+    lowGranularitySources,
+    ScrobbleThresholdResult
+} from "../common/infrastructure/Atomic";
 import {formatNumber} from "../utils";
-
-export const isPlayTemporallyClose = (existingPlay: PlayObject, candidatePlay: PlayObject, options: {
-    diffThreshold?: number,
-    fuzzyDuration?: boolean,
-    useListRanges?: boolean
-} = {}): boolean => {
-    return comparePlayTemporally(existingPlay, candidatePlay, options).close;
-}
+import {ScrobbleThresholds} from "../common/infrastructure/config/source";
+import {capitalize} from "../../core/StringUtils";
 
 export const temporalPlayComparisonSummary = (data: TemporalPlayComparison, existingPlay?: PlayObject, candidatePlay?: PlayObject) => {
     const parts: string[] = [];
@@ -19,7 +26,7 @@ export const temporalPlayComparisonSummary = (data: TemporalPlayComparison, exis
             parts.push(`Existing: ${existingPlay.data.playDate.toISOString()} - Candidate: ${candidatePlay.data.playDate.toISOString()}`);
         }
     }
-    parts.push(`Close: ${data.close ? 'YES' : 'NO'}`);
+    parts.push(`Temporal Sameness: ${capitalize(temporalAccuracyToString(data.match))}`);
     if (data.date !== undefined) {
         parts.push(`Play Diff: ${formatNumber(data.date.diff, {toFixed: 0})}s (Needed <${data.date.threshold}s)`)
     }
@@ -33,10 +40,10 @@ export const temporalPlayComparisonSummary = (data: TemporalPlayComparison, exis
         if (data.range === false) {
             parts.push('Candidate not played during Existing tracked listening');
         } else {
-            parts.push(`Candidate played during tracked listening range from existing: ${data.range[0].timestamp.format('HH:mm:ssZ')} => ${data.range[1].timestamp.format('HH:mm:ssZ')}`);
+            parts.push(`Candidate played during tracked listening range from Existing ${data.range[0].timestamp.format('HH:mm:ssZ')} => ${data.range[1].timestamp.format('HH:mm:ssZ')}`);
         }
     } else {
-        parts.push('One or both Plays did not have have tracked listening to compare');
+        parts.push('Range Comparison N/A');
     }
     return parts.join(' | ');
 }
@@ -47,7 +54,7 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P
 } = {}): TemporalPlayComparison => {
 
     const result: TemporalPlayComparison = {
-        close: false
+        match: TA_NONE
     };
 
     const {
@@ -94,8 +101,10 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P
         diff: scrobblePlayDiff
     };
 
-    if (scrobblePlayDiff <= playDiffThreshold) {
-        result.close = true;
+    if(scrobblePlayDiff <= 1) {
+        result.match = TA_EXACT;
+    } else if (scrobblePlayDiff <= playDiffThreshold) {
+        result.match = TA_CLOSE;
     }
 
     if (useListRanges && existingRanges !== undefined) {
@@ -105,7 +114,9 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P
         for (const range of existingRanges) {
             if (newPlayDate.isBetween(range.start.timestamp, range.end.timestamp)) {
                 result.range = range;
-                result.close = true;
+                if(!temporalAccuracyIsAtLeast(TA_CLOSE, result.match)) {
+                    result.match = TA_CLOSE;
+                }
                 break;
             }
         }
@@ -116,21 +127,73 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P
 
     // if the source has a duration its possible one play was scrobbled at the beginning of the track and the other at the end
     // so check if the duration matches the diff between the two play dates
-    if (result.close === false && referenceDuration !== undefined && fuzzyDuration) {
+    if (result.match === TA_NONE && referenceDuration !== undefined) {
         result.date.fuzzyDurationDiff = Math.abs(scrobblePlayDiff - referenceDuration);
         if (result.date.fuzzyDurationDiff < 10) { // TODO use finer comparison for this?
-            result.close = true;
+            result.match = TA_FUZZY;
         }
     }
     // if the source has listened duration (maloja) it may differ from actual track duration
     // and its possible (spotify) the candidate play date is set at the end of this duration
     // so check if there is a close match between candidate play date and source + listened for
-    if (result.close === false && referenceListenedFor !== undefined && fuzzyDuration) {
+    if (result.match === TA_NONE && referenceListenedFor !== undefined && fuzzyDuration) {
         result.date.fuzzyListenedDiff = Math.abs(scrobblePlayDiff - referenceListenedFor);
         if (result.date.fuzzyListenedDiff < 10) { // TODO use finer comparison for this?
-            result.close = true;
+            result.match = TA_FUZZY
         }
     }
 
     return result;
 }
+export const timePassesScrobbleThreshold = (thresholds: ScrobbleThresholds, secondsTracked: number, playDuration?: number): ScrobbleThresholdResult => {
+    let durationPasses = undefined,
+        durationThreshold: number | null = thresholds.duration ?? DEFAULT_SCROBBLE_DURATION_THRESHOLD,
+        percentPasses = undefined,
+        percentThreshold: number | null = thresholds.percent ?? DEFAULT_SCROBBLE_PERCENT_THRESHOLD,
+        percent: number | undefined;
+
+    if (percentThreshold !== null && playDuration !== undefined && playDuration !== 0) {
+        percent = Math.round(((secondsTracked / playDuration) * 100));
+        percentPasses = percent >= percentThreshold;
+    }
+    if (durationThreshold !== null || percentPasses === undefined) {
+        durationPasses = secondsTracked >= durationThreshold;
+    }
+
+    return {
+        passes: (durationPasses ?? false) || (percentPasses ?? false),
+        duration: {
+            passes: durationPasses,
+            threshold: durationThreshold,
+            value: secondsTracked
+        },
+        percent: {
+            passes: percentPasses,
+            value: percent,
+            threshold: percentThreshold
+        }
+    }
+}
+
+export const temporalAccuracyIsAtLeast = (expected: TemporalAccuracy, found: TemporalAccuracy): boolean => {
+    if(typeof expected === 'number') {
+        if(typeof found === 'number') {
+            return found <= expected;
+        }
+        return false;
+    }
+    return found === false;
+}
+
+export const temporalAccuracyToString = (acc: TemporalAccuracy): string => {
+    switch(acc) {
+        case 1:
+            return 'exact';
+        case 2:
+            return 'close';
+        case 3:
+            return 'fuzzy';
+        case false:
+            return 'no correlation';
+    }
+}