WIP testing and improvements for artist astring parsing

* Implement faker data generation for artist string
* Add test suite for parsing different styles of artist strings

Author: FoxxMD

src/backend/tests/listenbrainz/listenbrainz.test.ts

@@ -9,18 +9,18 @@ import { UpstreamError } from "../../common/errors/UpstreamError.js";
 import { ListenbrainzApiClient, ListenResponse } from "../../common/vendor/ListenbrainzApiClient.js";
 import { ExpectedResults } from "../utils/interfaces.js";
 import { withRequestInterception } from "../utils/networking.js";
-import artistWithProperJoiner from './correctlyMapped/artistProperHasJoinerInName.json';
+import artistWithProperJoiner from './correctlyMapped/artistProperHasJoinerInName.json' with { type: "json" };
 // correct mappings
-import multiArtistInArtistName from './correctlyMapped/multiArtistInArtistName.json';
-import multiArtistsInTrackName from './correctlyMapped/multiArtistInTrackName.json';
-import multiMappedArtistsWithSingleUserArtist from './correctlyMapped/multiArtistMappingWithSingleRecordedArtist.json';
-import noArtistMapping from './correctlyMapped/noArtistMapping.json';
-import normalizedValues from './correctlyMapped/normalizedName.json';
-import slightlyDifferentNames from './correctlyMapped/trackNameSlightlyDifferent.json';
+import multiArtistInArtistName from './correctlyMapped/multiArtistInArtistName.json' with { type: "json" };
+import multiArtistsInTrackName from './correctlyMapped/multiArtistInTrackName.json' with { type: "json" };
+import multiMappedArtistsWithSingleUserArtist from './correctlyMapped/multiArtistMappingWithSingleRecordedArtist.json' with { type: "json" };
+import noArtistMapping from './correctlyMapped/noArtistMapping.json' with { type: "json" };
+import normalizedValues from './correctlyMapped/normalizedName.json' with { type: "json" };
+import slightlyDifferentNames from './correctlyMapped/trackNameSlightlyDifferent.json' with { type: "json" };
 
 // incorrect mappings
-import incorrectMultiArtistsTrackName from './incorrectlyMapped/multiArtistsInTrackName.json';
-import veryWrong from './incorrectlyMapped/veryWrong.json';
+import incorrectMultiArtistsTrackName from './incorrectlyMapped/multiArtistsInTrackName.json' with { type: "json" };
+import veryWrong from './incorrectlyMapped/veryWrong.json' with { type: "json" };
 
 interface LZTestFixture {
     data: ListenResponse

src/backend/tests/plays/playParsing.test.ts

@@ -0,0 +1,67 @@
+import { loggerTest, loggerDebug, childLogger } from "@foxxmd/logging";
+import chai, { assert, expect } from 'chai';
+import asPromised from 'chai-as-promised';
+import { after, before, describe, it } from 'mocha';
+
+import { asPlays, generateArtistsStr, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js";
+import { parseArtistCredits, parseCredits } from "../../utils/StringUtils.js";
+
+describe('Parsing Artists from String', function() {
+
+    it('Parses Artists from an Artist-like string', function () {
+        for(const i of Array(20)) {
+            const [str, primaries, secondaries] = generateArtistsStr();
+            const credits = parseArtistCredits(str);
+            const allArtists = primaries.concat(secondaries);
+            const parsed = [credits.primary].concat(credits.secondary ?? [])
+            expect(primaries.concat(secondaries),`
+'${str}'
+Expected => ${allArtists.join(' || ')}
+Found    => ${parsed.join(' || ')}`)
+.eql(parsed)
+        }
+    });
+
+    it('Parses singlar Artist with wrapped vs multiple', function () {
+        const [str, primaries, secondaries] = generateArtistsStr({primary: 1, secondary: {num: 2, ft: 'vs', joiner: '/', ftWrap: true}});
+        const credits = parseArtistCredits(str);
+        const moreCredits = parseCredits(str);
+        expect(true).eq(true);
+    });
+
+    describe('When joiner is known', function () {
+
+        it('Parses many primary artists', function () {
+            for(const i of Array(10)) {
+                const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, joiner: '/'}, secondary: 0});
+                const credits = parseArtistCredits(str, ['/']);
+                const allArtists = primaries.concat(secondaries);
+                const parsed = [credits.primary].concat(credits.secondary ?? [])
+                expect(primaries.concat(secondaries),`
+'${str}'
+Expected => ${allArtists.join(' || ')}
+Found    => ${parsed.join(' || ')}`)
+        .eql(parsed)
+            }
+        });
+
+        it('Parses many secondary artists', function () {
+            // fails on -- Peso Pluma / Lil Baby / R. Kelly (featuring TOMORROW X TOGETHER / AC/DC / DaVido)
+            for(const i of Array(10)) {
+                const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, joiner: '/'}, secondary: {joiner: '/', finalJoiner: false}});
+                const credits = parseArtistCredits(str, ['/']);
+                const allArtists = primaries.concat(secondaries);
+                const parsed = [credits.primary].concat(credits.secondary ?? [])
+                expect(primaries.concat(secondaries),`
+'${str}'
+Expected => ${allArtists.join(' || ')}
+Found    => ${parsed.join(' || ')}`)
+        .eql(parsed)
+            }
+        });
+
+
+    });
+
+
+});

src/backend/tests/utils/PlayTestUtils.ts

@@ -5,8 +5,9 @@ import isBetween from "dayjs/plugin/isBetween.js";
 import relativeTime from "dayjs/plugin/relativeTime.js";
 import timezone from "dayjs/plugin/timezone.js";
 import utc from "dayjs/plugin/utc.js";
-import { JsonPlayObject, ObjectPlayData, PlayMeta, PlayObject } from "../../../core/Atomic.js";
+import { FEAT, JOINERS, JOINERS_FINAL, JsonPlayObject, ObjectPlayData, PlayMeta, PlayObject } from "../../../core/Atomic.js";
 import { sortByNewestPlayDate } from "../../utils.js";
+import { arrayListAnd } from '../../../core/StringUtils.js';
 
 dayjs.extend(utc)
 dayjs.extend(isBetween);
@@ -154,3 +155,92 @@ export const generatePlay = (data: ObjectPlayData = {}, meta: PlayMeta = {}): Pl
 export const generatePlays = (numberOfPlays: number, data: ObjectPlayData = {}, meta: PlayMeta = {}): PlayObject[] => {
     return Array.from(Array(numberOfPlays), () => generatePlay(data, meta));
 }
+
+export const generateArtist = () => faker.music.artist;
+
+export const generateArtists = (num?: number, max: number = 3) => {
+    // if(num !== undefined) {
+    //     return Array(num).map(x => faker.music.artist);
+    // }
+    if(num === 0 || max === 0) {
+        return [];
+    }
+    return faker.helpers.multiple(faker.music.artist, {count: {min: num ?? 1, max: num ?? max}});
+}
+
+export interface ArtistGenerateOptions {
+    num?: number
+    max?: number
+    joiner?: string
+    finalJoiner?: false | string
+    spacedJoiners?: boolean
+}
+
+export interface SecondaryArtistGenerateOptions extends ArtistGenerateOptions {
+    ft?: string
+    ftWrap?: boolean
+}
+
+export interface CompoundArtistGenerateOptions {
+    primary?: number | ArtistGenerateOptions
+    secondary?: number | SecondaryArtistGenerateOptions
+}
+
+export const generateArtistsStr = (options: CompoundArtistGenerateOptions = {}): [string, string[], string[]] => {
+
+    const {primary = {}, secondary = {}} = options;
+
+    const primaryOpts: ArtistGenerateOptions = typeof primary === 'number' ? {num: primary} : primary;
+    const secondaryOpts: SecondaryArtistGenerateOptions = typeof secondary === 'number' ? {num: secondary} : secondary;
+
+    const primaryArt = generateArtists(primaryOpts.num, primaryOpts.max)
+    const secondaryArt = generateArtists(secondaryOpts.num, secondaryOpts.max);
+
+
+    const joinerPrimary: string = primaryOpts.joiner ?? faker.helpers.arrayElement(JOINERS);
+    let finalJoinerPrimary: string = joinerPrimary;
+    if(primaryOpts.finalJoiner !== false) {
+        if(primaryOpts.finalJoiner === undefined) {
+            if(joinerPrimary === ',') {
+                finalJoinerPrimary = faker.helpers.arrayElement(JOINERS_FINAL);
+            }
+        
+        } else {
+            finalJoinerPrimary = primaryOpts.finalJoiner;
+        }
+    }
+
+    const primaryStr = arrayListAnd(primaryArt, joinerPrimary, finalJoinerPrimary, primaryOpts.spacedJoiners);
+
+    if(secondaryArt.length === 0) {
+        return [primaryStr, primaryArt, []];
+    }
+
+    const joinerSecondary: string = secondaryOpts.joiner ?? faker.helpers.arrayElement(JOINERS);
+    let finalJoinerSecondary: string = joinerSecondary;
+    if(secondaryOpts.finalJoiner !== false) {
+        if(secondaryOpts.finalJoiner === undefined) {
+            if(joinerSecondary === ',') {
+                finalJoinerSecondary = faker.helpers.arrayElement(JOINERS_FINAL);
+            }
+        } else {
+            finalJoinerSecondary = secondaryOpts.finalJoiner;
+        }
+    }
+
+    const secondaryStr = arrayListAnd(secondaryArt, joinerSecondary, finalJoinerSecondary, secondaryOpts.spacedJoiners);
+    const ft = secondaryOpts.ft ?? faker.helpers.arrayElement(FEAT);
+    let sec = `${ft} ${secondaryStr}`;
+    let wrap: boolean;
+    if(secondaryOpts.ftWrap !== undefined) {
+        wrap = secondaryOpts.ftWrap;
+    } else {
+        wrap = faker.datatype.boolean();
+    }
+    if(wrap) {
+        sec = `(${sec})`;
+    }
+    const artistStr = `${primaryStr} ${sec}`;
+
+    return [artistStr, primaryArt, secondaryArt];
+}

src/core/Atomic.ts

@@ -207,7 +207,7 @@ export interface SourcePlayerObj {
     play: PlayObject,
     playFirstSeenAt?: string,
     playLastUpdatedAt?: string,
-    playerLastUpdatedAt: string
+    playerLastUpdatedAt: strin
     position?: Second
     listenedDuration: Second
     status: {
@@ -273,4 +273,13 @@ export interface URLData {
     url: URL
     normal: string
     port: number
-}
+}
+
+export type Joiner = ',' | '&' | '/' | '\\' | string;
+export const JOINERS: Joiner[] = [',','&','/','\\'];
+
+export type FinalJoiners = '&';
+export const JOINERS_FINAL: FinalJoiners[] = ['&'];
+
+export type Feat = 'ft' | 'feat' | 'vs' | 'ft.' | 'feat.' | 'vs.' | 'featuring'
+export const FEAT: Feat[] = ['ft','feat','vs','ft.','feat.','vs.','featuring'];

src/core/StringUtils.ts

@@ -196,3 +196,29 @@ export const combinePartsToString = (parts: any[], glue: string = '-'): string |
     }
     return undefined;
 }
+
+export const arrayListOxfordAnd = (list: string[], joiner: string, finalJoiner: string, spaced: boolean = true): string => {
+    if(list.length === 1) {
+        return list[0];
+    }
+    const start = list.slice(0, list.length - 1);
+    const end = list.slice(list.length - 1);
+
+    const joinerProper = joiner === ',' ? ', ' : (spaced ? ` ${joiner} ` : joiner);
+    const finalProper = spaced ? ` ${finalJoiner} ` : finalJoiner;
+
+    return [start.join(joinerProper), end].join(joiner === ',' && spaced ? `,${finalProper}` : finalProper);
+}
+
+export const arrayListAnd = (list: string[], joiner: string, finalJoiner: string, spaced: boolean = true): string => {
+    if(list.length === 1) {
+        return list[0];
+    }
+    const start = list.slice(0, list.length - 1);
+    const end = list.slice(list.length - 1);
+
+    const joinerProper = joiner === ',' ? ', ' : (spaced ? ` ${joiner} ` : joiner);
+    const finalProper = spaced ? ` ${finalJoiner} ` : finalJoiner;
+
+    return [start.join(joinerProper), end].join(finalProper);
+}