From 3a89125d55cbb2c0e9d4b15b572f8724556dba52 Mon Sep 17 00:00:00 2001 From: DerLev Date: Wed, 27 Mar 2024 14:59:37 +0100 Subject: [PATCH] adding fixSongMismatch function --- README.md | 2 +- database/project/apiKey.json | 3 + .../ProjectCredentials.ts | 4 + .../src/firestoreDocumentTypes/SpotifyApi.ts | 50 +++++ functions/src/helpers/validateAuthToken.ts | 39 ++++ .../src/playlistScraping/fixSongMismatch.ts | 191 ++++++++++++++++++ functions/src/playlistScraping/index.ts | 1 + 7 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 database/project/apiKey.json create mode 100644 functions/src/helpers/validateAuthToken.ts create mode 100644 functions/src/playlistScraping/fixSongMismatch.ts diff --git a/README.md b/README.md index ab4b87b..c7f8956 100644 --- a/README.md +++ b/README.md @@ -9,4 +9,4 @@ An application for creating playlists from the German radio station 1LIVE - [ ] Add a webapp (most likely React/Next.js) to give users more insights on the playlists (show gathered data) - [x] Add a job for filling up the backlog *(already done by hand)* - [ ] Expand to other 1LIVE plalists (Neu für den Sektor, DIGGI) *(added NfdS)* -- [ ] Add function to replace tracks that have been wrongly mapped +- [x] Add function to replace tracks that have been wrongly mapped diff --git a/database/project/apiKey.json b/database/project/apiKey.json new file mode 100644 index 0000000..4f5a56f --- /dev/null +++ b/database/project/apiKey.json @@ -0,0 +1,3 @@ +{ + "invocationsKey": "Bearer auth token allowed to invoke HTTP functions" +} diff --git a/functions/src/firestoreDocumentTypes/ProjectCredentials.ts b/functions/src/firestoreDocumentTypes/ProjectCredentials.ts index 276b691..3d90695 100644 --- a/functions/src/firestoreDocumentTypes/ProjectCredentials.ts +++ b/functions/src/firestoreDocumentTypes/ProjectCredentials.ts @@ -22,3 +22,7 @@ export type PlaylistsDocument = { weeklyTop100: string newReleases: string } + +export type ApiKeyDocument = { + invocationsKey: string +} diff --git a/functions/src/firestoreDocumentTypes/SpotifyApi.ts b/functions/src/firestoreDocumentTypes/SpotifyApi.ts index 36db090..b9efbf7 100644 --- a/functions/src/firestoreDocumentTypes/SpotifyApi.ts +++ b/functions/src/firestoreDocumentTypes/SpotifyApi.ts @@ -171,3 +171,53 @@ export type UpdatePlaylistItems = { export type RemovePlaylistItems = UpdatePlaylistItems export type AddPlaylistItems = UpdatePlaylistItems + +export type GetTrack = { + album: { + album_type: string + total_tracks: number + available_markets: string[] + external_urls: { + spotify: string + } + href: string + id: string + images: ImageObject[] + name: string + release_date: string + release_date_precision: "year" | "month" | "day" + restrictions?: { + reason: string + } + type: "album" + uri: string + artists: SimplifiedArtistObject[] + } + artists: ArtistObject[] + available_markets: string[] + disc_number: number + duration_ms: number + explicit: boolean + external_ids?: { + isrc?: string + ean?: string + upc?: string + } + external_urls: { + spotify: string + } + href: string + id: string + is_playable: boolean + linked_from?: object + restrictions?: { + reason: string + } + name: string + populartiy: number + preview_url: string | null + track_number: number + type: "track" + uri: string + is_local: boolean +} diff --git a/functions/src/helpers/validateAuthToken.ts b/functions/src/helpers/validateAuthToken.ts new file mode 100644 index 0000000..d471cb1 --- /dev/null +++ b/functions/src/helpers/validateAuthToken.ts @@ -0,0 +1,39 @@ +import {HttpsError, type Request} from "firebase-functions/v2/https"; +import type { + ApiKeyDocument, +} from "../firestoreDocumentTypes/ProjectCredentials"; +import type {Response} from "express"; +import {db} from "./firebase"; +import firestoreConverter from "./firestoreConverter"; + +const validateAuthToken = async (req: Request, res: Response) => { + const apiKeyDoc = (await db.collection("project").doc("apiKey") + .withConverter(firestoreConverter()).get()).data(); + + if (apiKeyDoc === undefined) { + res.status(500).json({code: 500, message: "ApiKeyDoc does not exist"}); + throw new HttpsError("internal", "ApiKeyDoc does not exist"); + } + + const reqApiKey = req.headers["authorization"]?.split(" ")[1]; + + if (!reqApiKey?.length) { + res.status(401).json({ + code: 401, + message: "Add an API key to the Auth header", + }); + throw new HttpsError("unauthenticated", "No API key in Auth header"); + } + + if (reqApiKey !== apiKeyDoc.invocationsKey) { + res.status(401).json({ + code: 401, + message: "The supplied API key does not exist", + }); + throw new HttpsError("unauthenticated", "Supplied API key does not exist"); + } + + return; +}; + +export default validateAuthToken; diff --git a/functions/src/playlistScraping/fixSongMismatch.ts b/functions/src/playlistScraping/fixSongMismatch.ts new file mode 100644 index 0000000..4c238b7 --- /dev/null +++ b/functions/src/playlistScraping/fixSongMismatch.ts @@ -0,0 +1,191 @@ +import {HttpsError, onRequest} from "firebase-functions/v2/https"; +import onlyAllowMethods from "../helpers/onlyAllowMethods"; +import validateAuthToken from "../helpers/validateAuthToken"; +import {db} from "../helpers/firebase"; +import firestoreConverter from "../helpers/firestoreConverter"; +import type {SongsCollection} from "../firestoreDocumentTypes/SongsCollection"; +import type { + TracksSubcollection, +} from "../firestoreDocumentTypes/PlaylistsCollection"; +import type { + ArtistsCollection, +} from "../firestoreDocumentTypes/ArtistsCollection"; +import {getClientToken} from "../helpers/spotifyTokens"; +import type { + GetTrack, + SpotifyError, +} from "../firestoreDocumentTypes/SpotifyApi"; +import {FieldValue, Timestamp} from "firebase-admin/firestore"; +import generateSearchString from "../helpers/generateSearchString"; + +export const fixSongMismatch = onRequest(async (req, res) => { + onlyAllowMethods(req, res, ["PATCH"]); + + await validateAuthToken(req, res); + + const queryUid = req.query.uid?.toString(); + const queryTrackUri = req.query.newTrackUri?.toString(); + + if (!queryUid?.length || !queryTrackUri?.length) { + res.status(400).json({ + code: 400, + message: "uid and newTrackUri must be supplied", + }); + throw new HttpsError( + "invalid-argument", + "uid and newTrackUri must be supplied" + ); + } + + const spotifyAccessToken = await getClientToken(); + + const songsCollection = db.collection("songs") + .withConverter(firestoreConverter()); + + const songDoc = await songsCollection.doc(queryUid).get(); + const songDocData = songDoc.data(); + + if (songDocData === undefined) { + res.status(404).json({ + code: 404, + message: "The SongDoc could not be found", + }); + throw new HttpsError("not-found", "SongDoc not found"); + } + + const affectedTracks = (await db.collectionGroup("tracks") + .withConverter(firestoreConverter()) + .where("trackUid", "==", queryUid).get()).docs; + + const songNewUriQuery = await songsCollection + .where("spotifyTrackUri", "==", queryTrackUri).limit(1).get(); + + if (songNewUriQuery.empty) { + /* If the song was just badly mapped replace the doc */ + const queryTrackId = queryTrackUri.split(":")[2]; + + /* Calling Spotify API */ + const result = await fetch( + "https://api.spotify.com/v1/tracks/" + queryTrackId, { + method: "GET", + headers: { + "Authorization": "Bearer " + spotifyAccessToken, + }, + }).then((res) => res.json()) as GetTrack | SpotifyError; + + /* Handle errors returned by the Spotify Web API */ + if ("error" in result) { + throw new Error( + "Spotify API returned an error: " + result.error.message + ); + } + + /* Get the unix epoch mills of the track release */ + /* Spotify returns "2024", "2024-03", or "2024-03-26" */ + const trackReleaseDateArr = result.album.release_date + .split("-"); + const trackReleaseDateMills = Date.parse(trackReleaseDateArr[0] + "-" + + (trackReleaseDateArr[1] || "01") + "-" + + (trackReleaseDateArr[2] || "01")); + + /* Isolate the artists and only get their names */ + const artists = result.artists; + const artistNames = artists.map((item) => { + return item.name; + }); + + const artistsCollection = db.collection("artists") + .withConverter(firestoreConverter()); + + /* Get the artists from db -> return doc ref */ + const artistRefsPromises = artists.map(async (item) => { + /* Get the artist's doc by Spotify URI */ + const artistQuery = await artistsCollection + .where("spotifyUri", "==", item.uri).limit(1).get(); + + if (artistQuery.empty) { + /* Create artist if not in db */ + const newDoc = await artistsCollection.add({ + genres: [], + lastUpdated: FieldValue.serverTimestamp(), + name: item.name, + spotifyUri: item.uri, + searchString: generateSearchString(item.name), + }); + + const newDocRef = (await newDoc.get()).ref; + return newDocRef; + } else { + return artistQuery.docs[0].ref; + } + }); + + /* Execute artist search */ + const artistRefs = await Promise.all(artistRefsPromises); + + /* Add song to db */ + await songDoc.ref.update({ + title: result.name, + coverArt: result.album.images, + duration: result.duration_ms, + explicit: result.explicit, + released: Timestamp.fromMillis(trackReleaseDateMills), + releasePrecision: result.album.release_date_precision, + spotifyTrackUri: result.uri, + artists: artistRefs, + searchString: generateSearchString( + result.name, + artistNames.join(" & ") + ), + }); + + const fixAffectedTracksPromsies = affectedTracks.map(async (track) => { + return await track.ref.update({ + artists: artistNames, + duration: result.duration_ms, + explicit: result.explicit, + spotifyTrackUri: result.uri, + title: result.name, + }); + }); + + await Promise.all(fixAffectedTracksPromsies); + } else { + /* If the song was badly mapped but exists in db */ + const songFromDbDoc = songNewUriQuery.docs[0]; + const songFromDbData = songFromDbDoc.data(); + + const songFromDbArtistsPromises = songFromDbData.artists.map( + async (artist) => { + return (await artist.get()).data(); + } + ); + + const songFromDbArtists = (await Promise.all(songFromDbArtistsPromises)) + .filter((item) => item !== undefined) as ArtistsCollection[]; + + const fixAffectedTracksPromsies = affectedTracks.map(async (track) => { + return await track.ref.update({ + artists: songFromDbArtists.map((artist) => artist.name), + duration: songFromDbData.duration, + explicit: songFromDbData.explicit, + ref: songFromDbDoc.ref, + spotifyTrackUri: songFromDbData.spotifyTrackUri, + title: songFromDbData.title, + trackUid: songFromDbDoc.id, + }); + }); + + await Promise.all(fixAffectedTracksPromsies); + + await songFromDbDoc.ref.update({ + searchString: songDocData.searchString, + }); + + if (songFromDbDoc.id !== songDoc.id) { + await songDoc.ref.delete(); + } + } + + res.json({code: 200, message: "Track URI replaced!"}); +}); diff --git a/functions/src/playlistScraping/index.ts b/functions/src/playlistScraping/index.ts index ceb2cce..ed3d462 100644 --- a/functions/src/playlistScraping/index.ts +++ b/functions/src/playlistScraping/index.ts @@ -1,2 +1,3 @@ export * from "./scrapeSchedule"; export * from "./scrapeNewReleasesProgramm"; +export * from "./fixSongMismatch";