adding daily playlist from 1LIVE DIGGI station

DerLev · Mar 27, 2024 · e1e1172 · e1e1172
1 parent 3a89125
commit e1e1172
Show file tree

Hide file tree

Showing 7 changed files with 171 additions and 86 deletions.
diff --git a/README.md b/README.md
@@ -8,5 +8,6 @@ An application for creating playlists from the German radio station 1LIVE
 - [x] Add logic to interface with the Spotify API and create pre-made playlists on my personal account
 - [ ] Add a webapp (most likely React/Next.js) to give users more insights on the playlists (show gathered data)
 - [x] Add a job for filling up the backlog *(already done by hand)*
-- [ ] Expand to other 1LIVE plalists (Neu für den Sektor, DIGGI) *(added NfdS)*
+- [x] Expand to other 1LIVE plalists (Neu für den Sektor, DIGGI)
 - [x] Add function to replace tracks that have been wrongly mapped
+- [ ] Add a public facing API for everyone to use
diff --git a/database/project/playlists.json b/database/project/playlists.json
@@ -1,4 +1,5 @@
 {
   "weeklyTop100": "Spotify playlist id",
-  "newReleases": "Spotify playlist id"
+  "newReleases": "Spotify playlist id",
+  "diggiWeeklyTop100": "Spotify playlist id"
 }
diff --git a/functions/src/firestoreDocumentTypes/ProjectCredentials.ts b/functions/src/firestoreDocumentTypes/ProjectCredentials.ts
@@ -21,6 +21,7 @@ export type ProjectCredentials = {
 export type PlaylistsDocument = {
   weeklyTop100: string
   newReleases: string
+  diggiWeeklyTop100: string
 }
 
 export type ApiKeyDocument = {

diff --git a/functions/src/helpers/get1LiveHour.ts b/functions/src/helpers/get1LiveHour.ts
@@ -1,14 +1,22 @@
 import {JSDOM} from "jsdom";
 
-const get1LiveHour = async (date: string, hour: number) => {
+const get1LiveHour = async (
+  date: string,
+  hour: number,
+  station: "1live" | "1liveDiggi" = "1live"
+) => {
   const body = new URLSearchParams();
   body.append("playlistSearch_date", date);
   body.append("playlistSearch_hours", hour.toString().padStart(2, "0"));
   body.append("playlistSearch_minutes", "30");
   body.append("submit", "suchen");
 
+  const requestUrl = station === "1live" ?
+    "https://www1.wdr.de/radio/1live/musik/playlist/index.jsp" :
+    "https://www1.wdr.de/radio/1live-diggi/onair/1live-diggi-playlist/index.jsp";
+
   const response = await fetch(
-    "https://www1.wdr.de/radio/1live/musik/playlist/index.jsp",
+    requestUrl,
     {
       method: "POST",
       body: body,

diff --git a/functions/src/playlistScraping/scrapeBackfill.ts b/functions/src/playlistScraping/scrapeBackfill.ts
@@ -13,31 +13,39 @@ import type {
 } from "../firestoreDocumentTypes/CategoriesCollection";
 import {HttpsError, onRequest} from "firebase-functions/v2/https";
 import onlyAllowMethods from "../helpers/onlyAllowMethods";
+import validateAuthToken from "../helpers/validateAuthToken";
 
 export const scrapeBackfill = onRequest(async (req, res) => {
   onlyAllowMethods(req, res, ["POST"]);
 
-  const queryHour = req.query.hour?.toString();
-  if (!queryHour?.match(/\d{2}/) || Number.isNaN(Number(queryHour))) {
-    res.status(400).json({code: 400, message: "hour must be 2 digit integer"});
+  await validateAuthToken(req, res);
+
+  const queryDate = req.query.date?.toString();
+  if (!queryDate?.match(/\d{4}-\d{2}-\d{2}/)) {
+    res.status(400).json({code: 400, message: "Date must be a valid date"});
     throw new HttpsError(
       "invalid-argument",
-      "Hour must be a valid 2 digit integer"
+      "Date must be a valid a valid date"
     );
   }
 
-  const currentDateTime = new Date();
-
-  /* If CET/CEST is between 7:00 and 20:00 continue */
   const spotifyApiToken = await getClientToken();
 
-  const fetchFromHour = Number(queryHour);
-  const fetchFromDay = currentDateTime.getFullYear() + "-" +
-    (currentDateTime.getMonth() + 1).toString().padStart(2, "0") + "-" +
-    currentDateTime.getDate().toString().padStart(2, "0");
+  const fetchFromDay = queryDate;
 
-  const hourPlaylistResult = await get1LiveHour(fetchFromDay, fetchFromHour);
-  const convertedResult = hourPlaylistResult.map(
+  const stationPlaylistResultPromises = Array.from(Array(13).keys())
+    .map(async (_, index) => {
+      return await get1LiveHour(fetchFromDay, index + 6, "1liveDiggi")
+        .catch(() => undefined);
+    });
+  const stationPlaylistResult = (await Promise
+    .all(stationPlaylistResultPromises)).flat()
+    .filter((item) => item !== undefined) as {
+      title: string
+      artist: string
+      played: Date
+    }[];
+  const convertedResult = stationPlaylistResult.map(
     (track) => ({...track, played: Timestamp.fromDate(track.played)})
   );
 
@@ -52,32 +60,33 @@ export const scrapeBackfill = onRequest(async (req, res) => {
   /* Hard coded category */
   const categoryRef = (await db.collection("categories")
     .withConverter(firestoreConverter<CategoriesCollection>())
-    .doc("zTTb3AvkFPz0aUuyo02c").get()).ref;
+    .doc("kVxWJAElj0IliGqSKdof").get()).ref;
 
   /* Unix epoch mills for playlist creation date */
   const createdDateMills = Date.parse(fetchFromDay);
 
-  const playlistQuery = await playlistsCollection.where(
-    "name",
-    "==",
-    "1LIVE playlist - " + fetchFromDay
-  ).where("createdBy", "==", "system").limit(1).get();
+  // const playlistQuery = await playlistsCollection.where(
+  //   "name",
+  //   "==",
+  //   "1LIVE DIGGI playlist - " + fetchFromDay
+  // ).where("createdBy", "==", "system").where("category", "==", categoryRef)
+  //   .limit(1).get();
 
-  let playlistDoc: FirebaseFirestore
-    .DocumentReference<PlaylistsCollection> | undefined = undefined;
+  // let playlistDoc: FirebaseFirestore
+  //   .DocumentReference<PlaylistsCollection> | undefined = undefined;
 
   /* Create the playlist doc if it does not exist */
-  if (playlistQuery.empty) {
-    playlistDoc = await playlistsCollection.add({
-      category: categoryRef,
-      createdBy: "system",
-      lastUpdate: FieldValue.serverTimestamp(),
-      name: "1LIVE playlist - " + fetchFromDay,
-      date: Timestamp.fromMillis(createdDateMills),
-    });
-  } else {
-    playlistDoc = playlistQuery.docs[0].ref;
-  }
+  // if (playlistQuery.empty) {
+  const playlistDoc = await playlistsCollection.add({
+    category: categoryRef,
+    createdBy: "system",
+    lastUpdate: FieldValue.serverTimestamp(),
+    name: "1LIVE DIGGI playlist - " + fetchFromDay,
+    date: Timestamp.fromMillis(createdDateMills),
+  });
+  // } else {
+  //   playlistDoc = playlistQuery.docs[0].ref;
+  // }
 
   /* Add the tracks to the playlist */
   const tracksSubcollection = playlistDoc.collection("tracks")
@@ -89,5 +98,5 @@ export const scrapeBackfill = onRequest(async (req, res) => {
 
   await Promise.all(tracksPromises);
 
-  res.json({code: 200, message: "Backfill completed"});
+  res.json({code: 200, message: "Backfill completed", date: queryDate});
 });
diff --git a/functions/src/playlistScraping/scrapeSchedule.ts b/functions/src/playlistScraping/scrapeSchedule.ts
@@ -14,25 +14,19 @@ import type {
 } from "../firestoreDocumentTypes/CategoriesCollection";
 import {getTimezoneOffset} from "date-fns-tz";
 
-/* NOTE: Scheduler triggers on UTC time */
-export const scrapeSchedule = onSchedule("0 5-19 * * *", async () => {
-  /* Evaluate whether the function should trigger */
-  /* NOTE: needed for TZ's daylight savings time */
-  const centralEuropeOffset = getTimezoneOffset("Europe/Berlin", new Date());
-  const currentDateTime = new Date(Date.now() + centralEuropeOffset);
-  if (
-    !(7 <= currentDateTime.getHours()) || !(currentDateTime.getHours() <= 19)
-  ) return;
-
-  /* If CET/CEST is between 7:00 and 19:00 continue */
-  const spotifyApiToken = await getClientToken();
-
-  const fetchFromHour = currentDateTime.getHours() - 1;
-  const fetchFromDay = currentDateTime.getFullYear() + "-" +
-    (currentDateTime.getMonth() + 1).toString().padStart(2, "0") + "-" +
-    currentDateTime.getDate().toString().padStart(2, "0");
-
-  const hourPlaylistResult = await get1LiveHour(fetchFromDay, fetchFromHour);
+const scrapePlaylist = async (
+  spotifyApiToken: string,
+  fetchFromDay: string,
+  fetchFromHour: number,
+  station: "1live" | "1liveDiggi",
+  resultCategory: string,
+  namePrefix: string
+) => {
+  const hourPlaylistResult = await get1LiveHour(
+    fetchFromDay,
+    fetchFromHour,
+    station
+  );
   const convertedResult = hourPlaylistResult.map(
     (track) => ({...track, played: Timestamp.fromDate(track.played)})
   );
@@ -48,15 +42,15 @@ export const scrapeSchedule = onSchedule("0 5-19 * * *", async () => {
   /* Hard coded category */
   const categoryRef = (await db.collection("categories")
     .withConverter(firestoreConverter<CategoriesCollection>())
-    .doc("zTTb3AvkFPz0aUuyo02c").get()).ref;
+    .doc(resultCategory).get()).ref;
 
   /* Unix epoch mills for playlist creation date */
   const createdDateMills = Date.parse(fetchFromDay);
 
   const playlistQuery = await playlistsCollection.where(
     "name",
     "==",
-    "1LIVE playlist - " + fetchFromDay
+    namePrefix + " - " + fetchFromDay
   ).where("createdBy", "==", "system").limit(1).get();
 
   let playlistDoc: FirebaseFirestore
@@ -68,7 +62,7 @@ export const scrapeSchedule = onSchedule("0 5-19 * * *", async () => {
       category: categoryRef,
       createdBy: "system",
       lastUpdate: FieldValue.serverTimestamp(),
-      name: "1LIVE playlist - " + fetchFromDay,
+      name: namePrefix + " - " + fetchFromDay,
       date: Timestamp.fromMillis(createdDateMills),
     });
   } else {
@@ -88,6 +82,45 @@ export const scrapeSchedule = onSchedule("0 5-19 * * *", async () => {
   });
 
   await Promise.all(tracksPromises);
+};
+
+/* NOTE: Scheduler triggers on UTC time */
+export const scrapeSchedule = onSchedule("0 5-19 * * *", async () => {
+  /* Evaluate whether the function should trigger */
+  /* NOTE: needed for TZ's daylight savings time */
+  const centralEuropeOffset = getTimezoneOffset("Europe/Berlin", new Date());
+  const currentDateTime = new Date(Date.now() + centralEuropeOffset);
+  if (
+    !(7 <= currentDateTime.getHours()) || !(currentDateTime.getHours() <= 19)
+  ) return;
+
+  /* If CET/CEST is between 7:00 and 19:00 continue */
+  const spotifyApiToken = await getClientToken();
+
+  const fetchFromHour = currentDateTime.getHours() - 1;
+  const fetchFromDay = currentDateTime.getFullYear() + "-" +
+    (currentDateTime.getMonth() + 1).toString().padStart(2, "0") + "-" +
+    currentDateTime.getDate().toString().padStart(2, "0");
+
+  /* Scrape from 1LIVE */
+  await scrapePlaylist(
+    spotifyApiToken,
+    fetchFromDay,
+    fetchFromHour,
+    "1live",
+    "zTTb3AvkFPz0aUuyo02c",
+    "1LIVE playlist"
+  );
+
+  /* Scrape from 1LIVE DIGGI */
+  await scrapePlaylist(
+    spotifyApiToken,
+    fetchFromDay,
+    fetchFromHour,
+    "1liveDiggi",
+    "kVxWJAElj0IliGqSKdof",
+    "1LIVE DIGGI playlist"
+  );
 
   return;
 });