From f1664c8dd691d4e11d0c14479c0f6644c033df5b Mon Sep 17 00:00:00 2001 From: Nikkel Mollenhauer Date: Tue, 19 Sep 2023 22:56:57 +0100 Subject: [PATCH 01/32] Updated heading --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c5ffba5..1ff9a3e4 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Did you find any bugs with the version you tested? Please let me know by [openin - The bundled extension files will be placed in the `dist/` directories. - You can load the extension in your browser by following the instructions below. -#### Chrome/Chromium +#### Chromium - Open the Extension Management page by navigating to `chrome://extensions`. - Make sure that you have enabled developer mode. From 5473ac956c22197887d961b6a90e485a9c771bb7 Mon Sep 17 00:00:00 2001 From: Nikkel Mollenhauer Date: Tue, 19 Sep 2023 22:57:21 +0100 Subject: [PATCH 02/32] Updated changelog --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ffc0f4..23e64118 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ # Changelog -## v2.2.3 +## v2.3.0 (Unreleased) + + +## v2.2.3 - Error messages now include the current channel ID to help with reporting an issue. - Some changes in how data that is sent to the database is handled, to prepare for a future security update. - ## v2.2.2 From f3e116fa4211d934aa1ead52b93b5517f1cfaafa Mon Sep 17 00:00:00 2001 From: Nikkel Mollenhauer <57323886+NikkelM@users.noreply.github.com> Date: Wed, 20 Sep 2023 00:03:16 +0100 Subject: [PATCH 03/32] WIP for multiple video types in local playlistInfo --- src/shuffleVideo.js | 86 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index 8afade71..777b8685 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -89,7 +89,8 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE (!databaseSharing && ((playlistInfo["lastAccessedLocally"] ?? new Date(0).toISOString()) < addHours(new Date(), -48).toISOString()))) { console.log(`Local uploads playlist for this channel may be outdated.${databaseSharing ? " Updating from the database..." : ""}`); - playlistInfo = databaseSharing ? await tryGetPlaylistFromDB(uploadsPlaylistId) : {}; + // Try to get an updated version of the playlist, but keep the information about locally known videos and shorts + playlistInfo = databaseSharing ? await tryGetPlaylistFromDB(uploadsPlaylistId, playlistInfo) : {}; // The playlist does not exist in the database (==it was deleted since the user last fetched it). Get it from the API. // With the current functionality and db rules, this shouldn't happen, except if the user has opted out of database sharing. @@ -110,10 +111,8 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE // Update the remaining user quota in the configSync await setSyncStorageValue("userQuotaRemainingToday", Math.max(0, userQuotaRemainingToday)); - // To prevent potential TypeErrors later on, assign an empty newVideos object if it doesn't exist - if (!playlistInfo["newVideos"]) { - playlistInfo["newVideos"] = {}; - } + // Validate that all required keys exist in the playlistInfo object + validatePlaylistInfo(playlistInfo); let chosenVideos, encounteredDeletedVideos; ({ chosenVideos, playlistInfo, shouldUpdateDatabase, encounteredDeletedVideos } = await chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpdateDatabase)); @@ -127,14 +126,14 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE // If any videos need to be deleted, this should be the union of videos, newvideos, minus the videos to delete if (encounteredDeletedVideos) { console.log("Some videos need to be deleted from the database. All current videos will be uploaded to the database..."); - videosToDatabase = Object.assign({}, playlistInfo["videos"], playlistInfo["newVideos"] ?? {}); + videosToDatabase = Object.assign({}, getAllVideosFromLocalPlaylist(playlistInfo), playlistInfo["newVideos"] ?? {}); } else { // Otherwise, we want to only upload new videos. If there are no "newVideos", we upload all videos, as this is the first time we are uploading the playlist console.log("Uploading new video IDs to the database..."); if (getLength(playlistInfo["newVideos"] ?? {}) > 0) { videosToDatabase = playlistInfo["newVideos"]; } else { - videosToDatabase = playlistInfo["videos"]; + videosToDatabase = getAllVideosFromLocalPlaylist(playlistInfo); } } @@ -148,7 +147,10 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE console.log("Saving playlist to local storage..."); // We can now join the new videos with the old ones - playlistInfo["videos"] = Object.assign({}, playlistInfo["videos"], playlistInfo["newVideos"] ?? {}); + // TODO: This is where we add the new videos to the "unknownType" key, instead of joining + // playlistInfo["videos"] = Object.assign({}, playlistInfo["videos"], playlistInfo["newVideos"] ?? {}); + // TODO: Make sure the unknownType key exists here + playlistInfo["videos"]["unknownType"] = Object.assign({}, playlistInfo["videos"]["unknownType"], playlistInfo["newVideos"] ?? {}); // Only save the wanted keys const playlistInfoForLocalStorage = { @@ -156,9 +158,13 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE "lastAccessedLocally": new Date().toISOString(), "lastFetchedFromDB": playlistInfo["lastFetchedFromDB"] ?? new Date(0).toISOString(), "lastVideoPublishedAt": playlistInfo["lastVideoPublishedAt"] ?? new Date(0).toISOString().slice(0, 19) + 'Z', + // TODO: New shorts format here (separate keys) - it should work like this "videos": playlistInfo["videos"] ?? {} }; + // TODO: Remove debug code + console.log(playlistInfoForLocalStorage); + return; await savePlaylistToLocalStorage(uploadsPlaylistId, playlistInfoForLocalStorage); await setSyncStorageValue("numShuffledVideosTotal", configSync.numShuffledVideosTotal + 1); @@ -175,7 +181,7 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE // --------------- Private --------------- // ---------- Database ---------- // Try to get the playlist from the database. If it does not exist, return an empty dictionary. -async function tryGetPlaylistFromDB(playlistId) { +async function tryGetPlaylistFromDB(playlistId, localPlaylistInfo = null) { const msg = { command: "getPlaylistFromDB", data: playlistId @@ -217,6 +223,28 @@ async function tryGetPlaylistFromDB(playlistId) { playlistInfo["lastFetchedFromDB"] = new Date().toISOString(); + if (!localPlaylistInfo) { + // Since we just fetched the playlist, we do not have any info locally, so all videos are "unknownType" + const videosCopy = playlistInfo["videos"]; + playlistInfo["videos"] = {}; + playlistInfo["videos"]["unknownType"] = videosCopy; + playlistInfo["videos"]["knownVideos"] = {}; + playlistInfo["videos"]["knownShorts"] = {}; + } else { + // Else, we need to find out which videos are new, and add them to the unknownType key + // It is also possible that there are videos in the localPlaylistInfo which do not exist in the database anymore, so we need to delete them + // If a video is in localPlaylistInfo but not in the database, it needs to be removed from the localPlaylistInfo + // If a video is in the database but not in localPlaylistInfo, it needs to be added to the localPlaylistInfo, in the unknownType key + // If a video is in both, we don't need to do anything + // To get all videos that exist in only the database, we can join the local video types and get the difference to the database videos + // To get all videos that exist in only the local storage, we do the same but the other way around + + // Get all videos that exist in only the database + // TODO: Use a set to speed this lookup up + const videosOnlyInDatabase = Object.keys(playlistInfo["videos"]).filter((videoId) => !Object.keys(getAllVideosFromLocalPlaylist(localPlaylistInfo)).includes(videoId)); + // TODO: Get the difference between the local videos and the database videos in a more efficient way + } + return playlistInfo; } @@ -275,6 +303,8 @@ async function getPlaylistFromAPI(playlistId, useAPIKeyAtIndex, userQuotaRemaini } let playlistInfo = {}; + playlistInfo["videos"] = {}; + let pageToken = ""; let apiResponse = null; @@ -308,7 +338,7 @@ async function getPlaylistFromAPI(playlistId, useAPIKeyAtIndex, userQuotaRemaini } // For each video, add an entry in the form of videoId: uploadTime - playlistInfo["videos"] = Object.fromEntries(apiResponse["items"].map((video) => [video["contentDetails"]["videoId"], video["contentDetails"]["videoPublishedAt"].substring(0, 10)])); + playlistInfo["videos"]["unknownType"] = Object.fromEntries(apiResponse["items"].map((video) => [video["contentDetails"]["videoId"], video["contentDetails"]["videoPublishedAt"].substring(0, 10)])); // We also want to get the uploadTime of the most recent video playlistInfo["lastVideoPublishedAt"] = apiResponse["items"][0]["contentDetails"]["videoPublishedAt"]; @@ -323,7 +353,7 @@ async function getPlaylistFromAPI(playlistId, useAPIKeyAtIndex, userQuotaRemaini progressTextElement.innerText = `\xa0Fetching: ${Math.round(resultsFetchedCount / totalResults * 100)}%`; // For each video, add an entry in the form of videoId: uploadTime - playlistInfo["videos"] = Object.assign(playlistInfo["videos"], Object.fromEntries(apiResponse["items"].map((video) => [video["contentDetails"]["videoId"], video["contentDetails"]["videoPublishedAt"].substring(0, 10)]))); + playlistInfo["videos"]["unknownType"] = Object.assign(playlistInfo["videos"]["unknownType"], Object.fromEntries(apiResponse["items"].map((video) => [video["contentDetails"]["videoId"], video["contentDetails"]["videoPublishedAt"].substring(0, 10)]))); pageToken = apiResponse["nextPageToken"] ? apiResponse["nextPageToken"] : null; } @@ -358,7 +388,7 @@ async function updatePlaylistFromAPI(playlistInfo, playlistId, useAPIKeyAtIndex, const totalNumVideosOnChannel = apiResponse["pageInfo"]["totalResults"]; // If the channel has already reached the API cap, we don't know how many new videos there are, so we put an estimate to show the user something // The difference could be negative if there are more videos saved in the database than exist in the playlist, e.g videos were deleted - const numLocallyKnownVideos = getLength(playlistInfo["videos"]); + const numLocallyKnownVideos = getLength(getAllVideosFromLocalPlaylist(playlistInfo)); const totalExpectedNewResults = totalNumVideosOnChannel > 19999 ? 1000 : Math.max(totalNumVideosOnChannel - numLocallyKnownVideos, 0); // If there are more results we need to fetch than the user has quota remaining (+leeway) and the user is not using a custom API key, we need to throw an error @@ -662,6 +692,9 @@ async function chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpd } // Sort all videos by date + // TODO: How to retain information for which subdictionary a video belongs to? + // Do we maybe sort all three dicts separately and always choose from a random one of them? + // What is the overhead? let allVideos = Object.assign({}, playlistInfo["videos"], playlistInfo["newVideos"] ?? {}); let videosByDate = Object.keys(allVideos).sort((a, b) => { @@ -698,6 +731,7 @@ async function chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpd do { // Remove the video from the local playlist object // It will always be in the "videos" object, as we have just fetched the "newVideos" from the YouTube API + // TODO: Delete from the correct subdictionary delete playlistInfo["videos"][randomVideo]; // Remove the deleted video from the videosToShuffle array and choose a new random video @@ -925,6 +959,34 @@ async function aprilFoolsJoke() { } } +// ---------- Helper functions ---------- +// Join all videos into one object, useful for database interaction and filtering +function getAllVideosFromLocalPlaylist(playlistInfo) { + return Object.assign({}, playlistInfo["videos"]["knownVideos"], playlistInfo["videos"]["knownShorts"], playlistInfo["videos"]["unknownType"]); +} + +function validatePlaylistInfo(playlistInfo) { + // The playlistInfo object must contain lastVideoPublishedAt, lastFetchedFromDB and videos + // The videos subkey must contain knownVideos, knownShorts and unknownType + // If the newVideos key is missing, add it as an empty object + // TODO: Remove debug code + console.log(Object.keys(playlistInfo)); + if (!playlistInfo["lastVideoPublishedAt"] || !playlistInfo["lastFetchedFromDB"] + || !playlistInfo["videos"] || !playlistInfo["videos"]["knownVideos"] || !playlistInfo["videos"]["knownShorts"] || !playlistInfo["videos"]["unknownType"]) { + throw new RandomYoutubeVideoError( + { + code: "RYV-10", + message: `The playlistInfo object is missing one or more required keys (Got: ${Object.keys(playlistInfo)}).`, + solveHint: "Please try again and inform the developer if the error is not resolved.", + showTrace: false + } + ); + } + if (!playlistInfo["newVideos"]) { + playlistInfo["newVideos"] = {}; + } +} + // ---------- Local storage ---------- // Tries to fetch the playlist from local storage. If it is not present, returns an empty dictionary async function tryGetPlaylistFromLocalStorage(playlistId) { From b82c4f966ac6316f11c42d8e1ef070ce9f78a191 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sun, 22 Oct 2023 16:39:01 +0100 Subject: [PATCH 04/32] Fixed wrong return value if database value is falsy --- src/background.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/background.js b/src/background.js index 3a55d217..704b8198 100644 --- a/src/background.js +++ b/src/background.js @@ -324,7 +324,7 @@ async function openVideoInTabWithId(tabId, videoUrl) { // ---------- Local storage ---------- async function getFromLocalStorage(key) { return await chrome.storage.local.get([key]).then((result) => { - if (result[key]) { + if (result[key] !== undefined) { return result[key]; } return null; From 3fcab5e02ba58ffc57b94fac344828d1a1100a83 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sun, 22 Oct 2023 16:40:06 +0100 Subject: [PATCH 05/32] Updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23e64118..d7f23c51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## v2.3.0 (Unreleased) +- Shuffling when excluding shorts is now a lot quicker if you have shuffled from the channel before, as the extension will remember which videos are shorts. +- Fixed a rare data inconsistency bug occurring with specific database values. ## v2.2.3 From a86a7076802358faab6ecedb43772a045545d7dd Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sun, 22 Oct 2023 17:17:40 +0100 Subject: [PATCH 06/32] Updated variables, logs --- src/background.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/background.js b/src/background.js index 704b8198..936f5ec3 100644 --- a/src/background.js +++ b/src/background.js @@ -12,13 +12,13 @@ chrome.runtime.onStartup.addListener(async function () { console.log(`${((utilizedStorage / maxLocalStorage) * 100).toFixed(2)}% of local storage is used. (${utilizedStorage}/${maxLocalStorage} bytes)`); if (maxLocalStorage * 0.9 < utilizedStorage) { - console.log("Local storage is over 90% utilized. Removing playlists that have not been accessed the longest..."); + console.log("Local storage is over 90% utilized. Removing playlists that have not been accessed the longest to free up some space..."); // Get all playlists from local storage const localStorageContents = await chrome.storage.local.get(); // We only need the keys that hold playlists, which is signified by the existence of the "videos" sub-key - const allPlaylists = Object.fromEntries(Object.entries(localStorageContents).filter(([k, v]) => v["videos"])); + const allPlaylists = Object.fromEntries(Object.entries(localStorageContents).filter(([key, value]) => value["videos"])); // Sort the playlists by lastAccessedLocally value const sortedPlaylists = Object.entries(allPlaylists).sort((a, b) => { From d84ad4d8e5222fd1793c7dbbf0721e9c9126f07c Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sun, 22 Oct 2023 17:18:06 +0100 Subject: [PATCH 07/32] Added logic for comparing local and database video data sets --- src/shuffleVideo.js | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index 777b8685..803ec59d 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -165,11 +165,11 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE // TODO: Remove debug code console.log(playlistInfoForLocalStorage); return; - await savePlaylistToLocalStorage(uploadsPlaylistId, playlistInfoForLocalStorage); + // await savePlaylistToLocalStorage(uploadsPlaylistId, playlistInfoForLocalStorage); - await setSyncStorageValue("numShuffledVideosTotal", configSync.numShuffledVideosTotal + 1); + // await setSyncStorageValue("numShuffledVideosTotal", configSync.numShuffledVideosTotal + 1); - await playVideo(chosenVideos, firedFromPopup); + // await playVideo(chosenVideos, firedFromPopup); } catch (error) { await setSyncStorageValue("userQuotaRemainingToday", Math.max(0, configSync.userQuotaRemainingToday - 1)); throw error; @@ -231,18 +231,23 @@ async function tryGetPlaylistFromDB(playlistId, localPlaylistInfo = null) { playlistInfo["videos"]["knownVideos"] = {}; playlistInfo["videos"]["knownShorts"] = {}; } else { - // Else, we need to find out which videos are new, and add them to the unknownType key - // It is also possible that there are videos in the localPlaylistInfo which do not exist in the database anymore, so we need to delete them - // If a video is in localPlaylistInfo but not in the database, it needs to be removed from the localPlaylistInfo - // If a video is in the database but not in localPlaylistInfo, it needs to be added to the localPlaylistInfo, in the unknownType key - // If a video is in both, we don't need to do anything - // To get all videos that exist in only the database, we can join the local video types and get the difference to the database videos - // To get all videos that exist in only the local storage, we do the same but the other way around - - // Get all videos that exist in only the database - // TODO: Use a set to speed this lookup up - const videosOnlyInDatabase = Object.keys(playlistInfo["videos"]).filter((videoId) => !Object.keys(getAllVideosFromLocalPlaylist(localPlaylistInfo)).includes(videoId)); - // TODO: Get the difference between the local videos and the database videos in a more efficient way + const allVideosInLocalPlaylist = getAllVideosFromLocalPlaylist(localPlaylistInfo); + const allVideosInLocalPlaylistAsSet = new Set(Object.keys(allVideosInLocalPlaylist)); + const allVideosInDatabaseAsSet = new Set(Object.keys(playlistInfo["videos"])); + + // Add videos that are new from the database to the local playlist + const videosOnlyInDatabase = Object.keys(playlistInfo["videos"].filter((videoId) => !allVideosInLocalPlaylistAsSet.has(videoId))); + playlistInfo["videos"]["unknownType"] = Object.assign({}, playlistInfo["videos"]["unknownType"], Object.fromEntries(videosOnlyInDatabase.map((videoId) => [videoId, playlistInfo["videos"][videoId]]))); + + // Remove videos from the local playlist object that no longer exist in the database + const videoTypes = ["knownVideos", "knownShorts", "unknownType"]; + for (const type of videoTypes) { + for (const videoId in localPlaylistInfo["videos"][type]) { + if (!allVideosInDatabaseAsSet.has(videoId)) { + delete localPlaylistInfo["videos"][type][videoId]; + } + } + } } return playlistInfo; From fd06527f55df739232352bb53510c0aadf4e1674 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sun, 22 Oct 2023 17:24:28 +0100 Subject: [PATCH 08/32] Added better comments, resolved TODO's --- src/shuffleVideo.js | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index 803ec59d..b0ed0ad6 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -23,16 +23,17 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE try { // While chooseRandomVideo is running, we need to keep the service worker alive // Otherwise, it will get stopped after 30 seconds and we will get an error if fetching the videos takes longer + // So every 25 seconds, we send a message to the service worker to keep it alive var keepServiceWorkerAlive = setInterval(() => { chrome.runtime.sendMessage({ command: "connectionTest" }); }, 25000); /* c8 ignore stop */ // Each user has a set amount of quota they can use per day. - // If they exceed it, they need to provide a custom API key, or wait until the quota resets the next day. + // If they exceed it, they need to provide a custom API key, or wait until the quota resets the next day let userQuotaRemainingToday = await getUserQuotaRemainingToday(); - // If we somehow update the playlist info and want to send it to the database in the end, this variable indicates it + // If we update the playlist info in any way and want to send it to the database in the end, this variable indicates it let shouldUpdateDatabase = false; // User preferences @@ -147,10 +148,8 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE console.log("Saving playlist to local storage..."); // We can now join the new videos with the old ones - // TODO: This is where we add the new videos to the "unknownType" key, instead of joining - // playlistInfo["videos"] = Object.assign({}, playlistInfo["videos"], playlistInfo["newVideos"] ?? {}); - // TODO: Make sure the unknownType key exists here - playlistInfo["videos"]["unknownType"] = Object.assign({}, playlistInfo["videos"]["unknownType"], playlistInfo["newVideos"] ?? {}); + // We do not yet know if the new videos are videos or shorts, so we put them in the "unknownType" key + playlistInfo["videos"]["unknownType"] = Object.assign({}, playlistInfo["videos"]["unknownType"] ?? {}, playlistInfo["newVideos"] ?? {}); // Only save the wanted keys const playlistInfoForLocalStorage = { @@ -158,18 +157,18 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE "lastAccessedLocally": new Date().toISOString(), "lastFetchedFromDB": playlistInfo["lastFetchedFromDB"] ?? new Date(0).toISOString(), "lastVideoPublishedAt": playlistInfo["lastVideoPublishedAt"] ?? new Date(0).toISOString().slice(0, 19) + 'Z', - // TODO: New shorts format here (separate keys) - it should work like this + // TODO: New shorts format here (separate keys) - it should work like this -> Test it "videos": playlistInfo["videos"] ?? {} }; // TODO: Remove debug code console.log(playlistInfoForLocalStorage); return; - // await savePlaylistToLocalStorage(uploadsPlaylistId, playlistInfoForLocalStorage); + await savePlaylistToLocalStorage(uploadsPlaylistId, playlistInfoForLocalStorage); - // await setSyncStorageValue("numShuffledVideosTotal", configSync.numShuffledVideosTotal + 1); + await setSyncStorageValue("numShuffledVideosTotal", configSync.numShuffledVideosTotal + 1); - // await playVideo(chosenVideos, firedFromPopup); + await playVideo(chosenVideos, firedFromPopup); } catch (error) { await setSyncStorageValue("userQuotaRemainingToday", Math.max(0, configSync.userQuotaRemainingToday - 1)); throw error; @@ -973,9 +972,6 @@ function getAllVideosFromLocalPlaylist(playlistInfo) { function validatePlaylistInfo(playlistInfo) { // The playlistInfo object must contain lastVideoPublishedAt, lastFetchedFromDB and videos // The videos subkey must contain knownVideos, knownShorts and unknownType - // If the newVideos key is missing, add it as an empty object - // TODO: Remove debug code - console.log(Object.keys(playlistInfo)); if (!playlistInfo["lastVideoPublishedAt"] || !playlistInfo["lastFetchedFromDB"] || !playlistInfo["videos"] || !playlistInfo["videos"]["knownVideos"] || !playlistInfo["videos"]["knownShorts"] || !playlistInfo["videos"]["unknownType"]) { throw new RandomYoutubeVideoError( From 01fab15f601f8963f27d4a949457d0fceacfd1ac Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Fri, 27 Oct 2023 17:27:26 +0100 Subject: [PATCH 09/32] Filtering WIP --- src/shuffleVideo.js | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index b0ed0ad6..ca44da6d 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -696,14 +696,25 @@ async function chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpd } // Sort all videos by date + // TODO: WIP Start // TODO: How to retain information for which subdictionary a video belongs to? // Do we maybe sort all three dicts separately and always choose from a random one of them? // What is the overhead? - let allVideos = Object.assign({}, playlistInfo["videos"], playlistInfo["newVideos"] ?? {}); + // Problem with current below approach (22/10/2023): The applyShuffleFilter function needs to get all videos at once to correctly apply the percentageOption filter + let allUnknownTypeVideos = Object.assign({}, playlistInfo["videos"]["unknownType"], playlistInfo["newVideos"] ?? {}); + let allKnownVideos = playlistInfo["videos"]["knownVideos"]; + let allKnownShorts = playlistInfo["videos"]["knownShorts"]; - let videosByDate = Object.keys(allVideos).sort((a, b) => { - return new Date(allVideos[b]) - new Date(allVideos[a]); + let unknownTypeVideosByDate = Object.keys(allUnknownTypeVideos).sort((a, b) => { + return new Date(allUnknownTypeVideos[b]) - new Date(allUnknownTypeVideos[a]); }); + let knownVideosByDate = Object.keys(playlistInfo["videos"]["knownVideos"]).sort((a, b) => { + return new Date(allKnownVideos[b]) - new Date(allKnownVideos[a]); + }); + let knownShortsByDate = Object.keys(playlistInfo["videos"]["knownShorts"]).sort((a, b) => { + return new Date(allKnownShorts[b]) - new Date(allKnownShorts[a]); + }); + // TODO: WIP End // Error handling for videosToShuffle being undefined/empty is done in applyShuffleFilter() let videosToShuffle = applyShuffleFilter(allVideos, videosByDate, activeShuffleFilterOption, activeOptionValue); @@ -767,6 +778,7 @@ async function chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpd } // If the user does not want to shuffle from shorts, and we do not yet know the type of the chosen video, we check if it is a short + // TODO: Whenever we get to know the type of a video, we should move it to the correct subdictionary if (configSync.shuffleIgnoreShortsOption) { const videoIsShort = await isShort(randomVideo); From 0d184cf04a2c7c17261a8140a49a65ea86f8d05f Mon Sep 17 00:00:00 2001 From: Nikkel Mollenhauer <57323886+NikkelM@users.noreply.github.com> Date: Thu, 2 Nov 2023 18:42:49 +0000 Subject: [PATCH 10/32] Use correct filtering --- src/shuffleVideo.js | 85 ++++++++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 32 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index ca44da6d..c8963d43 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -157,7 +157,6 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE "lastAccessedLocally": new Date().toISOString(), "lastFetchedFromDB": playlistInfo["lastFetchedFromDB"] ?? new Date(0).toISOString(), "lastVideoPublishedAt": playlistInfo["lastVideoPublishedAt"] ?? new Date(0).toISOString().slice(0, 19) + 'Z', - // TODO: New shorts format here (separate keys) - it should work like this -> Test it "videos": playlistInfo["videos"] ?? {} }; @@ -308,6 +307,9 @@ async function getPlaylistFromAPI(playlistId, useAPIKeyAtIndex, userQuotaRemaini let playlistInfo = {}; playlistInfo["videos"] = {}; + playlistInfo["videos"]["unknownType"] = {}; + playlistInfo["videos"]["knownVideos"] = {}; + playlistInfo["videos"]["knownShorts"] = {}; let pageToken = ""; let apiResponse = null; @@ -362,6 +364,8 @@ async function getPlaylistFromAPI(playlistId, useAPIKeyAtIndex, userQuotaRemaini pageToken = apiResponse["nextPageToken"] ? apiResponse["nextPageToken"] : null; } + playlistInfo["lastFetchedFromDB"] = new Date().toISOString(); + return { playlistInfo, userQuotaRemainingToday }; } @@ -696,27 +700,20 @@ async function chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpd } // Sort all videos by date - // TODO: WIP Start - // TODO: How to retain information for which subdictionary a video belongs to? - // Do we maybe sort all three dicts separately and always choose from a random one of them? - // What is the overhead? - // Problem with current below approach (22/10/2023): The applyShuffleFilter function needs to get all videos at once to correctly apply the percentageOption filter - let allUnknownTypeVideos = Object.assign({}, playlistInfo["videos"]["unknownType"], playlistInfo["newVideos"] ?? {}); + let allUnknownType = Object.assign({}, playlistInfo["videos"]["unknownType"], playlistInfo["newVideos"] ?? {}); let allKnownVideos = playlistInfo["videos"]["knownVideos"]; let allKnownShorts = playlistInfo["videos"]["knownShorts"]; - let unknownTypeVideosByDate = Object.keys(allUnknownTypeVideos).sort((a, b) => { - return new Date(allUnknownTypeVideos[b]) - new Date(allUnknownTypeVideos[a]); - }); - let knownVideosByDate = Object.keys(playlistInfo["videos"]["knownVideos"]).sort((a, b) => { - return new Date(allKnownVideos[b]) - new Date(allKnownVideos[a]); - }); - let knownShortsByDate = Object.keys(playlistInfo["videos"]["knownShorts"]).sort((a, b) => { - return new Date(allKnownShorts[b]) - new Date(allKnownShorts[a]); + let allVideos = Object.assign({}, allUnknownType, allKnownVideos); + // Only if we are not ignoring shorts, we want to include them in the shuffle + if (!configSync.shuffleIgnoreShortsOption) { + allVideos = Object.assign({}, allVideos, allKnownShorts); + } + + let videosByDate = Object.keys(allVideos).sort((a, b) => { + return new Date(allVideos[b]) - new Date(allVideos[a]); }); - // TODO: WIP End - // Error handling for videosToShuffle being undefined/empty is done in applyShuffleFilter() let videosToShuffle = applyShuffleFilter(allVideos, videosByDate, activeShuffleFilterOption, activeOptionValue); let chosenVideos = []; @@ -745,9 +742,7 @@ async function chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpd shouldUpdateDatabase = true; do { // Remove the video from the local playlist object - // It will always be in the "videos" object, as we have just fetched the "newVideos" from the YouTube API - // TODO: Delete from the correct subdictionary - delete playlistInfo["videos"][randomVideo]; + delete playlistInfo[getVideoType(randomVideo, playlistInfo)][randomVideo]; // Remove the deleted video from the videosToShuffle array and choose a new random video videosToShuffle.splice(videosToShuffle.indexOf(randomVideo), 1); @@ -778,21 +773,34 @@ async function chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpd } // If the user does not want to shuffle from shorts, and we do not yet know the type of the chosen video, we check if it is a short - // TODO: Whenever we get to know the type of a video, we should move it to the correct subdictionary if (configSync.shuffleIgnoreShortsOption) { - const videoIsShort = await isShort(randomVideo); + if (playlistInfo["videos"]["unknownType"][randomVideo] !== undefined) { + const videoIsShort = await isShort(randomVideo); - if (videoIsShort) { - console.log('A chosen video was a short, but shorts are ignored. Choosing a new random video.'); + if (videoIsShort) { + console.log('A chosen video was a short, but shorts are ignored. Choosing a new random video.'); - // Remove the video from videosToShuffle to not choose it again - // Do not remove it from the playlistInfo object, as we do not want to delete it from the database - videosToShuffle.splice(videosToShuffle.indexOf(randomVideo), 1); + // Move the video to the knownShorts subdictionary + playlistInfo["videos"]["knownShorts"][randomVideo] = playlistInfo["videos"]["unknownType"][randomVideo]; + delete playlistInfo["videos"]["unknownType"][randomVideo]; - // We need to decrement i, as we did not choose a video in this iteration - i--; + // Remove the video from videosToShuffle to not choose it again + // Do not remove it from the playlistInfo object, as we do not want to delete it from the database + videosToShuffle.splice(videosToShuffle.indexOf(randomVideo), 1); + + // We need to decrement i, as we did not choose a video in this iteration + i--; + } else { + // Move the video to the knownVideos subdictionary + playlistInfo["videos"]["knownVideos"][randomVideo] = playlistInfo["videos"]["unknownType"][randomVideo]; + delete playlistInfo["videos"]["unknownType"][randomVideo]; + + // The video is not a short, so add it to the list of chosen videos and remove it from the pool of videos to choose from + chosenVideos.push(randomVideo); + videosToShuffle.splice(videosToShuffle.indexOf(randomVideo), 1); + } } else { - // The video is not a short, so add it to the list of chosen videos and remove it from the pool of videos to choose from + // Otherwise, the video must be a knownVideo, as we do not include knownShorts in allVideos above chosenVideos.push(randomVideo); videosToShuffle.splice(videosToShuffle.indexOf(randomVideo), 1); } @@ -819,6 +827,18 @@ async function chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpd return { chosenVideos, playlistInfo, shouldUpdateDatabase, encounteredDeletedVideos }; } +function getVideoType(videoId, playlistInfo) { + if (playlistInfo["videos"]["unknownType"][videoId]) { + return "unknownType"; + } else if (playlistInfo["videos"]["knownVideos"][videoId]) { + return "knownVideos"; + } else if (playlistInfo["videos"]["knownShorts"][videoId]) { + return "knownShorts"; + } else { + return null; + } +} + // Applies a filter to the playlist object, based on the setting set in the popup function applyShuffleFilter(allVideos, videosByDate, activeShuffleFilterOption, activeOptionValue) { let videosToShuffle; @@ -882,7 +902,7 @@ function applyShuffleFilter(allVideos, videosByDate, activeShuffleFilterOption, throw new RandomYoutubeVideoError( { code: "RYV-8D", - message: `The percentage you specified (${activeOptionValue}) should be between 1 and 100. Normally, you should not be able to set such a value.`, + message: `The percentage you specified (${activeOptionValue}%) should be between 1 and 100. Normally, you should not be able to set such a value.`, solveHint: "Please fix the percentage in the popup.", showTrace: false } @@ -986,7 +1006,8 @@ function validatePlaylistInfo(playlistInfo) { // The videos subkey must contain knownVideos, knownShorts and unknownType if (!playlistInfo["lastVideoPublishedAt"] || !playlistInfo["lastFetchedFromDB"] || !playlistInfo["videos"] || !playlistInfo["videos"]["knownVideos"] || !playlistInfo["videos"]["knownShorts"] || !playlistInfo["videos"]["unknownType"]) { - throw new RandomYoutubeVideoError( + console.log(playlistInfo); + throw new RandomYoutubeVideoError( { code: "RYV-10", message: `The playlistInfo object is missing one or more required keys (Got: ${Object.keys(playlistInfo)}).`, From 2f936fb24b2421c800d462999bd18612b5ed212d Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 11:51:21 +0000 Subject: [PATCH 11/32] Removed debug code --- src/shuffleVideo.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index c8963d43..3d2a9636 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -160,9 +160,6 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE "videos": playlistInfo["videos"] ?? {} }; - // TODO: Remove debug code - console.log(playlistInfoForLocalStorage); - return; await savePlaylistToLocalStorage(uploadsPlaylistId, playlistInfoForLocalStorage); await setSyncStorageValue("numShuffledVideosTotal", configSync.numShuffledVideosTotal + 1); @@ -800,7 +797,7 @@ async function chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpd videosToShuffle.splice(videosToShuffle.indexOf(randomVideo), 1); } } else { - // Otherwise, the video must be a knownVideo, as we do not include knownShorts in allVideos above + // Otherwise, the video must be a knownVideo, as we do not include knownShorts in allVideos chosenVideos.push(randomVideo); videosToShuffle.splice(videosToShuffle.indexOf(randomVideo), 1); } @@ -1006,8 +1003,7 @@ function validatePlaylistInfo(playlistInfo) { // The videos subkey must contain knownVideos, knownShorts and unknownType if (!playlistInfo["lastVideoPublishedAt"] || !playlistInfo["lastFetchedFromDB"] || !playlistInfo["videos"] || !playlistInfo["videos"]["knownVideos"] || !playlistInfo["videos"]["knownShorts"] || !playlistInfo["videos"]["unknownType"]) { - console.log(playlistInfo); - throw new RandomYoutubeVideoError( + throw new RandomYoutubeVideoError( { code: "RYV-10", message: `The playlistInfo object is missing one or more required keys (Got: ${Object.keys(playlistInfo)}).`, From 68c3bce937b5a47a421b9e3f258093eb8632cc26 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 11:53:00 +0000 Subject: [PATCH 12/32] Added button message if ignoring shorts --- src/content.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/content.js b/src/content.js index 11a0bec0..6a65586b 100644 --- a/src/content.js +++ b/src/content.js @@ -141,6 +141,9 @@ async function shuffleVideos() { var hasBeenShuffled = false; setDOMTextWithDelay(shuffleButtonTextElement, "\xa0Shuffling...", 1000, () => { return (shuffleButtonTextElement.innerText === "\xa0Shuffle" && !hasBeenShuffled); }); setDOMTextWithDelay(shuffleButtonTextElement, "\xa0Still on it...", 5000, () => { return (shuffleButtonTextElement.innerText === "\xa0Shuffling..." && !hasBeenShuffled); }); + if (configSync.shuffleIgnoreShortsOption) { + setDOMTextWithDelay(shuffleButtonTextElement, "\xa0Sorting shorts...", 8000, () => { return (shuffleButtonTextElement.innerText === "\xa0Still on it..." && !hasBeenShuffled); }); + } await chooseRandomVideo(channelId, false, shuffleButtonTextElement); hasBeenShuffled = true; From fc84178aa249d8eb62823b1b47303ee3913d0842 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 11:54:20 +0000 Subject: [PATCH 13/32] Updated version, changelog --- CHANGELOG.md | 1 + package.json | 2 +- static/manifest.json | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2d7743a..af5bc104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Shuffling when excluding shorts is now a lot quicker if you have shuffled from the channel before, as the extension will remember which videos are shorts. +- Added an additional message to the shuffle button if shuffling takes longer due to ignoring shorts. - Fixed a rare data inconsistency bug occurring with specific database values. diff --git a/package.json b/package.json index 613e8f34..326bdefb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "random-youtube-video", - "version": "2.2.4", + "version": "2.3.0", "description": "Play a random video uploaded on the current YouTube channel.", "scripts": { "dev": "concurrently \"npm run dev:chromium\" \"npm run dev:firefox\"", diff --git a/static/manifest.json b/static/manifest.json index e7b24a6e..ef7b6621 100644 --- a/static/manifest.json +++ b/static/manifest.json @@ -1,7 +1,7 @@ { "name": "Random YouTube Video", "description": "Play a random video uploaded on the current YouTube channel.", - "version": "2.2.4", + "version": "2.3.0", "manifest_version": 3, "content_scripts": [ { From bd65eae3b578b74a2d46b99e7938c03b1236ca71 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 12:51:28 +0000 Subject: [PATCH 14/32] Renamed variables --- test/playlistPermutations.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/playlistPermutations.js b/test/playlistPermutations.js index ce6d7e06..cead952a 100644 --- a/test/playlistPermutations.js +++ b/test/playlistPermutations.js @@ -471,23 +471,23 @@ for (let i = 0; i < playlistModifiers[0].length; i++) { } // ----- Locally stored playlists ----- -export const localPlaylistPermutations = playlistPermutations.reduce((acc, playlist) => { +export const localPlaylistPermutations = playlistPermutations.reduce((localPlaylists, playlist) => { if (playlist.playlistModifiers.lastAccessedLocally !== "LocalPlaylistDoesNotExist") { const playlistCopy = deepCopy(playlist); const { playlistId, lastAccessedLocally, lastFetchedFromDB, localLastVideoPublishedAt, localVideos, localDeletedVideos } = playlistCopy; - acc[playlistId] = { lastAccessedLocally, lastFetchedFromDB, lastVideoPublishedAt: localLastVideoPublishedAt, videos: deepCopy({ ...localVideos, ...localDeletedVideos }) }; + localPlaylists[playlistId] = { lastAccessedLocally, lastFetchedFromDB, lastVideoPublishedAt: localLastVideoPublishedAt, videos: deepCopy({ ...localVideos, ...localDeletedVideos }) }; } - return acc; + return localPlaylists; }, {}); // ----- Database ----- -export const databasePermutations = playlistPermutations.reduce((acc, playlist) => { +export const databasePermutations = playlistPermutations.reduce((databasePlaylists, playlist) => { if (playlist.playlistModifiers.lastUpdatedDBAt !== "DBEntryDoesNotExist") { const playlistCopy = deepCopy(playlist); const { playlistId, lastUpdatedDBAt, dbLastVideoPublishedAt, dbVideos, dbDeletedVideos } = playlistCopy; - acc[playlistId] = { lastUpdatedDBAt, lastVideoPublishedAt: dbLastVideoPublishedAt, videos: deepCopy({ ...dbVideos, ...dbDeletedVideos }) }; + databasePlaylists[playlistId] = { lastUpdatedDBAt, lastVideoPublishedAt: dbLastVideoPublishedAt, videos: deepCopy({ ...dbVideos, ...dbDeletedVideos }) }; } - return acc; + return databasePlaylists; }, {}); // ----- Utility functions ----- From c1afe2d9917fb8eff38cf03dee01d2d7658a0fe1 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 13:10:38 +0000 Subject: [PATCH 15/32] Test playlists now use the new localPlaylist format --- test/playlistPermutations.js | 334 ++++++++++++++++++++--------------- 1 file changed, 189 insertions(+), 145 deletions(-) diff --git a/test/playlistPermutations.js b/test/playlistPermutations.js index cead952a..9fecb133 100644 --- a/test/playlistPermutations.js +++ b/test/playlistPermutations.js @@ -231,6 +231,11 @@ const playlistModifiers = [ 'DBContainsVideosNotInLocalPlaylist', 'DBContainsNoVideosNotInLocalPlaylist', 'DBContainsDeletedVideos' + ], + // localPlaylistVideoKnowledge: If the local playlist already has sorted videos into knownShorts and knownVideos, or if all videos are of unknownType + [ + 'LocalPlaylistContainsKnownShortsAndVideos', + 'LocalPlaylistContainsOnlyUnknownVideos' ] ]; @@ -301,7 +306,8 @@ let playlistId, localVideos, localDeletedVideos, dbVideos, - dbDeletedVideos; + dbDeletedVideos, + localPlaylistVideoKnowledge; for (let i = 0; i < playlistModifiers[0].length; i++) { for (let j = 0; j < playlistModifiers[1].length; j++) { @@ -309,160 +315,174 @@ for (let i = 0; i < playlistModifiers[0].length; i++) { for (let l = 0; l < playlistModifiers[3].length; l++) { for (let m = 0; m < playlistModifiers[4].length; m++) { for (let n = 0; n < playlistModifiers[5].length; n++) { - // Skip permutations that are not possible - // If the local playlist recently fetched from the DB, it does not matter when the DB entry was last updated or if it contains any videos not in the local playlist, and we know the local playlist was recently accessed - if (playlistModifiers[0][i] === "LocalPlaylistFetchedDBRecently" && (playlistModifiers[1][j] !== "DBEntryIsUpToDate" - || playlistModifiers[2][k] !== "LocalPlaylistRecentlyAccessed" || playlistModifiers[5][n] !== "DBContainsNoVideosNotInLocalPlaylist")) { - continue; - } - // If the local playlist does not exist, it cannot contain deleted videos, only shorts, or have been updated from the DB recently - if ((playlistModifiers[2][k] === "LocalPlaylistDoesNotExist") && (playlistModifiers[3][l] === "LocalPlaylistContainsDeletedVideos" - || playlistModifiers[3][l] === "LocalPlaylistContainsOnlyShorts" || playlistModifiers[0][i] === "LocalPlaylistFetchedDBRecently")) { - continue; - } - // If the DB entry does not exist, it cannot contain videos not in the local playlist - if ((playlistModifiers[1][j] === "DBEntryDoesNotExist") && (playlistModifiers[5][n] !== "DBContainsNoVideosNotInLocalPlaylist")) { - continue; - } - // If the DB entry is up-to-date or the local playlist is up-to-date, it does not matter if there are new videos uploaded - if ((playlistModifiers[1][j] === "DBEntryIsUpToDate" || playlistModifiers[0][i] === "LocalPlaylistFetchedDBRecently") - && (playlistModifiers[4][m] !== "NoNewVideoUploaded")) { - continue; - } - // We only need one permutation in total that has only shorts saved locally, which is - // UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsOnlyShorts_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist - // This is because the other permutations are covered by the other tests - if (playlistModifiers[3][l] === "LocalPlaylistContainsOnlyShorts") { - // Discard all permutations that are not the one described above - if (playlistModifiers[0][i] !== "LocalPlaylistFetchedDBRecently" || playlistModifiers[1][j] !== "DBEntryIsUpToDate" - || playlistModifiers[2][k] !== "LocalPlaylistRecentlyAccessed" || playlistModifiers[4][m] !== "NoNewVideoUploaded" - || playlistModifiers[5][n] !== "DBContainsNoVideosNotInLocalPlaylist") { + for (let o = 0; o < playlistModifiers[6].length; o++) { + // Skip permutations that are not possible + // If the local playlist recently fetched from the DB, it does not matter when the DB entry was last updated or if it contains any videos not in the local playlist, and we know the local playlist was recently accessed + if (playlistModifiers[0][i] === "LocalPlaylistFetchedDBRecently" && (playlistModifiers[1][j] !== "DBEntryIsUpToDate" + || playlistModifiers[2][k] !== "LocalPlaylistRecentlyAccessed" || playlistModifiers[5][n] !== "DBContainsNoVideosNotInLocalPlaylist")) { continue; } - } + // If the local playlist does not exist, it cannot contain deleted videos, only shorts, have been updated from the DB recently, or have sorted videos into types + if ((playlistModifiers[2][k] === "LocalPlaylistDoesNotExist") && (playlistModifiers[3][l] === "LocalPlaylistContainsDeletedVideos" + || playlistModifiers[3][l] === "LocalPlaylistContainsOnlyShorts" || playlistModifiers[0][i] === "LocalPlaylistFetchedDBRecently" + || playlistModifiers[6][o] === "LocalPlaylistContainsKnownShortsAndVideos")) { + continue; + } + // If the DB entry does not exist, it cannot contain videos not in the local playlist + if ((playlistModifiers[1][j] === "DBEntryDoesNotExist") && (playlistModifiers[5][n] !== "DBContainsNoVideosNotInLocalPlaylist")) { + continue; + } + // If the DB entry is up-to-date or the local playlist is up-to-date, it does not matter if there are new videos uploaded + if ((playlistModifiers[1][j] === "DBEntryIsUpToDate" || playlistModifiers[0][i] === "LocalPlaylistFetchedDBRecently") + && (playlistModifiers[4][m] !== "NoNewVideoUploaded")) { + continue; + } + // We only need one permutation in total that has only shorts saved locally, which is + // UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsOnlyShorts_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos + // This is because the other permutations are covered by the other tests + if (playlistModifiers[3][l] === "LocalPlaylistContainsOnlyShorts") { + // Discard all permutations that are not the one described above + if (playlistModifiers[0][i] !== "LocalPlaylistFetchedDBRecently" || playlistModifiers[1][j] !== "DBEntryIsUpToDate" + || playlistModifiers[2][k] !== "LocalPlaylistRecentlyAccessed" || playlistModifiers[4][m] !== "NoNewVideoUploaded" + || playlistModifiers[5][n] !== "DBContainsNoVideosNotInLocalPlaylist" || playlistModifiers[6][o] !== "LocalPlaylistContainsOnlyUnknownVideos") { + continue; + } + } - // The playlist ID always exists - playlistId = (`UU_${playlistModifiers[0][i]}_${playlistModifiers[1][j]}_${playlistModifiers[2][k]}_${playlistModifiers[3][l]}_${playlistModifiers[4][m]}_${playlistModifiers[5][n]}`); - channelId = playlistId.replace("UU", "UC"); - - // When was the playlist last accessed locally - if (playlistModifiers[2][k] === "LocalPlaylistDoesNotExist") { - lastAccessedLocally = null; - } else if (playlistModifiers[2][k] === "LocalPlaylistRecentlyAccessed") { - lastAccessedLocally = zeroDaysAgo; - } else if (playlistModifiers[2][k] === "LocalPlaylistNotRecentlyAccessed") { - lastAccessedLocally = fourteenDaysAgo; - } else { - throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[2][k]}`); - } + // The playlist ID always exists + playlistId = (`UU_${playlistModifiers[0][i]}_${playlistModifiers[1][j]}_${playlistModifiers[2][k]}_${playlistModifiers[3][l]}_${playlistModifiers[4][m]}_${playlistModifiers[5][n]}_${playlistModifiers[6][o]}`); + channelId = playlistId.replace("UU", "UC"); + + // When was the playlist last accessed locally + if (playlistModifiers[2][k] === "LocalPlaylistDoesNotExist") { + lastAccessedLocally = null; + } else if (playlistModifiers[2][k] === "LocalPlaylistRecentlyAccessed") { + lastAccessedLocally = zeroDaysAgo; + } else if (playlistModifiers[2][k] === "LocalPlaylistNotRecentlyAccessed") { + lastAccessedLocally = fourteenDaysAgo; + } else { + throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[2][k]}`); + } - // When was the playlist last updated in the database, if it exists - if (playlistModifiers[1][j] === "DBEntryDoesNotExist") { - lastUpdatedDBAt = null; - } else if (playlistModifiers[1][j] === "DBEntryIsUpToDate") { - lastUpdatedDBAt = zeroDaysAgo; - } else if (playlistModifiers[1][j] === "DBEntryIsNotUpToDate") { - lastUpdatedDBAt = fourteenDaysAgo; - } else { - throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[1][j]}`); - } + // When was the playlist last updated in the database, if it exists + if (playlistModifiers[1][j] === "DBEntryDoesNotExist") { + lastUpdatedDBAt = null; + } else if (playlistModifiers[1][j] === "DBEntryIsUpToDate") { + lastUpdatedDBAt = zeroDaysAgo; + } else if (playlistModifiers[1][j] === "DBEntryIsNotUpToDate") { + lastUpdatedDBAt = fourteenDaysAgo; + } else { + throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[1][j]}`); + } - // How long ago until we last fetched the playlist from the database - if (playlistModifiers[0][i] === "LocalPlaylistFetchedDBRecently") { - lastFetchedFromDB = zeroDaysAgo; - } else if (playlistModifiers[0][i] === "LocalPlaylistDidNotFetchDBRecently") { - lastFetchedFromDB = fourteenDaysAgo; - } else if (playlistModifiers[2][k] === "LocalPlaylistDoesNotExist") { - lastFetchedFromDB = null; - } else { - throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[0][i]}`); - } + // How long ago until we last fetched the playlist from the database + if (playlistModifiers[0][i] === "LocalPlaylistFetchedDBRecently") { + lastFetchedFromDB = zeroDaysAgo; + } else if (playlistModifiers[0][i] === "LocalPlaylistDidNotFetchDBRecently") { + lastFetchedFromDB = fourteenDaysAgo; + } else if (playlistModifiers[2][k] === "LocalPlaylistDoesNotExist") { + lastFetchedFromDB = null; + } else { + throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[0][i]}`); + } - // When was the last locally known video published - localLastVideoPublishedAt = threeDaysAgo.slice(0, 19) + 'Z'; - - if (playlistModifiers[3][l] === "LocalPlaylistContainsDeletedVideos") { - localVideos = deepCopy(defaultLocalVideos); - localDeletedVideos = deepCopy(defaultLocalDeletedVideos); - } else if (playlistModifiers[3][l] === "LocalPlaylistContainsNoDeletedVideos") { - localVideos = deepCopy(defaultLocalVideos); - localDeletedVideos = null; - } else if (playlistModifiers[2][k] === "LocalPlaylistDoesNotExist") { - localVideos = null; - localDeletedVideos = null; - } else if (playlistModifiers[3][l] === "LocalPlaylistContainsOnlyShorts") { - localVideos = deepCopy(defaultLocalShorts); - localDeletedVideos = null; - } else { - throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[3][l]}`); - } + // When was the last locally known video published + localLastVideoPublishedAt = threeDaysAgo.slice(0, 19) + 'Z'; + + if (playlistModifiers[3][l] === "LocalPlaylistContainsDeletedVideos") { + localVideos = deepCopy(defaultLocalVideos); + localDeletedVideos = deepCopy(defaultLocalDeletedVideos); + } else if (playlistModifiers[3][l] === "LocalPlaylistContainsNoDeletedVideos") { + localVideos = deepCopy(defaultLocalVideos); + localDeletedVideos = {}; + } else if (playlistModifiers[2][k] === "LocalPlaylistDoesNotExist") { + localVideos = {}; + localDeletedVideos = {}; + } else if (playlistModifiers[3][l] === "LocalPlaylistContainsOnlyShorts") { + localVideos = deepCopy(defaultLocalShorts); + localDeletedVideos = {}; + } else { + throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[3][l]}`); + } - // Does the db contain videos unknown to the local playlist - if (playlistModifiers[5][n] === "DBContainsVideosNotInLocalPlaylist") { - dbVideos = deepCopy({ ...defaultLocalVideos, ...defaultDBVideos }); - dbDeletedVideos = null; - dbLastVideoPublishedAt = twoDaysAgo.slice(0, 19) + 'Z'; - } else if (playlistModifiers[5][n] === "DBContainsNoVideosNotInLocalPlaylist") { - dbVideos = playlistModifiers[3][l] === "LocalPlaylistContainsOnlyShorts" - ? deepCopy(defaultLocalShorts) - : deepCopy(defaultLocalVideos); - dbDeletedVideos = null; - dbLastVideoPublishedAt = localLastVideoPublishedAt; - } else if (playlistModifiers[5][n] === "DBContainsDeletedVideos") { - dbVideos = deepCopy(defaultLocalVideos); - dbDeletedVideos = deepCopy(defaultDBDeletedVideos); - dbLastVideoPublishedAt = localLastVideoPublishedAt; - } else { - throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[5][n]}`); - } + // Does the db contain videos unknown to the local playlist + if (playlistModifiers[5][n] === "DBContainsVideosNotInLocalPlaylist") { + dbVideos = deepCopy({ ...defaultLocalVideos, ...defaultDBVideos }); + dbDeletedVideos = null; + dbLastVideoPublishedAt = twoDaysAgo.slice(0, 19) + 'Z'; + } else if (playlistModifiers[5][n] === "DBContainsNoVideosNotInLocalPlaylist") { + dbVideos = playlistModifiers[3][l] === "LocalPlaylistContainsOnlyShorts" + ? deepCopy(defaultLocalShorts) + : deepCopy(defaultLocalVideos); + dbDeletedVideos = null; + dbLastVideoPublishedAt = localLastVideoPublishedAt; + } else if (playlistModifiers[5][n] === "DBContainsDeletedVideos") { + dbVideos = deepCopy(defaultLocalVideos); + dbDeletedVideos = deepCopy(defaultDBDeletedVideos); + dbLastVideoPublishedAt = localLastVideoPublishedAt; + } else { + throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[5][n]}`); + } - // Was a new video uploaded since the last time we fetched data from the YouTube API - // newLastVideoPublishedAt is the new date that should be in the database and locally after the update - if (playlistModifiers[1][j] !== "DBEntryIsUpToDate") { - if (playlistModifiers[4][m] === "OneNewVideoUploaded") { - newUploadedVideos = deepCopy(oneNewYTAPIVideo); - newLastVideoPublishedAt = zeroDaysAgo.slice(0, 19) + 'Z'; - } else if (playlistModifiers[4][m] === "MultipleNewVideosUploaded") { - newUploadedVideos = deepCopy(multipleNewYTAPIVideos); - newLastVideoPublishedAt = zeroDaysAgo.slice(0, 19) + 'Z'; - } else if (playlistModifiers[4][m] === "NoNewVideoUploaded") { + // Was a new video uploaded since the last time we fetched data from the YouTube API + // newLastVideoPublishedAt is the new date that should be in the database and locally after the update + if (playlistModifiers[1][j] !== "DBEntryIsUpToDate") { + if (playlistModifiers[4][m] === "OneNewVideoUploaded") { + newUploadedVideos = deepCopy(oneNewYTAPIVideo); + newLastVideoPublishedAt = zeroDaysAgo.slice(0, 19) + 'Z'; + } else if (playlistModifiers[4][m] === "MultipleNewVideosUploaded") { + newUploadedVideos = deepCopy(multipleNewYTAPIVideos); + newLastVideoPublishedAt = zeroDaysAgo.slice(0, 19) + 'Z'; + } else if (playlistModifiers[4][m] === "NoNewVideoUploaded") { + newUploadedVideos = null; + newLastVideoPublishedAt = dbLastVideoPublishedAt; + } else { + throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[4][m]}`); + } + } else { newUploadedVideos = null; newLastVideoPublishedAt = dbLastVideoPublishedAt; + } + + // Does the local playlist already have sorted videos into knownShorts and knownVideos, or are all videos of unknownType + if (playlistModifiers[6][o] === "LocalPlaylistContainsKnownShortsAndVideos") { + localPlaylistVideoKnowledge = "knownShortsAndVideos"; + } else if (playlistModifiers[6][o] === "LocalPlaylistContainsOnlyUnknownVideos") { + localPlaylistVideoKnowledge = "unknownType"; } else { - throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[4][m]}`); + throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[6][o]}`); } - } else { - newUploadedVideos = null; - newLastVideoPublishedAt = dbLastVideoPublishedAt; - } - playlistPermutations.push({ - // Also add the modifiers to the object - playlistModifiers: { - lastFetchedFromDB: playlistModifiers[0][i], - lastUpdatedDBAt: playlistModifiers[1][j], - lastAccessedLocally: playlistModifiers[2][k], - containsDeletedVideos: playlistModifiers[3][l], - newUploadedVideos: playlistModifiers[4][m], - dbContainsNewVideos: playlistModifiers[5][n] - }, - playlistId, - channelId, - // Local - lastAccessedLocally, - lastFetchedFromDB, - localVideos, - localDeletedVideos, - localLastVideoPublishedAt, - // DB - dbVideos, - dbDeletedVideos, - lastUpdatedDBAt, - dbLastVideoPublishedAt, - // "YT API" (actually DB) - newUploadedVideos, - newLastVideoPublishedAt - }); + playlistPermutations.push({ + // Also add the modifiers to the object + playlistModifiers: { + lastFetchedFromDB: playlistModifiers[0][i], + lastUpdatedDBAt: playlistModifiers[1][j], + lastAccessedLocally: playlistModifiers[2][k], + containsDeletedVideos: playlistModifiers[3][l], + newUploadedVideos: playlistModifiers[4][m], + dbContainsNewVideos: playlistModifiers[5][n], + localPlaylistVideoKnowledge: playlistModifiers[6][o] + }, + playlistId, + channelId, + // Local + lastAccessedLocally, + lastFetchedFromDB, + localVideos, + localDeletedVideos, + localLastVideoPublishedAt, + localPlaylistVideoKnowledge, + // DB + dbVideos, + dbDeletedVideos, + lastUpdatedDBAt, + dbLastVideoPublishedAt, + // "YT API" (actually DB) + newUploadedVideos, + newLastVideoPublishedAt + }); + } } } } @@ -471,11 +491,13 @@ for (let i = 0; i < playlistModifiers[0].length; i++) { } // ----- Locally stored playlists ----- +// TODO: Make this adhere to the new format export const localPlaylistPermutations = playlistPermutations.reduce((localPlaylists, playlist) => { if (playlist.playlistModifiers.lastAccessedLocally !== "LocalPlaylistDoesNotExist") { const playlistCopy = deepCopy(playlist); - const { playlistId, lastAccessedLocally, lastFetchedFromDB, localLastVideoPublishedAt, localVideos, localDeletedVideos } = playlistCopy; - localPlaylists[playlistId] = { lastAccessedLocally, lastFetchedFromDB, lastVideoPublishedAt: localLastVideoPublishedAt, videos: deepCopy({ ...localVideos, ...localDeletedVideos }) }; + const { playlistId, lastAccessedLocally, lastFetchedFromDB, localLastVideoPublishedAt, localVideos, localDeletedVideos, localPlaylistVideoKnowledge } = playlistCopy; + const videosWithTypes = makeLocalPlaylistFromVideos(deepCopy(localVideos), deepCopy(localDeletedVideos), localPlaylistVideoKnowledge); + localPlaylists[playlistId] = { lastAccessedLocally, lastFetchedFromDB, lastVideoPublishedAt: localLastVideoPublishedAt, videos: videosWithTypes }; } return localPlaylists; }, {}); @@ -496,6 +518,29 @@ function daysAgo(x) { return new Date(Date.now() - x * 24 * 60 * 60 * 1000).toISOString(); } +function makeLocalPlaylistFromVideos(localVideos, localDeletedVideos, localPlaylistVideoKnowledge) { + const localPlaylist = { + knownShorts: {}, + knownVideos: {}, + unknownType: {} + }; + + if (localPlaylistVideoKnowledge == "knownShortsAndVideos") { + // Assign depending on the type, signified by the prefix of the video id + // Do this for both the localVideos and localDeletedVideos + localPlaylist.knownShorts = Object.fromEntries(Object.entries(localVideos).filter(([key, value]) => key.startsWith("LOC_S_"))); + localPlaylist.knownShorts = Object.assign(localPlaylist.knownShorts ?? {}, Object.fromEntries(Object.entries(localDeletedVideos).filter(([key, value]) => key.startsWith("DEL_LOC_S_")))); + localPlaylist.knownVideos = Object.fromEntries(Object.entries(localVideos).filter(([key, value]) => key.startsWith("LOC_V_"))); + localPlaylist.knownVideos = Object.assign(localPlaylist.knownVideos ?? {}, Object.fromEntries(Object.entries(localDeletedVideos).filter(([key, value]) => key.startsWith("DEL_LOC_V_")))); + } else if (localPlaylistVideoKnowledge == "unknownType") { + localPlaylist.unknownType = Object.assign({}, localVideos, localDeletedVideos); + } else { + throw new Error(`Invalid localPlaylistVideoKnowledge: ${localPlaylistVideoKnowledge}`); + } + + return localPlaylist; +} + // Create a deep copy of an object export function deepCopy(obj) { return JSON.parse(JSON.stringify(obj)); @@ -517,5 +562,4 @@ export function needsYTAPIInteraction(permutation, configSync = configSyncDefaul } else { return (permutation.playlistModifiers.lastAccessedLocally === 'LocalPlaylistDoesNotExist' || permutation.playlistModifiers.lastAccessedLocally === 'LocalPlaylistNotRecentlyAccessed'); } - } \ No newline at end of file From a5727ebdd1b43ce11fc0d1168e0628c63398376b Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 13:13:12 +0000 Subject: [PATCH 16/32] Updated playlist names --- test/shuffleVideo.test.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/shuffleVideo.test.js b/test/shuffleVideo.test.js index a7779395..f8939195 100644 --- a/test/shuffleVideo.test.js +++ b/test/shuffleVideo.test.js @@ -473,21 +473,21 @@ describe('shuffleVideo', function () { // Choose a number of playlists for which to test different user setting combinations const playlists = [ // Playlist that does not exist locally, DB is outdated - deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsNotUpToDate_LocalPlaylistDoesNotExist_LocalPlaylistContainsNoDeletedVideos_MultipleNewVideosUploaded_DBContainsNoVideosNotInLocalPlaylist')), + deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsNotUpToDate_LocalPlaylistDoesNotExist_LocalPlaylistContainsNoDeletedVideos_MultipleNewVideosUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), // Playlist that does not exist locally, DB is up-to-date - deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsUpToDate_LocalPlaylistDoesNotExist_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist')), + deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsUpToDate_LocalPlaylistDoesNotExist_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), // Locally up-to-date playlist with deleted videos - deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist')), + deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), // Locally up-to-date playlist without deleted videos - deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist')), + deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), // Playlist that has to be updated from the database, but not from the YT API - deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsUpToDate_LocalPlaylistNotRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist')), + deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsUpToDate_LocalPlaylistNotRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), // Playlist that has to be updated from the YT API as well, YT API has no new videos - deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsNotUpToDate_LocalPlaylistNotRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist')), + deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsNotUpToDate_LocalPlaylistNotRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), // Playlist that has to be updated from the YT API as well, YT API has new videos - deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsNotUpToDate_LocalPlaylistNotRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_MultipleNewVideosUploaded_DBContainsNoVideosNotInLocalPlaylist')), + deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsNotUpToDate_LocalPlaylistNotRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_MultipleNewVideosUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), // Playlist that only has shorts saved locally - deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsOnlyShorts_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist')), + deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsOnlyShorts_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), ]; setupChannelSettings(configSyncPermutations.channelSettingsPermutations, playlists); From c86f3d0587c0d9736795267959902424d754bfec Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 13:55:48 +0000 Subject: [PATCH 17/32] Fixed local storage access, test --- src/shuffleVideo.js | 6 +++--- test/playlistPermutations.js | 1 - test/shuffleVideo.test.js | 19 ++++++++++++------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index 3d2a9636..e9cf288b 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -1006,7 +1006,7 @@ function validatePlaylistInfo(playlistInfo) { throw new RandomYoutubeVideoError( { code: "RYV-10", - message: `The playlistInfo object is missing one or more required keys (Got: ${Object.keys(playlistInfo)}).`, + message: `The playlistInfo object is missing one or more required keys (Got: ${Object.keys(playlistInfo)}, videos key: ${Object.keys(playlistInfo["videos"]) ?? "No keys"}).`, solveHint: "Please try again and inform the developer if the error is not resolved.", showTrace: false } @@ -1021,10 +1021,10 @@ function validatePlaylistInfo(playlistInfo) { // Tries to fetch the playlist from local storage. If it is not present, returns an empty dictionary async function tryGetPlaylistFromLocalStorage(playlistId) { return await chrome.storage.local.get([playlistId]).then(async (result) => { - if (result[playlistId]) { + if (result[playlistId] !== undefined) { /* c8 ignore start */ // To fix a bug introduced in v2.2.1 - if (!result[playlistId]["videos"]) { + if (result[playlistId]["videos"] === undefined) { // Remove from localStorage await chrome.storage.local.remove([playlistId]); return {} diff --git a/test/playlistPermutations.js b/test/playlistPermutations.js index 9fecb133..97100f6f 100644 --- a/test/playlistPermutations.js +++ b/test/playlistPermutations.js @@ -491,7 +491,6 @@ for (let i = 0; i < playlistModifiers[0].length; i++) { } // ----- Locally stored playlists ----- -// TODO: Make this adhere to the new format export const localPlaylistPermutations = playlistPermutations.reduce((localPlaylists, playlist) => { if (playlist.playlistModifiers.lastAccessedLocally !== "LocalPlaylistDoesNotExist") { const playlistCopy = deepCopy(playlist); diff --git a/test/shuffleVideo.test.js b/test/shuffleVideo.test.js index f8939195..5b39bc90 100644 --- a/test/shuffleVideo.test.js +++ b/test/shuffleVideo.test.js @@ -10,7 +10,10 @@ import { deepCopy, configSyncPermutations, playlistPermutations, needsDBInteract // Utility to get the contents of localStorage at a certain key async function getKeyFromLocalStorage(key) { return await chrome.storage.local.get([key]).then((result) => { - return result[key]; + if (result[key] !== undefined) { + return result[key]; + } + return null; }); } @@ -144,22 +147,24 @@ describe('shuffleVideo', function () { it('should throw an error if the channel has no uploads', async function () { // Take a playlist with deleted videos - const testedPlaylist = await getKeyFromLocalStorage('UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist'); + const playlistId = "UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos"; + const channelId = playlistId.replace("UU", "UC"); + const testedPlaylist = await getKeyFromLocalStorage(playlistId); - const newVideos = Object.keys(testedPlaylist.videos).reduce((acc, videoId) => { + const newVideos = Object.keys(testedPlaylist.videos).reduce((newPlaylist, videoId) => { if (videoId.includes('DEL')) { - acc[videoId] = testedPlaylist.videos[videoId]; + newPlaylist[videoId] = testedPlaylist.videos[videoId]; } - return acc; + return newPlaylist; }, {}); - testedPlaylist.videos = newVideos; + testedPlaylist.videos.unknownType = newVideos; await chrome.storage.local.set({ [testedPlaylist.playlistId]: testedPlaylist }); setUpMockResponses(videoExistenceMockResponses); try { - await chooseRandomVideo("UC_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist", false, domElement); + await chooseRandomVideo(channelId, false, domElement); } catch (error) { expect(error).to.be.a(RandomYoutubeVideoError); expect(error.code).to.be("RYV-6B"); From 7c4a3cffb826a6807f22d4153702d67ba2d89565 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 14:09:02 +0000 Subject: [PATCH 18/32] Assigned videos to the correct playlist --- src/shuffleVideo.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index e9cf288b..eccb479b 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -231,8 +231,8 @@ async function tryGetPlaylistFromDB(playlistId, localPlaylistInfo = null) { const allVideosInDatabaseAsSet = new Set(Object.keys(playlistInfo["videos"])); // Add videos that are new from the database to the local playlist - const videosOnlyInDatabase = Object.keys(playlistInfo["videos"].filter((videoId) => !allVideosInLocalPlaylistAsSet.has(videoId))); - playlistInfo["videos"]["unknownType"] = Object.assign({}, playlistInfo["videos"]["unknownType"], Object.fromEntries(videosOnlyInDatabase.map((videoId) => [videoId, playlistInfo["videos"][videoId]]))); + const videosOnlyInDatabase = Object.keys(playlistInfo["videos"]).filter((videoId) => !allVideosInLocalPlaylistAsSet.has(videoId)); + localPlaylistInfo["videos"]["unknownType"] = Object.assign({}, localPlaylistInfo["videos"]["unknownType"], Object.fromEntries(videosOnlyInDatabase.map((videoId) => [videoId, playlistInfo["videos"][videoId]]))); // Remove videos from the local playlist object that no longer exist in the database const videoTypes = ["knownVideos", "knownShorts", "unknownType"]; From 2a60c242666a95a9751d7583ce110a19d9514724 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 14:47:43 +0000 Subject: [PATCH 19/32] Correctly merge playlist objects after fetching from DB --- src/shuffleVideo.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index eccb479b..15a3acbb 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -218,9 +218,9 @@ async function tryGetPlaylistFromDB(playlistId, localPlaylistInfo = null) { playlistInfo["lastFetchedFromDB"] = new Date().toISOString(); + const videosCopy = JSON.parse(JSON.stringify(playlistInfo["videos"])); if (!localPlaylistInfo) { // Since we just fetched the playlist, we do not have any info locally, so all videos are "unknownType" - const videosCopy = playlistInfo["videos"]; playlistInfo["videos"] = {}; playlistInfo["videos"]["unknownType"] = videosCopy; playlistInfo["videos"]["knownVideos"] = {}; @@ -232,7 +232,11 @@ async function tryGetPlaylistFromDB(playlistId, localPlaylistInfo = null) { // Add videos that are new from the database to the local playlist const videosOnlyInDatabase = Object.keys(playlistInfo["videos"]).filter((videoId) => !allVideosInLocalPlaylistAsSet.has(videoId)); - localPlaylistInfo["videos"]["unknownType"] = Object.assign({}, localPlaylistInfo["videos"]["unknownType"], Object.fromEntries(videosOnlyInDatabase.map((videoId) => [videoId, playlistInfo["videos"][videoId]]))); + + playlistInfo["videos"] = {}; + playlistInfo["videos"]["unknownType"] = Object.assign({}, localPlaylistInfo["videos"]["unknownType"], Object.fromEntries(videosOnlyInDatabase.map((videoId) => [videoId, videosCopy[videoId]]))); + playlistInfo["videos"]["knownVideos"] = localPlaylistInfo["videos"]["knownVideos"] ?? {}; + playlistInfo["videos"]["knownShorts"] = localPlaylistInfo["videos"]["knownShorts"] ?? {}; // Remove videos from the local playlist object that no longer exist in the database const videoTypes = ["knownVideos", "knownShorts", "unknownType"]; From 6d460d9359f8bd31c4245c4e33b66c491efde9ea Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 14:49:20 +0000 Subject: [PATCH 20/32] Fixed playlist access --- src/shuffleVideo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index 15a3acbb..d65b8349 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -743,7 +743,7 @@ async function chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpd shouldUpdateDatabase = true; do { // Remove the video from the local playlist object - delete playlistInfo[getVideoType(randomVideo, playlistInfo)][randomVideo]; + delete playlistInfo["videos"][getVideoType(randomVideo, playlistInfo)][randomVideo]; // Remove the deleted video from the videosToShuffle array and choose a new random video videosToShuffle.splice(videosToShuffle.indexOf(randomVideo), 1); From 8ae92d2da44194699b780816210d300584e00ea7 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 15:02:49 +0000 Subject: [PATCH 21/32] Join videos before shuffling --- src/shuffleVideo.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index d65b8349..b113afd8 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -115,6 +115,10 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE // Validate that all required keys exist in the playlistInfo object validatePlaylistInfo(playlistInfo); + // Join the new videos with the old ones to be able to use them when shuffling + // Do not delete the newVideos key as it may be needed when updating the database + playlistInfo["videos"]["unknownType"] = Object.assign({}, playlistInfo["videos"]["unknownType"] ?? {}, playlistInfo["newVideos"] ?? {}); + let chosenVideos, encounteredDeletedVideos; ({ chosenVideos, playlistInfo, shouldUpdateDatabase, encounteredDeletedVideos } = await chooseRandomVideosFromPlaylist(playlistInfo, channelId, shouldUpdateDatabase)); @@ -147,10 +151,6 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE // Update the playlist locally console.log("Saving playlist to local storage..."); - // We can now join the new videos with the old ones - // We do not yet know if the new videos are videos or shorts, so we put them in the "unknownType" key - playlistInfo["videos"]["unknownType"] = Object.assign({}, playlistInfo["videos"]["unknownType"] ?? {}, playlistInfo["newVideos"] ?? {}); - // Only save the wanted keys const playlistInfoForLocalStorage = { // Remember the last time the playlist was accessed locally (==now) From 24ca7e2f5282d7a2b95605749cf2045bc85631cd Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 15:03:24 +0000 Subject: [PATCH 22/32] More diversity in choosing playlists for testing user settings --- test/shuffleVideo.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/shuffleVideo.test.js b/test/shuffleVideo.test.js index 5b39bc90..b2fcc8c1 100644 --- a/test/shuffleVideo.test.js +++ b/test/shuffleVideo.test.js @@ -482,15 +482,15 @@ describe('shuffleVideo', function () { // Playlist that does not exist locally, DB is up-to-date deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsUpToDate_LocalPlaylistDoesNotExist_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), // Locally up-to-date playlist with deleted videos - deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), + deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsKnownShortsAndVideos')), // Locally up-to-date playlist without deleted videos deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), // Playlist that has to be updated from the database, but not from the YT API - deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsUpToDate_LocalPlaylistNotRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), + deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsUpToDate_LocalPlaylistNotRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsKnownShortsAndVideos')), // Playlist that has to be updated from the YT API as well, YT API has no new videos deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsNotUpToDate_LocalPlaylistNotRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), // Playlist that has to be updated from the YT API as well, YT API has new videos - deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsNotUpToDate_LocalPlaylistNotRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_MultipleNewVideosUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), + deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsNotUpToDate_LocalPlaylistNotRecentlyAccessed_LocalPlaylistContainsNoDeletedVideos_MultipleNewVideosUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsKnownShortsAndVideos')), // Playlist that only has shorts saved locally deepCopy(playlistPermutations.find((playlist) => playlist.playlistId === 'UU_LocalPlaylistFetchedDBRecently_DBEntryIsUpToDate_LocalPlaylistRecentlyAccessed_LocalPlaylistContainsOnlyShorts_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos')), ]; From a4ec5ed06ac5f6ee13c68d10a86be6aa8ed7625d Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 16:49:26 +0000 Subject: [PATCH 23/32] Fixed some tests --- test/shuffleVideo.test.js | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/test/shuffleVideo.test.js b/test/shuffleVideo.test.js index b2fcc8c1..880ef937 100644 --- a/test/shuffleVideo.test.js +++ b/test/shuffleVideo.test.js @@ -7,6 +7,7 @@ import { chooseRandomVideo } from '../src/shuffleVideo.js'; import { configSync, setSyncStorageValue } from '../src/chromeStorage.js'; import { deepCopy, configSyncPermutations, playlistPermutations, needsDBInteraction, needsYTAPIInteraction } from './playlistPermutations.js'; +// ---------- Utility functions ---------- // Utility to get the contents of localStorage at a certain key async function getKeyFromLocalStorage(key) { return await chrome.storage.local.get([key]).then((result) => { @@ -67,6 +68,11 @@ function checkPlaylistsUploadedToDB(messages, input) { }); } +function getAllVideosAsOneObject(playlistInfo) { + return Object.assign({}, playlistInfo.videos.unknownType, playlistInfo.videos.knownShorts, playlistInfo.videos.knownVideos); +} + +// ---------- Tests ---------- describe('shuffleVideo', function () { beforeEach(function () { @@ -943,10 +949,10 @@ describe('shuffleVideo', function () { expect(testedPlaylistLocally.lastFetchedFromDB).to.be(input.lastFetchedFromDB); expect(testedPlaylistLocally.lastVideoPublishedAt).to.be(input.localLastVideoPublishedAt); expect(testedPlaylistLocally.videos).to.be.an('object'); - expect(testedPlaylistLocally.videos).to.eql({ ...input.localVideos, ...input.localDeletedVideos }); + expect(getAllVideosAsOneObject(testedPlaylistLocally)).to.eql({ ...input.localVideos, ...input.localDeletedVideos }); } else { // For non-existent playlists, the local storage should be empty - expect(testedPlaylistLocally).to.be(undefined); + expect(testedPlaylistLocally).to.be(null); } }); @@ -1030,7 +1036,7 @@ describe('shuffleVideo', function () { expect(commands).to.contain('getCurrentTabId'); const numDeletedVideosBefore = Object.keys(input.dbDeletedVideos).filter(videoId => videoId.includes('DEL')).length; - const numDeletedVideosAfter = Object.keys(playlistInfoAfter.videos).filter(videoId => videoId.includes('DEL')).length; + const numDeletedVideosAfter = Object.keys(getAllVideosAsOneObject(playlistInfoAfter)).filter(videoId => videoId.includes('DEL')).length; // Callcount: // 6 if we need to fetch from the YT API, with update or overwrite depending on if a video was deleted @@ -1139,7 +1145,6 @@ describe('shuffleVideo', function () { const playlistInfoBefore = await getKeyFromLocalStorage(input.playlistId); await chooseRandomVideo(input.channelId, false, domElement); const playlistInfoAfter = await getKeyFromLocalStorage(input.playlistId); - const videosAfter = Object.keys(playlistInfoAfter.videos); if (input.playlistModifiers.lastAccessedLocally !== 'LocalPlaylistDoesNotExist') { expect(playlistInfoAfter.lastAccessedLocally).to.be.greaterThan(playlistInfoBefore.lastAccessedLocally); @@ -1147,7 +1152,7 @@ describe('shuffleVideo', function () { if (!needsDBInteraction(input)) { expect(playlistInfoAfter.lastFetchedFromDB).to.be(playlistInfoBefore.lastFetchedFromDB); expect(playlistInfoAfter.lastVideoPublishedAt).to.be(playlistInfoBefore.lastVideoPublishedAt); - expect(playlistInfoBefore.videos).to.have.keys(videosAfter); + expect(playlistInfoBefore.videos).to.have.keys(Object.keys(playlistInfoAfter.videos)); // If the database contains videos not in the local playlist } else if (input.playlistModifiers.dbContainsNewVideos === 'DBContainsVideosNotInLocalPlaylist' || input.playlistModifiers.dbContainsNewVideos === 'DBContainsDeletedVideos') { @@ -1159,16 +1164,16 @@ describe('shuffleVideo', function () { } if (input.localDeletedVideos) { // By fetching the videos from the database, any locally deleted videos should have been removed from the local playlist - expect(playlistInfoAfter.videos).to.not.have.keys(Object.keys(input.localDeletedVideos)); + expect(getAllVideosAsOneObject(playlistInfoAfter)).to.not.have.keys(Object.keys(input.localDeletedVideos)); } - expect({ ...input.localVideos, ...input.dbVideos, ...input.dbDeletedVideos }).to.have.keys(videosAfter); + expect({ ...input.localVideos, ...input.dbVideos, ...input.dbDeletedVideos }).to.have.keys(Object.keys(getAllVideosAsOneObject(playlistInfoAfter))); // The database and local playlist are in sync already } else { expect(playlistInfoAfter.lastFetchedFromDB).to.be.greaterThan(playlistInfoBefore.lastFetchedFromDB); expect(playlistInfoAfter.lastVideoPublishedAt).to.be(playlistInfoBefore.lastVideoPublishedAt); // By fetching the videos from the database, any deleted videos should have been removed from the local playlist // We also know that here, the db did not contain any deleted videos - expect(playlistInfoAfter.videos).to.have.keys(Object.keys({ ...input.localVideos, ...input.dbVideos })); + expect(getAllVideosAsOneObject(playlistInfoAfter)).to.have.keys(Object.keys({ ...input.localVideos, ...input.dbVideos })); } } else { // If the playlist did not exist locally before, it should now @@ -1177,7 +1182,7 @@ describe('shuffleVideo', function () { expect(playlistInfoAfter.lastAccessedLocally).to.be.greaterThan(timeBefore); expect(playlistInfoAfter.lastFetchedFromDB).to.be.greaterThan(timeBefore); expect(playlistInfoAfter.lastVideoPublishedAt).to.be(input.dbLastVideoPublishedAt); - expect({ ...input.dbVideos, ...input.dbDeletedVideos }).to.have.keys(videosAfter) + expect({ ...input.dbVideos, ...input.dbDeletedVideos }).to.have.keys(Object.keys(getAllVideosAsOneObject(playlistInfoAfter))) } }); // For playlists that need to interact with the YouTube API @@ -1187,7 +1192,6 @@ describe('shuffleVideo', function () { const playlistInfoBefore = await getKeyFromLocalStorage(input.playlistId); await chooseRandomVideo(input.channelId, false, domElement); const playlistInfoAfter = await getKeyFromLocalStorage(input.playlistId); - const videosAfter = Object.keys(playlistInfoAfter.videos); if (!needsDBInteraction(input)) { throw new Error('This test should not be run for playlists that do not need to interact with the database. If they are, this means we are now using different configSync objects.'); @@ -1200,19 +1204,19 @@ describe('shuffleVideo', function () { if (input.playlistModifiers.dbContainsNewVideos === 'DBContainsNoVideosNotInLocalPlaylist' && input.playlistModifiers.newUploadedVideos === 'NoNewVideoUploaded') { expect(playlistInfoAfter.lastVideoPublishedAt.substring(0, 10)).to.be(playlistInfoBefore.lastVideoPublishedAt.substring(0, 10)); if (input.playlistModifiers.containsDeletedVideos === 'LocalPlaylistContainsDeletedVideos') { - expect(playlistInfoAfter.videos).to.have.keys(Object.keys({ ...input.localVideos, ...input.dbVideos })); + expect(getAllVideosAsOneObject(playlistInfoAfter)).to.have.keys(Object.keys({ ...input.localVideos, ...input.dbVideos })); } else { expect(playlistInfoAfter.videos).to.eql(playlistInfoBefore.videos); } // If there are deleted videos, but no new videos } else if (input.playlistModifiers.dbContainsNewVideos === 'DBContainsDeletedVideos' && input.playlistModifiers.newUploadedVideos === 'NoNewVideoUploaded') { expect(playlistInfoAfter.lastVideoPublishedAt.substring(0, 10)).to.be(playlistInfoBefore.lastVideoPublishedAt.substring(0, 10)); - expect(playlistInfoAfter.videos).to.have.keys(Object.keys(input.localVideos)); - expect({ ...playlistInfoAfter.videos, ...input.dbDeletedVideos }).to.have.keys(Object.keys(playlistInfoAfter.videos)); + expect(getAllVideosAsOneObject(playlistInfoAfter)).to.have.keys(Object.keys(input.localVideos)); + expect({ ...getAllVideosAsOneObject(playlistInfoAfter), ...input.dbDeletedVideos }).to.have.keys(Object.keys(getAllVideosAsOneObject(playlistInfoAfter))); // If there were new videos, either in the DB or uploaded } else { expect(playlistInfoAfter.lastVideoPublishedAt).to.be.greaterThan(playlistInfoBefore.lastVideoPublishedAt); - expect({ ...input.localVideos, ...input.dbVideos, ...input.dbDeletedVideos, ...input.newUploadedVideos }).to.have.keys(videosAfter); + expect({ ...input.localVideos, ...input.dbVideos, ...input.dbDeletedVideos, ...input.newUploadedVideos }).to.have.keys(Object.keys(getAllVideosAsOneObject(playlistInfoAfter))); } } else { // If the playlist did not exist locally before, it should now @@ -1221,7 +1225,7 @@ describe('shuffleVideo', function () { expect(playlistInfoAfter.lastAccessedLocally).to.be.greaterThan(timeBefore); expect(playlistInfoAfter.lastFetchedFromDB).to.be.greaterThan(timeBefore); expect(playlistInfoAfter.lastVideoPublishedAt.substring(0, 10)).to.be(input.newLastVideoPublishedAt.substring(0, 10)); - expect({ ...input.localVideos, ...input.dbVideos, ...input.dbDeletedVideos, ...input.newUploadedVideos }).to.have.keys(videosAfter) + expect({ ...input.localVideos, ...input.dbVideos, ...input.dbDeletedVideos, ...input.newUploadedVideos }).to.have.keys(Object.keys(getAllVideosAsOneObject(playlistInfoAfter))) } }); From 7d4b236fe000a004a4d190f1ccd021bcf96c654c Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 16:49:42 +0000 Subject: [PATCH 24/32] Fixed some tests --- test/playlistPermutations.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/playlistPermutations.js b/test/playlistPermutations.js index 97100f6f..fb2c448a 100644 --- a/test/playlistPermutations.js +++ b/test/playlistPermutations.js @@ -394,13 +394,13 @@ for (let i = 0; i < playlistModifiers[0].length; i++) { localDeletedVideos = deepCopy(defaultLocalDeletedVideos); } else if (playlistModifiers[3][l] === "LocalPlaylistContainsNoDeletedVideos") { localVideos = deepCopy(defaultLocalVideos); - localDeletedVideos = {}; + localDeletedVideos = null; } else if (playlistModifiers[2][k] === "LocalPlaylistDoesNotExist") { - localVideos = {}; - localDeletedVideos = {}; + localVideos = null; + localDeletedVideos = null; } else if (playlistModifiers[3][l] === "LocalPlaylistContainsOnlyShorts") { localVideos = deepCopy(defaultLocalShorts); - localDeletedVideos = {}; + localDeletedVideos = null; } else { throw new Error(`Invalid playlist modifier combination: ${playlistModifiers[3][l]}`); } @@ -495,7 +495,7 @@ export const localPlaylistPermutations = playlistPermutations.reduce((localPlayl if (playlist.playlistModifiers.lastAccessedLocally !== "LocalPlaylistDoesNotExist") { const playlistCopy = deepCopy(playlist); const { playlistId, lastAccessedLocally, lastFetchedFromDB, localLastVideoPublishedAt, localVideos, localDeletedVideos, localPlaylistVideoKnowledge } = playlistCopy; - const videosWithTypes = makeLocalPlaylistFromVideos(deepCopy(localVideos), deepCopy(localDeletedVideos), localPlaylistVideoKnowledge); + const videosWithTypes = makeLocalPlaylistFromVideos(deepCopy(localVideos ?? {}), deepCopy(localDeletedVideos ?? {}), localPlaylistVideoKnowledge); localPlaylists[playlistId] = { lastAccessedLocally, lastFetchedFromDB, lastVideoPublishedAt: localLastVideoPublishedAt, videos: videosWithTypes }; } return localPlaylists; From 9d05232266487b6570977d9a1e950546bd4902c1 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 17:47:16 +0000 Subject: [PATCH 25/32] Fixed tests, bugs, removed unnecessary assignment --- src/shuffleVideo.js | 7 ++++--- test/testSetup.js | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index b113afd8..eedb7edd 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -131,7 +131,7 @@ export async function chooseRandomVideo(channelId, firedFromPopup, progressTextE // If any videos need to be deleted, this should be the union of videos, newvideos, minus the videos to delete if (encounteredDeletedVideos) { console.log("Some videos need to be deleted from the database. All current videos will be uploaded to the database..."); - videosToDatabase = Object.assign({}, getAllVideosFromLocalPlaylist(playlistInfo), playlistInfo["newVideos"] ?? {}); + videosToDatabase = getAllVideosFromLocalPlaylist(playlistInfo); } else { // Otherwise, we want to only upload new videos. If there are no "newVideos", we upload all videos, as this is the first time we are uploading the playlist console.log("Uploading new video IDs to the database..."); @@ -182,7 +182,8 @@ async function tryGetPlaylistFromDB(playlistId, localPlaylistInfo = null) { data: playlistId }; - let playlistInfo = await chrome.runtime.sendMessage(msg); + // Some of the tests break if we do not create a deepCopy here, as they run on the same object + let playlistInfo = JSON.parse(JSON.stringify(await chrome.runtime.sendMessage(msg))); /* c8 ignore start - These are legacy conversions we don't want to test */ // In case the playlist is still in the old Array format (before v1.0.0) in the database, convert it to the new format @@ -243,7 +244,7 @@ async function tryGetPlaylistFromDB(playlistId, localPlaylistInfo = null) { for (const type of videoTypes) { for (const videoId in localPlaylistInfo["videos"][type]) { if (!allVideosInDatabaseAsSet.has(videoId)) { - delete localPlaylistInfo["videos"][type][videoId]; + delete playlistInfo["videos"][type][videoId]; } } } diff --git a/test/testSetup.js b/test/testSetup.js index 849981fc..8cbc863c 100644 --- a/test/testSetup.js +++ b/test/testSetup.js @@ -64,7 +64,7 @@ chrome.runtime.sendMessage.callsFake((request) => { // Only for the tests case "setKeyInDB": mockedDatabase[request.data.key] = request.data.val; - return "Key was removed from database."; + return "Key was set in the database (mocked for tests)."; case "getAPIKey": return getAPIKey(false, request.data.useAPIKeyAtIndex); From 8fc1343a5332eda8eeee1156d8fc32baebb13a16 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 17:56:01 +0000 Subject: [PATCH 26/32] Fixed, updated tests --- test/shuffleVideo.test.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/shuffleVideo.test.js b/test/shuffleVideo.test.js index 880ef937..8635ed76 100644 --- a/test/shuffleVideo.test.js +++ b/test/shuffleVideo.test.js @@ -1205,8 +1205,12 @@ describe('shuffleVideo', function () { expect(playlistInfoAfter.lastVideoPublishedAt.substring(0, 10)).to.be(playlistInfoBefore.lastVideoPublishedAt.substring(0, 10)); if (input.playlistModifiers.containsDeletedVideos === 'LocalPlaylistContainsDeletedVideos') { expect(getAllVideosAsOneObject(playlistInfoAfter)).to.have.keys(Object.keys({ ...input.localVideos, ...input.dbVideos })); - } else { + } else if(input.playlistModifiers.lastUpdatedDBAt !== 'DBEntryDoesNotExist') { expect(playlistInfoAfter.videos).to.eql(playlistInfoBefore.videos); + } else { + // If the DB entry does not exist, we overwrite possible local knowledge about known videos and shorts + expect(playlistInfoAfter.videos.knownVideos).to.eql({}); + expect(playlistInfoAfter.videos.knownShorts).to.eql({}); } // If there are deleted videos, but no new videos } else if (input.playlistModifiers.dbContainsNewVideos === 'DBContainsDeletedVideos' && input.playlistModifiers.newUploadedVideos === 'NoNewVideoUploaded') { From fdd432bd99587930b0ac0950bdb7061ba42d16cf Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 17:56:07 +0000 Subject: [PATCH 27/32] Formatting --- test/shuffleVideo.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/shuffleVideo.test.js b/test/shuffleVideo.test.js index 8635ed76..0d0a925b 100644 --- a/test/shuffleVideo.test.js +++ b/test/shuffleVideo.test.js @@ -1205,7 +1205,7 @@ describe('shuffleVideo', function () { expect(playlistInfoAfter.lastVideoPublishedAt.substring(0, 10)).to.be(playlistInfoBefore.lastVideoPublishedAt.substring(0, 10)); if (input.playlistModifiers.containsDeletedVideos === 'LocalPlaylistContainsDeletedVideos') { expect(getAllVideosAsOneObject(playlistInfoAfter)).to.have.keys(Object.keys({ ...input.localVideos, ...input.dbVideos })); - } else if(input.playlistModifiers.lastUpdatedDBAt !== 'DBEntryDoesNotExist') { + } else if (input.playlistModifiers.lastUpdatedDBAt !== 'DBEntryDoesNotExist') { expect(playlistInfoAfter.videos).to.eql(playlistInfoBefore.videos); } else { // If the DB entry does not exist, we overwrite possible local knowledge about known videos and shorts From a731387b6bf7a9458b1bea047399fa2d3d69c1ff Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 22:47:35 +0000 Subject: [PATCH 28/32] Fixed behaviour and added test for no video data in DB --- src/shuffleVideo.js | 10 +--------- test/shuffleVideo.test.js | 40 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index eedb7edd..45a6ebd9 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -205,15 +205,7 @@ async function tryGetPlaylistFromDB(playlistId, localPlaylistInfo = null) { } /* c8 ignore stop */ - if (!playlistInfo) { - return {}; - } - - if (!playlistInfo["videos"]) { - // Due to some mistake, there is no video data in the database - // Overwrite the playlist with an empty one - console.log("The playlist was found in the database, but it is empty. Removing..."); - await uploadPlaylistToDatabase({}, {}, playlistId, true); + if (!playlistInfo || !playlistInfo["videos"]) { return {}; } diff --git a/test/shuffleVideo.test.js b/test/shuffleVideo.test.js index 0d0a925b..200ec428 100644 --- a/test/shuffleVideo.test.js +++ b/test/shuffleVideo.test.js @@ -480,7 +480,7 @@ describe('shuffleVideo', function () { }); }); - context('various user settings', async function () { + context('user settings', async function () { // Choose a number of playlists for which to test different user setting combinations const playlists = [ // Playlist that does not exist locally, DB is outdated @@ -995,7 +995,7 @@ describe('shuffleVideo', function () { context('database interaction', function () { if (!needsDBInteraction(input)) { - it('should only interact with the database to remove deleted videos', async function () { + it('should only interact with the database to remove deleted videos if the local playlist is up-to-date', async function () { await chooseRandomVideo(input.channelId, false, domElement); const messages = chrome.runtime.sendMessage.args; @@ -1096,7 +1096,6 @@ describe('shuffleVideo', function () { } const numDeletedVideosAfter = Object.keys(playlistInfoAfter.videos).filter(videoId => videoId.includes('DEL')).length; - expect(numDeletedVideosAfter).to.be(0); }); } else if (input.playlistModifiers.lastUpdatedDBAt === 'DBEntryIsUpToDate') { @@ -1135,6 +1134,41 @@ describe('shuffleVideo', function () { checkPlaylistsUploadedToDB(updateMessages, input); }); } + // Test special case where the database has an entry but no video data + if (input.playlistId === 'UU_LocalPlaylistDidNotFetchDBRecently_DBEntryIsUpToDate_LocalPlaylistDoesNotExist_LocalPlaylistContainsNoDeletedVideos_NoNewVideoUploaded_DBContainsNoVideosNotInLocalPlaylist_LocalPlaylistContainsOnlyUnknownVideos') { + it('should correctly handle the case that the database entry exists but has no videos', async function () { + // Remove the video data from the database entry + let newPlaylistInfoInDB = deepCopy(await chrome.runtime.sendMessage({ command: 'getPlaylistFromDB', data: input.playlistId })); + delete newPlaylistInfoInDB["videos"]; + await chrome.runtime.sendMessage({ command: 'overwritePlaylistInfoInDB', data: { key: input.playlistId, val: newPlaylistInfoInDB } }); + + await chooseRandomVideo(input.channelId, false, domElement); + + const messages = chrome.runtime.sendMessage.args; + const commands = messages.map(arg => arg[0].command); + + expect(chrome.runtime.sendMessage.callCount).to.be(8); + + // First two are from the test setup + expect(commands).to.contain('overwritePlaylistInfoInDB'); + const getPlaylistFromDBCount = commands.filter(command => command === 'getPlaylistFromDB').length; + expect(getPlaylistFromDBCount).to.equal(2); + expect(commands).to.contain('connectionTest'); + expect(commands).to.contain('getAPIKey'); + expect(commands).to.contain('updatePlaylistInfoInDB'); + const updateMessages = messages.filter(arg => arg[0].command === 'updatePlaylistInfoInDB'); + checkPlaylistsUploadedToDB(updateMessages, input); + expect(commands).to.contain('getAllYouTubeTabs'); + expect(commands).to.contain('getCurrentTabId'); + + // Get the entry from the database + const playlistInfoInDB = await chrome.runtime.sendMessage({ command: 'getPlaylistFromDB', data: input.playlistId }); + expect(playlistInfoInDB).to.not.be(null); + expect(Object.keys(playlistInfoInDB.videos).length).to.equal(10); + // These are the videos that should be in the playlist, but due to some mistake were missing from the DB + expect(playlistInfoInDB.videos).to.eql({ ...input.dbVideos }); + }); + } }); context('locally stored playlist', function () { From b5c1467047498a31de0489783f5c435b8b200cbe Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 22:56:42 +0000 Subject: [PATCH 29/32] Exclude some code from coverage, change some wording --- src/shuffleVideo.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index 45a6ebd9..7970ae85 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -600,7 +600,7 @@ async function testVideoExistence(videoId) { videoExists = true; } } catch (error) { - console.log(`Video doesn't exist: ${videoId}`); + console.log(`An error was encountered and it is assumed the video does not exist: ${videoId}`); videoExists = false; } @@ -828,9 +828,17 @@ function getVideoType(videoId, playlistInfo) { return "knownVideos"; } else if (playlistInfo["videos"]["knownShorts"][videoId]) { return "knownShorts"; - } else { - return null; } + /* c8 ignore start - This will only happen if we forget to implement something here, so we do not need to test it */ + throw new RandomYoutubeVideoError( + { + code: "RYV-11", + message: `The video that was tested does not exist in the local playlist ${videoId}, so it's type could not be found.`, + solveHint: "Please contact the developer, as this should not happen.", + showTrace: false + } + ); + /* c8 ignore stop */ } // Applies a filter to the playlist object, based on the setting set in the popup @@ -884,7 +892,7 @@ function applyShuffleFilter(allVideos, videosByDate, activeShuffleFilterOption, { code: "RYV-8C", message: `There are no videos that were released after the specified video ID (${activeOptionValue}), or the newest video has not yet been added to the database.`, - solveHint: "The extension will update playlists every 48 hours, so please wait for an update, change the video ID or use a different shuffle filter option.", + solveHint: "The extension updates playlists every 48 hours, so please wait for an update, change the video ID used in the filer or use a different filter option.", showTrace: false } ); @@ -995,6 +1003,7 @@ function getAllVideosFromLocalPlaylist(playlistInfo) { return Object.assign({}, playlistInfo["videos"]["knownVideos"], playlistInfo["videos"]["knownShorts"], playlistInfo["videos"]["unknownType"]); } +/* c8 ignore start - We do not test this function as it will only trigger if we make a mistake programming somewhere, which will get caught by the test suite */ function validatePlaylistInfo(playlistInfo) { // The playlistInfo object must contain lastVideoPublishedAt, lastFetchedFromDB and videos // The videos subkey must contain knownVideos, knownShorts and unknownType @@ -1013,6 +1022,7 @@ function validatePlaylistInfo(playlistInfo) { playlistInfo["newVideos"] = {}; } } +/* c8 ignore stop */ // ---------- Local storage ---------- // Tries to fetch the playlist from local storage. If it is not present, returns an empty dictionary From a9e0e21a85aedd3f8eaa0493c4da1092245fa7cd Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 22:57:31 +0000 Subject: [PATCH 30/32] Updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af5bc104..a3a592f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## v2.3.0 (Unreleased) +## v2.3.0 - Shuffling when excluding shorts is now a lot quicker if you have shuffled from the channel before, as the extension will remember which videos are shorts. From 460c54658fcea2b2c231692115e45ee811eb9096 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 23:04:01 +0000 Subject: [PATCH 31/32] Updated comments --- src/shuffleVideo.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shuffleVideo.js b/src/shuffleVideo.js index 7970ae85..0cd3201a 100644 --- a/src/shuffleVideo.js +++ b/src/shuffleVideo.js @@ -182,7 +182,7 @@ async function tryGetPlaylistFromDB(playlistId, localPlaylistInfo = null) { data: playlistId }; - // Some of the tests break if we do not create a deepCopy here, as they run on the same object + // Some of the tests break if we do not create a deepCopy here, as the local and database object somehow get linked let playlistInfo = JSON.parse(JSON.stringify(await chrome.runtime.sendMessage(msg))); /* c8 ignore start - These are legacy conversions we don't want to test */ @@ -600,7 +600,7 @@ async function testVideoExistence(videoId) { videoExists = true; } } catch (error) { - console.log(`An error was encountered and it is assumed the video does not exist: ${videoId}`); + console.log(`An error was encountered while checking for video existence, so it is assumed the video does not exist: ${videoId}`); videoExists = false; } From 9088aa0ddde58961274683f830aea07d178612e6 Mon Sep 17 00:00:00 2001 From: NikkelM <57323886+NikkelM@users.noreply.github.com> Date: Sat, 4 Nov 2023 23:05:07 +0000 Subject: [PATCH 32/32] Updated changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3a592f6..a964bfb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,8 @@ ## v2.3.0 -- Shuffling when excluding shorts is now a lot quicker if you have shuffled from the channel before, as the extension will remember which videos are shorts. -- Added an additional message to the shuffle button if shuffling takes longer due to ignoring shorts. +- Shuffling when excluding shorts is now a lot quicker if you have shuffled from the channel before, as the extension will remember which videos are shorts and skip them automatically. +- Added an additional message to the shuffle button if shuffling takes a bit longer due to ignoring shorts. - Fixed a rare data inconsistency bug occurring with specific database values.