From 4baee92fb132b91948c9015bf9d3d41a2ac62894 Mon Sep 17 00:00:00 2001 From: Mikal Stordal Date: Mon, 7 Oct 2024 06:15:45 +0200 Subject: [PATCH] feat: add vfs preview endpoint - Added a new VFS preview endpoint for debugging purposes. ~~A debug utility may or may not appear in the plugin soon-_ish_.~~ --- .../Resolvers/Models/LinkGenerationResult.cs | 7 + .../Resolvers/VirtualFileSystemService.cs | 184 +++++++++++++----- Shokofin/Web/Models/VfsLibraryPreview.cs | 67 +++++++ Shokofin/Web/UtilityApiController.cs | 49 +++++ 4 files changed, 255 insertions(+), 52 deletions(-) create mode 100644 Shokofin/Web/Models/VfsLibraryPreview.cs create mode 100644 Shokofin/Web/UtilityApiController.cs diff --git a/Shokofin/Resolvers/Models/LinkGenerationResult.cs b/Shokofin/Resolvers/Models/LinkGenerationResult.cs index 51d89070..ade9b8f5 100644 --- a/Shokofin/Resolvers/Models/LinkGenerationResult.cs +++ b/Shokofin/Resolvers/Models/LinkGenerationResult.cs @@ -11,6 +11,8 @@ public class LinkGenerationResult public ConcurrentBag Paths { get; init; } = []; + public ConcurrentBag RemovedPaths { get; init; } = []; + public int Total => TotalVideos + TotalSubtitles; @@ -81,10 +83,15 @@ public void Print(ILogger logger, string path) foreach (var path in b.Paths) a.Paths.Add(path); + var removedPaths = a.RemovedPaths; + foreach (var path in b.RemovedPaths) + removedPaths.Add(path); + return new() { CreatedAt = a.CreatedAt, Paths = paths, + RemovedPaths = removedPaths, CreatedVideos = a.CreatedVideos + b.CreatedVideos, FixedVideos = a.FixedVideos + b.FixedVideos, SkippedVideos = a.SkippedVideos + b.SkippedVideos, diff --git a/Shokofin/Resolvers/VirtualFileSystemService.cs b/Shokofin/Resolvers/VirtualFileSystemService.cs index 98b79e0c..7978461c 100644 --- a/Shokofin/Resolvers/VirtualFileSystemService.cs +++ b/Shokofin/Resolvers/VirtualFileSystemService.cs @@ -101,6 +101,52 @@ public void Clear() DataCache.Clear(); } + #region Preview Structure + + public async Task<(HashSet filesBefore, HashSet filesAfter, VirtualFolderInfo? virtualFolder, LinkGenerationResult? result, string vfsPath)> PreviewChangesForLibrary(Guid libraryId) + { + // Don't allow starting a preview if a library scan is running. + + var virtualFolders = LibraryManager.GetVirtualFolders(); + var selectedFolder = virtualFolders.FirstOrDefault(folder => Guid.TryParse(folder.ItemId, out var guid) && guid == libraryId); + if (selectedFolder is null) + return ([], [], null, null, string.Empty); + + if (LibraryManager.FindByPath(selectedFolder.Locations[0], true) is not Folder mediaFolder) + return ([], [], selectedFolder, null, string.Empty); + + var collectionType = selectedFolder.CollectionType.ConvertToCollectionType(); + var (vfsPath, _, mediaConfigs, _) = await ConfigurationService.GetMediaFoldersForLibraryInVFS(mediaFolder, collectionType, config => config.IsVirtualFileSystemEnabled); + if (string.IsNullOrEmpty(vfsPath) || mediaConfigs.Count is 0) + return ([], [], selectedFolder, null, string.Empty); + + if (LibraryManager.IsScanRunning) + return ([], [], selectedFolder, null, string.Empty); + + // Only allow the preview to run once per caching cycle. + return await DataCache.GetOrCreateAsync($"preview-changes:{vfsPath}", async () => { + var allPaths = GetPathsForMediaFolders(mediaConfigs); + var allFiles = GetFilesForImportFolders(mediaConfigs, allPaths); + var result = await GenerateStructure(collectionType, vfsPath, allFiles, preview: true); + result += CleanupStructure(vfsPath, vfsPath, result.Paths.ToArray(), preview: true); + + // This call will be slow depending on the size of your collection. + var existingPaths = FileSystem.DirectoryExists(vfsPath) + ? FileSystem.GetFilePaths(vfsPath, true).ToHashSet() + : []; + + // Alter the paths to match the new structure. + var alteredPaths = existingPaths + .Concat(result.Paths.ToArray()) + .Except(result.RemovedPaths.ToArray()) + .ToHashSet(); + + return (existingPaths, alteredPaths, selectedFolder, result, vfsPath); + }); + } + + #endregion + #region Generate Structure /// @@ -626,7 +672,7 @@ private HashSet GetPathsForMediaFolders(IReadOnlyList GenerateStructure(CollectionType? collectionType, string vfsPath, IEnumerable<(string sourceLocation, string fileId, string seriesId)> allFiles) + private async Task GenerateStructure(CollectionType? collectionType, string vfsPath, IEnumerable<(string sourceLocation, string fileId, string seriesId)> allFiles, bool preview = false) { var result = new LinkGenerationResult(); var maxTotalExceptions = Plugin.Instance.Configuration.VFS_MaxTotalExceptionsBeforeAbort; @@ -652,7 +698,7 @@ await Task.WhenAll(allFiles.Select(async (tuple) => { if (symbolicLinks.Length == 0 || !importedAt.HasValue) return; - var subResult = GenerateSymbolicLinks(sourceLocation, symbolicLinks, importedAt.Value); + var subResult = GenerateSymbolicLinks(sourceLocation, symbolicLinks, importedAt.Value, preview); // Combine the current results with the overall results. lock (semaphore) { @@ -804,11 +850,11 @@ file.Shoko.AniDBData is not null return (symbolicLinks, (file.Shoko.ImportedAt ?? file.Shoko.CreatedAt).ToLocalTime()); } - public LinkGenerationResult GenerateSymbolicLinks(string sourceLocation, string[] symbolicLinks, DateTime importedAt) + public LinkGenerationResult GenerateSymbolicLinks(string sourceLocation, string[] symbolicLinks, DateTime importedAt, bool preview = false) { try { var result = new LinkGenerationResult(); - if (Plugin.Instance.Configuration.VFS_ResolveLinks) { + if (Plugin.Instance.Configuration.VFS_ResolveLinks && !preview) { Logger.LogTrace("Attempting to resolve link for {Path}", sourceLocation); try { if (File.ResolveLinkTarget(sourceLocation, true) is { } linkTarget) { @@ -832,10 +878,12 @@ public LinkGenerationResult GenerateSymbolicLinks(string sourceLocation, string[ result.Paths.Add(symbolicLink); if (!File.Exists(symbolicLink)) { result.CreatedVideos++; - Logger.LogDebug("Linking {Link} → {LinkTarget}", symbolicLink, sourceLocation); - File.CreateSymbolicLink(symbolicLink, sourceLocation); - // Mock the creation date to fake the "date added" order in Jellyfin. - File.SetCreationTime(symbolicLink, importedAt); + if (!preview) { + Logger.LogDebug("Linking {Link} → {LinkTarget}", symbolicLink, sourceLocation); + File.CreateSymbolicLink(symbolicLink, sourceLocation); + // Mock the creation date to fake the "date added" order in Jellyfin. + File.SetCreationTime(symbolicLink, importedAt); + } } else { var shouldFix = false; @@ -843,26 +891,29 @@ public LinkGenerationResult GenerateSymbolicLinks(string sourceLocation, string[ var nextTarget = File.ResolveLinkTarget(symbolicLink, false); if (!string.Equals(sourceLocation, nextTarget?.FullName)) { shouldFix = true; - - Logger.LogWarning("Fixing broken symbolic link {Link} → {LinkTarget} (RealTarget={RealTarget})", symbolicLink, sourceLocation, nextTarget?.FullName); + if (!preview) + Logger.LogWarning("Fixing broken symbolic link {Link} → {LinkTarget} (RealTarget={RealTarget})", symbolicLink, sourceLocation, nextTarget?.FullName); } var date = File.GetCreationTime(symbolicLink).ToLocalTime(); if (date != importedAt) { shouldFix = true; - - Logger.LogWarning("Fixing broken symbolic link {Link} with incorrect date.", symbolicLink); + if (!preview) + Logger.LogWarning("Fixing broken symbolic link {Link} with incorrect date.", symbolicLink); } } catch (Exception ex) { - Logger.LogError(ex, "Encountered an error trying to resolve symbolic link {Link}", symbolicLink); shouldFix = true; + if (!preview) + Logger.LogError(ex, "Encountered an error trying to resolve symbolic link {Link}", symbolicLink); } if (shouldFix) { - File.Delete(symbolicLink); - File.CreateSymbolicLink(symbolicLink, sourceLocation); - // Mock the creation date to fake the "date added" order in Jellyfin. - File.SetCreationTime(symbolicLink, importedAt); result.FixedVideos++; + if (!preview) { + File.Delete(symbolicLink); + File.CreateSymbolicLink(symbolicLink, sourceLocation); + // Mock the creation date to fake the "date added" order in Jellyfin. + File.SetCreationTime(symbolicLink, importedAt); + } } else { result.SkippedVideos++; @@ -878,8 +929,10 @@ public LinkGenerationResult GenerateSymbolicLinks(string sourceLocation, string[ result.Paths.Add(subtitleLink); if (!File.Exists(subtitleLink)) { result.CreatedSubtitles++; - Logger.LogDebug("Linking {Link} → {LinkTarget}", subtitleLink, subtitleSource); - File.CreateSymbolicLink(subtitleLink, subtitleSource); + if (!preview) { + Logger.LogDebug("Linking {Link} → {LinkTarget}", subtitleLink, subtitleSource); + File.CreateSymbolicLink(subtitleLink, subtitleSource); + } } else { var shouldFix = false; @@ -887,18 +940,21 @@ public LinkGenerationResult GenerateSymbolicLinks(string sourceLocation, string[ var nextTarget = File.ResolveLinkTarget(subtitleLink, false); if (!string.Equals(subtitleSource, nextTarget?.FullName)) { shouldFix = true; - - Logger.LogWarning("Fixing broken symbolic link {Link} → {LinkTarget} (RealTarget={RealTarget})", subtitleLink, subtitleSource, nextTarget?.FullName); + if (!preview) + Logger.LogWarning("Fixing broken symbolic link {Link} → {LinkTarget} (RealTarget={RealTarget})", subtitleLink, subtitleSource, nextTarget?.FullName); } } catch (Exception ex) { - Logger.LogError(ex, "Encountered an error trying to resolve symbolic link {Link} for {LinkTarget}", subtitleLink, subtitleSource); shouldFix = true; + if (!preview) + Logger.LogError(ex, "Encountered an error trying to resolve symbolic link {Link} for {LinkTarget}", subtitleLink, subtitleSource); } if (shouldFix) { - File.Delete(subtitleLink); - File.CreateSymbolicLink(subtitleLink, subtitleSource); result.FixedSubtitles++; + if (!preview) { + File.Delete(subtitleLink); + File.CreateSymbolicLink(subtitleLink, subtitleSource); + } } else { result.SkippedSubtitles++; @@ -943,9 +999,10 @@ private List FindSubtitlesForPath(string sourcePath) return externalPaths; } - private LinkGenerationResult CleanupStructure(string vfsPath, string directoryToClean, IReadOnlyList allKnownPaths) + private LinkGenerationResult CleanupStructure(string vfsPath, string directoryToClean, IReadOnlyList allKnownPaths, bool preview = false) { - Logger.LogDebug("Looking for files to remove in folder at {Path}", directoryToClean); + if (!preview) + Logger.LogDebug("Looking for files to remove in folder at {Path}", directoryToClean); var start = DateTime.Now; var previousStep = start; var result = new LinkGenerationResult(); @@ -953,59 +1010,79 @@ private LinkGenerationResult CleanupStructure(string vfsPath, string directoryTo var toBeRemoved = FileSystem.GetFilePaths(directoryToClean, true) .Select(path => (path, extName: Path.GetExtension(path))) .Where(tuple => !string.IsNullOrEmpty(tuple.extName) && searchFiles.Contains(tuple.extName)) - .ExceptBy(allKnownPaths.ToHashSet(), tuple => tuple.path) + .ExceptBy(allKnownPaths, tuple => tuple.path) .ToList(); var nextStep = DateTime.Now; - Logger.LogDebug("Found {FileCount} files to remove in {DirectoryToClean} in {TimeSpent}", toBeRemoved.Count, directoryToClean, nextStep - previousStep); + if (!preview) + Logger.LogDebug("Found {FileCount} files to remove in {DirectoryToClean} in {TimeSpent}", toBeRemoved.Count, directoryToClean, nextStep - previousStep); previousStep = nextStep; foreach (var (location, extName) in toBeRemoved) { if (extName is ".nfo") { - try { - Logger.LogTrace("Removing NFO file at {Path}", location); - File.Delete(location); - } - catch (Exception ex) { - Logger.LogError(ex, "Encountered an error trying to remove {FilePath}", location); - continue; + if (!preview) { + try { + Logger.LogTrace("Removing NFO file at {Path}", location); + File.Delete(location); + } + catch (Exception ex) { + Logger.LogError(ex, "Encountered an error trying to remove {FilePath}", location); + continue; + } } + result.RemovedPaths.Add(location); result.RemovedNfos++; } else if (NamingOptions.SubtitleFileExtensions.Contains(extName)) { - if (TryMoveSubtitleFile(allKnownPaths, location)) { - result.FixedSubtitles++; + if (TryMoveSubtitleFile(allKnownPaths, location, preview)) { + result.Paths.Add(location); + if (preview) { + result.SkippedSubtitles++; + } + else { + result.FixedSubtitles++; + } continue; } - try { - Logger.LogTrace("Removing subtitle file at {Path}", location); - File.Delete(location); - } - catch (Exception ex) { - Logger.LogError(ex, "Encountered an error trying to remove {FilePath}", location); - continue; + if (!preview) { + try { + Logger.LogTrace("Removing subtitle file at {Path}", location); + File.Delete(location); + } + catch (Exception ex) { + Logger.LogError(ex, "Encountered an error trying to remove {FilePath}", location); + continue; + } } + result.RemovedPaths.Add(location); result.RemovedSubtitles++; } else { if (ShouldIgnoreVideo(vfsPath, location)) { + result.Paths.Add(location); result.SkippedVideos++; continue; } - try { - Logger.LogTrace("Removing video file at {Path}", location); - File.Delete(location); - } - catch (Exception ex) { - Logger.LogError(ex, "Encountered an error trying to remove {FilePath}", location); - continue; + if (!preview) { + try { + Logger.LogTrace("Removing video file at {Path}", location); + File.Delete(location); + } + catch (Exception ex) { + Logger.LogError(ex, "Encountered an error trying to remove {FilePath}", location); + continue; + } } + result.RemovedPaths.Add(location); result.RemovedVideos++; } } + if (preview) + return result; + nextStep = DateTime.Now; Logger.LogTrace("Removed {FileCount} files in {DirectoryToClean} in {TimeSpent} (Total={TotalSpent})", result.Removed, directoryToClean, nextStep - previousStep, nextStep - start); previousStep = nextStep; @@ -1047,7 +1124,7 @@ private LinkGenerationResult CleanupStructure(string vfsPath, string directoryTo return result; } - private static bool TryMoveSubtitleFile(IReadOnlyList allKnownPaths, string subtitlePath) + private static bool TryMoveSubtitleFile(IReadOnlyList allKnownPaths, string subtitlePath, bool preview) { if (!TryGetIdsForPath(subtitlePath, out var seriesId, out var fileId)) return false; @@ -1069,6 +1146,9 @@ private static bool TryMoveSubtitleFile(IReadOnlyList allKnownPaths, str if (string.IsNullOrEmpty(realTarget)) return false; + if (preview) + return true; + var realSubtitlePath = realTarget[..^Path.GetExtension(realTarget).Length] + extName; if (!File.Exists(realSubtitlePath)) File.Move(subtitlePath, realSubtitlePath); diff --git a/Shokofin/Web/Models/VfsLibraryPreview.cs b/Shokofin/Web/Models/VfsLibraryPreview.cs new file mode 100644 index 00000000..c0636430 --- /dev/null +++ b/Shokofin/Web/Models/VfsLibraryPreview.cs @@ -0,0 +1,67 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using MediaBrowser.Model.Entities; +using Shokofin.Resolvers.Models; + +namespace Shokofin.Web.Models; + +public class VfsLibraryPreview(HashSet filesBefore, HashSet filesAfter, VirtualFolderInfo virtualFolder, LinkGenerationResult? result, string vfsPath) +{ + public string LibraryId = virtualFolder.ItemId; + + public string LibraryName { get; } = virtualFolder.Name; + + public string CollectionType { get; } = virtualFolder.CollectionType.ConvertToCollectionType()?.ToString() ?? "-"; + + public string VfsRoot { get; } = Plugin.Instance.VirtualRoot; + + public bool IsSuccess = result is not null; + + public IReadOnlyList FilesBeforeChanges { get; } = filesBefore + .Select(path => path.Replace(vfsPath, string.Empty).Replace(Path.DirectorySeparatorChar, '/')) + .OrderBy(path => path) + .ToList(); + + public IReadOnlyList FilesAfterChanges { get; } = filesAfter + .Select(path => path.Replace(vfsPath, string.Empty).Replace(Path.DirectorySeparatorChar, '/')) + .OrderBy(path => path) + .ToList(); + + public VfsLibraryPreviewStats Stats { get; } = new(result); + + public class VfsLibraryPreviewStats(LinkGenerationResult? result) + { + public int Total { get; } = result?.Total ?? 0; + + public int Created { get; } = result?.Created ?? 0; + + public int Fixed { get; } = result?.Fixed ?? 0; + + public int Skipped { get; } = result?.Skipped ?? 0; + + public int Removed { get; } = result?.Removed ?? 0; + + public int TotalVideos { get; } = result?.TotalVideos ?? 0; + + public int CreatedVideos { get; } = result?.CreatedVideos ?? 0; + + public int FixedVideos { get; } = result?.FixedVideos ?? 0; + + public int SkippedVideos { get; } = result?.SkippedVideos ?? 0; + + public int RemovedVideos { get; } = result?.RemovedVideos ?? 0; + + public int TotalSubtitles { get; } = result?.TotalSubtitles ?? 0; + + public int CreatedSubtitles { get; } = result?.CreatedSubtitles ?? 0; + + public int FixedSubtitles { get; } = result?.FixedSubtitles ?? 0; + + public int SkippedSubtitles { get; } = result?.SkippedSubtitles ?? 0; + + public int RemovedSubtitles { get; } = result?.RemovedSubtitles ?? 0; + + public int RemovedNfos { get; } = result?.RemovedNfos ?? 0; + } +} \ No newline at end of file diff --git a/Shokofin/Web/UtilityApiController.cs b/Shokofin/Web/UtilityApiController.cs new file mode 100644 index 00000000..b1db91e6 --- /dev/null +++ b/Shokofin/Web/UtilityApiController.cs @@ -0,0 +1,49 @@ +using System; +using System.Net.Mime; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Shokofin.Configuration; +using Shokofin.Resolvers; +using Shokofin.Web.Models; + +namespace Shokofin.Web; + +/// +/// Shoko Utility Web Controller. +/// +/// +/// Initializes a new instance of the class. +/// +[ApiController] +[Route("Plugin/Shokofin/Utility")] +[Produces(MediaTypeNames.Application.Json)] +public class UtilityApiController(ILogger logger, MediaFolderConfigurationService mediaFolderConfigurationService, VirtualFileSystemService virtualFileSystemService) : ControllerBase +{ + private readonly ILogger Logger = logger; + + private readonly MediaFolderConfigurationService ConfigurationService = mediaFolderConfigurationService; + + private readonly VirtualFileSystemService VirtualFileSystemService = virtualFileSystemService; + + /// + /// Previews the VFS structure for the given library. + /// + /// The id of the library to preview. + /// A or if the library is not found. + [HttpPost("VFS/Library/{libraryId}/Preview")] + public async Task> PreviewVFS(Guid libraryId) + { + var trackerId = Plugin.Instance.Tracker.Add("Preview VFS"); + try { + var (filesBefore, filesAfter, virtualFolder, result, vfsPath) = await VirtualFileSystemService.PreviewChangesForLibrary(libraryId); + if (virtualFolder is null) + return NotFound("Unable to find library with the given id."); + + return new VfsLibraryPreview(filesBefore, filesAfter, virtualFolder, result, vfsPath); + } + finally { + Plugin.Instance.Tracker.Remove(trackerId); + } + } +} \ No newline at end of file