Updates for mod mirrors and 3.7.3.0

CHANGELOG.md

@@ -1,5 +1,9 @@
 ### Changelog
 
+#### Version - 3.7.3.0 - 9/19/2024
+* Downloads now start in a random order, to reduce the amount of lag in the UI when initially starting
+* Wabbajack may now redirect some downloads to mirrors hosted on Nexus Mods (by request of the mod authors)
+
 #### Version - 3.7.2.1 - 9/1/2024
 * Fixed a bug with the html reports when in a folder with a space in the name
 

Wabbajack.CLI.Builder/CommandLineBuilder.cs

@@ -32,6 +32,10 @@ public async Task<int> Run(string[] args)
             typeof(string),
             d => new Option<string>(d.Aliases, description: d.Description)
         },
+        {
+            typeof(int),
+            d => new Option<int>(d.Aliases, description: d.Description)
+        },
         {
             typeof(AbsolutePath),
             d => new Option<AbsolutePath>(d.Aliases, description: d.Description, parseArgument: d => d.Tokens.Single().Value.ToAbsolutePath())

Wabbajack.CLI/Program.cs

@@ -41,6 +41,7 @@ private static async Task<int> Main(string[] args)
                 services.AddSingleton<Client>();
                 services.AddSingleton<Networking.WabbajackClientApi.Client>();
                 services.AddSingleton(s => new GitHubClient(new ProductHeaderValue("wabbajack")));
+                services.AddSingleton<TemporaryFileManager>();
 
                 services.AddOSIntegrated();
                 services.AddServerLib();

Wabbajack.CLI/VerbRegistration.cs

@@ -1,4 +1,4 @@
-
+
 using Microsoft.Extensions.DependencyInjection;
 namespace Wabbajack.CLI;
 using Wabbajack.CLI.Verbs;
@@ -27,6 +27,8 @@ public static void AddCLIVerbs(this IServiceCollection services) {
 services.AddSingleton<HashFile>();
 CommandLineBuilder.RegisterCommand<HashUrlString>(HashUrlString.Definition, c => ((HashUrlString)c).Run);
 services.AddSingleton<HashUrlString>();
+CommandLineBuilder.RegisterCommand<IndexNexusMod>(IndexNexusMod.Definition, c => ((IndexNexusMod)c).Run);
+services.AddSingleton<IndexNexusMod>();
 CommandLineBuilder.RegisterCommand<Install>(Install.Definition, c => ((Install)c).Run);
 services.AddSingleton<Install>();
 CommandLineBuilder.RegisterCommand<InstallCompileInstallVerify>(InstallCompileInstallVerify.Definition, c => ((InstallCompileInstallVerify)c).Run);

Wabbajack.CLI/Verbs/IndexNexusMod.cs

@@ -0,0 +1,93 @@
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.CLI.Builder;
+using Wabbajack.Downloaders;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.DownloadStates;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Hashing.xxHash64;
+using Wabbajack.Networking.NexusApi;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+using AbsolutePath = Wabbajack.Paths.AbsolutePath;
+
+namespace Wabbajack.CLI.Verbs;
+
+public class IndexNexusMod
+{
+    private readonly ILogger<IndexNexusMod> _logger;
+    private readonly NexusApi _client;
+    private readonly TemporaryFileManager _manager;
+    private readonly DownloadDispatcher _downloadDispatcher;
+    private readonly DTOSerializer _serializer;
+
+    public IndexNexusMod(ILogger<IndexNexusMod> logger, NexusApi nexusClient, TemporaryFileManager manager, DownloadDispatcher downloadDispatcher, DTOSerializer serializer)
+    {
+        _logger = logger;
+        _client = nexusClient;
+        _manager = manager;
+        _downloadDispatcher = downloadDispatcher;
+        _serializer = serializer;
+    }
+
+    public static VerbDefinition Definition = new VerbDefinition("index-nexus-mod",
+        "Downloads all files for a mod and creates a mirror.json entry for the files", new[]
+        {
+            new OptionDefinition(typeof(string), "g", "game", "Game Domain"),
+            new OptionDefinition(typeof(int), "m", "mod-id", "Nexus mod ID"),
+            new OptionDefinition(typeof(AbsolutePath), "o", "output", "Output mirror.json file")
+        });
+
+    public async Task<int> Run(string game, int modId, AbsolutePath output, CancellationToken token)
+    {
+        var gameInstance = GameRegistry.GetByFuzzyName(game);
+        var modFiles = await _client.ModFiles(game, modId, token);
+        _logger.LogInformation("Found {Count} files", modFiles.info.Files.Length);
+
+        var files = new List<Archive>();
+        foreach (var file in modFiles.info.Files)
+        {
+            _logger.LogInformation("Downloading {File}", file.FileName);
+            await using var path = _manager.CreateFile();
+            var archive = new Archive()
+            {
+                Name = file.FileName,
+                State = new Nexus
+                {
+                    FileID = file.FileId,
+                    Game = gameInstance.Game,
+                    ModID = modId,
+                    Description = file.Description,
+                    Name = file.Name,
+                    Version = file.Version
+                }
+            };
+
+            Hash hash;
+            try
+            {
+                hash = await _downloadDispatcher.Download(archive, path.Path.ToString().ToAbsolutePath(), token);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Failed to download {File}", file.FileName);
+                continue;
+            }
+
+            _logger.LogInformation("Downloaded {File} with hash {Hash}", file.FileName, hash);
+            archive.Hash = hash;
+            archive.Size = file.SizeInBytes!.Value;
+            files.Add(archive);
+        }
+
+        await using var stream = output.Open(FileMode.Create, FileAccess.Write, FileShare.None);
+        await _serializer.Serialize(files, stream, true);
+        return 0;
+    }
+}

Wabbajack.Compiler/MO2Compiler.cs

@@ -206,9 +206,18 @@ private async Task RunValidation(ModList modList)
     {
         NextStep("Finalizing", "Validating Archives", modList.Archives.Length);
         var allowList = await _wjClient.LoadDownloadAllowList();
+        var mirrors = (await _wjClient.LoadMirrors()).ToLookup(a => a.Hash);
         foreach (var archive in modList.Archives)
         {
             UpdateProgress(1);
+            var matchedHashes = mirrors[archive.Hash].ToArray();
+            if (matchedHashes.Any())
+            {
+                _logger.LogInformation("Replacing {name}, {primaryKeyString} with {mirror}", archive.Name,
+                    archive.State.PrimaryKeyString, matchedHashes.First().Name);
+                archive.State = matchedHashes.First().State;
+            }
+
             if (!_dispatcher.IsAllowed(archive, allowList))
             {
                 _logger.LogCritical("Archive {name}, {primaryKeyString} is not allowed", archive.Name,

Wabbajack.Installer/AInstaller.cs

@@ -324,8 +324,22 @@ public async Task DownloadArchives(CancellationToken token)
 
         _logger.LogInformation("Downloading validation data");
         var validationData = await _wjClient.LoadDownloadAllowList();
+        var mirrors = (await _wjClient.LoadMirrors()).ToLookup(m => m.Hash);
 
         _logger.LogInformation("Validating Archives");
+
+        foreach (var archive in missing)
+        {
+            var matches = mirrors[archive.Hash].ToArray();
+            if (!matches.Any()) continue;
+
+            archive.State = matches.First().State;
+            _ = _wjClient.SendMetric("rerouted", archive.Hash.ToString());
+            _logger.LogInformation("Rerouted {Archive} to {Mirror}", archive.Name,
+                matches.First().State.PrimaryKeyString);
+        }
+
+
         foreach (var archive in missing.Where(archive =>
                      !_downloadDispatcher.Downloader(archive).IsAllowed(validationData, archive.State)))
         {
@@ -358,7 +372,7 @@ public async Task DownloadMissingArchives(List<Archive> missing, CancellationTok
         }
 
         await missing
-            .OrderBy(a => a.Size)
+            .Shuffle()
             .Where(a => a.State is not Manual)
             .PDoAll(async archive =>
             {

Wabbajack.Networking.WabbajackClientApi/Client.cs

@@ -106,6 +106,12 @@ public async Task<ServerAllowList> LoadDownloadAllowList()
         return d.Deserialize<ServerAllowList>(str);
     }
 
+    public async Task<Archive[]> LoadMirrors()
+    {
+        var str = await _client.GetStringAsync(_configuration.MirrorList);
+        return JsonSerializer.Deserialize<Archive[]>(str, _dtos.Options) ?? [];
+    }
+
     public async Task<ServerAllowList> LoadMirrorAllowList()
     {
         var str = await _client.GetStringAsync(_configuration.MirrorAllowList);

Wabbajack.Networking.WabbajackClientApi/Configuration.cs

@@ -15,6 +15,9 @@ public class Configuration
 
     public Uri ServerAllowList { get; set; } =
         new("https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml");
+
+    public Uri MirrorList { get; set; } =
+        new("https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/mirrors.json");
 
     public Uri MirrorAllowList { get; set; } =
         new("https://raw.githubusercontent.com/wabbajack-tools/allow-lists/main/allowed-mirrors.yaml");