Server manifest & Multi-fork support (#5)

* Directory.Packages.props

* Delete database request logger.

Design is bad, I had to turn it off on our servers ages ago due to perf issues. I doubt anybody cares.

* Fix migrations running twice.

Apparently??

* Style settings in Rider

* Multi-fork support, test project, library

Content versions are now stored per fork.

Created a downloader project that serves as a test of the CDN. Added a Robust.Cdn.Lib project that sharable code will be moved to. In the future the launcher will be able to use this project instead.

A bunch of misc shit.

* Replace most custom Zstd streams with SharpZstd

* Disable request log in dev

* Make content ingestion work with new fork system, move to proper job scheduling library

* CDN fork publishing system

* Fix typo causing incorrect hash calculation

* Fix broken file get query

* Infrastructure to notify watchdogs of new published versions.

* Fix System.CommandLine reference being in Robust.Cdn.Lib

Should be in Robust.Cdn.Downloader

* Add user agent to HttpClients

* Private manifest fork support

* Automatically prune old manifest builds.

* Fancy build list page.

* Fix dockerfile

.dockerignore

@@ -24,3 +24,8 @@
 LICENSE
 README.md
 **/appsettings.Development.json
+**/*.db
+*.DotSettings*
+*.editorconfig
+testData/
+

.editorconfig

@@ -8,5 +8,5 @@ indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[*.{csproj,xml,yml,dll.config,msbuildproj,targets,json}]
+[*.{csproj,xml,yml,dll.config,msbuildproj,targets,props,json}]
 indent_size = 2

.gitignore

@@ -5,5 +5,6 @@ riderModule.iml
 /_ReSharper.Caches/
 
 Robust.Cdn/content.db*
+Robust.Cdn/manifest.db*
 *.user
 testData/

Directory.Packages.props

@@ -0,0 +1,18 @@
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageVersion Include="Microsoft.Extensions.Hosting.Systemd" Version="8.0.0" />
+    <!--<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.3.0" />-->
+    <PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.1" />
+    <PackageVersion Include="Dapper" Version="2.1.28" />
+    <PackageVersion Include="Quartz" Version="3.9.0" />
+    <PackageVersion Include="Quartz.Extensions.DependencyInjection" Version="3.9.0" />
+    <PackageVersion Include="Quartz.Extensions.Hosting" Version="3.9.0" />
+    <PackageVersion Include="SpaceWizards.Sodium" Version="0.2.1" />
+    <PackageVersion Include="SharpZstd.Interop" Version="1.5.6" />
+    <PackageVersion Include="SharpZstd" Version="1.5.6" />
+    <PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+  </ItemGroup>
+</Project>

Dockerfile

@@ -6,9 +6,8 @@ EXPOSE 8080
 FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 ARG BUILD_CONFIGURATION=Release
 WORKDIR /src
-COPY ["Robust.Cdn/Robust.Cdn.csproj", "Robust.Cdn/"]
-RUN dotnet restore "Robust.Cdn/Robust.Cdn.csproj"
 COPY . .
+RUN dotnet restore "Robust.Cdn/Robust.Cdn.csproj"
 WORKDIR "/src/Robust.Cdn"
 RUN dotnet build "Robust.Cdn.csproj" -c $BUILD_CONFIGURATION -o /app/build
 
@@ -22,5 +21,7 @@ COPY --from=publish /app/publish .
 ENTRYPOINT ["dotnet", "Robust.Cdn.dll"]
 VOLUME /database
 ENV CDN__DatabaseFileName=/database/content.db
+VOLUME /manifest
+ENV Manifest__DatabaseFileName=/manifest/manifest.db
 VOLUME /builds
-ENV CDN__VersionDiskPath=/builds
+ENV Manifest__FileDiskPath=/builds

Robust.Cdn.Downloader/Program.cs

@@ -0,0 +1,28 @@
+using System.CommandLine;
+using Robust.Cdn.Lib;
+
+var rootCommand = new RootCommand();
+
+{
+    var downloadDestinationArgument = new Argument<FileInfo>("destination");
+    var downloadUrlArgument = new Argument<string>("url");
+    var downloadIndexArgument = new Argument<int>("index");
+    var downloadIndexFromUrlCommand = new Command("index-from-url");
+    downloadIndexFromUrlCommand.AddArgument(downloadUrlArgument);
+    downloadIndexFromUrlCommand.AddArgument(downloadIndexArgument);
+    downloadIndexFromUrlCommand.AddArgument(downloadDestinationArgument);
+    downloadIndexFromUrlCommand.SetHandler(async (url, index, destination) =>
+    {
+        using var httpClient = new HttpClient();
+        using var downloader = await Downloader.DownloadFilesAsync(httpClient, url, [index]);
+
+        using var file = destination.Create();
+
+        await downloader.ReadFileHeaderAsync();
+        await downloader.ReadFileContentsAsync(file);
+
+    }, downloadUrlArgument, downloadIndexArgument, downloadDestinationArgument);
+    rootCommand.AddCommand(downloadIndexFromUrlCommand);
+}
+
+await rootCommand.InvokeAsync(args);

Robust.Cdn.Downloader/Robust.Cdn.Downloader.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Robust.Cdn.Lib\Robust.Cdn.Lib.csproj" />
+      <PackageReference Include="System.CommandLine" />
+    </ItemGroup>
+
+</Project>

Robust.Cdn.Lib/Downloader.cs

@@ -0,0 +1,219 @@
+using System.Buffers;
+using System.Buffers.Binary;
+using System.Globalization;
+using System.Net.Http.Headers;
+using System.Net.Mime;
+using SharpZstd;
+
+namespace Robust.Cdn.Lib;
+
+public static class Downloader
+{
+    // ReSharper disable once ConvertToConstant.Global
+    public static readonly int ManifestDownloadProtocolVersion = 1;
+
+    public static async Task<DownloadReader> DownloadFilesAsync(
+        HttpClient client,
+        string downloadUrl,
+        IEnumerable<int> downloadIndices,
+        CancellationToken cancel = default)
+    {
+        var request = new HttpRequestMessage(HttpMethod.Post, downloadUrl);
+        request.Content = new ByteArrayContent(BuildRequestBody(downloadIndices, out var totalFiles));
+        request.Content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypeNames.Application.Octet);
+        request.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("zstd"));
+        request.Headers.Add(
+            "X-Robust-Download-Protocol",
+            ManifestDownloadProtocolVersion.ToString(CultureInfo.InvariantCulture));
+
+        var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancel);
+        try
+        {
+            response.EnsureSuccessStatusCode();
+
+            var stream = await response.Content.ReadAsStreamAsync(cancel);
+            if (response.Content.Headers.ContentEncoding.Contains("zstd"))
+                stream = new ZstdDecodeStream(stream, leaveOpen: false);
+
+            try
+            {
+                var header = await ReadStreamHeaderAsync(stream, cancel);
+
+                return new DownloadReader(response, stream, header, totalFiles);
+            }
+            catch
+            {
+                await stream.DisposeAsync();
+                throw;
+            }
+        }
+        catch
+        {
+            response.Dispose();
+            throw;
+        }
+    }
+
+    private static byte[] BuildRequestBody(IEnumerable<int> indices, out int totalFiles)
+    {
+        var toDownload = indices.ToArray();
+        var requestBody = new byte[toDownload.Length * 4];
+        var reqI = 0;
+        foreach (var idx in toDownload)
+        {
+            BinaryPrimitives.WriteInt32LittleEndian(requestBody.AsSpan(reqI, 4), idx);
+            reqI += 4;
+        }
+
+        totalFiles = toDownload.Length;
+        return requestBody;
+    }
+
+    private static async Task<DownloadStreamHeaderData> ReadStreamHeaderAsync(Stream stream, CancellationToken cancel)
+    {
+        var streamHeader = await stream.ReadExactAsync(4, cancel);
+        var streamFlags = (DownloadStreamHeaderFlags)BinaryPrimitives.ReadInt32LittleEndian(streamHeader);
+
+        return new DownloadStreamHeaderData
+        {
+            Flags = streamFlags
+        };
+    }
+}
+
+[Flags]
+public enum DownloadStreamHeaderFlags
+{
+    None = 0,
+
+    /// <summary>
+    /// If this flag is set on the download stream, individual files have been pre-compressed by the server.
+    /// This means each file has a compression header, and the launcher should not attempt to compress files itself.
+    /// </summary>
+    PreCompressed = 1 << 0
+}
+
+public sealed class DownloadStreamHeaderData
+{
+    public DownloadStreamHeaderFlags Flags { get; init; }
+
+    public bool PreCompressed => (Flags & DownloadStreamHeaderFlags.PreCompressed) != 0;
+}
+
+public sealed class DownloadReader : IDisposable
+{
+    private readonly Stream _stream;
+    private readonly HttpResponseMessage _httpResponse;
+    private readonly int _totalFileCount;
+    private readonly byte[] _headerReadBuffer;
+    public DownloadStreamHeaderData Data { get; }
+
+    private int _filesRead;
+    private State _state = State.ReadFileHeader;
+    private FileHeaderData _currentHeader;
+
+    internal DownloadReader(
+        HttpResponseMessage httpResponse,
+        Stream stream,
+        DownloadStreamHeaderData data,
+        int totalFileCount)
+    {
+        _stream = stream;
+        Data = data;
+        _totalFileCount = totalFileCount;
+        _httpResponse = httpResponse;
+        _headerReadBuffer = new byte[data.PreCompressed ? 8 : 4];
+    }
+
+    public async ValueTask<FileHeaderData?> ReadFileHeaderAsync(CancellationToken cancel = default)
+    {
+        CheckState(State.ReadFileHeader);
+
+        if (_filesRead >= _totalFileCount)
+            return null;
+
+        await _stream.ReadExactlyAsync(_headerReadBuffer, cancel);
+
+        var length = BinaryPrimitives.ReadInt32LittleEndian(_headerReadBuffer.AsSpan(0, 4));
+        var compressedLength = 0;
+
+        if (Data.PreCompressed)
+            compressedLength = BinaryPrimitives.ReadInt32LittleEndian(_headerReadBuffer.AsSpan(4, 4));
+
+        _currentHeader = new FileHeaderData
+        {
+            DataLength = length,
+            CompressedLength = compressedLength
+        };
+
+        _state = State.ReadFileContents;
+        _filesRead += 1;
+
+        return _currentHeader;
+    }
+
+    public async ValueTask ReadRawFileContentsAsync(Memory<byte> buffer, CancellationToken cancel = default)
+    {
+        CheckState(State.ReadFileContents);
+
+        var size = _currentHeader.IsPreCompressed ? _currentHeader.CompressedLength : _currentHeader.DataLength;
+        if (size > buffer.Length)
+            throw new ArgumentException("Provided buffer is not large enough to fit entire data size");
+
+        await _stream.ReadExactlyAsync(buffer, cancel);
+
+        _state = State.ReadFileHeader;
+    }
+
+    public async ValueTask ReadFileContentsAsync(Stream destination, CancellationToken cancel = default)
+    {
+        CheckState(State.ReadFileContents);
+
+        if (_currentHeader.IsPreCompressed)
+        {
+            // TODO: Buffering can be avoided here.
+            var compressedBuffer = ArrayPool<byte>.Shared.Rent(_currentHeader.CompressedLength);
+
+            await _stream.ReadExactlyAsync(compressedBuffer, cancel);
+
+            var ms = new MemoryStream(compressedBuffer, writable: false);
+            await using var decompress = new ZstdDecodeStream(ms, false);
+
+            await decompress.CopyToAsync(destination, cancel);
+
+            ArrayPool<byte>.Shared.Return(compressedBuffer);
+        }
+        else
+        {
+            await _stream.CopyAmountToAsync(destination, _currentHeader.DataLength, 4096, cancel);
+        }
+
+        _state = State.ReadFileHeader;
+    }
+
+    private void CheckState(State expectedState)
+    {
+        if (expectedState != _state)
+            throw new InvalidOperationException($"Invalid state! Expected {expectedState}, but was {_state}");
+    }
+
+    public enum State : byte
+    {
+        ReadFileHeader,
+        ReadFileContents
+    }
+
+    public struct FileHeaderData
+    {
+        public int DataLength;
+        public int CompressedLength;
+
+        public bool IsPreCompressed => CompressedLength > 0;
+    }
+
+    public void Dispose()
+    {
+        _stream.Dispose();
+        _httpResponse.Dispose();
+    }
+}

Robust.Cdn.Lib/Robust.Cdn.Lib.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="SharpZstd.Interop" />
+    <PackageReference Include="SharpZstd" />
+  </ItemGroup>
+</Project>

Robust.Cdn.Lib/StreamHelper.cs

@@ -0,0 +1,42 @@
+using System.Buffers;
+
+namespace Robust.Cdn.Lib;
+
+internal static class StreamHelper
+{
+    public static async ValueTask<byte[]> ReadExactAsync(this Stream stream, int amount, CancellationToken cancel)
+    {
+        var data = new byte[amount];
+        await stream.ReadExactlyAsync(data, cancel);
+        return data;
+    }
+
+    public static async Task CopyAmountToAsync(
+        this Stream stream,
+        Stream to,
+        int amount,
+        int bufferSize,
+        CancellationToken cancel)
+    {
+        var buffer = ArrayPool<byte>.Shared.Rent(bufferSize);
+
+        while (amount > 0)
+        {
+            Memory<byte> readInto = buffer;
+            if (amount < readInto.Length)
+                readInto = readInto[..amount];
+
+            var read = await stream.ReadAsync(readInto, cancel);
+            if (read == 0)
+                throw new EndOfStreamException();
+
+            amount -= read;
+
+            readInto = readInto[..read];
+
+            await to.WriteAsync(readInto, cancel);
+        }
+
+        ArrayPool<byte>.Shared.Return(buffer);
+    }
+}