Try to fix broken danbooru

Source/WetPicsRebirth/Infrastructure/EngineFactory.cs

@@ -7,6 +7,7 @@
 using WetPicsRebirth.Data.Entities;
 using WetPicsRebirth.Infrastructure.Engines;
 using WetPicsRebirth.Infrastructure.Engines.Pixiv;
+using IHttpClientFactory = System.Net.Http.IHttpClientFactory;
 
 namespace WetPicsRebirth.Infrastructure;
 
@@ -16,21 +17,27 @@ public class EngineFactory : IEngineFactory
     private readonly HttpClient _httpClient;
     private readonly IPixivApiClient _pixivApiClient;
     private readonly ILoggerFactory _loggerFactory;
+    private readonly IHttpClientFactory _httpClientFactory;
 
     public EngineFactory(
         HttpClient httpClient,
         IOptions<DanbooruConfiguration> danbooruConfiguration,
         IPixivApiClient pixivApiClient,
-        ILoggerFactory loggerFactory)
+        ILoggerFactory loggerFactory,
+        IHttpClientFactory httpClientFactory)
     {
         _httpClient = httpClient;
         _pixivApiClient = pixivApiClient;
         _loggerFactory = loggerFactory;
+        _httpClientFactory = httpClientFactory;
         _danbooruConfiguration = danbooruConfiguration.Value;
     }
 
     public IPopularListLoaderEngine Get(ImageSource imageSource)
     {
+        var danbooruHttpClient = _httpClientFactory.CreateClient();
+        danbooruHttpClient.DefaultRequestHeaders.Add("User-Agent", _danbooruConfiguration.BotUserAgent);
+
         return imageSource switch
         {
             ImageSource.Yandere => new BooruEngine(
@@ -39,7 +46,7 @@ public IPopularListLoaderEngine Get(ImageSource imageSource)
                     Options.Create(new YandereSettings())), 
                 _httpClient,
                 _loggerFactory.CreateLogger<BooruEngine>()),
-            ImageSource.Danbooru => new BooruEngine(
+            ImageSource.Danbooru => new FallbackToGelbooruDanbooruEngine(
                 new DanbooruApiLoader(
                     new PerBaseUrlFlurlClientFactory(),
                     Options.Create(new DanbooruSettings()
@@ -49,8 +56,14 @@ public IPopularListLoaderEngine Get(ImageSource imageSource)
                         PauseBetweenRequestsInMs = _danbooruConfiguration.Delay,
                         BotUserAgent = _danbooruConfiguration.BotUserAgent
                     })),
-                _httpClient,
-                _loggerFactory.CreateLogger<BooruEngine>()),
+                new GelbooruApiLoader(
+                    new PerBaseUrlFlurlClientFactory(),
+                    Options.Create(new GelbooruSettings()
+                    {
+                        PauseBetweenRequestsInMs = _danbooruConfiguration.Delay,
+                    })),
+                danbooruHttpClient,
+                _loggerFactory.CreateLogger<FallbackToGelbooruDanbooruEngine>()),
             ImageSource.Rule34 => new BooruEngine(
                 new Rule34ApiLoader(
                     new PerBaseUrlFlurlClientFactory(),

Source/WetPicsRebirth/Infrastructure/Engines/BooruEngine.cs

@@ -28,7 +28,7 @@ public async Task<IReadOnlyCollection<PostHeader>> LoadPopularList(string option
         return popular.Results.Select(x => new PostHeader(x.Id, x.Md5Hash)).ToList();
     }
 
-    public async Task<LoadedPost> LoadPost(PostHeader postHeader)
+    public virtual async Task<LoadedPost> LoadPost(PostHeader postHeader)
     {
         var postId = postHeader.Id;
         var mediaUrl = string.Empty;

Source/WetPicsRebirth/Infrastructure/Engines/FallbackToGelbooruDanbooruEngine.cs

@@ -0,0 +1,91 @@
+using System.Net;
+using Imouto.BooruParser;
+using WetPicsRebirth.Infrastructure.Models;
+using Post = WetPicsRebirth.Infrastructure.Models.Post;
+
+namespace WetPicsRebirth.Infrastructure.Engines;
+
+/// <summary>
+/// This is temporary fix for blocked bots in Danbooru. 
+/// </summary>
+public class FallbackToGelbooruDanbooruEngine : BooruEngine
+{
+    private readonly IBooruApiLoader _gelbooruLoader;
+    private readonly HttpClient _httpClient;
+    private readonly ILogger<FallbackToGelbooruDanbooruEngine> _logger;
+
+    public FallbackToGelbooruDanbooruEngine(
+        IBooruApiLoader danbooruLoader,
+        IBooruApiLoader gelbooruLoader,
+        HttpClient httpClient,
+        ILogger<FallbackToGelbooruDanbooruEngine> logger) 
+        : base(danbooruLoader, httpClient, logger)
+    {
+        _gelbooruLoader = gelbooruLoader;
+        _httpClient = httpClient;
+        _logger = logger;
+    }
+
+    public override async Task<LoadedPost> LoadPost(PostHeader postHeader)
+    {
+        try
+        {
+            return await base.LoadPost(postHeader);
+        }
+        catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.Forbidden)
+        {
+            // ignore
+        }
+
+        var postId = postHeader.Id;
+        var mediaUrl = string.Empty;
+
+        try
+        {
+            return await LoadPostCore();
+        }
+        catch (Exception e) when(string.IsNullOrWhiteSpace(mediaUrl))
+        {
+            _logger.LogWarning(e, "Failed to load post {PostId} with media url {MediaUrl}", postId, mediaUrl);
+            throw;
+        }        
+        catch (Exception e)
+        {
+            _logger.LogError(e, "Failed to load post {PostId} with media url {MediaUrl}", postId, mediaUrl);
+            throw;
+        }        
+
+        async Task<LoadedPost> LoadPostCore()
+        {
+            if (postHeader.Md5Hash == null)
+                throw new InvalidOperationException("Md5 hash is null");
+
+            var post = await _gelbooruLoader.GetPostByMd5Async(postHeader.Md5Hash!);
+
+            if (post == null)
+                throw new InvalidOperationException($"Gelbooru post with md5: {postHeader.Md5Hash} not found");
+
+            mediaUrl = GetMediaUrl(post);
+
+            var response = await _httpClient.GetAsync(mediaUrl);
+            response.EnsureSuccessStatusCode();
+
+            var stream = await response.Content.ReadAsStreamAsync();
+            var length = response.Content.Headers.ContentLength;
+
+            if (!length.HasValue)
+                throw new("Unexpected length");
+
+            var resultPost = new Post(postHeader, mediaUrl, stream, length.Value);
+            var requireModeration = CheckForModeration(post);
+
+            return new(resultPost, requireModeration);
+        }
+    }
+
+    private static string GetMediaUrl(Imouto.BooruParser.Post post)
+        => post.OriginalUrl?.EndsWith(".zip") == true ? post.SampleUrl! : post.OriginalUrl!;
+
+    private static bool CheckForModeration(Imouto.BooruParser.Post post)
+        => post.Rating == Rating.Explicit && post.Tags.Any(x => x.Name is "loli" or "shota");
+}