feat(tex): make optimization smarter

include/btu/tex/formats.hpp

@@ -54,10 +54,19 @@ enum class AllowCompressed : std::uint8_t
 
 NLOHMANN_JSON_SERIALIZE_ENUM(AllowCompressed, {{AllowCompressed::Yes, "yes"}, {AllowCompressed::No, "no"}});
 
+enum class ForceAlpha : std::uint8_t
+{
+    Yes,
+    No
+};
+
+NLOHMANN_JSON_SERIALIZE_ENUM(ForceAlpha, {{ForceAlpha::Yes, "yes"}, {ForceAlpha::No, "no"}});
+
 /// Based on https://forums.nexusmods.com/index.php?/topic/476227-skyrim-nif-files-with-underscores/
 auto guess_texture_type(std::u8string_view path) noexcept -> std::optional<TextureType>;
 
-auto guess_best_format(const Texture &tex,
+auto guess_best_format(DXGI_FORMAT current_format,
                        BestFormatFor formats,
-                       AllowCompressed allow_compressed = AllowCompressed::Yes) noexcept -> DXGI_FORMAT;
+                       AllowCompressed allow_compressed = AllowCompressed::Yes,
+                       ForceAlpha force_alpha           = ForceAlpha::No) noexcept -> DXGI_FORMAT;
 } // namespace btu::tex

src/tex/formats.cpp

@@ -1,6 +1,5 @@
 #include <btu/common/string.hpp>
 #include <btu/tex/formats.hpp>
-#include <btu/tex/texture.hpp>
 
 namespace btu::tex {
 
@@ -43,13 +42,15 @@ auto guess_texture_type(std::u8string_view path) noexcept -> std::optional<Textu
     return std::nullopt;
 }
 
-auto guess_best_format(const Texture &tex,
-                       BestFormatFor formats,
-                       AllowCompressed allow_compressed) noexcept -> DXGI_FORMAT
+auto guess_best_format(const DXGI_FORMAT current_format,
+                       const BestFormatFor formats,
+                       const AllowCompressed allow_compressed,
+                       const ForceAlpha force_alpha) noexcept -> DXGI_FORMAT
 {
-    const bool compressed = allow_compressed == AllowCompressed::Yes
-                            && DirectX::IsCompressed(tex.get().GetMetadata().format);
-    const bool alpha = DirectX::IsCompressed(tex.get().GetMetadata().format);
+    // allow compression if user requested it, or it was already compressed
+    const bool compressed = allow_compressed == AllowCompressed::Yes || DirectX::IsCompressed(current_format);
+    // provide an alpha channel if there was already one
+    const bool alpha = DirectX::HasAlpha(current_format) || force_alpha == ForceAlpha::Yes;
     if (compressed && alpha)
         return formats.compressed;
     if (compressed)

src/tex/optimize.cpp

@@ -31,17 +31,27 @@ auto optimize(Texture &&file, OptimizationSteps sets, CompressionDevice &dev) no
 
     // We have uncompressed the texture. If it was compressed, it's best to convert it to a better format
     const auto cur_format_is_same_as_best = res && res->get().GetMetadata().format == sets.best_format;
+
     if ((sets.convert || compressed) && !cur_format_is_same_as_best)
-        res = std::move(res).and_then(
-            [&](Texture &&tex) { return convert(std::move(tex), sets.best_format, dev); });
+    {
+        auto out = sets.best_format;
+        res      = std::move(res)
+                  // // safety check: make sure we don't remove the alpha
+                  .and_then([&](Texture &&tex) -> Result {
+                      if (!DirectX::HasAlpha(out) && !tex.get().IsAlphaAllOpaque())
+                          return tl::make_unexpected(Error(TextureErr::BadInput));
+                      return std::move(tex);
+                  })
+                  .and_then([&](Texture &&tex) { return convert(std::move(tex), out, dev); });
+    }
 
     return res;
 }
 
 /// SSE landscape textures uses alpha channel as specularity
 /// Textures with opaque alpha are thus rendered shiny
 /// To fix this, alpha has to be made transparent
-auto can_be_optimized_landscape(const Texture &file, const Settings &sets) -> bool
+auto transparent_alpha_required(const Texture &file, const Settings &sets) -> bool
 {
     const auto &tex         = file.get();
     const auto path         = canonize_path(file.get_load_path());
@@ -91,12 +101,14 @@ auto can_be_optimized_landscape(const Texture &file, const Settings &sets) -> bo
     return bad || is_tga(file);
 }
 
-[[nodiscard]] auto best_output_format(const Texture &file, const Settings &sets) noexcept -> DXGI_FORMAT
+[[nodiscard]] auto best_output_format(const Texture &file,
+                                      const Settings &sets,
+                                      ForceAlpha force_alpha) noexcept -> DXGI_FORMAT
 {
     const auto &info = file.get().GetMetadata();
 
     const auto allow_compressed = can_be_compressed(info) ? AllowCompressed::Yes : AllowCompressed::No;
-    return guess_best_format(file, sets.output_format, allow_compressed);
+    return guess_best_format(info.format, sets.output_format, allow_compressed, force_alpha);
 }
 
 auto compute_optimization_steps(const Texture &file, const Settings &sets) noexcept -> OptimizationSteps
@@ -106,8 +118,6 @@ auto compute_optimization_steps(const Texture &file, const Settings &sets) noexc
 
     auto res = OptimizationSteps{};
 
-    res.best_format = best_output_format(file, sets);
-
     res.convert = conversion_required(file, sets);
     // do not recompress if original is already compressed
     if (sets.compress && res.best_format != info.format && !DirectX::IsCompressed(info.format))
@@ -128,13 +138,18 @@ auto compute_optimization_steps(const Texture &file, const Settings &sets) noexc
         res.resize = target_dim.value();
 
     if (sets.game == Game::SSE)
-        if (can_be_optimized_landscape(file, sets))
+        if (transparent_alpha_required(file, sets))
             res.add_transparent_alpha = true;
 
     const bool opt_mip = optimal_mip_count(file.get_dimension()) == info.mipLevels;
     if (sets.mipmaps && (!opt_mip || res.resize)) // resize removes mips
         res.mipmaps = true;
 
+    // I prefer to keep steps independent, but this one has to depend on add_transparent_alpha. If we add an alpha, the output format must have alpha
+    res.best_format = best_output_format(file,
+                                         sets,
+                                         res.add_transparent_alpha ? ForceAlpha::Yes : ForceAlpha::No);
+
     return res;
 }
 
@@ -152,7 +167,7 @@ auto Settings::get(Game game) noexcept -> const Settings &
             .output_format        = {.uncompressed               = DXGI_FORMAT_R8G8B8A8_UNORM,
                                      .uncompressed_without_alpha = DXGI_FORMAT_R8G8B8A8_UNORM,
                                      .compressed                 = DXGI_FORMAT_BC3_UNORM,
-                                     .compressed_without_alpha   = DXGI_FORMAT_BC1_UNORM},
+                                     .compressed_without_alpha   = DXGI_FORMAT_BC5_UNORM},
             .landscape_textures   = {}, // Unknown
         };
     }();
@@ -200,8 +215,9 @@ auto Settings::get(Game game) noexcept -> const Settings &
                     DXGI_FORMAT_BC1_UNORM,
                     DXGI_FORMAT_R8G8B8A8_UNORM,
                 };
-                sets.output_format.compressed = DXGI_FORMAT_BC7_UNORM;
-                sets.game                     = Game::SSE;
+                sets.output_format.compressed               = DXGI_FORMAT_BC7_UNORM;
+                sets.output_format.compressed_without_alpha = DXGI_FORMAT_BC7_UNORM;
+                sets.game                                   = Game::SSE;
                 return sets;
             }();
             return sse_sets;

tests/tex/formats.cpp

@@ -2,6 +2,9 @@
 
 #include <catch.hpp>
 
+using btu::tex::BestFormatFor, btu::tex::guess_best_format;
+using enum btu::tex::AllowCompressed;
+
 TEST_CASE("guess_texture_type", "[src]")
 {
     using btu::tex::guess_texture_type, btu::tex::TextureType;
@@ -39,3 +42,44 @@ TEST_CASE("guess_texture_type", "[src]")
         CHECK(guess_texture_type(u8"some_name_m.dds") == TextureType::EnvironmentMask);
     }
 }
+
+constexpr auto k_best_formats = BestFormatFor{.uncompressed               = DXGI_FORMAT_R8G8B8A8_UNORM,
+                                              .uncompressed_without_alpha = DXGI_FORMAT_R8G8_UNORM,
+                                              .compressed                 = DXGI_FORMAT_BC7_UNORM,
+                                              .compressed_without_alpha   = DXGI_FORMAT_BC5_UNORM};
+
+TEST_CASE("guess_best_format compressed with alpha", "[src]")
+{
+    auto result = guess_best_format(DXGI_FORMAT_BC3_UNORM, k_best_formats, Yes);
+    CHECK(result == DXGI_FORMAT_BC7_UNORM);
+}
+
+TEST_CASE("guess_best_format compressed without alpha", "[src]")
+{
+    auto result = guess_best_format(DXGI_FORMAT_BC5_UNORM, k_best_formats, Yes);
+    CHECK(result == DXGI_FORMAT_BC5_UNORM);
+}
+
+TEST_CASE("guess_best_format uncompressed with alpha", "[src]")
+{
+    auto result = guess_best_format(DXGI_FORMAT_R8G8B8A8_UNORM, k_best_formats, No);
+    CHECK(result == DXGI_FORMAT_R8G8B8A8_UNORM);
+}
+
+TEST_CASE("guess_best_format uncompressed without alpha", "[src]")
+{
+    auto result = guess_best_format(DXGI_FORMAT_R8G8_B8G8_UNORM, k_best_formats, No);
+    CHECK(result == DXGI_FORMAT_R8G8_UNORM);
+}
+
+TEST_CASE("guess_best_format already compressed with allow compressed no", "[src]")
+{
+    auto result = guess_best_format(DXGI_FORMAT_BC3_UNORM, k_best_formats, No);
+    CHECK(result == DXGI_FORMAT_BC7_UNORM);
+}
+
+TEST_CASE("guess_best_format non compressed with allow compressed yes", "[src]")
+{
+    auto result = guess_best_format(DXGI_FORMAT_R8G8B8A8_UNORM, k_best_formats, Yes);
+    CHECK(result == DXGI_FORMAT_BC7_UNORM);
+}