From 2465130f2405ee96f414c9941d5e0aa01fc32218 Mon Sep 17 00:00:00 2001 From: Julian Waller Date: Tue, 21 Jan 2025 22:35:38 +0000 Subject: [PATCH] feat: perform less cpu conversions with the image producer --- src/modules/image/producer/image_producer.cpp | 15 +-- .../image/producer/image_scroll_producer.cpp | 12 +-- src/modules/image/util/image_loader.cpp | 91 +++++++++++++------ src/modules/image/util/image_loader.h | 17 +++- 4 files changed, 92 insertions(+), 43 deletions(-) diff --git a/src/modules/image/producer/image_producer.cpp b/src/modules/image/producer/image_producer.cpp index c90effab38..d38710a57e 100644 --- a/src/modules/image/producer/image_producer.cpp +++ b/src/modules/image/producer/image_producer.cpp @@ -72,7 +72,7 @@ struct image_producer : public core::frame_producer , frame_factory_(frame_factory) , length_(length) { - load(load_image(description_), scale_mode); + load(load_image(description_, true), scale_mode); CASPAR_LOG(info) << print() << L" Initialized"; } @@ -86,21 +86,22 @@ struct image_producer : public core::frame_producer , frame_factory_(frame_factory) , length_(length) { - load(load_png_from_memory(png_data, size), scale_mode); + load(load_png_from_memory(png_data, size, true), scale_mode); CASPAR_LOG(info) << print() << L" Initialized"; } - void load(const std::shared_ptr& bitmap, core::frame_geometry::scale_mode scale_mode) + void load(const loaded_image& image, core::frame_geometry::scale_mode scale_mode) { - FreeImage_FlipVertical(bitmap.get()); core::pixel_format_desc desc(core::pixel_format::bgra); - desc.planes.emplace_back(FreeImage_GetWidth(bitmap.get()), FreeImage_GetHeight(bitmap.get()), 4); + desc.is_straight_alpha = image.is_straight; + desc.planes.emplace_back( + FreeImage_GetWidth(image.bitmap.get()), FreeImage_GetHeight(image.bitmap.get()), image.stride, image.depth); auto frame = frame_factory_->create_frame(this, desc); - frame.geometry() = core::frame_geometry::get_default(scale_mode); + frame.geometry() = core::frame_geometry::get_default_vflip(scale_mode); - std::copy_n(FreeImage_GetBits(bitmap.get()), frame.image_data(0).size(), frame.image_data(0).begin()); + std::copy_n(FreeImage_GetBits(image.bitmap.get()), frame.image_data(0).size(), frame.image_data(0).begin()); frame_ = core::draw_frame(std::move(frame)); } diff --git a/src/modules/image/producer/image_scroll_producer.cpp b/src/modules/image/producer/image_scroll_producer.cpp index 0fed215cac..4f1d2cf9d7 100644 --- a/src/modules/image/producer/image_scroll_producer.cpp +++ b/src/modules/image/producer/image_scroll_producer.cpp @@ -141,11 +141,11 @@ struct image_scroll_producer : public core::frame_producer if (end_time_) speed = -1.0; - auto bitmap = load_image(filename_); - FreeImage_FlipVertical(bitmap.get()); + auto image = load_image(filename_, false); + FreeImage_FlipVertical(image.bitmap.get()); - width_ = FreeImage_GetWidth(bitmap.get()); - height_ = FreeImage_GetHeight(bitmap.get()); + width_ = FreeImage_GetWidth(image.bitmap.get()); + height_ = FreeImage_GetHeight(image.bitmap.get()); bool vertical = width_ == format_desc_.width; bool horizontal = height_ == format_desc_.height; @@ -170,7 +170,7 @@ struct image_scroll_producer : public core::frame_producer speed_ = speed_tweener(speed, speed, 0, tweener(L"linear")); - auto bytes = FreeImage_GetBits(bitmap.get()); + auto bytes = FreeImage_GetBits(image.bitmap.get()); auto count = width_ * height_ * 4; image_view original_view(bytes, width_, height_); @@ -194,7 +194,7 @@ struct image_scroll_producer : public core::frame_producer caspar::tweener blur_tweener(L"easeInQuad"); blur(original_view, blurred_view, angle, motion_blur_px, blur_tweener); bytes = blurred_copy.get(); - bitmap.reset(); + image.bitmap.reset(); } if (vertical) { diff --git a/src/modules/image/util/image_loader.cpp b/src/modules/image/util/image_loader.cpp index 859539c912..9fb6913fb0 100644 --- a/src/modules/image/util/image_loader.cpp +++ b/src/modules/image/util/image_loader.cpp @@ -42,9 +42,68 @@ #include "image_algorithms.h" #include "image_view.h" +#if FREEIMAGE_COLORORDER == FREEIMAGE_COLORORDER_BGR +#define IMAGE_BGRA_FORMAT core::pixel_format::bgra +#define IMAGE_BGR_FORMAT core::pixel_format::bgr +#else +#define IMAGE_BGRA_FORMAT core::pixel_format::rgba +#define IMAGE_BGR_FORMAT core::pixel_format::rgb +#endif + namespace caspar { namespace image { -std::shared_ptr load_image(const std::wstring& filename) +loaded_image prepare_loaded_image(FREE_IMAGE_FORMAT fif, std::shared_ptr bitmap, bool allow_all_formats) +{ + core::pixel_format format; + int stride; + common::bit_depth depth = common::bit_depth::bit8; + + unsigned int bpp = FreeImage_GetBPP(bitmap.get()); + + if (bpp == 32) { + format = IMAGE_BGRA_FORMAT; + stride = 4; + } else if (allow_all_formats && bpp == 24) { + format = IMAGE_BGR_FORMAT; + stride = 3; + } else if (allow_all_formats && bpp == 64) { + // freeimage appears to ignore endianness + format = core::pixel_format::rgba; + stride = 4; + depth = common::bit_depth::bit16; + } else if (allow_all_formats && bpp == 48) { + // freeimage appears to ignore endianness + format = core::pixel_format::rgb; + stride = 3; + depth = common::bit_depth::bit16; + } else if (allow_all_formats && !FreeImage_IsTransparent(bitmap.get())) { + format = IMAGE_BGR_FORMAT; + stride = 3; + + bitmap = std::shared_ptr(FreeImage_ConvertTo24Bits(bitmap.get()), FreeImage_Unload); + + } else { + format = IMAGE_BGRA_FORMAT; + stride = 4; + + bitmap = std::shared_ptr(FreeImage_ConvertTo32Bits(bitmap.get()), FreeImage_Unload); + } + + if (!bitmap) + CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info("Unsupported image format.")); + + // PNG-images need to be premultiplied with their alpha + bool is_straight = fif == FIF_PNG && (format == core::pixel_format::bgra || format == core::pixel_format::rgba); + if (!allow_all_formats && is_straight) { + image_view original_view( + FreeImage_GetBits(bitmap.get()), FreeImage_GetWidth(bitmap.get()), FreeImage_GetHeight(bitmap.get())); + premultiply(original_view); + } + + return {std::move(bitmap), format, stride, depth, is_straight}; +} + +loaded_image load_image(const std::wstring& filename, bool allow_all_formats) { if (!boost::filesystem::exists(filename)) CASPAR_THROW_EXCEPTION(file_not_found() << boost::errinfo_file_name(u8(filename))); @@ -72,23 +131,10 @@ std::shared_ptr load_image(const std::wstring& filename) std::shared_ptr(FreeImage_Load(fif, u8(filename).c_str(), JPEG_EXIFROTATE), FreeImage_Unload); #endif - if (FreeImage_GetBPP(bitmap.get()) != 32) { - bitmap = std::shared_ptr(FreeImage_ConvertTo32Bits(bitmap.get()), FreeImage_Unload); - if (!bitmap) - CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info("Unsupported image format.")); - } - - // PNG-images need to be premultiplied with their alpha - if (fif == FIF_PNG) { - image_view original_view( - FreeImage_GetBits(bitmap.get()), FreeImage_GetWidth(bitmap.get()), FreeImage_GetHeight(bitmap.get())); - premultiply(original_view); - } - - return bitmap; + return prepare_loaded_image(fif, std::move(bitmap), allow_all_formats); } -std::shared_ptr load_png_from_memory(const void* memory_location, size_t size) +loaded_image load_png_from_memory(const void* memory_location, size_t size, bool allow_all_formats) { FREE_IMAGE_FORMAT fif = FIF_PNG; @@ -98,18 +144,7 @@ std::shared_ptr load_png_from_memory(const void* memory_location, size auto bitmap = std::shared_ptr(FreeImage_LoadFromMemory(fif, memory.get(), JPEG_EXIFROTATE), FreeImage_Unload); - if (FreeImage_GetBPP(bitmap.get()) != 32) { - bitmap = std::shared_ptr(FreeImage_ConvertTo32Bits(bitmap.get()), FreeImage_Unload); - - if (!bitmap) - CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info("Unsupported image format.")); - } - - // PNG-images need to be premultiplied with their alpha - image_view original_view( - FreeImage_GetBits(bitmap.get()), FreeImage_GetWidth(bitmap.get()), FreeImage_GetHeight(bitmap.get())); - premultiply(original_view); - return bitmap; + return prepare_loaded_image(fif, std::move(bitmap), allow_all_formats); } bool is_valid_file(const boost::filesystem::path& filename) diff --git a/src/modules/image/util/image_loader.h b/src/modules/image/util/image_loader.h index 756d180df9..46d55c6f23 100644 --- a/src/modules/image/util/image_loader.h +++ b/src/modules/image/util/image_loader.h @@ -21,6 +21,10 @@ #pragma once +#include + +#include + #include #include #include @@ -31,8 +35,17 @@ struct FIBITMAP; namespace caspar { namespace image { -std::shared_ptr load_image(const std::wstring& filename); -std::shared_ptr load_png_from_memory(const void* memory_location, size_t size); +struct loaded_image +{ + std::shared_ptr bitmap; + core::pixel_format format; + int stride; + common::bit_depth depth; + bool is_straight; +}; + +loaded_image load_image(const std::wstring& filename, bool allow_all_formats); +loaded_image load_png_from_memory(const void* memory_location, size_t size, bool allow_all_formats); bool is_valid_file(const boost::filesystem::path& filename);