From 0ff6ac4e5c34f665e3557c93d66d0dee56967ca4 Mon Sep 17 00:00:00 2001 From: MistEO Date: Fri, 29 Mar 2024 15:31:51 +0800 Subject: [PATCH] refactor: draws of recognizer --- source/MaaFramework/Task/Recognizer.cpp | 129 +++++++++++++--------- source/MaaFramework/Task/Recognizer.h | 24 ++-- source/MaaFramework/Vision/VisionBase.cpp | 34 +----- source/MaaFramework/Vision/VisionBase.h | 9 +- 4 files changed, 98 insertions(+), 98 deletions(-) diff --git a/source/MaaFramework/Task/Recognizer.cpp b/source/MaaFramework/Task/Recognizer.cpp index 48956c91d..913643ad4 100644 --- a/source/MaaFramework/Task/Recognizer.cpp +++ b/source/MaaFramework/Task/Recognizer.cpp @@ -3,6 +3,7 @@ #include "Instance/InstanceStatus.h" #include "Option/GlobalOptionMgr.h" #include "Resource/ResourceMgr.h" +#include "Utils/ImageIo.h" #include "Utils/Logger.h" #include "Vision/ColorMatcher.h" #include "Vision/CustomRecognizer.h" @@ -31,51 +32,51 @@ std::optional return std::nullopt; } - std::optional result; + ResultAndDraws result_and_draws; switch (task_data.rec_type) { case Type::DirectHit: - result = direct_hit(task_data.name); + result_and_draws = direct_hit(task_data.name); break; case Type::TemplateMatch: - result = template_match( + result_and_draws = template_match( image, std::get(task_data.rec_param), task_data.name); break; case Type::FeatureMatch: - result = feature_match( + result_and_draws = feature_match( image, std::get(task_data.rec_param), task_data.name); break; case Type::ColorMatch: - result = + result_and_draws = color_match(image, std::get(task_data.rec_param), task_data.name); break; case Type::OCR: - result = ocr(image, std::get(task_data.rec_param), task_data.name); + result_and_draws = ocr(image, std::get(task_data.rec_param), task_data.name); break; case Type::NeuralNetworkClassify: - result = nn_classify( + result_and_draws = nn_classify( image, std::get(task_data.rec_param), task_data.name); break; case Type::NeuralNetworkDetect: - result = nn_detect( + result_and_draws = nn_detect( image, std::get(task_data.rec_param), task_data.name); break; case Type::Custom: - result = custom_recognize( + result_and_draws = custom_recognize( image, std::get(task_data.rec_param), task_data.name); @@ -87,28 +88,32 @@ std::optional return std::nullopt; } - if (result) { - status()->set_rec_box(task_data.name, result->box); - status()->set_rec_detail(task_data.name, result->detail); + save_draws(result_and_draws.draws, task_data.name); - show_hit_draw(image, *result, task_data.name); + if (result_and_draws.result) { + status()->set_rec_box(task_data.name, result_and_draws.result->box); + status()->set_rec_detail(task_data.name, result_and_draws.result->detail); + + show_hit_draw(image, *result_and_draws.result, task_data.name); } if (task_data.inverse) { LogDebug << "task_data.inverse is true, reverse the result" << VAR(task_data.name) - << VAR(result.has_value()); - return result ? std::nullopt : std::make_optional(Result { .box = cv::Rect() }); + << VAR(result_and_draws.result.has_value()); + return result_and_draws.result ? std::nullopt + : std::make_optional(Result { .box = cv::Rect() }); } - return result; + + return result_and_draws.result; } -std::optional Recognizer::direct_hit(const std::string& name) +Recognizer::ResultAndDraws Recognizer::direct_hit(const std::string& name) { LogTrace << name; - return Result { .box = cv::Rect(), .detail = json::array() }; + return ResultAndDraws { .result = Result { .box = cv::Rect(), .detail = json::array() } }; } -std::optional Recognizer::template_match( +Recognizer::ResultAndDraws Recognizer::template_match( const cv::Mat& image, const MAA_VISION_NS::TemplateMatcherParam& param, const std::string& name) @@ -117,7 +122,7 @@ std::optional Recognizer::template_match( if (!resource()) { LogError << "Resource not binded"; - return std::nullopt; + return {}; } std::vector> templates; @@ -136,15 +141,16 @@ std::optional Recognizer::template_match( size_t index = matcher.preferred_index(); auto draws = std::move(matcher).draws(); + ResultAndDraws rd { .draws = std::move(draws) }; if (index >= results.size()) { - return std::nullopt; + return rd; } - const cv::Rect& box = results[index].box; - return Result { .box = box, .detail = std::move(results), .draws = std::move(draws) }; + rd.result = { .box = results[index].box, .detail = std::move(results) }; + return rd; } -std::optional Recognizer::feature_match( +Recognizer::ResultAndDraws Recognizer::feature_match( const cv::Mat& image, const MAA_VISION_NS::FeatureMatcherParam& param, const std::string& name) @@ -153,7 +159,7 @@ std::optional Recognizer::feature_match( if (!resource()) { LogError << "Resource not binded"; - return std::nullopt; + return {}; } std::vector> templates; @@ -172,15 +178,16 @@ std::optional Recognizer::feature_match( size_t index = matcher.preferred_index(); auto draws = std::move(matcher).draws(); + ResultAndDraws rd { .draws = std::move(draws) }; if (index >= results.size()) { - return std::nullopt; + return rd; } - const cv::Rect& box = results[index].box; - return Result { .box = box, .detail = std::move(results), .draws = std::move(draws) }; + rd.result = { .box = results[index].box, .detail = std::move(results) }; + return rd; } -std::optional Recognizer::color_match( +Recognizer::ResultAndDraws Recognizer::color_match( const cv::Mat& image, const MAA_VISION_NS::ColorMatcherParam& param, const std::string& name) @@ -189,7 +196,7 @@ std::optional Recognizer::color_match( if (!resource()) { LogError << "Resource not binded"; - return std::nullopt; + return {}; } ColorMatcher matcher(image, param, name); @@ -198,15 +205,16 @@ std::optional Recognizer::color_match( size_t index = matcher.preferred_index(); auto draws = std::move(matcher).draws(); + ResultAndDraws rd { .draws = std::move(draws) }; if (index >= results.size()) { - return std::nullopt; + return rd; } - const cv::Rect& box = results[index].box; - return Result { .box = box, .detail = std::move(results), .draws = std::move(draws) }; + rd.result = { .box = results[index].box, .detail = std::move(results) }; + return rd; } -std::optional Recognizer::ocr( +Recognizer::ResultAndDraws Recognizer::ocr( const cv::Mat& image, const MAA_VISION_NS::OCRerParam& param, const std::string& name) @@ -215,7 +223,7 @@ std::optional Recognizer::ocr( if (!resource() || !status()) { LogError << "Resource not binded or status is null" << VAR(resource()) << VAR(status()); - return std::nullopt; + return {}; } auto det_session = resource()->ocr_res().deter(param.model); @@ -228,15 +236,16 @@ std::optional Recognizer::ocr( size_t index = ocrer.preferred_index(); auto draws = std::move(ocrer).draws(); + ResultAndDraws rd { .draws = std::move(draws) }; if (index >= results.size()) { - return std::nullopt; + return rd; } - const cv::Rect& box = results[index].box; - return Result { .box = box, .detail = std::move(results), .draws = std::move(draws) }; + rd.result = { .box = results[index].box, .detail = std::move(results) }; + return rd; } -std::optional Recognizer::nn_classify( +Recognizer::ResultAndDraws Recognizer::nn_classify( const cv::Mat& image, const MAA_VISION_NS::NeuralNetworkClassifierParam& param, const std::string& name) @@ -245,7 +254,7 @@ std::optional Recognizer::nn_classify( if (!resource()) { LogError << "Resource not binded"; - return std::nullopt; + return {}; } auto session = resource()->onnx_res().classifier(param.model); @@ -256,15 +265,16 @@ std::optional Recognizer::nn_classify( size_t index = classifier.preferred_index(); auto draws = std::move(classifier).draws(); + ResultAndDraws rd { .draws = std::move(draws) }; if (index >= results.size()) { - return std::nullopt; + return rd; } - const cv::Rect& box = results[index].box; - return Result { .box = box, .detail = std::move(results), .draws = std::move(draws) }; + rd.result = { .box = results[index].box, .detail = std::move(results) }; + return rd; } -std::optional Recognizer::nn_detect( +Recognizer::ResultAndDraws Recognizer::nn_detect( const cv::Mat& image, const MAA_VISION_NS::NeuralNetworkDetectorParam& param, const std::string& name) @@ -273,7 +283,7 @@ std::optional Recognizer::nn_detect( if (!resource()) { LogError << "Resource not binded"; - return std::nullopt; + return {}; } auto session = resource()->onnx_res().detector(param.model); @@ -284,15 +294,16 @@ std::optional Recognizer::nn_detect( size_t index = detector.preferred_index(); auto draws = std::move(detector).draws(); + ResultAndDraws rd { .draws = std::move(draws) }; if (index >= results.size()) { - return std::nullopt; + return rd; } - const cv::Rect& box = results[index].box; - return Result { .box = box, .detail = std::move(results), .draws = std::move(draws) }; + rd.result = { .box = results[index].box, .detail = std::move(results) }; + return rd; } -std::optional Recognizer::custom_recognize( +Recognizer::ResultAndDraws Recognizer::custom_recognize( const cv::Mat& image, const MAA_VISION_NS::CustomRecognizerParam& param, const std::string& name) @@ -301,13 +312,13 @@ std::optional Recognizer::custom_recognize( if (!inst_) { LogError << "Instance not binded"; - return std::nullopt; + return {}; } auto* session = inst_->custom_recognizer_session(param.name); if (!session) { LogError << "Custom recognizer not found:" << param.name; - return std::nullopt; + return {}; } CustomRecognizer recognizer(image, param, *session, inst_, name); @@ -315,11 +326,23 @@ std::optional Recognizer::custom_recognize( bool ret = recognizer.ret(); if (!ret) { - return std::nullopt; + return {}; } const cv::Rect& box = results.box; - return Result { .box = box, .detail = std::move(results) }; + return ResultAndDraws { Result { .box = box, .detail = std::move(results) } }; +} + +void Recognizer::save_draws(const std::vector& draws, const std::string& task_name) const +{ + auto dir = GlobalOptionMgr::get_instance().log_dir() / "vision"; + + for (size_t i = 0; i < draws.size(); ++i) { + std::string filename = std::format("{}_{}_{}.png", task_name, i, format_now_for_filename()); + auto filepath = dir / path(filename); + imwrite(filepath, draws.at(i)); + LogDebug << "save draw to" << filepath; + } } void Recognizer::show_hit_draw( diff --git a/source/MaaFramework/Task/Recognizer.h b/source/MaaFramework/Task/Recognizer.h index e7395ec42..49409f8ed 100644 --- a/source/MaaFramework/Task/Recognizer.h +++ b/source/MaaFramework/Task/Recognizer.h @@ -22,7 +22,6 @@ class Recognizer { cv::Rect box {}; json::value detail; - std::vector draws; }; public: @@ -32,34 +31,41 @@ class Recognizer std::optional recognize(const cv::Mat& image, const TaskData& task_data); private: - std::optional direct_hit(const std::string& name); - std::optional template_match( + struct ResultAndDraws + { + std::optional result = std::nullopt; + std::vector draws; + }; + + ResultAndDraws direct_hit(const std::string& name); + ResultAndDraws template_match( const cv::Mat& image, const MAA_VISION_NS::TemplateMatcherParam& param, const std::string& name); - std::optional feature_match( + ResultAndDraws feature_match( const cv::Mat& image, const MAA_VISION_NS::FeatureMatcherParam& param, const std::string& name); - std::optional color_match( + ResultAndDraws color_match( const cv::Mat& image, const MAA_VISION_NS::ColorMatcherParam& param, const std::string& name); - std::optional + ResultAndDraws ocr(const cv::Mat& image, const MAA_VISION_NS::OCRerParam& param, const std::string& name); - std::optional nn_classify( + ResultAndDraws nn_classify( const cv::Mat& image, const MAA_VISION_NS::NeuralNetworkClassifierParam& param, const std::string& name); - std::optional nn_detect( + ResultAndDraws nn_detect( const cv::Mat& image, const MAA_VISION_NS::NeuralNetworkDetectorParam& param, const std::string& name); - std::optional custom_recognize( + ResultAndDraws custom_recognize( const cv::Mat& image, const MAA_VISION_NS::CustomRecognizerParam& param, const std::string& name); + void save_draws(const std::vector& draws, const std::string& task_name) const; void show_hit_draw(const cv::Mat& image, const Result& res, const std::string& task_name) const; private: diff --git a/source/MaaFramework/Vision/VisionBase.cpp b/source/MaaFramework/Vision/VisionBase.cpp index e2209dcfa..05ded5240 100644 --- a/source/MaaFramework/Vision/VisionBase.cpp +++ b/source/MaaFramework/Vision/VisionBase.cpp @@ -3,10 +3,7 @@ #include "Utils/NoWarningCV.hpp" #include "Option/GlobalOptionMgr.h" -#include "Utils/ImageIo.h" #include "Utils/Logger.h" -#include "Utils/StringMisc.hpp" -#include "Utils/Time.hpp" #include "VisionUtils.hpp" MAA_VISION_NS_BEGIN @@ -15,7 +12,7 @@ VisionBase::VisionBase(cv::Mat image, std::string name) : image_(std::move(image)) , name_(std::move(name)) { - init_debug_draw(); + init_draw(); } cv::Mat VisionBase::image_with_roi(const cv::Rect& roi) const @@ -55,10 +52,6 @@ cv::Mat VisionBase::draw_roi(const cv::Rect& roi, const cv::Mat& base) const void VisionBase::handle_draw(const cv::Mat& draw) { draws_.emplace_back(draw); - - if (save_draw_) { - draw_paths_.emplace_back(save_image(draw)); - } } void VisionBase::handle_index(size_t total, int index) @@ -72,29 +65,14 @@ void VisionBase::handle_index(size_t total, int index) preferred_index_ = *index_opt; } -std::filesystem::path VisionBase::save_image(const cv::Mat& image) const -{ - std::string filename = std::format("{}_{}.png", name_, format_now_for_filename()); - auto filepath = GlobalOptionMgr::get_instance().log_dir() / "vision" / path(filename); - MAA_NS::imwrite(filepath, image); - LogDebug << "save image to" << filepath; - return filepath; -} - -void VisionBase::init_debug_draw() +void VisionBase::init_draw() { - save_draw_ = GlobalOptionMgr::get_instance().save_draw(); - - if (save_draw_) { - debug_draw_ = true; - } - else { #ifdef MAA_DEBUG - debug_draw_ = true; + debug_draw_ = true; #else - debug_draw_ = false; + const auto& option = GlobalOptionMgr::get_instance(); + debug_draw_ = option.save_draw() || option.debug_message(); #endif - } } -MAA_VISION_NS_END \ No newline at end of file +MAA_VISION_NS_END diff --git a/source/MaaFramework/Vision/VisionBase.h b/source/MaaFramework/Vision/VisionBase.h index c3819b188..c82c56305 100644 --- a/source/MaaFramework/Vision/VisionBase.h +++ b/source/MaaFramework/Vision/VisionBase.h @@ -17,10 +17,6 @@ class VisionBase std::vector draws() && { return std::move(draws_); } - const std::vector& draw_paths() const& { return draw_paths_; } - - std::vector draw_paths() && { return std::move(draw_paths_); } - size_t preferred_index() const { return preferred_index_; } protected: @@ -38,13 +34,10 @@ class VisionBase bool debug_draw_ = false; private: - void init_debug_draw(); - std::filesystem::path save_image(const cv::Mat& image) const; + void init_draw(); private: - bool save_draw_ = false; std::vector draws_; - std::vector draw_paths_; size_t preferred_index_ = SIZE_MAX; };