From 719e65bdaad3f32e18decbc68f20d834da1ee92e Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 14:03:25 -0400 Subject: [PATCH 01/19] Add depdency install through taskfile --- dep-tasks.yaml | 105 +++++++++++++++++++++++++++++++++++++++++++++++++ taskfile.yaml | 5 +++ 2 files changed, 110 insertions(+) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index a5cb8e17..e75b4343 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -3,6 +3,13 @@ version: "3" vars: G_SCRIPT_DIR: "{{.ROOT_DIR}}/tools/scripts" + G_CATCH2_LIB_NAME: "Catch2" + G_CATCH2_WORK_DIR: "{{.G_DEPS_DIR}}/{{.G_CATCH2_LIB_NAME}}" + G_OUTCOME_LIB_NAME: "outcome" + G_OUTCOME_WORK_DIR: "{{.G_DEPS_DIR}}/{{.G_OUTCOME_LIB_NAME}}" + G_QUICKCPPLIB_LIB_NAME: "quickcpplib" + G_QUICKCPPLIB_WORK_DIR: "{{.G_DEPS_DIR}}/{{.G_QUICKCPPLIB_LIB_NAME}}" + tasks: lib_install: @@ -21,3 +28,101 @@ tasks: dir: "{{.G_SCRIPT_DIR}}/lib_install/linux" cmds: - "./install-lib.sh" + + dep_install: + cmds: + - task: "install-all-init" + - task: "install-all-run" + - task: "install-all-finish" + + install-all-init: + internal: true + cmds: + - "rm -rf {{.G_DEPS_CMAKE_SETTINGS_DIR}}" + - "mkdir -p {{.G_DEPS_CMAKE_SETTINGS_DIR}}" + + install-all-run: + internal: true + deps: + - "install-Catch2" + - "install-outcome" + + install-all-finish: + internal: true + cmds: + - >- + for file in {{.G_DEPS_CMAKE_SETTINGS_DIR}}/*.cmake; do + if [ "$file" != "{{.G_DEPS_CMAKE_SETTINGS_FILE}}" ]; then + echo "include(\"$file\")" >> "{{.G_DEPS_CMAKE_SETTINGS_FILE}}"; + fi + done + + add-package-root-to-cmake-settings: + internal: true + requires: + vars: + - "NAME" + - "INSTALL_PREFIX" + cmds: + - >- + echo "set( + {{.NAME}}_ROOT + \"{{.INSTALL_PREFIX}}\" + CACHE PATH + \"Path to {{.NAME}} settings\" + )" >> "{{.G_DEPS_CMAKE_SETTINGS_DIR}}/{{.NAME}}.cmake" + + install-Catch2: + internal: true + run: "once" + cmds: + - task: ":utils:cmake:install-remote-tar" + vars: + NAME: "{{.G_CATCH2_LIB_NAME}}" + WORK_DIR: "{{.G_CATCH2_WORK_DIR}}" + FILE_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" + URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" + - task: "add-package-root-to-cmake-settings" + vars: + NAME: "{{.G_CATCH2_LIB_NAME}}" + INSTALL_PREFIX: "{{.G_CATCH2_WORK_DIR}}/{{.G_CATCH2_LIB_NAME}}-install" + + install-outcome: + internal: true + run: "once" + deps: + - "install-quickcpplib" + cmds: + - task: ":utils:cmake:install-remote-tar" + vars: + NAME: "{{.G_OUTCOME_LIB_NAME}}" + WORK_DIR: "{{.G_OUTCOME_WORK_DIR}}" + FILE_SHA256: "0382248cbb00806ce4b5f3ce6939797dc3b597c85fd3531614959e31ef488b39" + URL: "https://github.com/ned14/outcome/archive/refs/tags/v2.2.11.tar.gz" + GEN_ARGS: + - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/{{.G_QUICKCPPLIB_LIB_NAME}}.cmake" + - "-DBUILD_TESTING=OFF" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" + - task: "add-package-root-to-cmake-settings" + vars: + NAME: "{{.G_OUTCOME_LIB_NAME}}" + INSTALL_PREFIX: "{{.G_OUTCOME_WORK_DIR}}/{{.G_OUTCOME_LIB_NAME}}-install" + + install-quickcpplib: + internal: true + run: "once" + cmds: + - task: ":utils:cmake:install-remote-tar" + vars: + NAME: "{{.G_QUICKCPPLIB_LIB_NAME}}" + WORK_DIR: "{{.G_QUICKCPPLIB_WORK_DIR}}" + FILE_SHA256: "5d4c9b2d6fa177d3fb14f3fe3086867e43b44f4a7a944eb10ee4616b2b0f3c05" + URL: "https://github.com/ned14/quickcpplib/archive/f3e452e.tar.gz" + GEN_ARGS: + - "-DBUILD_TESTING=OFF" + - "-DCMAKE_BUILD_TYPE=Release" + - task: "add-package-root-to-cmake-settings" + vars: + NAME: "{{.G_QUICKCPPLIB_LIB_NAME}}" + INSTALL_PREFIX: "{{.G_QUICKCPPLIB_WORK_DIR}}/{{.G_QUICKCPPLIB_LIB_NAME}}-install" diff --git a/taskfile.yaml b/taskfile.yaml index cd5884ad..556ef348 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,11 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" + # These should be kept in-sync with its usage in CMakeLists.txt + G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" + G_DEPS_CMAKE_SETTINGS_FILE: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}/settings.cmake" + tasks: clean: cmds: From a1a829c6b6a5f0b37e9696d1ecb0e544d79ad9e5 Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 14:04:50 -0400 Subject: [PATCH 02/19] Remove catch2 submodule --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index f8ea5ed0..73630b7f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule "tools/yscope-dev-utils"] path = tools/yscope-dev-utils url = https://github.com/y-scope/yscope-dev-utils.git -[submodule "submodules/Catch2"] - path = submodules/Catch2 - url = https://github.com/catchorg/Catch2.git [submodule "submodules/abseil-cpp"] path = submodules/abseil-cpp url = https://github.com/abseil/abseil-cpp.git From ed93185131cedbc9f80d8201bed5eb59b3b2492a Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 14:21:46 -0400 Subject: [PATCH 03/19] Update cmake to use dependencies installed through tasks --- CMakeLists.txt | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d69104d..636430ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,13 +156,27 @@ else() message(FATAL_ERROR "Could not find msgpack-cxx") endif() +# Include dependency settings. +include("build/deps/cmake-settings/settings.cmake") + +find_package(outcome REQUIRED) +if(outcome_FOUND) + message(STATUS "Found outcome.") +else() + message(FATAL_ERROR "Could not find libraries for outcome.") +endif() + +find_package(Catch2 3.8.0 REQUIRED) +if(Catch2_FOUND) + message(STATUS "Found Catch2 ${Catch2_VERSION}.") +else() + message(FATAL_ERROR "Could not find libraries for Catch2.") +endif() + # Add abseil-cpp set(ABSL_PROPAGATE_CXX_STD ON) add_subdirectory(submodules/abseil-cpp) -# Add catch2 -add_subdirectory(submodules/Catch2) - find_package(Threads REQUIRED) add_subdirectory(src/spider) From c03700d11bcbcbb0aa12c2fd5009278216de1a3a Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 14:24:31 -0400 Subject: [PATCH 04/19] Add dep_install in lib_install to satisfy GH workflow --- dep-tasks.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index e75b4343..644a4251 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -21,6 +21,7 @@ tasks: dir: "{{.G_SCRIPT_DIR}}/lib_install/macOS" cmds: - "./install-lib.sh" + - task: "dep_install" lib_install_linux: internal: true @@ -28,6 +29,7 @@ tasks: dir: "{{.G_SCRIPT_DIR}}/lib_install/linux" cmds: - "./install-lib.sh" + - task: "dep_install" dep_install: cmds: From c7de69c23a3f390feda9f6700637eac46681b5e8 Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 14:37:47 -0400 Subject: [PATCH 05/19] Add abseil in deps tasks --- dep-tasks.yaml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 644a4251..c7c4b9f5 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -3,6 +3,8 @@ version: "3" vars: G_SCRIPT_DIR: "{{.ROOT_DIR}}/tools/scripts" + G_ABSEIL_LIB_NAME: "abseil" + G_ABSEIL_WORK_DIR: "{{.G_DEPS_DIR}}/{{.G_ABSEIL_LIB_NAME}}" G_CATCH2_LIB_NAME: "Catch2" G_CATCH2_WORK_DIR: "{{.G_DEPS_DIR}}/{{.G_CATCH2_LIB_NAME}}" G_OUTCOME_LIB_NAME: "outcome" @@ -46,8 +48,9 @@ tasks: install-all-run: internal: true deps: - - "install-Catch2" - - "install-outcome" + - task: "install-abseil" + - task: "install-Catch2" + - task: "install-outcome" install-all-finish: internal: true @@ -74,6 +77,21 @@ tasks: \"Path to {{.NAME}} settings\" )" >> "{{.G_DEPS_CMAKE_SETTINGS_DIR}}/{{.NAME}}.cmake" + install-abseil: + internal: true + run: "once" + cmds: + - task: ":utils:cmake:install-remote-tar" + vars: + NAME: "{{.G_ABSEIL_LIB_NAME}}" + WORK_DIR: "{{.G_ABSEIL_WORK_DIR}}" + FILE_SHA256: "b396401fd29e2e679cace77867481d388c807671dc2acc602a0259eeb79b7811" + URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20250127.1.tar.gz" + - task: "add-package-root-to-cmake-settings" + vars: + NAME: "{{.G_ABSEIL_LIB_NAME}}" + INSTALL_PREFIX: "{{.G_ABSEIL_WORK_DIR}}/{{.G_ABSEIL_LIB_NAME}}-install" + install-Catch2: internal: true run: "once" From ad3a40ac89b15361de3e9a31a773af8b93b6801f Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 14:46:28 -0400 Subject: [PATCH 06/19] Remove abseil from submodule and submodules directory --- .gitmodules | 3 --- submodules/Catch2 | 1 - submodules/abseil-cpp | 1 - 3 files changed, 5 deletions(-) delete mode 160000 submodules/Catch2 delete mode 160000 submodules/abseil-cpp diff --git a/.gitmodules b/.gitmodules index 73630b7f..2d96d0d8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "tools/yscope-dev-utils"] path = tools/yscope-dev-utils url = https://github.com/y-scope/yscope-dev-utils.git -[submodule "submodules/abseil-cpp"] - path = submodules/abseil-cpp - url = https://github.com/abseil/abseil-cpp.git diff --git a/submodules/Catch2 b/submodules/Catch2 deleted file mode 160000 index a6ee7e20..00000000 --- a/submodules/Catch2 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a6ee7e20cd4011129816df7992c1a9db2ef4b58f diff --git a/submodules/abseil-cpp b/submodules/abseil-cpp deleted file mode 160000 index 07fff76e..00000000 --- a/submodules/abseil-cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 07fff76ed619672bb7bf00939231ba45aef2b043 From 1d7d607f1549b93e3ee4a515a54a5cc543e9d131 Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 14:46:59 -0400 Subject: [PATCH 07/19] Rename abseil lib name --- dep-tasks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index c7c4b9f5..c01eea26 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -3,7 +3,7 @@ version: "3" vars: G_SCRIPT_DIR: "{{.ROOT_DIR}}/tools/scripts" - G_ABSEIL_LIB_NAME: "abseil" + G_ABSEIL_LIB_NAME: "absl" G_ABSEIL_WORK_DIR: "{{.G_DEPS_DIR}}/{{.G_ABSEIL_LIB_NAME}}" G_CATCH2_LIB_NAME: "Catch2" G_CATCH2_WORK_DIR: "{{.G_DEPS_DIR}}/{{.G_CATCH2_LIB_NAME}}" From cff7adeb9f973fdc95f7db33d432fb3917914029 Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 14:47:39 -0400 Subject: [PATCH 08/19] Update cmake for abseil installed by tasks --- CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 636430ec..1018aff9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,9 +173,14 @@ else() message(FATAL_ERROR "Could not find libraries for Catch2.") endif() -# Add abseil-cpp +# Add abseil set(ABSL_PROPAGATE_CXX_STD ON) -add_subdirectory(submodules/abseil-cpp) +find_package(absl REQUIRED) +if(absl_FOUND) + message(STATUS "Found abseil ${absl_VERSION}.") +else() + message(FATAL_ERROR "Could not find libraries for abseil.") +endif() find_package(Threads REQUIRED) From b94aff56494445fa52551a8b3693d629bf9267d7 Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 15:02:15 -0400 Subject: [PATCH 09/19] Add PIC in dependency --- dep-tasks.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index c01eea26..399ef008 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -87,6 +87,8 @@ tasks: WORK_DIR: "{{.G_ABSEIL_WORK_DIR}}" FILE_SHA256: "b396401fd29e2e679cace77867481d388c807671dc2acc602a0259eeb79b7811" URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20250127.1.tar.gz" + GEN_ARGS: + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - task: "add-package-root-to-cmake-settings" vars: NAME: "{{.G_ABSEIL_LIB_NAME}}" @@ -102,6 +104,8 @@ tasks: WORK_DIR: "{{.G_CATCH2_WORK_DIR}}" FILE_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" + GEN_ARGS: + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - task: "add-package-root-to-cmake-settings" vars: NAME: "{{.G_CATCH2_LIB_NAME}}" @@ -124,6 +128,7 @@ tasks: - "-DBUILD_TESTING=OFF" - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - task: "add-package-root-to-cmake-settings" vars: NAME: "{{.G_OUTCOME_LIB_NAME}}" @@ -142,6 +147,7 @@ tasks: GEN_ARGS: - "-DBUILD_TESTING=OFF" - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - task: "add-package-root-to-cmake-settings" vars: NAME: "{{.G_QUICKCPPLIB_LIB_NAME}}" From 7995d1d133a2eecd9fe67a540697fa3e33adfaf3 Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 16:21:36 -0400 Subject: [PATCH 10/19] Add task to download ystdlib source --- dep-tasks.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 399ef008..6f481fd7 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -11,6 +11,8 @@ vars: G_OUTCOME_WORK_DIR: "{{.G_DEPS_DIR}}/{{.G_OUTCOME_LIB_NAME}}" G_QUICKCPPLIB_LIB_NAME: "quickcpplib" G_QUICKCPPLIB_WORK_DIR: "{{.G_DEPS_DIR}}/{{.G_QUICKCPPLIB_LIB_NAME}}" + G_YSTDLIB_LIB_NAME: "ystdlib" + G_YSTDLIB_WORK_DIR: "{{.G_DEPS_DIR}}/{{.G_YSTDLIB_LIB_NAME}}" tasks: @@ -51,6 +53,7 @@ tasks: - task: "install-abseil" - task: "install-Catch2" - task: "install-outcome" + - task: "download-ystdlib" install-all-finish: internal: true @@ -152,3 +155,17 @@ tasks: vars: NAME: "{{.G_QUICKCPPLIB_LIB_NAME}}" INSTALL_PREFIX: "{{.G_QUICKCPPLIB_WORK_DIR}}/{{.G_QUICKCPPLIB_LIB_NAME}}-install" + + download-ystdlib: + internal: true + run: "once" + cmds: + - task: ":utils:remote:download-and-extract-tar" + vars: + FILE_SHA256: "d3fc9804eacb3ee4f156ae0ca37151cb04847580" + OUTPUT_DIR: "{{.G_YSTDLIB_WORK_DIR}}/{{.G_YSTDLIB_LIB_NAME}}-src" + URL: "https://github.com/y-scope/ystdlib-cpp/archive/d3fc980.tar.gz" + - >- + echo "set( + SPIDER_YSTDLIB_SOURCE_DIRECTORY \"{{.G_YSTDLIB_WORK_DIR}}/{{.G_YSTDLIB_LIB_NAME}}-src\" + )" >> "{{.G_DEPS_CMAKE_SETTINGS_DIR}}/{{.G_YSTDLIB_LIB_NAME}}.cmake" From e02310cf37e91756418c6e832ff90516c820de4c Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 16:32:38 -0400 Subject: [PATCH 11/19] Add ystdlib-cpp in cmake --- CMakeLists.txt | 7 +++++++ src/spider/CMakeLists.txt | 1 + 2 files changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1018aff9..b8ac1147 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -182,6 +182,13 @@ else() message(FATAL_ERROR "Could not find libraries for abseil.") endif() +# Add ystdlib-cpp +add_subdirectory( + "${SPIDER_YSTDLIB_SOURCE_DIRECTORY}" + "${CMAKE_BINARY_DIR}/ystdlib" + EXCLUDE_FROM_ALL +) + find_package(Threads REQUIRED) add_subdirectory(src/spider) diff --git a/src/spider/CMakeLists.txt b/src/spider/CMakeLists.txt index db0a773c..b8f51c73 100644 --- a/src/spider/CMakeLists.txt +++ b/src/spider/CMakeLists.txt @@ -52,6 +52,7 @@ target_link_libraries( MariaDBClientCpp::MariaDBClientCpp msgpack-cxx spdlog::spdlog + ystdlib::error_handling ) target_link_libraries(spider_core PRIVATE fmt::fmt) From 1f6c808bd5e96b94abb227fc21b2ab5c346e5d22 Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 23:09:29 -0400 Subject: [PATCH 12/19] Update clang format with ystdlib library --- src/spider/.clang-format | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spider/.clang-format b/src/spider/.clang-format index 06150eea..18897e34 100644 --- a/src/spider/.clang-format +++ b/src/spider/.clang-format @@ -10,7 +10,7 @@ IncludeCategories: # Ex: # - Regex: "<(fmt|spdlog)" # Priority: 3 - - Regex: "^<(absl|boost|catch2|fmt|mariadb|msgpack|spdlog)" + - Regex: "^<(absl|boost|catch2|fmt|mariadb|msgpack|spdlog|ystdlib)" Priority: 3 # C system headers - Regex: "^<.+\\.h>" From 388b6c2e61f456ab1544473517f59598631510db Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 23:09:50 -0400 Subject: [PATCH 13/19] Add worker error code --- src/spider/CMakeLists.txt | 2 ++ src/spider/worker/WorkerErrorCode.cpp | 40 +++++++++++++++++++++++++++ src/spider/worker/WorkerErrorCode.hpp | 25 +++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 src/spider/worker/WorkerErrorCode.cpp create mode 100644 src/spider/worker/WorkerErrorCode.hpp diff --git a/src/spider/CMakeLists.txt b/src/spider/CMakeLists.txt index b8f51c73..dad6d3ec 100644 --- a/src/spider/CMakeLists.txt +++ b/src/spider/CMakeLists.txt @@ -59,6 +59,8 @@ target_link_libraries(spider_core PRIVATE fmt::fmt) set(SPIDER_WORKER_SOURCES worker/DllLoader.hpp worker/DllLoader.cpp + worker/WorkerErrorCode.cpp + worker/WorkerErrorCode.hpp worker/Process.hpp worker/Process.cpp worker/TaskExecutor.hpp diff --git a/src/spider/worker/WorkerErrorCode.cpp b/src/spider/worker/WorkerErrorCode.cpp new file mode 100644 index 00000000..0ed4ff50 --- /dev/null +++ b/src/spider/worker/WorkerErrorCode.cpp @@ -0,0 +1,40 @@ +#include "WorkerErrorCode.hpp" + +#include +#include + +#include + +namespace spider::worker { +using WorkerErrorCategory = ystdlib::error_handling::ErrorCategory; +} // namespace spider::worker + +constexpr std::string_view cWorkerErrorCategoryName = "Worker Error Code"; + +template <> +auto spider::worker::WorkerErrorCategory::name() const noexcept -> char const* { + return cWorkerErrorCategoryName.data(); +} + +template <> +auto spider::worker::WorkerErrorCategory::message(spider::worker::WorkerErrorCodeEnum code) const + -> std::string { + switch (code) { + case spider::worker::WorkerErrorCodeEnum::Success: + return "Success"; + case spider::worker::WorkerErrorCodeEnum::CmdLineArgumentInvalid: + return "Invalid command line argument"; + case spider::worker::WorkerErrorCodeEnum::TaskArgumentInvalid: + return "Invalid task argument"; + case spider::worker::WorkerErrorCodeEnum::TaskFailed: + return "Task failed"; + case spider::worker::WorkerErrorCodeEnum::TaskOutputUnavailable: + return "Task output unavailable"; + case spider::worker::WorkerErrorCodeEnum::TaskOutputInvalid: + return "Task output invalid"; + case spider::worker::WorkerErrorCodeEnum::StorageError: + return "Storage error"; + default: + return "Unknown error"; + } +} diff --git a/src/spider/worker/WorkerErrorCode.hpp b/src/spider/worker/WorkerErrorCode.hpp new file mode 100644 index 00000000..15552a3c --- /dev/null +++ b/src/spider/worker/WorkerErrorCode.hpp @@ -0,0 +1,25 @@ +#ifndef SPIDER_WORKER_ERROR_CODE_HPP +#define SPIDER_WORKER_ERROR_CODE_HPP + +#include + +#include + +namespace spider::worker { +enum class WorkerErrorCodeEnum : uint8_t { + Success = 0, + CmdLineArgumentInvalid = 1, + TaskArgumentInvalid = 2, + TaskFailed = 3, + TaskOutputUnavailable = 4, + TaskOutputInvalid = 5, + // Storage related errors will be removed and use storage error codes instead + StorageError = 6, +}; + +using WorkerErrorCode = ystdlib::error_handling::ErrorCode; +} // namespace spider::worker + +YSTDLIB_ERROR_HANDLING_MARK_AS_ERROR_CODE_ENUM(spider::worker::WorkerErrorCodeEnum); + +#endif From e478ac50a7db695d710290435a2acffcd34188e1 Mon Sep 17 00:00:00 2001 From: sitao Date: Tue, 15 Apr 2025 23:52:23 -0400 Subject: [PATCH 14/19] [WIP] Update worker code to use error code. Not compiling --- src/spider/worker/worker.cpp | 68 +++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/src/spider/worker/worker.cpp b/src/spider/worker/worker.cpp index f3d1c4ce..fdfb3dce 100644 --- a/src/spider/worker/worker.cpp +++ b/src/spider/worker/worker.cpp @@ -28,6 +28,8 @@ #include #include // IWYU pragma: keep #include +#include +#include #include "../core/Data.hpp" #include "../core/Driver.hpp" @@ -44,6 +46,7 @@ #include "../utils/StopToken.hpp" #include "TaskExecutor.hpp" #include "WorkerClient.hpp" +#include "WorkerErrorCode.hpp" constexpr int cCmdArgParseErr = 1; constexpr int cWorkerAddrErr = 2; @@ -157,7 +160,7 @@ fetch_task(spider::worker::WorkerClient& client, std::optional const& storage_factory, std::shared_ptr const& metadata_store, spider::core::TaskInstance const& instance, spider::core::Task& task -) -> std::optional> { +) -> ystdlib::error_handling:: + Result, spider::worker::WorkerErrorCode> { std::variant, spider::core::StorageErr> conn_result = storage_factory->provide_storage_connection(); if (std::holds_alternative(conn_result)) { @@ -181,7 +185,7 @@ auto setup_task( "Failed to connect to storage: {}", std::get(conn_result).description ); - return std::nullopt; + return spider::worker::WorkerErrorCodeEnum::StorageError; } auto conn = std::move(std::get>(conn_result)); @@ -189,21 +193,30 @@ auto setup_task( spider::core::StorageErr const err = metadata_store->get_task(*conn, instance.task_id, &task); if (!err.success()) { spdlog::error("Failed to fetch task detail: {}", err.description); - return std::nullopt; + return spider::worker::WorkerErrorCodeEnum::StorageError; } std::optional> optional_arg_buffers = task.get_arg_buffers(); if (!optional_arg_buffers.has_value()) { spdlog::error("Failed to fetch task arguments"); metadata_store->task_fail(*conn, instance, fmt::format("Failed to fetch task arguments")); - return std::nullopt; + return spider::worker::WorkerErrorCodeEnum::TaskArgumentInvalid; } - return optional_arg_buffers; + return std::move(optional_arg_buffers.value()); } +/** + * Parses the task outputs from the result buffers. + * + * @param task The task that was executed. + * @param result_buffers The result buffers containing the outputs. + * @return A vector of TaskOutput objects if parsing is successful. + * @return WorkerErrorCode::TaskOutputInvalid if any failure occurs. + */ auto parse_outputs(spider::core::Task const& task, std::vector const& result_buffers) - -> std::optional> { + -> ystdlib::error_handling:: + Result, spider::worker::WorkerErrorCode> { std::vector outputs; outputs.reserve(task.get_num_outputs()); for (size_t i = 0; i < task.get_num_outputs(); ++i) { @@ -221,7 +234,7 @@ parse_outputs(spider::core::Task const& task, std::vector cons "Task {} failed to parse result as data id", task.get_function_name() ); - return std::nullopt; + return spider::worker::WorkerErrorCodeEnum::TaskOutputInvalid; } } else { msgpack::sbuffer const& buffer = result_buffers[i]; @@ -240,7 +253,8 @@ parse_outputs(spider::core::Task const& task, std::vector cons * @param instance Task instance that was executed. * @param task The task that was executed. * @param executor The executor that ran the task. - * @return true if results were successfully handled, false if any errors occurred. + * @return ystdlib::error_handling::success() if successful. + * @return WorkerErrorCode if any failure occurs. */ auto handle_executor_result( std::shared_ptr const& storage_factory, @@ -248,7 +262,7 @@ auto handle_executor_result( spider::core::TaskInstance const& instance, spider::core::Task const& task, spider::worker::TaskExecutor& executor -) -> bool { +) -> ystdlib::error_handling::Result { std::variant, spider::core::StorageErr> conn_result = storage_factory->provide_storage_connection(); if (std::holds_alternative(conn_result)) { @@ -256,7 +270,7 @@ auto handle_executor_result( "Failed to connect to storage: {}", std::get(conn_result).description ); - return false; + return spider::worker::WorkerErrorCodeEnum::StorageError; } auto conn = std::move(std::get>(conn_result)); @@ -267,7 +281,7 @@ auto handle_executor_result( instance, fmt::format("Task {} failed", task.get_function_name()) ); - return false; + return spider::worker::WorkerErrorCodeEnum::TaskFailed; } // Parse result @@ -280,12 +294,14 @@ auto handle_executor_result( instance, fmt::format("Task {} failed to parse result into buffers", task.get_function_name()) ); - return false; + return spider::worker::WorkerErrorCodeEnum::TaskOutputUnavailable; } std::vector const& result_buffers = optional_result_buffers.value(); - std::optional> const optional_outputs + ystdlib::error_handling::Result< + std::vector, + spider::worker::WorkerErrorCode> const output_result = parse_outputs(task, result_buffers); - if (!optional_outputs.has_value()) { + if (!output_result.has_value()) { metadata_store->task_fail( *conn, instance, @@ -294,10 +310,10 @@ auto handle_executor_result( task.get_function_name() ) ); - return false; + return output_result.error(); } - std::vector const& outputs = optional_outputs.value(); + std::vector const& outputs = output_result.value(); // Submit result spdlog::debug("Submitting result for task {}", boost::uuids::to_string(task.get_id())); spider::core::StorageErr err; @@ -313,9 +329,9 @@ auto handle_executor_result( } if (!err.success()) { spdlog::error("Submit task {} fails: {}", task.get_function_name(), err.description); - return false; + return spider::worker::WorkerErrorCodeEnum::StorageError; } - return true; + return ystdlib::error_handling::success(); } // NOLINTBEGIN(clang-analyzer-unix.BlockInCriticalSection) @@ -339,14 +355,16 @@ auto task_loop( spdlog::debug("Fetched task {}", boost::uuids::to_string(task_id)); // Fetch task detail from metadata storage spider::core::Task task{""}; - std::optional> optional_arg_buffers + ystdlib::error_handling:: + Result, spider::worker::WorkerErrorCode> + arg_buffers_result = setup_task(storage_factory, metadata_store, instance, task); - if (!optional_arg_buffers.has_value()) { + if (!arg_buffers_result.has_value()) { spdlog::error("Failed to setup task {}", task.get_function_name()); fail_task_id = task.get_id(); continue; } - std::vector const& arg_buffers = optional_arg_buffers.value(); + std::vector const& arg_buffers = arg_buffers_result.value(); // Execute task spider::worker::TaskExecutor executor{ @@ -362,7 +380,9 @@ auto task_loop( context.run(); executor.wait(); - if (handle_executor_result(storage_factory, metadata_store, instance, task, executor)) { + if (handle_executor_result(storage_factory, metadata_store, instance, task, executor) + .has_value()) + { fail_task_id = std::nullopt; } else { fail_task_id = task.get_id(); From a01a1d3061b986a98518403af1b0faa577441377 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 24 Apr 2025 14:47:40 -0400 Subject: [PATCH 15/19] Fix missing headers after merge --- src/spider/worker/worker.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/spider/worker/worker.cpp b/src/spider/worker/worker.cpp index 8f1bd325..e0304b60 100644 --- a/src/spider/worker/worker.cpp +++ b/src/spider/worker/worker.cpp @@ -51,6 +51,7 @@ #include "ChildPid.hpp" #include "TaskExecutor.hpp" #include "WorkerClient.hpp" +#include "WorkerErrorCode.hpp" constexpr int cCmdArgParseErr = 1; constexpr int cSignalHandleErr = 2; From cff9eb365661e13e28499a7fca2abd7d25358586 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 24 Apr 2025 15:40:02 -0400 Subject: [PATCH 16/19] Remove unused header include --- src/spider/worker/worker.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/spider/worker/worker.cpp b/src/spider/worker/worker.cpp index e0304b60..ceb72afa 100644 --- a/src/spider/worker/worker.cpp +++ b/src/spider/worker/worker.cpp @@ -32,7 +32,6 @@ #include #include // IWYU pragma: keep #include -#include #include #include "../core/Data.hpp" From fb00b886ce8f200c818721009ac0bce798d8d49d Mon Sep 17 00:00:00 2001 From: sitao Date: Thu, 24 Apr 2025 19:58:12 -0400 Subject: [PATCH 17/19] Add temporary fix for quickcpplib --- src/spider/worker/worker.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/spider/worker/worker.cpp b/src/spider/worker/worker.cpp index ceb72afa..c1594edc 100644 --- a/src/spider/worker/worker.cpp +++ b/src/spider/worker/worker.cpp @@ -32,6 +32,10 @@ #include #include // IWYU pragma: keep #include +// Temporarily disable quickcpplib to include valgrind headers to avoid +// conflicts with abseil valgrind headers, until the issue is resolved at +// https://github.com/y-scope/ystdlib-cpp/issues/59. +#undef QUICKCPPLIB_ENABLE_VALGRIND #include #include "../core/Data.hpp" From b30ed9a1c6daf0190550917fd629323e1bf52e3e Mon Sep 17 00:00:00 2001 From: sitao Date: Thu, 24 Apr 2025 20:14:05 -0400 Subject: [PATCH 18/19] Move constexpr into namespace --- src/spider/worker/WorkerErrorCode.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spider/worker/WorkerErrorCode.cpp b/src/spider/worker/WorkerErrorCode.cpp index 0ed4ff50..735c35fa 100644 --- a/src/spider/worker/WorkerErrorCode.cpp +++ b/src/spider/worker/WorkerErrorCode.cpp @@ -7,13 +7,13 @@ namespace spider::worker { using WorkerErrorCategory = ystdlib::error_handling::ErrorCategory; -} // namespace spider::worker constexpr std::string_view cWorkerErrorCategoryName = "Worker Error Code"; +} // namespace spider::worker template <> auto spider::worker::WorkerErrorCategory::name() const noexcept -> char const* { - return cWorkerErrorCategoryName.data(); + return spider::worker::cWorkerErrorCategoryName.data(); } template <> From 10d5f8453fe735e9e039de75a2900045ac562568 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 25 Apr 2025 11:42:40 -0400 Subject: [PATCH 19/19] Improve todo comment for storage error code. Co-authored-by: davidlion --- src/spider/worker/WorkerErrorCode.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spider/worker/WorkerErrorCode.hpp b/src/spider/worker/WorkerErrorCode.hpp index 15552a3c..010919cd 100644 --- a/src/spider/worker/WorkerErrorCode.hpp +++ b/src/spider/worker/WorkerErrorCode.hpp @@ -13,7 +13,7 @@ enum class WorkerErrorCodeEnum : uint8_t { TaskFailed = 3, TaskOutputUnavailable = 4, TaskOutputInvalid = 5, - // Storage related errors will be removed and use storage error codes instead + // TODO: Move storage related errors to an ErrorCode in the storage namespace. StorageError = 6, };