From 250d36defc99644fc39432e9f38e8c10731e8f7a Mon Sep 17 00:00:00 2001 From: root Date: Wed, 4 Oct 2023 04:35:12 -0400 Subject: [PATCH 1/4] YDA-5243 - Created non-persistent checksum microservice --- CMakeLists.txt | 4 ++ src/msi_dataObj_checksum.cpp | 120 +++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 src/msi_dataObj_checksum.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d215c34..e4e5e51 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,7 @@ link_libraries(c++abi) include_directories("${IRODS_EXTERNALS_FULLPATH_CLANG}/include/c++/v1") include_directories("${IRODS_EXTERNALS_FULLPATH_JSON}/include") include_directories("/usr/include/irods") +include_directories("/opt/irods-externals/fmt6.1.2-1/include") set(BOOST_ROOT "${IRODS_EXTERNALS_FULLPATH_BOOST}") @@ -66,6 +67,7 @@ add_library(msi_json_arrayops SHARED src/msi_json_arrayops.cc) add_library(msi_json_objops SHARED src/msi_json_objops.cc) add_library(msi_add_avu SHARED src/msi_add_avu.cpp) add_library(msi_rmw_avu SHARED src/msi_rmw_avu.cpp) +add_library(msi_dataObj_checksum SHARED src/msi_dataObj_checksum.cpp) target_link_libraries(msiRegisterEpicPID LINK_PUBLIC ${CURL_LIBRARIES} ${JANSSON_LIBRARIES} ${UUID_LIBRARIES}) target_link_libraries(msiArchiveCreate LINK_PUBLIC ${LibArchive_LIBRARIES} ${JANSSON_LIBRARIES} ) @@ -75,6 +77,7 @@ target_link_libraries(msi_json_arrayops LINK_PUBLIC ${JANSSON_LIBRARIES} target_link_libraries(msi_json_objops LINK_PUBLIC ${JANSSON_LIBRARIES} ${Boost_LIBRARIES}) target_link_libraries(msi_add_avu LINK_PUBLIC ${Boost_LIBRARIES}) target_link_libraries(msi_rmw_avu LINK_PUBLIC ${Boost_LIBRARIES}) +target_link_libraries(msi_dataObj_checksum LINK_PUBLIC ${Boost_LIBRARIES} ${LIB_NAME} ${CMAKE_DL_LIBS}) install(TARGETS msiRegisterEpicPID @@ -85,6 +88,7 @@ install(TARGETS msi_json_objops msi_add_avu msi_rmw_avu + msi_dataObj_checksum DESTINATION /usr/lib/irods/plugins/microservices) diff --git a/src/msi_dataObj_checksum.cpp b/src/msi_dataObj_checksum.cpp new file mode 100644 index 0000000..0ee6c33 --- /dev/null +++ b/src/msi_dataObj_checksum.cpp @@ -0,0 +1,120 @@ +#include "irods/SHA256Strategy.hpp" +#include "irods/checksum.h" +#include "rsGenQuery.hpp" +#include "irods/rcMisc.h" +#include "irods/rods.h" +#include "irods/dataObjInpOut.h" + +#include "irods_includes.hh" + +#include +#include +#include +#include + + +int calculate_checksum(rsComm_t *rsComm, std::string const& dataObjInp, std::string replNum, char* outChksum) { + char pathQCond[MAX_NAME_LEN]; + char replQCond[MAX_NAME_LEN]; + genQueryInp_t genQueryInp; + genQueryOut_t *genQueryOut; + char myColl[MAX_NAME_LEN], myData[MAX_NAME_LEN]; + sqlResult_t *path; + std::string phyPath; + int status; + int ret = 0; + + memset(myColl, 0, MAX_NAME_LEN); + memset(myData, 0, MAX_NAME_LEN); + + if ((status = splitPathByKey(dataObjInp.c_str(), myColl, MAX_NAME_LEN, myData, MAX_NAME_LEN, '/')) < 0) { + ret = OBJ_PATH_DOES_NOT_EXIST; + } + + memset(&genQueryInp, '\0', sizeof(genQueryInp_t)); + + snprintf(pathQCond, MAX_NAME_LEN, "='%s'", myColl); + addInxVal(&genQueryInp.sqlCondInp, COL_COLL_NAME, pathQCond); + snprintf(pathQCond, MAX_NAME_LEN, "='%s'", myData); + addInxVal(&genQueryInp.sqlCondInp, COL_DATA_NAME, pathQCond); + snprintf(replQCond, MAX_NAME_LEN, "='%s'", replNum.c_str()); + addInxVal(&genQueryInp.sqlCondInp, COL_DATA_REPL_NUM, replQCond); + + addInxIval(&genQueryInp.selectInp, COL_D_DATA_PATH, 1); + genQueryInp.maxRows = 1; + genQueryOut = NULL; + status = rsGenQuery(rsComm, &genQueryInp, &genQueryOut); + + if ( status >= 0 ) { + if (genQueryOut->rowCnt != 1) { + rodsLog( LOG_ERROR, "_msi_dataObj_checksum: Unknown File"); + ret = CAT_UNKNOWN_FILE; + } else { + if (( path = getSqlResultByInx( genQueryOut, COL_D_DATA_PATH ) ) == NULL) { + rodsLog( LOG_ERROR, "_msi_dataObj_checksum: getSqlResultByInx for COL_D_DATA_PATH failed"); + ret = UNMATCHED_KEY_OR_INDEX; + } + else { + phyPath = path->value; + } + } + } + + + clearGenQueryInp(&genQueryInp); + + freeGenQueryOut(&genQueryOut); + + int retCode = chksumLocFile(phyPath.c_str(), outChksum, irods::SHA256_NAME.c_str()); + + if (retCode < 0) { + rodsLog( LOG_ERROR, "_msi_dataObj_checksum: Failed to calculate checksum for file: [{}], status = {}", myData, retCode); + ret = retCode; + } + + return ret; +} + + +extern "C" { + +int msidataObjchecksum( + msParam_t* dataObjInp, + msParam_t* replNum, + msParam_t* statusOut, + ruleExecInfo_t* rei ) { + + std::string repl_num = parseMspForStr( replNum ); + std::string dataObj = parseMspForStr( dataObjInp ); + char chksum[NAME_LEN]; + + int ret = calculate_checksum(rei->rsComm, dataObj, repl_num, chksum); + + if (ret < 0) { + rodsLog( LOG_ERROR, "_msi_dataObj_checksum: Failed to calculate checksum"); + } + else{ + fillStrInMsParam(statusOut, chksum); + } + + return ret; + + } + +irods::ms_table_entry* plugin_factory() { + irods::ms_table_entry* msvc = new irods::ms_table_entry(3); + + msvc->add_operation< + msParam_t*, + msParam_t*, + msParam_t*, + ruleExecInfo_t*>("msidataObjchecksum", + std::function(msidataObjchecksum)); + return msvc; +} + +} \ No newline at end of file From f91c55c94186088099b1abfac221a6a114748ed1 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Fri, 8 Mar 2024 14:31:37 +0100 Subject: [PATCH 2/4] YDA-5243: format with clang-format --- src/msi_dataObj_checksum.cpp | 73 ++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/src/msi_dataObj_checksum.cpp b/src/msi_dataObj_checksum.cpp index 0ee6c33..a0c6c20 100644 --- a/src/msi_dataObj_checksum.cpp +++ b/src/msi_dataObj_checksum.cpp @@ -1,25 +1,25 @@ #include "irods/SHA256Strategy.hpp" #include "irods/checksum.h" -#include "rsGenQuery.hpp" +#include "irods/dataObjInpOut.h" #include "irods/rcMisc.h" #include "irods/rods.h" -#include "irods/dataObjInpOut.h" +#include "rsGenQuery.hpp" #include "irods_includes.hh" -#include -#include #include #include +#include +#include - -int calculate_checksum(rsComm_t *rsComm, std::string const& dataObjInp, std::string replNum, char* outChksum) { +int calculate_checksum(rsComm_t* rsComm, std::string const& dataObjInp, std::string replNum, char* outChksum) +{ char pathQCond[MAX_NAME_LEN]; char replQCond[MAX_NAME_LEN]; genQueryInp_t genQueryInp; - genQueryOut_t *genQueryOut; + genQueryOut_t* genQueryOut; char myColl[MAX_NAME_LEN], myData[MAX_NAME_LEN]; - sqlResult_t *path; + sqlResult_t* path; std::string phyPath; int status; int ret = 0; @@ -45,13 +45,16 @@ int calculate_checksum(rsComm_t *rsComm, std::string const& dataObjInp, std::str genQueryOut = NULL; status = rsGenQuery(rsComm, &genQueryInp, &genQueryOut); - if ( status >= 0 ) { + if (status >= 0) { if (genQueryOut->rowCnt != 1) { - rodsLog( LOG_ERROR, "_msi_dataObj_checksum: Unknown File"); + rodsLog(LOG_ERROR, "_msi_dataObj_checksum: Unknown File"); ret = CAT_UNKNOWN_FILE; - } else { - if (( path = getSqlResultByInx( genQueryOut, COL_D_DATA_PATH ) ) == NULL) { - rodsLog( LOG_ERROR, "_msi_dataObj_checksum: getSqlResultByInx for COL_D_DATA_PATH failed"); + } + else { + if ((path = getSqlResultByInx(genQueryOut, COL_D_DATA_PATH)) == NULL) { + rodsLog(LOG_ERROR, + "_msi_dataObj_checksum: getSqlResultByInx for " + "COL_D_DATA_PATH failed"); ret = UNMATCHED_KEY_OR_INDEX; } else { @@ -60,7 +63,6 @@ int calculate_checksum(rsComm_t *rsComm, std::string const& dataObjInp, std::str } } - clearGenQueryInp(&genQueryInp); freeGenQueryOut(&genQueryOut); @@ -68,53 +70,44 @@ int calculate_checksum(rsComm_t *rsComm, std::string const& dataObjInp, std::str int retCode = chksumLocFile(phyPath.c_str(), outChksum, irods::SHA256_NAME.c_str()); if (retCode < 0) { - rodsLog( LOG_ERROR, "_msi_dataObj_checksum: Failed to calculate checksum for file: [{}], status = {}", myData, retCode); + rodsLog(LOG_ERROR, + "_msi_dataObj_checksum: Failed to calculate checksum for file: " + "[{}], status = {}", + myData, + retCode); ret = retCode; } return ret; } - extern "C" { -int msidataObjchecksum( - msParam_t* dataObjInp, - msParam_t* replNum, - msParam_t* statusOut, - ruleExecInfo_t* rei ) { - - std::string repl_num = parseMspForStr( replNum ); - std::string dataObj = parseMspForStr( dataObjInp ); +int msidataObjchecksum(msParam_t* dataObjInp, msParam_t* replNum, msParam_t* statusOut, ruleExecInfo_t* rei) +{ + std::string repl_num = parseMspForStr(replNum); + std::string dataObj = parseMspForStr(dataObjInp); char chksum[NAME_LEN]; int ret = calculate_checksum(rei->rsComm, dataObj, repl_num, chksum); if (ret < 0) { - rodsLog( LOG_ERROR, "_msi_dataObj_checksum: Failed to calculate checksum"); + rodsLog(LOG_ERROR, "_msi_dataObj_checksum: Failed to calculate checksum"); } - else{ + else { fillStrInMsParam(statusOut, chksum); } return ret; - - } +} -irods::ms_table_entry* plugin_factory() { +irods::ms_table_entry* plugin_factory() +{ irods::ms_table_entry* msvc = new irods::ms_table_entry(3); - msvc->add_operation< - msParam_t*, - msParam_t*, - msParam_t*, - ruleExecInfo_t*>("msidataObjchecksum", - std::function(msidataObjchecksum)); + msvc->add_operation( + "msidataObjchecksum", + std::function(msidataObjchecksum)); return msvc; } - } \ No newline at end of file From 3e59f423110468af1dbbbde60757f24422dcd303 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Fri, 8 Mar 2024 14:53:27 +0100 Subject: [PATCH 3/4] YDA-5243: add license, improve variable names and add some comments --- src/msi_dataObj_checksum.cpp | 74 +++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/src/msi_dataObj_checksum.cpp b/src/msi_dataObj_checksum.cpp index a0c6c20..0a92339 100644 --- a/src/msi_dataObj_checksum.cpp +++ b/src/msi_dataObj_checksum.cpp @@ -1,3 +1,26 @@ +/** + * \file + * \brief iRODS microservice to compute SHA256 checksum of a data object replica + * \author Sirjan Kaur + * \author Lazlo Westerhof + * \copyright Copyright (c) 2024, Utrecht University + * + * This file is part of irods-uu-microservices. + * + * irods-uu-microservices is free software: you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the License, + * or (at your option) any later version. + * + * irods-uu-microservices is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser + * General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with irods-uu-microservices. + * If not, see . + */ #include "irods/SHA256Strategy.hpp" #include "irods/checksum.h" #include "irods/dataObjInpOut.h" @@ -12,49 +35,48 @@ #include #include -int calculate_checksum(rsComm_t* rsComm, std::string const& dataObjInp, std::string replNum, char* outChksum) +int calculate_checksum(rsComm_t* rsComm, std::string const& dataObjInp, std::string replNumInp, char* checksumOut) { char pathQCond[MAX_NAME_LEN]; char replQCond[MAX_NAME_LEN]; genQueryInp_t genQueryInp; genQueryOut_t* genQueryOut; - char myColl[MAX_NAME_LEN], myData[MAX_NAME_LEN]; + char coll[MAX_NAME_LEN], dataObj[MAX_NAME_LEN]; sqlResult_t* path; std::string phyPath; int status; int ret = 0; - memset(myColl, 0, MAX_NAME_LEN); - memset(myData, 0, MAX_NAME_LEN); + memset(coll, 0, MAX_NAME_LEN); + memset(dataObj, 0, MAX_NAME_LEN); - if ((status = splitPathByKey(dataObjInp.c_str(), myColl, MAX_NAME_LEN, myData, MAX_NAME_LEN, '/')) < 0) { + if ((status = splitPathByKey(dataObjInp.c_str(), coll, MAX_NAME_LEN, dataObj, MAX_NAME_LEN, '/')) < 0) { ret = OBJ_PATH_DOES_NOT_EXIST; } memset(&genQueryInp, '\0', sizeof(genQueryInp_t)); - snprintf(pathQCond, MAX_NAME_LEN, "='%s'", myColl); + // Build query to find data object replica. + snprintf(pathQCond, MAX_NAME_LEN, "='%s'", coll); addInxVal(&genQueryInp.sqlCondInp, COL_COLL_NAME, pathQCond); - snprintf(pathQCond, MAX_NAME_LEN, "='%s'", myData); + snprintf(pathQCond, MAX_NAME_LEN, "='%s'", dataObj); addInxVal(&genQueryInp.sqlCondInp, COL_DATA_NAME, pathQCond); - snprintf(replQCond, MAX_NAME_LEN, "='%s'", replNum.c_str()); + snprintf(replQCond, MAX_NAME_LEN, "='%s'", replNumInp.c_str()); addInxVal(&genQueryInp.sqlCondInp, COL_DATA_REPL_NUM, replQCond); - addInxIval(&genQueryInp.selectInp, COL_D_DATA_PATH, 1); + + // Execute query to find data object replica. genQueryInp.maxRows = 1; genQueryOut = NULL; status = rsGenQuery(rsComm, &genQueryInp, &genQueryOut); - if (status >= 0) { if (genQueryOut->rowCnt != 1) { - rodsLog(LOG_ERROR, "_msi_dataObj_checksum: Unknown File"); + rodsLog(LOG_ERROR, "msi_dataObj_checksum: unkown file"); ret = CAT_UNKNOWN_FILE; } else { if ((path = getSqlResultByInx(genQueryOut, COL_D_DATA_PATH)) == NULL) { - rodsLog(LOG_ERROR, - "_msi_dataObj_checksum: getSqlResultByInx for " - "COL_D_DATA_PATH failed"); + rodsLog(LOG_ERROR, "msi_dataObj_checksum: getSqlResultByInx for COL_D_DATA_PATH failed"); ret = UNMATCHED_KEY_OR_INDEX; } else { @@ -64,16 +86,15 @@ int calculate_checksum(rsComm_t* rsComm, std::string const& dataObjInp, std::str } clearGenQueryInp(&genQueryInp); - freeGenQueryOut(&genQueryOut); - int retCode = chksumLocFile(phyPath.c_str(), outChksum, irods::SHA256_NAME.c_str()); + // Compute checksum of data object replica. + int retCode = chksumLocFile(phyPath.c_str(), checksumOut, irods::SHA256_NAME.c_str()); if (retCode < 0) { rodsLog(LOG_ERROR, - "_msi_dataObj_checksum: Failed to calculate checksum for file: " - "[{}], status = {}", - myData, + "msi_dataObj_checksum: failed to calculate checksum for file: [{}], status = {}", + dataObj, retCode); ret = retCode; } @@ -83,19 +104,20 @@ int calculate_checksum(rsComm_t* rsComm, std::string const& dataObjInp, std::str extern "C" { -int msidataObjchecksum(msParam_t* dataObjInp, msParam_t* replNum, msParam_t* statusOut, ruleExecInfo_t* rei) +int msidataObjchecksum(msParam_t* dataObjInp, msParam_t* replNumInp, msParam_t* statusOut, ruleExecInfo_t* rei) { - std::string repl_num = parseMspForStr(replNum); + // Parse input parameters. + std::string repl_num = parseMspForStr(replNumInp); std::string dataObj = parseMspForStr(dataObjInp); - char chksum[NAME_LEN]; - int ret = calculate_checksum(rei->rsComm, dataObj, repl_num, chksum); + char checksum[NAME_LEN]; + int ret = calculate_checksum(rei->rsComm, dataObj, repl_num, checksum); if (ret < 0) { - rodsLog(LOG_ERROR, "_msi_dataObj_checksum: Failed to calculate checksum"); + rodsLog(LOG_ERROR, "msi_dataObj_checksum: failed to calculate checksum"); } else { - fillStrInMsParam(statusOut, chksum); + fillStrInMsParam(statusOut, checksum); } return ret; @@ -110,4 +132,4 @@ irods::ms_table_entry* plugin_factory() std::function(msidataObjchecksum)); return msvc; } -} \ No newline at end of file +} From 315569363bcd888096b892673314fbb6b4380b94 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Tue, 2 Apr 2024 08:23:56 +0200 Subject: [PATCH 4/4] YDA-5243: fix typo --- src/msi_dataObj_checksum.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/msi_dataObj_checksum.cpp b/src/msi_dataObj_checksum.cpp index 0a92339..66437fa 100644 --- a/src/msi_dataObj_checksum.cpp +++ b/src/msi_dataObj_checksum.cpp @@ -71,7 +71,7 @@ int calculate_checksum(rsComm_t* rsComm, std::string const& dataObjInp, std::str status = rsGenQuery(rsComm, &genQueryInp, &genQueryOut); if (status >= 0) { if (genQueryOut->rowCnt != 1) { - rodsLog(LOG_ERROR, "msi_dataObj_checksum: unkown file"); + rodsLog(LOG_ERROR, "msi_dataObj_checksum: unknown file"); ret = CAT_UNKNOWN_FILE; } else {