From 66220cee76ce5acb011aa066916adae7a2068322 Mon Sep 17 00:00:00 2001 From: Jonathan Hollocombe Date: Mon, 9 Dec 2024 11:31:42 +0000 Subject: [PATCH 1/6] Adding chunk downloading to BYTES plugin. --- CMakeLists.txt | 2 +- source/plugins/bytes/bytesPlugin.cfg.in | 1 + source/plugins/bytes/bytesPlugin.cpp | 203 +++++++++++++----- .../plugins/bytes/readBytesNonOptimally.cpp | 160 +++++--------- source/plugins/bytes/readBytesNonOptimally.h | 30 +-- 5 files changed, 211 insertions(+), 185 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b5b28aa..665e89ca 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required( VERSION 3.0 ) +cmake_minimum_required( VERSION 3.12 ) ######################################################################################################################## # Macro used to convert Windows path to Unix diff --git a/source/plugins/bytes/bytesPlugin.cfg.in b/source/plugins/bytes/bytesPlugin.cfg.in index e69de29b..3bc67c9e 100755 --- a/source/plugins/bytes/bytesPlugin.cfg.in +++ b/source/plugins/bytes/bytesPlugin.cfg.in @@ -0,0 +1 @@ +export UDA_BYTES_PLUGIN_ALLOWED_PATHS= \ No newline at end of file diff --git a/source/plugins/bytes/bytesPlugin.cpp b/source/plugins/bytes/bytesPlugin.cpp index 78fd060d..ea83983a 100644 --- a/source/plugins/bytes/bytesPlugin.cpp +++ b/source/plugins/bytes/bytesPlugin.cpp @@ -20,24 +20,51 @@ namespace filesystem = std::filesystem; #include -static int do_help(IDAM_PLUGIN_INTERFACE* plugin_interface); +#define BYTEFILEOPENERROR 100004 +#define BYTEFILEHEAPERROR 100005 -static int do_version(IDAM_PLUGIN_INTERFACE* plugin_interface); - -static int do_builddate(IDAM_PLUGIN_INTERFACE* plugin_interface); +class BytesPlugin +{ +public: + void init(IDAM_PLUGIN_INTERFACE* plugin_interface) + { + REQUEST_DATA* request = plugin_interface->request_data; + if (!init_ + || STR_IEQUALS(request->function, "init") + || STR_IEQUALS(request->function, "initialise")) { + reset(plugin_interface); + // Initialise plugin + init_ = true; + } + } + void reset(IDAM_PLUGIN_INTERFACE* plugin_interface) + { + if (!init_) { + // Not previously initialised: Nothing to do! + return; + } + // Free Heap & reset counters + init_ = false; + } -static int do_defaultmethod(IDAM_PLUGIN_INTERFACE* plugin_interface); + int help(IDAM_PLUGIN_INTERFACE* plugin_interface); + int version(IDAM_PLUGIN_INTERFACE* plugin_interface); + int build_date(IDAM_PLUGIN_INTERFACE* plugin_interface); + int default_method(IDAM_PLUGIN_INTERFACE* plugin_interface); + int max_interface_version(IDAM_PLUGIN_INTERFACE* plugin_interface); + int read(IDAM_PLUGIN_INTERFACE* plugin_interface); + int size(IDAM_PLUGIN_INTERFACE* plugin_interface); -static int do_maxinterfaceversion(IDAM_PLUGIN_INTERFACE* plugin_interface); +private: + using file_ptr = std::unique_ptr; -static int do_read(IDAM_PLUGIN_INTERFACE* plugin_interface); + bool init_ = false; + std::unordered_map file_map_ = {}; +}; int bytesPlugin(IDAM_PLUGIN_INTERFACE* plugin_interface) { - static int init = 0; - - //---------------------------------------------------------------------------------------- - // Standard v1 Plugin Interface + static BytesPlugin plugin = {}; if (plugin_interface->interfaceVersion > THISPLUGIN_MAX_INTERFACE_VERSION) { RAISE_PLUGIN_ERROR("Plugin Interface Version Unknown to this plugin: Unable to execute the request!"); @@ -45,38 +72,17 @@ int bytesPlugin(IDAM_PLUGIN_INTERFACE* plugin_interface) plugin_interface->pluginVersion = THISPLUGIN_VERSION; - //---------------------------------------------------------------------------------------- - // Heap Housekeeping - - // Plugin must maintain a list of open file handles and sockets: loop over and close all files and sockets - // Plugin must maintain a list of plugin functions called: loop over and reset state and free heap. - // Plugin must maintain a list of calls to other plugins: loop over and call each plugin with the housekeeping request - // Plugin must destroy lists at end of housekeeping - - // A plugin only has a single instance on a server. For multiple instances, multiple servers are needed. - // Plugins can maintain state so recursive calls (on the same server) must respect this. - // If the housekeeping action is requested, this must be also applied to all plugins called. - // A list must be maintained to register these plugin calls to manage housekeeping. - // Calls to plugins must also respect access policy and user authentication policy - REQUEST_DATA* request = plugin_interface->request_data; if (plugin_interface->housekeeping || STR_IEQUALS(request->function, "reset")) { - if (!init) return 0; // Not previously initialised: Nothing to do! - // Free Heap & reset counters - init = 0; + plugin.reset(plugin_interface); return 0; } - //---------------------------------------------------------------------------------------- - // Initialise - - if (!init || STR_IEQUALS(request->function, "init") + plugin.init(plugin_interface); + if (STR_IEQUALS(request->function, "init") || STR_IEQUALS(request->function, "initialise")) { - - init = 1; - if (STR_IEQUALS(request->function, "init") || STR_IEQUALS(request->function, "initialise")) - return 0; + return 0; } //---------------------------------------------------------------------------------------- @@ -87,17 +93,19 @@ int bytesPlugin(IDAM_PLUGIN_INTERFACE* plugin_interface) // Standard methods: version, builddate, defaultmethod, maxinterfaceversion if (STR_IEQUALS(request->function, "help")) { - return do_help(plugin_interface); + return plugin.help(plugin_interface); } else if (STR_IEQUALS(request->function, "version")) { - return do_version(plugin_interface); + return plugin.version(plugin_interface); } else if (STR_IEQUALS(request->function, "builddate")) { - return do_builddate(plugin_interface); + return plugin.build_date(plugin_interface); } else if (STR_IEQUALS(request->function, "defaultmethod")) { - return do_defaultmethod(plugin_interface); + return plugin.default_method(plugin_interface); } else if (STR_IEQUALS(request->function, "maxinterfaceversion")) { - return do_maxinterfaceversion(plugin_interface); + return plugin.max_interface_version(plugin_interface); } else if (STR_IEQUALS(request->function, "read")) { - return do_read(plugin_interface); + return plugin.read(plugin_interface); + } else if (STR_IEQUALS(request->function, "size")) { + return plugin.size(plugin_interface); } else { RAISE_PLUGIN_ERROR("Unknown function requested!"); } @@ -108,7 +116,7 @@ int bytesPlugin(IDAM_PLUGIN_INTERFACE* plugin_interface) * @param plugin_interface * @return */ -int do_help(IDAM_PLUGIN_INTERFACE* plugin_interface) +int BytesPlugin::help(IDAM_PLUGIN_INTERFACE* plugin_interface) { const char* help = "\nbytes: data reader to access files as a block of bytes without interpretation\n\n"; const char* desc = "bytes: help = description of this plugin"; @@ -121,7 +129,7 @@ int do_help(IDAM_PLUGIN_INTERFACE* plugin_interface) * @param plugin_interface * @return */ -int do_version(IDAM_PLUGIN_INTERFACE* plugin_interface) +int BytesPlugin::version(IDAM_PLUGIN_INTERFACE* plugin_interface) { return setReturnDataIntScalar(plugin_interface->data_block, THISPLUGIN_VERSION, "Plugin version number"); } @@ -131,7 +139,7 @@ int do_version(IDAM_PLUGIN_INTERFACE* plugin_interface) * @param plugin_interface * @return */ -int do_builddate(IDAM_PLUGIN_INTERFACE* plugin_interface) +int BytesPlugin::build_date(IDAM_PLUGIN_INTERFACE* plugin_interface) { return setReturnDataString(plugin_interface->data_block, __DATE__, "Plugin build date"); } @@ -141,7 +149,7 @@ int do_builddate(IDAM_PLUGIN_INTERFACE* plugin_interface) * @param plugin_interface * @return */ -int do_defaultmethod(IDAM_PLUGIN_INTERFACE* plugin_interface) +int BytesPlugin::default_method(IDAM_PLUGIN_INTERFACE* plugin_interface) { return setReturnDataString(plugin_interface->data_block, THISPLUGIN_DEFAULT_METHOD, "Plugin default method"); } @@ -151,7 +159,7 @@ int do_defaultmethod(IDAM_PLUGIN_INTERFACE* plugin_interface) * @param plugin_interface * @return */ -int do_maxinterfaceversion(IDAM_PLUGIN_INTERFACE* plugin_interface) +int BytesPlugin::max_interface_version(IDAM_PLUGIN_INTERFACE* plugin_interface) { return setReturnDataIntScalar(plugin_interface->data_block, THISPLUGIN_MAX_INTERFACE_VERSION, "Maximum Interface Version"); } @@ -189,20 +197,107 @@ int check_allowed_path(const char* expandedPath) { return 0; } -int do_read(IDAM_PLUGIN_INTERFACE* plugin_interface) +int check_path(const Environment* environment, const std::string& path) +{ + int err = 0; + + //---------------------------------------------------------------------- + // Block Access to External Users + + if (environment->external_user) { + err = 999; + addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "This Service is Disabled"); + UDA_LOG(UDA_LOG_DEBUG, "Disabled Service - Requested File: %s \n", path.c_str()); + return err; + } + + //---------------------------------------------------------------------- + // Test the filepath + + if (!IsLegalFilePath(path.c_str())) { + err = 999; + addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "The directory path has incorrect syntax"); + UDA_LOG(UDA_LOG_DEBUG, "The directory path has incorrect syntax [%s] \n", path.c_str()); + return err; + } + + //---------------------------------------------------------------------- + // Data Source Details + + UDA_LOG(UDA_LOG_DEBUG, "File Name : %s \n", path.c_str()); + + return err; +} + +int BytesPlugin::read(IDAM_PLUGIN_INTERFACE* plugin_interface) { - DATA_SOURCE* data_source = plugin_interface->data_source; - SIGNAL_DESC* signal_desc = plugin_interface->signal_desc; DATA_BLOCK* data_block = plugin_interface->data_block; const char* path; FIND_REQUIRED_STRING_VALUE(plugin_interface->request_data->nameValueList, path); - StringCopy(data_source->path, path, MAXPATH); + int max_bytes = -1; + FIND_INT_VALUE(plugin_interface->request_data->nameValueList, max_bytes); + + int offset = -1; + FIND_INT_VALUE(plugin_interface->request_data->nameValueList, offset); + + const char* checksum = nullptr; + FIND_STRING_VALUE(plugin_interface->request_data->nameValueList, checksum); + + char tmp_path[MAXPATH]; + StringCopy(tmp_path, path, MAXPATH); UDA_LOG(UDA_LOG_DEBUG, "expand_environment_variables! \n"); - expand_environment_variables(data_source->path); + expand_environment_variables(tmp_path); - check_allowed_path(data_source->path); + int rc = check_allowed_path(tmp_path); + if (rc != 0) { + return rc; + } + + if (checksum == nullptr) { + checksum = ""; + } + + rc = check_path(plugin_interface->environment, tmp_path); + if (rc != 0) { + return rc; + } + + errno = 0; + + FILE* file = nullptr; + if (file_map_.count(tmp_path) == 0) { + file_ptr ptr = {fopen(tmp_path, "rb"), fclose}; + file = ptr.get(); + file_map_.emplace(tmp_path, std::move(ptr)); + } else { + file = file_map_.at(tmp_path).get(); + } + + int serrno = errno; + + if (file == nullptr || ferror(file) || serrno != 0) { + int err = BYTEFILEOPENERROR; + if (serrno != 0) { + addIdamError(UDA_SYSTEM_ERROR_TYPE, "readBytes", serrno, ""); + } + addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "Unable to Open the File for Read Access"); + file_map_.erase(tmp_path); + return err; + } + + return readBytes(file, data_block, offset, max_bytes, checksum); +} + +int BytesPlugin::size(IDAM_PLUGIN_INTERFACE* plugin_interface) +{ + DATA_BLOCK* data_block = plugin_interface->data_block; + + const char* path = ""; + FIND_REQUIRED_STRING_VALUE(plugin_interface->request_data->nameValueList, path); + + size_t file_size = filesystem::file_size(path); - return readBytes(*data_source, *signal_desc, data_block, plugin_interface->environment); + return setReturnDataLongScalar(data_block, (long)file_size, nullptr); } diff --git a/source/plugins/bytes/readBytesNonOptimally.cpp b/source/plugins/bytes/readBytesNonOptimally.cpp index 0df0f26b..e19424ec 100755 --- a/source/plugins/bytes/readBytesNonOptimally.cpp +++ b/source/plugins/bytes/readBytesNonOptimally.cpp @@ -1,5 +1,5 @@ /*--------------------------------------------------------------- -* IDAM Plugin data Reader to Access Files as a Block of Bytes without Interpretation +* UDA Plugin data Reader to Access Files as a Block of Bytes without Interpretation * * Input Arguments: DATA_SOURCE data_source * SIGNAL_DESC signal_desc @@ -31,136 +31,88 @@ #include #include -int readBytes(DATA_SOURCE data_source, SIGNAL_DESC signal_desc, DATA_BLOCK* data_block, const ENVIRONMENT* environment) +#define BYTEFILEDOESNOTEXIST 100001 +#define BYTEFILEATTRIBUTEERROR 100002 +#define BYTEFILEISNOTREGULAR 100003 +#define BYTEFILEOPENERROR 100004 +#define BYTEFILEHEAPERROR 100005 +#define BYTEFILEMD5ERROR 100006 +#define BYTEFILEMD5DIFF 100007 + +int readBytes(FILE* fh, DATA_BLOCK* data_block, int offset, int max_bytes, const std::string& checksum) { int err = 0; - char md5file[2 * MD5_SIZE + 1] = ""; - char md5check[2 * MD5_SIZE + 1] = ""; - - //---------------------------------------------------------------------- - // Block Access to External Users + // Read File (Consider using memory mapped I/O & new type to avoid heap free at end if this is too slow!) - if (environment->external_user) { - err = 999; - addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "This Service is Disabled"); - UDA_LOG(UDA_LOG_DEBUG, "Disabled Service - Requested File: %s \n", data_source.path); - return err; + int nchar = 0; + int buf_offset = 0; + int buf_size = 1024 * 1024; + if (max_bytes > 0 && max_bytes < buf_size) { + buf_size = max_bytes; } - //---------------------------------------------------------------------- - // Test the filepath + data_block->data_n = buf_size; // 1 less than no. bytes read: Last Byte is an EOF - if (!IsLegalFilePath(data_source.path)) { - err = 999; - addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "The directory path has incorrect syntax"); - UDA_LOG(UDA_LOG_DEBUG, "The directory path has incorrect syntax [%s] \n", data_source.path); - return err; + if (offset >= 0) { + fseek(fh, offset, SEEK_SET); } - //---------------------------------------------------------------------- - // Data Source Details - - err = 0; - - UDA_LOG(UDA_LOG_DEBUG, "File Name : %s \n", data_source.path); - - //---------------------------------------------------------------------- - // File Attributes - - errno = 0; - - //---------------------------------------------------------------------- - // Open the File as a Binary Stream - - errno = 0; - FILE* fh = fopen(data_source.path, "rb"); - - int serrno = errno; - - if (fh == nullptr || ferror(fh) || serrno != 0) { - err = BYTEFILEOPENERROR; - if (serrno != 0) { - addIdamError(UDA_SYSTEM_ERROR_TYPE, "readBytes", serrno, ""); + char* bp = nullptr; + while (!feof(fh)) { + char* newp = (char*)realloc(bp, (size_t)data_block->data_n); + if (newp == nullptr) { + free(bp); + err = BYTEFILEHEAPERROR; + addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "Unable to Allocate Heap Memory for the File"); + break; } - addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "Unable to Open the File for Read Access"); - if (fh != nullptr) { - fclose(fh); + bp = newp; + int n_read = (int)fread(bp + buf_offset, sizeof(char), (size_t)buf_size, fh); + nchar += n_read; + if (nchar >= max_bytes) { + break; } - return err; + buf_offset = nchar; + data_block->data_n = nchar + buf_size + 1; } - //---------------------------------------------------------------------- - // Error Trap Loop - - do { - - // Read File (Consider using memory mapped I/O & new type to avoid heap free at end if this is too slow!) - - int nchar = 0; - int offset = 0; - int bufsize = 100 * 1024; - data_block->data_n = bufsize; // 1 less than no. bytes read: Last Byte is an EOF - - char* bp = nullptr; - while (!feof(fh)) { - if ((bp = (char*)realloc(bp, (size_t)data_block->data_n)) == nullptr) { - err = BYTEFILEHEAPERROR; - addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "Unable to Allocate Heap Memory for the File"); - break; - } - int nread = (int)fread(bp + offset, sizeof(char), (size_t)bufsize, fh); - nchar = nchar + nread; - offset = nchar; - data_block->data_n = nchar + bufsize + 1; - } - - if (err != 0) { - break; - } + if (err != 0) { + return err; + } - //nchar--; // Remove EOF Character from end of Byte Block - data_block->data_n = nchar; - data_block->data = bp; + //nchar--; // Remove EOF Character from end of Byte Block + data_block->data_n = nchar; + data_block->data = bp; + if (checksum == "md5") { //---------------------------------------------------------------------- // MD5 Checksum + char md5check[MD5_SIZE]; md5Sum(bp, data_block->data_n, md5check); - strcpy(data_block->data_desc, md5check); // Pass back the Checksum to the Client UDA_LOG(UDA_LOG_DEBUG, "File Size : %d \n", nchar); - UDA_LOG(UDA_LOG_DEBUG, "File Checksum : %s \n", md5file); UDA_LOG(UDA_LOG_DEBUG, "Read Checksum : %s \n", md5check); + } - // MD5 Difference? - - //---------------------------------------------------------------------- - // Fetch Dimensional Data - - data_block->rank = 1; - data_block->dims = (DIMS*)malloc(sizeof(DIMS)); - initDimBlock(data_block->dims); - - data_block->dims[0].data_type = UDA_TYPE_UNSIGNED_INT; - data_block->dims[0].dim_n = data_block->data_n; - data_block->dims[0].compressed = 1; - data_block->dims[0].dim0 = 0.0; - data_block->dims[0].diff = 1.0; - data_block->dims[0].method = 0; - - data_block->order = -1; // No Dimensions - data_block->data_type = UDA_TYPE_CHAR; + //---------------------------------------------------------------------- + // Fetch Dimensional Data - } while (0); + data_block->rank = 1; + data_block->dims = (DIMS*)malloc(sizeof(DIMS)); + initDimBlock(data_block->dims); - //---------------------------------------------------------------------- - // Housekeeping + data_block->dims[0].data_type = UDA_TYPE_UNSIGNED_INT; + data_block->dims[0].dim_n = data_block->data_n; + data_block->dims[0].compressed = 1; + data_block->dims[0].dim0 = 0.0; + data_block->dims[0].diff = 1.0; + data_block->dims[0].method = 0; -// if (err != 0) { -// freeDataBlock(data_block); -// } + data_block->order = -1; // No Dimensions + data_block->data_type = UDA_TYPE_CHAR; fclose(fh); // Close the File diff --git a/source/plugins/bytes/readBytesNonOptimally.h b/source/plugins/bytes/readBytesNonOptimally.h index 792f8c74..ad051702 100755 --- a/source/plugins/bytes/readBytesNonOptimally.h +++ b/source/plugins/bytes/readBytesNonOptimally.h @@ -1,30 +1,8 @@ -#ifndef UDA_PLUGIN_READBYTESNONOPTIMALLY_H -#define UDA_PLUGIN_READBYTESNONOPTIMALLY_H +#pragma once -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -LIBRARY_API int readBytes(DATA_SOURCE data_source, SIGNAL_DESC signal_desc, DATA_BLOCK *data_block, const ENVIRONMENT* environment); - -#ifndef NOBINARYPLUGIN +#include -#define BYTEFILEDOESNOTEXIST 100001 -#define BYTEFILEATTRIBUTEERROR 100002 -#define BYTEFILEISNOTREGULAR 100003 -#define BYTEFILEOPENERROR 100004 -#define BYTEFILEHEAPERROR 100005 -#define BYTEFILEMD5ERROR 100006 -#define BYTEFILEMD5DIFF 100007 - -#endif - -#ifdef __cplusplus -} -#endif +#include -#endif // UDA_PLUGIN_READBYTESNONOPTIMALLY_H +int readBytes(FILE* file, DATA_BLOCK* data_block, int offset, int max_bytes, const std::string& checksum); From b5b35db0101689c7ea22b2c123ee92aa38592084 Mon Sep 17 00:00:00 2001 From: Jonathan Hollocombe Date: Mon, 9 Dec 2024 15:59:56 +0000 Subject: [PATCH 2/6] Not closing file in readBytesNonOptimally.cpp --- source/plugins/bytes/CMakeLists.txt | 1 + source/plugins/bytes/md5Sum.cpp | 2 +- .../plugins/bytes/readBytesNonOptimally.cpp | 33 ++++++++++++------- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/source/plugins/bytes/CMakeLists.txt b/source/plugins/bytes/CMakeLists.txt index ac1e9de0..23904720 100644 --- a/source/plugins/bytes/CMakeLists.txt +++ b/source/plugins/bytes/CMakeLists.txt @@ -24,6 +24,7 @@ uda_plugin( EXAMPLE "BYTES::read()" LIBNAME bytes_plugin SOURCES bytesPlugin.cpp readBytesNonOptimally.cpp md5Sum.cpp + CONFIG_FILE bytesPlugin.cfg EXTRA_INCLUDE_DIRS ${LIBXML2_INCLUDE_DIR} EXTRA_LINK_LIBS diff --git a/source/plugins/bytes/md5Sum.cpp b/source/plugins/bytes/md5Sum.cpp index 5a778bf1..d6f52d90 100755 --- a/source/plugins/bytes/md5Sum.cpp +++ b/source/plugins/bytes/md5Sum.cpp @@ -34,7 +34,7 @@ void md5Sum(char *bp, int size, char *md5check) { md5check[2*MD5_SIZE]='\0'; - for(i=0; i #include -#define BYTEFILEDOESNOTEXIST 100001 -#define BYTEFILEATTRIBUTEERROR 100002 -#define BYTEFILEISNOTREGULAR 100003 -#define BYTEFILEOPENERROR 100004 -#define BYTEFILEHEAPERROR 100005 -#define BYTEFILEMD5ERROR 100006 -#define BYTEFILEMD5DIFF 100007 +#define BYTE_FILE_DOES_NOT_EXIST 100001 +#define BYTE_FILE_ATTRIBUTE_ERROR 100002 +#define BYTE_FILE_IS_NOT_REGULAR 100003 +#define BYTE_FILE_OPEN_ERROR 100004 +#define BYTE_FILE_HEAP_ERROR 100005 +#define BYTE_FILE_MD5_ERROR 100006 +#define BYTE_FILE_MD5DIFF 100007 +#define BYTE_FILE_READ_ERROR 100008 int readBytes(FILE* fh, DATA_BLOCK* data_block, int offset, int max_bytes, const std::string& checksum) { @@ -63,14 +64,26 @@ int readBytes(FILE* fh, DATA_BLOCK* data_block, int offset, int max_bytes, const char* newp = (char*)realloc(bp, (size_t)data_block->data_n); if (newp == nullptr) { free(bp); - err = BYTEFILEHEAPERROR; + err = BYTE_FILE_HEAP_ERROR; addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "Unable to Allocate Heap Memory for the File"); break; } bp = newp; + + errno = 0; int n_read = (int)fread(bp + buf_offset, sizeof(char), (size_t)buf_size, fh); + + int serrno = errno; + if (n_read == 0 || serrno != 0) { + int err = BYTE_FILE_READ_ERROR; + if (serrno != 0) { + addIdamError(UDA_SYSTEM_ERROR_TYPE, "readBytes", serrno, ""); + } + addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "Unable to Open the File for Read Access"); + } + nchar += n_read; - if (nchar >= max_bytes) { + if (max_bytes > 0 && nchar >= max_bytes) { break; } buf_offset = nchar; @@ -114,7 +127,5 @@ int readBytes(FILE* fh, DATA_BLOCK* data_block, int offset, int max_bytes, const data_block->order = -1; // No Dimensions data_block->data_type = UDA_TYPE_CHAR; - fclose(fh); // Close the File - return err; } From ecf2f20455b3bdc9e4e9504ec03444859d0b77a1 Mon Sep 17 00:00:00 2001 From: Jonathan Hollocombe Date: Tue, 10 Dec 2024 13:49:29 +0000 Subject: [PATCH 3/6] Bumping protocol version to 10 - CAPNP data is now transferred as raw bytes, and UDA_TYPE_OPAQUE is handled as raw bytes. --- source/client/udaClient.cpp | 2 +- source/clientserver/protocol.cpp | 4 ++-- source/clientserver/protocol2.cpp | 4 ++-- source/clientserver/xdrlib.cpp | 10 ++++++++-- source/clientserver/xdrlib.h | 2 +- source/plugins/bytes/readBytesNonOptimally.cpp | 2 +- source/server/fatServer.cpp | 4 ++-- source/server/udaServer.cpp | 4 ++-- source/server2/server.hpp | 2 +- 9 files changed, 20 insertions(+), 14 deletions(-) diff --git a/source/client/udaClient.cpp b/source/client/udaClient.cpp index 8169b0ce..47a0b6d1 100755 --- a/source/client/udaClient.cpp +++ b/source/client/udaClient.cpp @@ -41,7 +41,7 @@ #if !defined(FATCLIENT) || !defined(NOLIBMEMCACHED) //static int protocol_version = 9; #endif -int client_version = 9; // previous version +int client_version = 10; // previous version //---------------------------------------------------------------------------------------------------------------------- // FATCLIENT objects shared with server code diff --git a/source/clientserver/protocol.cpp b/source/clientserver/protocol.cpp index 5521f9e3..3b6b4d91 100755 --- a/source/clientserver/protocol.cpp +++ b/source/clientserver/protocol.cpp @@ -221,7 +221,7 @@ int protocol(XDR* xdrs, int protocol_id, int direction, int* token, LOGMALLOCLIS if ((err = allocData(data_block)) != 0) break; // Allocate Heap Memory - if (!xdr_data_block2(xdrs, data_block)) { + if (!xdr_data_block2(xdrs, data_block, protocolVersion)) { err = UDA_PROTOCOL_ERROR_62; break; } @@ -309,7 +309,7 @@ int protocol(XDR* xdrs, int protocol_id, int direction, int* token, LOGMALLOCLIS break; } - if (!xdr_data_block2(xdrs, data_block)) { + if (!xdr_data_block2(xdrs, data_block, protocolVersion)) { err = UDA_PROTOCOL_ERROR_62; break; } diff --git a/source/clientserver/protocol2.cpp b/source/clientserver/protocol2.cpp index bfd6d5dc..429fdbd4 100755 --- a/source/clientserver/protocol2.cpp +++ b/source/clientserver/protocol2.cpp @@ -742,7 +742,7 @@ static int handle_data_block(XDR* xdrs, int direction, const void* str, int prot if ((err = allocData(data_block)) != 0) break; // Allocate Heap Memory - if (!xdr_data_block2(xdrs, data_block)) { + if (!xdr_data_block2(xdrs, data_block, protocolVersion)) { err = UDA_PROTOCOL_ERROR_62; break; } @@ -838,7 +838,7 @@ static int handle_data_block(XDR* xdrs, int direction, const void* str, int prot break; } - if (!xdr_data_block2(xdrs, data_block)) { + if (!xdr_data_block2(xdrs, data_block, protocolVersion)) { err = UDA_PROTOCOL_ERROR_62; break; } diff --git a/source/clientserver/xdrlib.cpp b/source/clientserver/xdrlib.cpp index 67ef0b5a..1f10223d 100755 --- a/source/clientserver/xdrlib.cpp +++ b/source/clientserver/xdrlib.cpp @@ -616,7 +616,7 @@ bool_t xdr_data_block1(XDR* xdrs, DATA_BLOCK* str, int protocolVersion) return rc; } -bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str) +bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str, int protocolVersion) { switch (str->data_type) { case UDA_TYPE_FLOAT: @@ -662,7 +662,13 @@ bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str) return 1; // Nothing to send so retain good return code case UDA_TYPE_CAPNP: - return xdr_vector(xdrs, str->data, (u_int)str->data_n, sizeof(char), (xdrproc_t)xdr_char); + if (protocolVersion >= 10) { + return xdr_opaque(xdrs, str->data, (u_int)str->data_n * sizeof(char)); + } else { + return xdr_vector(xdrs, str->data, (u_int)str->data_n, sizeof(char), (xdrproc_t)xdr_char); + } + case UDA_TYPE_OPAQUE: + return xdr_opaque(xdrs, str->data, (u_int)str->data_n * sizeof(char)); default: return 0; diff --git a/source/clientserver/xdrlib.h b/source/clientserver/xdrlib.h index 59e71c69..68f08796 100755 --- a/source/clientserver/xdrlib.h +++ b/source/clientserver/xdrlib.h @@ -43,7 +43,7 @@ bool_t xdr_putdata_block1(XDR* xdrs, PUTDATA_BLOCK* str); bool_t xdr_putdata_block2(XDR* xdrs, PUTDATA_BLOCK* str); bool_t xdr_data_block_list(XDR* xdrs, DATA_BLOCK_LIST* str, int protocolVersion); bool_t xdr_data_block1(XDR* xdrs, DATA_BLOCK* str, int protocolVersion); -bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str); +bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str, int protocolVersion); bool_t xdr_data_block3(XDR* xdrs, DATA_BLOCK* str); bool_t xdr_data_block4(XDR* xdrs, DATA_BLOCK* str); bool_t xdr_data_dim1(XDR* xdrs, DATA_BLOCK* str); diff --git a/source/plugins/bytes/readBytesNonOptimally.cpp b/source/plugins/bytes/readBytesNonOptimally.cpp index 07ebd7f7..0c8e64c1 100755 --- a/source/plugins/bytes/readBytesNonOptimally.cpp +++ b/source/plugins/bytes/readBytesNonOptimally.cpp @@ -125,7 +125,7 @@ int readBytes(FILE* fh, DATA_BLOCK* data_block, int offset, int max_bytes, const data_block->dims[0].method = 0; data_block->order = -1; // No Dimensions - data_block->data_type = UDA_TYPE_CHAR; + data_block->data_type = UDA_TYPE_OPAQUE; // pass as raw unmodified bytes return err; } diff --git a/source/server/fatServer.cpp b/source/server/fatServer.cpp index c0a5979c..7328eb76 100755 --- a/source/server/fatServer.cpp +++ b/source/server/fatServer.cpp @@ -44,8 +44,8 @@ unsigned int clientFlags = 0; int malloc_source = UDA_MALLOC_SOURCE_NONE; unsigned int private_flags = 0; -int server_version = 8; -static int protocol_version = 8; +int server_version = 10; +static int protocol_version = 10; SOCKETLIST socket_list; diff --git a/source/server/udaServer.cpp b/source/server/udaServer.cpp index 64fa49fc..847af33d 100755 --- a/source/server/udaServer.cpp +++ b/source/server/udaServer.cpp @@ -42,8 +42,8 @@ //-------------------------------------------------------------------------------------- // static globals -constexpr int server_version = 9; -static int protocol_version = 9; +constexpr int server_version = 10; +static int protocol_version = 10; static int legacy_server_version = 6; static USERDEFINEDTYPELIST* user_defined_type_list = nullptr; // User Defined Structure Types from Data Files & Plugins diff --git a/source/server2/server.hpp b/source/server2/server.hpp index 23c70ef7..2193bd08 100644 --- a/source/server2/server.hpp +++ b/source/server2/server.hpp @@ -27,7 +27,7 @@ struct MetadataBlock { class Server { public: - constexpr static int ServerVersion = 8; + constexpr static int ServerVersion = 10; constexpr static int LegacyServerVersion = 6; LIBRARY_API Server(); From f5399bdc2dda05ebc39b4193d1a7865200107438 Mon Sep 17 00:00:00 2001 From: Jonathan Hollocombe Date: Tue, 10 Dec 2024 14:41:59 +0000 Subject: [PATCH 4/6] Undoing change to client protocol version for now. --- source/client/udaClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/udaClient.cpp b/source/client/udaClient.cpp index 47a0b6d1..8169b0ce 100755 --- a/source/client/udaClient.cpp +++ b/source/client/udaClient.cpp @@ -41,7 +41,7 @@ #if !defined(FATCLIENT) || !defined(NOLIBMEMCACHED) //static int protocol_version = 9; #endif -int client_version = 10; // previous version +int client_version = 9; // previous version //---------------------------------------------------------------------------------------------------------------------- // FATCLIENT objects shared with server code From ace4f64043e28658a44d8abb0d9361f1d109b970 Mon Sep 17 00:00:00 2001 From: Jonathan Hollocombe Date: Tue, 10 Dec 2024 14:56:29 +0000 Subject: [PATCH 5/6] Fixing bug and reverting back to protocol 10 in client. --- source/client/udaClient.cpp | 2 +- source/server/udaServer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/udaClient.cpp b/source/client/udaClient.cpp index 8169b0ce..47a0b6d1 100755 --- a/source/client/udaClient.cpp +++ b/source/client/udaClient.cpp @@ -41,7 +41,7 @@ #if !defined(FATCLIENT) || !defined(NOLIBMEMCACHED) //static int protocol_version = 9; #endif -int client_version = 9; // previous version +int client_version = 10; // previous version //---------------------------------------------------------------------------------------------------------------------- // FATCLIENT objects shared with server code diff --git a/source/server/udaServer.cpp b/source/server/udaServer.cpp index 847af33d..b87216bb 100755 --- a/source/server/udaServer.cpp +++ b/source/server/udaServer.cpp @@ -343,7 +343,7 @@ reportToClient(SERVER_BLOCK* server_block, DATA_BLOCK_LIST* data_block_list, CLI for (int i = 0; i < data_block_list->count; ++i) { DATA_BLOCK* data_block = &data_block_list->data[i]; - if (protocol_version < 10 && data_block->data_type == UDA_TYPE_COMPOUND && + if (data_block->data_type == UDA_TYPE_COMPOUND && data_block->opaque_type != UDA_OPAQUE_TYPE_UNKNOWN) { if (data_block->opaque_type == UDA_OPAQUE_TYPE_XML_DOCUMENT) { protocol_id = UDA_PROTOCOL_META; From 3939176c0dde144e3d6fb9f9e443dbe9e826aa17 Mon Sep 17 00:00:00 2001 From: sdixon Date: Wed, 11 Dec 2024 15:11:54 +0000 Subject: [PATCH 6/6] fixing issues with using UDA_TYPE_OPAQUE for returning binary data --- source/clientserver/udaTypes.cpp | 4 +++- source/wrappers/python/pyuda/cpyuda/types.pyx | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/source/clientserver/udaTypes.cpp b/source/clientserver/udaTypes.cpp index 7f5c0ae7..60a2badd 100755 --- a/source/clientserver/udaTypes.cpp +++ b/source/clientserver/udaTypes.cpp @@ -37,6 +37,8 @@ size_t getSizeOf(UDA_TYPE data_type) return sizeof(DCOMPLEX); case UDA_TYPE_CAPNP: return sizeof(char); + case UDA_TYPE_OPAQUE: + return sizeof(char); default: return 0; } @@ -78,4 +80,4 @@ size_t getPtrSizeOf(UDA_TYPE data_type) default: return 0; } -} \ No newline at end of file +} diff --git a/source/wrappers/python/pyuda/cpyuda/types.pyx b/source/wrappers/python/pyuda/cpyuda/types.pyx index 6a58c9be..c2a1088c 100644 --- a/source/wrappers/python/pyuda/cpyuda/types.pyx +++ b/source/wrappers/python/pyuda/cpyuda/types.pyx @@ -17,6 +17,7 @@ __uda2np_map = { 12: np.NPY_UINT64, # UDA_TYPE_UNSIGNED_LONG64 = 12, 13: np.NPY_COMPLEX64, # UDA_TYPE_COMPLEX = 13, 14: np.NPY_COMPLEX128, # UDA_TYPE_DCOMPLEX = 14, + 19: np.NPY_UINT8, # UDA_TYPE_OPAQUE } __np2uda_map = dict((__uda2np_map[i], i) for i in __uda2np_map)