diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 2d3d518f..5ce8f0e2 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -28,20 +28,20 @@ jobs: # which are only executed on push to tag. # Test wheels are also generated for the other platforms (and uploaded to testpypi) # on pushes to release branches and to main - - [ubuntu-latest, x86_64, manylinux2014_x86_64] - - [ubuntu-latest, x86_64, manylinux_2_28_x86_64] - - [ubuntu-latest, aarch64, manylinux_2_28_aarch64] - - [macos-13, x86_64, macosx_x86_64] - - [macos-14, arm64, macosx_arm64] - - [windows-latest, AMD64, win_amd64] + - [ubuntu-latest, x86_64, manylinux2014_x86_64, 0.0] + - [ubuntu-latest, x86_64, manylinux_2_28_x86_64, 0.0] + - [ubuntu-latest, aarch64, manylinux_2_28_aarch64, 0.0] + - [macos-13, x86_64, macosx_x86_64, 13.0] + - [macos-14, arm64, macosx_arm64, 14.0] + - [windows-latest, AMD64, win_amd64, 0.0] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # need git tags available for setuptools_scm to grab tags with: fetch-depth: 0 - - uses: actions/github-script@v6 + - uses: actions/github-script@v7 with: script: | core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); @@ -107,7 +107,7 @@ jobs: - name: build windows wheels if: runner.OS == 'Windows' - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.22.0 with: package-dir: ${{github.workspace}}/install/python_installer config-file: ${{github.workspace}}/install/python_installer/pyproject.toml @@ -115,7 +115,7 @@ jobs: CIBW_ARCHS: ${{matrix.build-platform[1]}} CIBW_PLATFORM: windows CIBW_BUILD: cp3*-${{matrix.build-platform[2]}} - CIBW_SKIP: cp36* cp37* + CIBW_SKIP: cp36* cp37* cp313* CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel} --add-path ${{github.workspace}}/install/lib" CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" @@ -125,7 +125,7 @@ jobs: startswith(matrix.build-platform[2], 'manylinux2014') && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.22.0 with: package-dir: ./source/wrappers/python config-file: ./source/wrappers/python/pyproject.toml @@ -176,7 +176,7 @@ jobs: startswith(matrix.build-platform[2], 'manylinux_2_28') && ! ( startswith(matrix.build-platform[1], 'aarch64') && !startsWith(github.ref, 'refs/tags/') ) - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.22.0 with: package-dir: ./source/wrappers/python config-file: ./source/wrappers/python/pyproject.toml @@ -189,7 +189,7 @@ jobs: CIBW_BEFORE_ALL: > dnf update -y && dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && - dnf install -y boost-devel openssl-devel libxml2-devel libtirpc-devel + dnf install -y boost1.78-devel openssl-devel libxml2-devel libtirpc-devel fmt fmt-devel spdlog spdlog-devel capnproto capnproto-devel && cd /project && cmake -B build @@ -203,6 +203,10 @@ jobs: - name: build uda on macos if: runner.os == 'macOS' + # note: it's recommended in cibuildwheel to build all deps from source in macos + # instead of using homebrew as this way the wheel will be back-compatible with + # other macos versions. MACOSX_DEPLOYMENT_TARGET will no longer be required if + # this change is made. run: > brew update-reset && brew install git @@ -222,18 +226,20 @@ jobs: -DCMAKE_INSTALL_PREFIX=$PWD/install -DCLIENT_ONLY=ON && cmake --build build -j --config Release --target install && - cp -r $PWD/install/python_installer/* ${{github.workspace}}/source/wrappers/python/ + cp -r $PWD/install/python_installer/* ${{github.workspace}}/source/wrappers/python/ - name: Build macos wheels if: runner.os == 'macOS' - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.22.0 with: package-dir: ./source/wrappers/python config-file: ./source/wrappers/python/pyproject.toml env: CIBW_ARCHS: ${{matrix.build-platform[1]}} CIBW_PLATFORM: macos + CIBW_SKIP: cp313* CIBW_BUILD: cp*-${{matrix.build-platform[2]}} + MACOSX_DEPLOYMENT_TARGET: ${{matrix.build-platform[3]}} - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index c8c7b381..883adc0e 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -18,7 +18,7 @@ jobs: build: strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-22.04, windows-latest, macos-latest] release: [Release] ssl: [ON, OFF] client-only: [ON, OFF] @@ -26,7 +26,7 @@ jobs: exclude: - os: windows-latest client-only: OFF - - os: ubuntu-latest + - os: ubuntu-22.04 client-only: ON - os: macos-latest client-only: ON @@ -47,7 +47,7 @@ jobs: core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Install linux dependencies - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-22.04' run: > sudo apt update && sudo apt install -y git @@ -67,7 +67,7 @@ jobs: python3-venv - name: Install Intel compiler - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-22.04' run: > sudo apt install -y wget && wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB && @@ -115,7 +115,7 @@ jobs: spdlog - name: Configure CMake (linux) - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-22.04' run: > cmake -G Ninja -B build -DBUILD_SHARED_LIBS=ON @@ -125,7 +125,7 @@ jobs: -DENABLE_CAPNP=${{ matrix.capnp }} - name: Configure CMake (linux Intel) - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-22.04' run: > source /opt/intel/oneapi/setvars.sh && CXX=icpx CC=icx cmake -G Ninja -B build-intel @@ -176,7 +176,7 @@ jobs: run: cmake --build build --config ${{ matrix.release }} - name: Build Intel - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-22.04' run: cmake --build build-intel --config ${{ matrix.release }} - name: Install @@ -188,7 +188,7 @@ jobs: run: cmake --install build --config ${{ matrix.release }} - name: Install pyuda - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-22.04' run: > cp -r /usr/local/python_installer ${{github.workspace}}/python_installer && python3 -m venv ${{github.workspace}}/venv && @@ -198,13 +198,13 @@ jobs: pip3 install ${{github.workspace}}/python_installer - name: Test pyuda import - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-22.04' run: > source ${{github.workspace}}/venv/bin/activate && python3 -c 'import pyuda; client=pyuda.Client()' - name: Run non-SSL system tests - if: matrix.os == 'ubuntu-latest' && matrix.ssl == 'OFF' + if: matrix.os == 'ubuntu-22.04' && matrix.ssl == 'OFF' run: > sudo cp /usr/local/etc/uda.socket /usr/local/etc/uda@.service /etc/systemd/system && sudo systemctl start uda.socket && @@ -216,7 +216,7 @@ jobs: ./build/test/plugins/plugin_test_testplugin - name: Run SSL system tests - if: matrix.os == 'ubuntu-latest' && matrix.ssl == 'ON' + if: matrix.os == 'ubuntu-22.04' && matrix.ssl == 'ON' run: > sudo cp /usr/local/etc/uda.socket /usr/local/etc/uda@.service /etc/systemd/system && sudo chown -R $USER:$USER /usr/local/etc && diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b5b28aa..665e89ca 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required( VERSION 3.0 ) +cmake_minimum_required( VERSION 3.12 ) ######################################################################################################################## # Macro used to convert Windows path to Unix diff --git a/docs/client_installation.md b/docs/client_installation.md index 1318dd3a..1134de79 100644 --- a/docs/client_installation.md +++ b/docs/client_installation.md @@ -76,6 +76,7 @@ Install dependencies from system package manager. ```sh sudo apt update && sudo apt install -y \ git \ +libtirpc-dev \ libboost-dev \ libboost-program-options-dev \ libssl-dev \ diff --git a/source/cache/CMakeLists.txt b/source/cache/CMakeLists.txt index f474dcc9..1ce3575d 100755 --- a/source/cache/CMakeLists.txt +++ b/source/cache/CMakeLists.txt @@ -24,6 +24,7 @@ set( SRC_FILES include_directories( ${OPENSSL_INCLUDE_DIR} + ${Boost_INCLUDE_DIRS} ${Boost_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/source ${LIBXML2_INCLUDE_DIR} diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 2474b1ef..3a285829 100755 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -4,6 +4,7 @@ find_package( OpenSSL REQUIRED ) find_package( fmt REQUIRED ) find_package( LibXml2 REQUIRED ) +find_package( Boost REQUIRED ) if( WIN32 OR MINGW ) find_package( XDR REQUIRED ) @@ -75,6 +76,7 @@ include_directories( ${OPENSSL_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR} ${fmt_SOURCE_DIR}/include + ${Boost_INCLUDE_DIRS} ) add_library( client-objects OBJECT ${SRC_FILES} ${HEADER_FILES} ) diff --git a/source/client/udaClient.cpp b/source/client/udaClient.cpp index 8169b0ce..61e33170 100755 --- a/source/client/udaClient.cpp +++ b/source/client/udaClient.cpp @@ -41,7 +41,7 @@ #if !defined(FATCLIENT) || !defined(NOLIBMEMCACHED) //static int protocol_version = 9; #endif -int client_version = 9; // previous version +int client_version = 10; // previous version //---------------------------------------------------------------------------------------------------------------------- // FATCLIENT objects shared with server code @@ -1042,6 +1042,8 @@ int idamClient(REQUEST_BLOCK* request_block, int* indices) } printDataBlock(*data_block); + data_received = true; + data_block_indices[i] = data_block_idx; } #endif // <========================== End of FatClient Code Only diff --git a/source/client2/CMakeLists.txt b/source/client2/CMakeLists.txt index 2bcadb38..18ac1ea4 100755 --- a/source/client2/CMakeLists.txt +++ b/source/client2/CMakeLists.txt @@ -4,6 +4,7 @@ find_package( OpenSSL REQUIRED ) find_package( LibXml2 REQUIRED ) find_package( fmt REQUIRED ) +find_package( Boost REQUIRED ) if( WIN32 OR MINGW ) find_package( XDR REQUIRED ) @@ -73,6 +74,7 @@ include_directories( ${OPENSSL_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR} ${fmt_SOURCE_DIR}/include + ${Boost_INCLUDE_DIRS} ) add_library( client2-objects OBJECT ${SRC_FILES} ${HEADER_FILES} ) diff --git a/source/clientserver/CMakeLists.txt b/source/clientserver/CMakeLists.txt index d3a22060..364cd2b4 100755 --- a/source/clientserver/CMakeLists.txt +++ b/source/clientserver/CMakeLists.txt @@ -6,6 +6,7 @@ find_package( fmt REQUIRED ) if( NOT CLIENT_ONLY ) find_package( LibXml2 REQUIRED ) endif() +find_package( Boost REQUIRED ) if( WIN32 OR MINGW ) find_package( XDR REQUIRED ) @@ -105,6 +106,7 @@ include_directories( ${OPENSSL_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR} ${fmt_SOURCE_DIR}/include + ${Boost_INCLUDE_DIRS} ) if( MINGW OR WIN32 ) diff --git a/source/clientserver/protocol.cpp b/source/clientserver/protocol.cpp index 5521f9e3..3b6b4d91 100755 --- a/source/clientserver/protocol.cpp +++ b/source/clientserver/protocol.cpp @@ -221,7 +221,7 @@ int protocol(XDR* xdrs, int protocol_id, int direction, int* token, LOGMALLOCLIS if ((err = allocData(data_block)) != 0) break; // Allocate Heap Memory - if (!xdr_data_block2(xdrs, data_block)) { + if (!xdr_data_block2(xdrs, data_block, protocolVersion)) { err = UDA_PROTOCOL_ERROR_62; break; } @@ -309,7 +309,7 @@ int protocol(XDR* xdrs, int protocol_id, int direction, int* token, LOGMALLOCLIS break; } - if (!xdr_data_block2(xdrs, data_block)) { + if (!xdr_data_block2(xdrs, data_block, protocolVersion)) { err = UDA_PROTOCOL_ERROR_62; break; } diff --git a/source/clientserver/protocol2.cpp b/source/clientserver/protocol2.cpp index bfd6d5dc..429fdbd4 100755 --- a/source/clientserver/protocol2.cpp +++ b/source/clientserver/protocol2.cpp @@ -742,7 +742,7 @@ static int handle_data_block(XDR* xdrs, int direction, const void* str, int prot if ((err = allocData(data_block)) != 0) break; // Allocate Heap Memory - if (!xdr_data_block2(xdrs, data_block)) { + if (!xdr_data_block2(xdrs, data_block, protocolVersion)) { err = UDA_PROTOCOL_ERROR_62; break; } @@ -838,7 +838,7 @@ static int handle_data_block(XDR* xdrs, int direction, const void* str, int prot break; } - if (!xdr_data_block2(xdrs, data_block)) { + if (!xdr_data_block2(xdrs, data_block, protocolVersion)) { err = UDA_PROTOCOL_ERROR_62; break; } diff --git a/source/clientserver/udaTypes.cpp b/source/clientserver/udaTypes.cpp index 7f5c0ae7..60a2badd 100755 --- a/source/clientserver/udaTypes.cpp +++ b/source/clientserver/udaTypes.cpp @@ -37,6 +37,8 @@ size_t getSizeOf(UDA_TYPE data_type) return sizeof(DCOMPLEX); case UDA_TYPE_CAPNP: return sizeof(char); + case UDA_TYPE_OPAQUE: + return sizeof(char); default: return 0; } @@ -78,4 +80,4 @@ size_t getPtrSizeOf(UDA_TYPE data_type) default: return 0; } -} \ No newline at end of file +} diff --git a/source/clientserver/xdrlib.cpp b/source/clientserver/xdrlib.cpp index 67ef0b5a..1f10223d 100755 --- a/source/clientserver/xdrlib.cpp +++ b/source/clientserver/xdrlib.cpp @@ -616,7 +616,7 @@ bool_t xdr_data_block1(XDR* xdrs, DATA_BLOCK* str, int protocolVersion) return rc; } -bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str) +bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str, int protocolVersion) { switch (str->data_type) { case UDA_TYPE_FLOAT: @@ -662,7 +662,13 @@ bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str) return 1; // Nothing to send so retain good return code case UDA_TYPE_CAPNP: - return xdr_vector(xdrs, str->data, (u_int)str->data_n, sizeof(char), (xdrproc_t)xdr_char); + if (protocolVersion >= 10) { + return xdr_opaque(xdrs, str->data, (u_int)str->data_n * sizeof(char)); + } else { + return xdr_vector(xdrs, str->data, (u_int)str->data_n, sizeof(char), (xdrproc_t)xdr_char); + } + case UDA_TYPE_OPAQUE: + return xdr_opaque(xdrs, str->data, (u_int)str->data_n * sizeof(char)); default: return 0; diff --git a/source/clientserver/xdrlib.h b/source/clientserver/xdrlib.h index 59e71c69..68f08796 100755 --- a/source/clientserver/xdrlib.h +++ b/source/clientserver/xdrlib.h @@ -43,7 +43,7 @@ bool_t xdr_putdata_block1(XDR* xdrs, PUTDATA_BLOCK* str); bool_t xdr_putdata_block2(XDR* xdrs, PUTDATA_BLOCK* str); bool_t xdr_data_block_list(XDR* xdrs, DATA_BLOCK_LIST* str, int protocolVersion); bool_t xdr_data_block1(XDR* xdrs, DATA_BLOCK* str, int protocolVersion); -bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str); +bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str, int protocolVersion); bool_t xdr_data_block3(XDR* xdrs, DATA_BLOCK* str); bool_t xdr_data_block4(XDR* xdrs, DATA_BLOCK* str); bool_t xdr_data_dim1(XDR* xdrs, DATA_BLOCK* str); diff --git a/source/logging/logging.h b/source/logging/logging.h index 3e450de6..bbd9dd23 100755 --- a/source/logging/logging.h +++ b/source/logging/logging.h @@ -22,7 +22,7 @@ strftime(uix_buf, 30, "%Y:%m:%dT%H:%M:%S", uix_tm); # define uix_printtime() { \ UIX_DEFINETIME \ - printf("%s.%dZ, ", uix_buf,(int32_t)uix_tmnow.tv_usec); \ + printf("%s.%06dZ, ", uix_buf,(int32_t)uix_tmnow.tv_usec); \ } # define UDA_LOG(LEVEL, FMT, ...) do{ if (LEVEL >= udaGetLogLevel()) {UIX_DEFINETIME udaLog(LEVEL, "%s.%dZ, %s:%d >> " FMT, uix_buf, (int32_t)uix_tmnow.tv_usec, basename((char *)__FILE__), __LINE__, ##__VA_ARGS__); }} while(0) #endif diff --git a/source/plugins/bytes/CMakeLists.txt b/source/plugins/bytes/CMakeLists.txt index ac1e9de0..23904720 100644 --- a/source/plugins/bytes/CMakeLists.txt +++ b/source/plugins/bytes/CMakeLists.txt @@ -24,6 +24,7 @@ uda_plugin( EXAMPLE "BYTES::read()" LIBNAME bytes_plugin SOURCES bytesPlugin.cpp readBytesNonOptimally.cpp md5Sum.cpp + CONFIG_FILE bytesPlugin.cfg EXTRA_INCLUDE_DIRS ${LIBXML2_INCLUDE_DIR} EXTRA_LINK_LIBS diff --git a/source/plugins/bytes/bytesPlugin.cfg.in b/source/plugins/bytes/bytesPlugin.cfg.in index e69de29b..3bc67c9e 100755 --- a/source/plugins/bytes/bytesPlugin.cfg.in +++ b/source/plugins/bytes/bytesPlugin.cfg.in @@ -0,0 +1 @@ +export UDA_BYTES_PLUGIN_ALLOWED_PATHS= \ No newline at end of file diff --git a/source/plugins/bytes/bytesPlugin.cpp b/source/plugins/bytes/bytesPlugin.cpp index 78fd060d..a565d3d8 100644 --- a/source/plugins/bytes/bytesPlugin.cpp +++ b/source/plugins/bytes/bytesPlugin.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include "readBytesNonOptimally.h" @@ -19,25 +21,54 @@ namespace filesystem = std::filesystem; #endif #include +#include +#include -static int do_help(IDAM_PLUGIN_INTERFACE* plugin_interface); +#define BYTEFILEOPENERROR 100004 +#define BYTEFILEHEAPERROR 100005 -static int do_version(IDAM_PLUGIN_INTERFACE* plugin_interface); - -static int do_builddate(IDAM_PLUGIN_INTERFACE* plugin_interface); +class BytesPlugin +{ +public: + void init(IDAM_PLUGIN_INTERFACE* plugin_interface) + { + REQUEST_DATA* request = plugin_interface->request_data; + if (!init_ + || STR_IEQUALS(request->function, "init") + || STR_IEQUALS(request->function, "initialise")) { + reset(plugin_interface); + // Initialise plugin + init_ = true; + } + } + void reset(IDAM_PLUGIN_INTERFACE* plugin_interface) + { + if (!init_) { + // Not previously initialised: Nothing to do! + return; + } + // Free Heap & reset counters + init_ = false; + } -static int do_defaultmethod(IDAM_PLUGIN_INTERFACE* plugin_interface); + int help(IDAM_PLUGIN_INTERFACE* plugin_interface); + int version(IDAM_PLUGIN_INTERFACE* plugin_interface); + int build_date(IDAM_PLUGIN_INTERFACE* plugin_interface); + int default_method(IDAM_PLUGIN_INTERFACE* plugin_interface); + int max_interface_version(IDAM_PLUGIN_INTERFACE* plugin_interface); + int read(IDAM_PLUGIN_INTERFACE* plugin_interface); + int size(IDAM_PLUGIN_INTERFACE* plugin_interface); -static int do_maxinterfaceversion(IDAM_PLUGIN_INTERFACE* plugin_interface); +private: + using file_ptr = std::unique_ptr; -static int do_read(IDAM_PLUGIN_INTERFACE* plugin_interface); + bool init_ = false; + std::unordered_map file_map_ = {}; +}; int bytesPlugin(IDAM_PLUGIN_INTERFACE* plugin_interface) { - static int init = 0; - - //---------------------------------------------------------------------------------------- - // Standard v1 Plugin Interface + static BytesPlugin plugin = {}; if (plugin_interface->interfaceVersion > THISPLUGIN_MAX_INTERFACE_VERSION) { RAISE_PLUGIN_ERROR("Plugin Interface Version Unknown to this plugin: Unable to execute the request!"); @@ -45,38 +76,17 @@ int bytesPlugin(IDAM_PLUGIN_INTERFACE* plugin_interface) plugin_interface->pluginVersion = THISPLUGIN_VERSION; - //---------------------------------------------------------------------------------------- - // Heap Housekeeping - - // Plugin must maintain a list of open file handles and sockets: loop over and close all files and sockets - // Plugin must maintain a list of plugin functions called: loop over and reset state and free heap. - // Plugin must maintain a list of calls to other plugins: loop over and call each plugin with the housekeeping request - // Plugin must destroy lists at end of housekeeping - - // A plugin only has a single instance on a server. For multiple instances, multiple servers are needed. - // Plugins can maintain state so recursive calls (on the same server) must respect this. - // If the housekeeping action is requested, this must be also applied to all plugins called. - // A list must be maintained to register these plugin calls to manage housekeeping. - // Calls to plugins must also respect access policy and user authentication policy - REQUEST_DATA* request = plugin_interface->request_data; if (plugin_interface->housekeeping || STR_IEQUALS(request->function, "reset")) { - if (!init) return 0; // Not previously initialised: Nothing to do! - // Free Heap & reset counters - init = 0; + plugin.reset(plugin_interface); return 0; } - //---------------------------------------------------------------------------------------- - // Initialise - - if (!init || STR_IEQUALS(request->function, "init") + plugin.init(plugin_interface); + if (STR_IEQUALS(request->function, "init") || STR_IEQUALS(request->function, "initialise")) { - - init = 1; - if (STR_IEQUALS(request->function, "init") || STR_IEQUALS(request->function, "initialise")) - return 0; + return 0; } //---------------------------------------------------------------------------------------- @@ -87,19 +97,21 @@ int bytesPlugin(IDAM_PLUGIN_INTERFACE* plugin_interface) // Standard methods: version, builddate, defaultmethod, maxinterfaceversion if (STR_IEQUALS(request->function, "help")) { - return do_help(plugin_interface); + return plugin.help(plugin_interface); } else if (STR_IEQUALS(request->function, "version")) { - return do_version(plugin_interface); + return plugin.version(plugin_interface); } else if (STR_IEQUALS(request->function, "builddate")) { - return do_builddate(plugin_interface); + return plugin.build_date(plugin_interface); } else if (STR_IEQUALS(request->function, "defaultmethod")) { - return do_defaultmethod(plugin_interface); + return plugin.default_method(plugin_interface); } else if (STR_IEQUALS(request->function, "maxinterfaceversion")) { - return do_maxinterfaceversion(plugin_interface); + return plugin.max_interface_version(plugin_interface); } else if (STR_IEQUALS(request->function, "read")) { - return do_read(plugin_interface); + return plugin.read(plugin_interface); + } else if (STR_IEQUALS(request->function, "size")) { + return plugin.size(plugin_interface); } else { - RAISE_PLUGIN_ERROR("Unknown function requested!"); + RAISE_PLUGIN_ERROR_AND_EXIT("Unknown function requested!", plugin_interface); } } @@ -108,7 +120,7 @@ int bytesPlugin(IDAM_PLUGIN_INTERFACE* plugin_interface) * @param plugin_interface * @return */ -int do_help(IDAM_PLUGIN_INTERFACE* plugin_interface) +int BytesPlugin::help(IDAM_PLUGIN_INTERFACE* plugin_interface) { const char* help = "\nbytes: data reader to access files as a block of bytes without interpretation\n\n"; const char* desc = "bytes: help = description of this plugin"; @@ -121,9 +133,55 @@ int do_help(IDAM_PLUGIN_INTERFACE* plugin_interface) * @param plugin_interface * @return */ -int do_version(IDAM_PLUGIN_INTERFACE* plugin_interface) +int BytesPlugin::version(IDAM_PLUGIN_INTERFACE* plugin_interface) { - return setReturnDataIntScalar(plugin_interface->data_block, THISPLUGIN_VERSION, "Plugin version number"); + /* + * byte encoded version number + * b00 00 00 00 00 00 00 00 + * major. minor. patch. (optional) post-tag commit + */ + std::string version_string = UDA_BUILD_VERSION; + unsigned int encoded_version = 0; + + std::regex r("([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))?"); + + std::smatch r_matches; + std::regex_match(version_string, r_matches, r); + int bitshift = 24; + for (std::size_t i = 1; i < r_matches.size(); ++i) + { + std::ssub_match sub_match = r_matches[i]; + std::string token = sub_match.str(); + if (i !=4 and !token.empty() and bitshift >=0) + { + encoded_version |= static_cast(std::stoi(token)) << bitshift; + bitshift -= 8; + } + } + + // ideally parsed version should always have at least 3 matches. Unexpected version string format or + // problem parsing if fewer matches than this + // if (bitshift > 0) + // { + // RAISE_PLUGIN_ERROR_AND_EXIT("unexpected plugin version format: expected at least MAJOR.MINOR.PATCH", plugin_interface); + // } + + // no set-return function for unsigned int type + DATA_BLOCK* data_block = plugin_interface->data_block; + initDataBlock(data_block); + + int* data = (int*)malloc(sizeof(unsigned int)); + data[0] = encoded_version; + + const char* description = "Byte-encoded plugin version number from MAJOR.MINOR.PATCH.POST string"; + strncpy(data_block->data_desc, description, STRING_LENGTH); + data_block->data_desc[STRING_LENGTH - 1] = '\0'; + + data_block->rank = 0; + data_block->data_type = UDA_TYPE_UNSIGNED_INT; + data_block->data = reinterpret_cast(data); + data_block->data_n = 1; + return 0; } /** @@ -131,7 +189,7 @@ int do_version(IDAM_PLUGIN_INTERFACE* plugin_interface) * @param plugin_interface * @return */ -int do_builddate(IDAM_PLUGIN_INTERFACE* plugin_interface) +int BytesPlugin::build_date(IDAM_PLUGIN_INTERFACE* plugin_interface) { return setReturnDataString(plugin_interface->data_block, __DATE__, "Plugin build date"); } @@ -141,7 +199,7 @@ int do_builddate(IDAM_PLUGIN_INTERFACE* plugin_interface) * @param plugin_interface * @return */ -int do_defaultmethod(IDAM_PLUGIN_INTERFACE* plugin_interface) +int BytesPlugin::default_method(IDAM_PLUGIN_INTERFACE* plugin_interface) { return setReturnDataString(plugin_interface->data_block, THISPLUGIN_DEFAULT_METHOD, "Plugin default method"); } @@ -151,7 +209,7 @@ int do_defaultmethod(IDAM_PLUGIN_INTERFACE* plugin_interface) * @param plugin_interface * @return */ -int do_maxinterfaceversion(IDAM_PLUGIN_INTERFACE* plugin_interface) +int BytesPlugin::max_interface_version(IDAM_PLUGIN_INTERFACE* plugin_interface) { return setReturnDataIntScalar(plugin_interface->data_block, THISPLUGIN_MAX_INTERFACE_VERSION, "Maximum Interface Version"); } @@ -189,20 +247,109 @@ int check_allowed_path(const char* expandedPath) { return 0; } -int do_read(IDAM_PLUGIN_INTERFACE* plugin_interface) +int check_path(const Environment* environment, const std::string& path) +{ + int err = 0; + + //---------------------------------------------------------------------- + // Block Access to External Users + + if (environment->external_user) { + err = 999; + addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "This Service is Disabled"); + UDA_LOG(UDA_LOG_DEBUG, "Disabled Service - Requested File: %s \n", path.c_str()); + return err; + } + + //---------------------------------------------------------------------- + // Test the filepath + + if (!IsLegalFilePath(path.c_str())) { + err = 999; + addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "The directory path has incorrect syntax"); + UDA_LOG(UDA_LOG_DEBUG, "The directory path has incorrect syntax [%s] \n", path.c_str()); + return err; + } + + //---------------------------------------------------------------------- + // Data Source Details + + UDA_LOG(UDA_LOG_DEBUG, "File Name : %s \n", path.c_str()); + + return err; +} + +int BytesPlugin::read(IDAM_PLUGIN_INTERFACE* plugin_interface) { - DATA_SOURCE* data_source = plugin_interface->data_source; - SIGNAL_DESC* signal_desc = plugin_interface->signal_desc; DATA_BLOCK* data_block = plugin_interface->data_block; const char* path; FIND_REQUIRED_STRING_VALUE(plugin_interface->request_data->nameValueList, path); - StringCopy(data_source->path, path, MAXPATH); + int max_bytes = -1; + FIND_INT_VALUE(plugin_interface->request_data->nameValueList, max_bytes); + + int offset = -1; + FIND_INT_VALUE(plugin_interface->request_data->nameValueList, offset); + + const char* checksum = nullptr; + FIND_STRING_VALUE(plugin_interface->request_data->nameValueList, checksum); + + bool opaque = findValue(&plugin_interface->request_data->nameValueList, "opaque"); + + char tmp_path[MAXPATH]; + StringCopy(tmp_path, path, MAXPATH); UDA_LOG(UDA_LOG_DEBUG, "expand_environment_variables! \n"); - expand_environment_variables(data_source->path); + expand_environment_variables(tmp_path); - check_allowed_path(data_source->path); + int rc = check_allowed_path(tmp_path); + if (rc != 0) { + return rc; + } + + if (checksum == nullptr) { + checksum = ""; + } + + rc = check_path(plugin_interface->environment, tmp_path); + if (rc != 0) { + return rc; + } + + errno = 0; + + FILE* file = nullptr; + if (file_map_.count(tmp_path) == 0) { + file_ptr ptr = {fopen(tmp_path, "rb"), fclose}; + file = ptr.get(); + file_map_.emplace(tmp_path, std::move(ptr)); + } else { + file = file_map_.at(tmp_path).get(); + } + + int serrno = errno; + + if (file == nullptr || ferror(file) || serrno != 0) { + int err = BYTEFILEOPENERROR; + if (serrno != 0) { + addIdamError(UDA_SYSTEM_ERROR_TYPE, "readBytes", serrno, ""); + } + addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "Unable to Open the File for Read Access"); + file_map_.erase(tmp_path); + return err; + } + + return readBytes(file, data_block, offset, max_bytes, checksum, opaque); +} + +int BytesPlugin::size(IDAM_PLUGIN_INTERFACE* plugin_interface) +{ + DATA_BLOCK* data_block = plugin_interface->data_block; + + const char* path = ""; + FIND_REQUIRED_STRING_VALUE(plugin_interface->request_data->nameValueList, path); + + size_t file_size = filesystem::file_size(path); - return readBytes(*data_source, *signal_desc, data_block, plugin_interface->environment); + return setReturnDataLongScalar(data_block, (long)file_size, nullptr); } diff --git a/source/plugins/bytes/md5Sum.cpp b/source/plugins/bytes/md5Sum.cpp index 5a778bf1..d6f52d90 100755 --- a/source/plugins/bytes/md5Sum.cpp +++ b/source/plugins/bytes/md5Sum.cpp @@ -34,7 +34,7 @@ void md5Sum(char *bp, int size, char *md5check) { md5check[2*MD5_SIZE]='\0'; - for(i=0; i #include -int readBytes(DATA_SOURCE data_source, SIGNAL_DESC signal_desc, DATA_BLOCK* data_block, const ENVIRONMENT* environment) +#define BYTE_FILE_DOES_NOT_EXIST 100001 +#define BYTE_FILE_ATTRIBUTE_ERROR 100002 +#define BYTE_FILE_IS_NOT_REGULAR 100003 +#define BYTE_FILE_OPEN_ERROR 100004 +#define BYTE_FILE_HEAP_ERROR 100005 +#define BYTE_FILE_MD5_ERROR 100006 +#define BYTE_FILE_MD5DIFF 100007 +#define BYTE_FILE_READ_ERROR 100008 + +int readBytes(FILE* fh, DATA_BLOCK* data_block, int offset, int max_bytes, const std::string& checksum, bool opaque) { int err = 0; - char md5file[2 * MD5_SIZE + 1] = ""; - char md5check[2 * MD5_SIZE + 1] = ""; - - //---------------------------------------------------------------------- - // Block Access to External Users + // Read File (Consider using memory mapped I/O & new type to avoid heap free at end if this is too slow!) - if (environment->external_user) { - err = 999; - addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "This Service is Disabled"); - UDA_LOG(UDA_LOG_DEBUG, "Disabled Service - Requested File: %s \n", data_source.path); - return err; + int nchar = 0; + int buf_offset = 0; + int buf_size = 1024 * 1024; + if (max_bytes > 0 && max_bytes < buf_size) { + buf_size = max_bytes; } - //---------------------------------------------------------------------- - // Test the filepath + data_block->data_n = buf_size; // 1 less than no. bytes read: Last Byte is an EOF - if (!IsLegalFilePath(data_source.path)) { - err = 999; - addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "The directory path has incorrect syntax"); - UDA_LOG(UDA_LOG_DEBUG, "The directory path has incorrect syntax [%s] \n", data_source.path); - return err; + if (offset >= 0) { + fseek(fh, offset, SEEK_SET); } - //---------------------------------------------------------------------- - // Data Source Details - - err = 0; - - UDA_LOG(UDA_LOG_DEBUG, "File Name : %s \n", data_source.path); - - //---------------------------------------------------------------------- - // File Attributes - - errno = 0; - - //---------------------------------------------------------------------- - // Open the File as a Binary Stream - - errno = 0; - FILE* fh = fopen(data_source.path, "rb"); - - int serrno = errno; - - if (fh == nullptr || ferror(fh) || serrno != 0) { - err = BYTEFILEOPENERROR; - if (serrno != 0) { - addIdamError(UDA_SYSTEM_ERROR_TYPE, "readBytes", serrno, ""); - } - addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "Unable to Open the File for Read Access"); - if (fh != nullptr) { - fclose(fh); + char* bp = nullptr; + while (!feof(fh)) { + char* newp = (char*)realloc(bp, (size_t)data_block->data_n); + if (newp == nullptr) { + free(bp); + err = BYTE_FILE_HEAP_ERROR; + addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "Unable to Allocate Heap Memory for the File"); + break; } - return err; - } + bp = newp; - //---------------------------------------------------------------------- - // Error Trap Loop + errno = 0; + int n_read = (int)fread(bp + buf_offset, sizeof(char), (size_t)buf_size, fh); - do { - - // Read File (Consider using memory mapped I/O & new type to avoid heap free at end if this is too slow!) - - int nchar = 0; - int offset = 0; - int bufsize = 100 * 1024; - data_block->data_n = bufsize; // 1 less than no. bytes read: Last Byte is an EOF - - char* bp = nullptr; - while (!feof(fh)) { - if ((bp = (char*)realloc(bp, (size_t)data_block->data_n)) == nullptr) { - err = BYTEFILEHEAPERROR; - addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "Unable to Allocate Heap Memory for the File"); - break; + int serrno = errno; + if (n_read == 0 || serrno != 0) { + int err = BYTE_FILE_READ_ERROR; + if (serrno != 0) { + addIdamError(UDA_SYSTEM_ERROR_TYPE, "readBytes", serrno, ""); } - int nread = (int)fread(bp + offset, sizeof(char), (size_t)bufsize, fh); - nchar = nchar + nread; - offset = nchar; - data_block->data_n = nchar + bufsize + 1; + addIdamError(UDA_CODE_ERROR_TYPE, "readBytes", err, "Unable to Open the File for Read Access"); } - if (err != 0) { + nchar += n_read; + if (max_bytes > 0 && nchar >= max_bytes) { break; } + buf_offset = nchar; + data_block->data_n = nchar + buf_size + 1; + } - //nchar--; // Remove EOF Character from end of Byte Block - data_block->data_n = nchar; - data_block->data = bp; + if (err != 0) { + return err; + } + //nchar--; // Remove EOF Character from end of Byte Block + data_block->data_n = nchar; + data_block->data = bp; + + if (checksum == "md5") { //---------------------------------------------------------------------- // MD5 Checksum + char md5check[MD5_SIZE]; md5Sum(bp, data_block->data_n, md5check); - strcpy(data_block->data_desc, md5check); // Pass back the Checksum to the Client UDA_LOG(UDA_LOG_DEBUG, "File Size : %d \n", nchar); - UDA_LOG(UDA_LOG_DEBUG, "File Checksum : %s \n", md5file); UDA_LOG(UDA_LOG_DEBUG, "Read Checksum : %s \n", md5check); - - // MD5 Difference? - - //---------------------------------------------------------------------- - // Fetch Dimensional Data - - data_block->rank = 1; - data_block->dims = (DIMS*)malloc(sizeof(DIMS)); - initDimBlock(data_block->dims); - - data_block->dims[0].data_type = UDA_TYPE_UNSIGNED_INT; - data_block->dims[0].dim_n = data_block->data_n; - data_block->dims[0].compressed = 1; - data_block->dims[0].dim0 = 0.0; - data_block->dims[0].diff = 1.0; - data_block->dims[0].method = 0; - - data_block->order = -1; // No Dimensions - data_block->data_type = UDA_TYPE_CHAR; - - } while (0); + } //---------------------------------------------------------------------- - // Housekeeping - -// if (err != 0) { -// freeDataBlock(data_block); -// } - - fclose(fh); // Close the File + // Fetch Dimensional Data + + data_block->rank = 1; + data_block->dims = (DIMS*)malloc(sizeof(DIMS)); + initDimBlock(data_block->dims); + + data_block->dims[0].data_type = UDA_TYPE_UNSIGNED_INT; + data_block->dims[0].dim_n = data_block->data_n; + data_block->dims[0].compressed = 1; + data_block->dims[0].dim0 = 0.0; + data_block->dims[0].diff = 1.0; + data_block->dims[0].method = 0; + + data_block->order = -1; // No Dimensions + if (opaque) { + data_block->data_type = UDA_TYPE_OPAQUE; // pass as raw unmodified bytes + } else { + data_block->data_type = UDA_TYPE_CHAR; + } return err; } diff --git a/source/plugins/bytes/readBytesNonOptimally.h b/source/plugins/bytes/readBytesNonOptimally.h index 792f8c74..402fccf6 100755 --- a/source/plugins/bytes/readBytesNonOptimally.h +++ b/source/plugins/bytes/readBytesNonOptimally.h @@ -1,30 +1,8 @@ -#ifndef UDA_PLUGIN_READBYTESNONOPTIMALLY_H -#define UDA_PLUGIN_READBYTESNONOPTIMALLY_H +#pragma once -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -LIBRARY_API int readBytes(DATA_SOURCE data_source, SIGNAL_DESC signal_desc, DATA_BLOCK *data_block, const ENVIRONMENT* environment); - -#ifndef NOBINARYPLUGIN +#include -#define BYTEFILEDOESNOTEXIST 100001 -#define BYTEFILEATTRIBUTEERROR 100002 -#define BYTEFILEISNOTREGULAR 100003 -#define BYTEFILEOPENERROR 100004 -#define BYTEFILEHEAPERROR 100005 -#define BYTEFILEMD5ERROR 100006 -#define BYTEFILEMD5DIFF 100007 - -#endif - -#ifdef __cplusplus -} -#endif +#include -#endif // UDA_PLUGIN_READBYTESNONOPTIMALLY_H +int readBytes(FILE* file, DATA_BLOCK* data_block, int offset, int max_bytes, const std::string& checksum, bool opaque); diff --git a/source/server/fatServer.cpp b/source/server/fatServer.cpp index c0a5979c..7328eb76 100755 --- a/source/server/fatServer.cpp +++ b/source/server/fatServer.cpp @@ -44,8 +44,8 @@ unsigned int clientFlags = 0; int malloc_source = UDA_MALLOC_SOURCE_NONE; unsigned int private_flags = 0; -int server_version = 8; -static int protocol_version = 8; +int server_version = 10; +static int protocol_version = 10; SOCKETLIST socket_list; diff --git a/source/server/udaServer.cpp b/source/server/udaServer.cpp index 64fa49fc..b87216bb 100755 --- a/source/server/udaServer.cpp +++ b/source/server/udaServer.cpp @@ -42,8 +42,8 @@ //-------------------------------------------------------------------------------------- // static globals -constexpr int server_version = 9; -static int protocol_version = 9; +constexpr int server_version = 10; +static int protocol_version = 10; static int legacy_server_version = 6; static USERDEFINEDTYPELIST* user_defined_type_list = nullptr; // User Defined Structure Types from Data Files & Plugins @@ -343,7 +343,7 @@ reportToClient(SERVER_BLOCK* server_block, DATA_BLOCK_LIST* data_block_list, CLI for (int i = 0; i < data_block_list->count; ++i) { DATA_BLOCK* data_block = &data_block_list->data[i]; - if (protocol_version < 10 && data_block->data_type == UDA_TYPE_COMPOUND && + if (data_block->data_type == UDA_TYPE_COMPOUND && data_block->opaque_type != UDA_OPAQUE_TYPE_UNKNOWN) { if (data_block->opaque_type == UDA_OPAQUE_TYPE_XML_DOCUMENT) { protocol_id = UDA_PROTOCOL_META; diff --git a/source/server2/server.hpp b/source/server2/server.hpp index 23c70ef7..2193bd08 100644 --- a/source/server2/server.hpp +++ b/source/server2/server.hpp @@ -27,7 +27,7 @@ struct MetadataBlock { class Server { public: - constexpr static int ServerVersion = 8; + constexpr static int ServerVersion = 10; constexpr static int LegacyServerVersion = 6; LIBRARY_API Server(); diff --git a/source/wrappers/python/pyproject.toml b/source/wrappers/python/pyproject.toml index d2e27b52..f02f0a9f 100644 --- a/source/wrappers/python/pyproject.toml +++ b/source/wrappers/python/pyproject.toml @@ -1,8 +1,8 @@ [build-system] requires = [ "setuptools>=42", - "oldest-supported-numpy; python_version < '3.13'", - "numpy >=1.7, <2; python_version >= '3.13'", + "oldest-supported-numpy; python_version < '3.12'", + "numpy >=1.7, <2; python_version >= '3.12'", "Cython>=0.29", "six" ] @@ -19,7 +19,7 @@ name = "uda" dynamic = ["version"] readme = {file = 'README.md', content-type = "text/markdown"} license = {text = "Apache-2.0 license"} -dependencies = ["numpy>1.7, <2", "six"] +dependencies = ["numpy>1.7, <2", "six", "progress"] requires-python = ">= 3.5" classifiers = [ diff --git a/source/wrappers/python/pyuda/_client.py b/source/wrappers/python/pyuda/_client.py index 07d177d3..6b4c7afc 100644 --- a/source/wrappers/python/pyuda/_client.py +++ b/source/wrappers/python/pyuda/_client.py @@ -11,7 +11,8 @@ from six import with_metaclass import logging -from collections import namedtuple +import math +from progress.bar import Bar from collections.abc import Iterable import sys try: @@ -85,18 +86,45 @@ def __init__(self, debug_level=logging.ERROR): except ImportError: pass - def get_file(self, source_file, output_file=None): + def get_file(self, source_file, output_file=None, chunk_size=1): """ Retrieve file using bytes plugin and write to file - :param source_file: the full path to the file - :param output_file: the name of the output file + :param str source_file: the full path to the file + :param str|None output_file: the name of the output file + :param int chunk_size: download chunk size in MB, set to 0 to download the file in one chunk :return: """ + if chunk_size < 0: + raise ValueError("chunk_size must not be negative") + + # bytes::size() function won't exist in some old servers, + # check for compatible plugin version. + # magic number is byte encoding for version number 1.8.0.31 + # when automatic versioning was introduced for bytes plugin + result = cpyuda.get_data("bytes::version()", "") + if result.data() < 34078751: + chunk_size = 0 + + if chunk_size: + result = cpyuda.get_data(f"bytes::size(path={source_file})", "") + size = result.data() + chunk_size = int(chunk_size * 1024 * 1024) + count = 0 + steps = math.ceil(size / chunk_size) + bar = Bar('Downloading', max=steps, suffix='%(percent)d%%') + with open(output_file, 'wb') as f_out: + while count < size: + result = cpyuda.get_data(f"bytes::read(path={source_file}, max_bytes={chunk_size}, offset={count}, /opaque)", "") + data = result.data() + count += data.size + data.tofile(f_out) + bar.next() + print(flush=True) + else: + result = cpyuda.get_data(f"bytes::read(path={source_file}, /opaque)", "") - result = cpyuda.get_data("bytes::read(path=%s)" % source_file, "") - - with open(output_file, 'wb') as f_out: - result.data().tofile(f_out) + with open(output_file, 'wb') as f_out: + result.data().tofile(f_out) return diff --git a/source/wrappers/python/pyuda/cpyuda/types.pyx b/source/wrappers/python/pyuda/cpyuda/types.pyx index 6a58c9be..c2a1088c 100644 --- a/source/wrappers/python/pyuda/cpyuda/types.pyx +++ b/source/wrappers/python/pyuda/cpyuda/types.pyx @@ -17,6 +17,7 @@ __uda2np_map = { 12: np.NPY_UINT64, # UDA_TYPE_UNSIGNED_LONG64 = 12, 13: np.NPY_COMPLEX64, # UDA_TYPE_COMPLEX = 13, 14: np.NPY_COMPLEX128, # UDA_TYPE_DCOMPLEX = 14, + 19: np.NPY_UINT8, # UDA_TYPE_OPAQUE } __np2uda_map = dict((__uda2np_map[i], i) for i in __uda2np_map) diff --git a/source/wrappers/python/setup.py.in b/source/wrappers/python/setup.py.in index 2c4da60c..e865cbc2 100644 --- a/source/wrappers/python/setup.py.in +++ b/source/wrappers/python/setup.py.in @@ -36,7 +36,8 @@ if build_fat: 'libfat@PROJECT_NAME@_client.' + stalib_ext, 'lib@PROJECT_NAME@_plugins.' + stalib_ext, ] + ['lib@PROJECT_NAME@_' + name + '.' + stalib_ext for name in uda_lib_names] - extra_libs = ['xml2', 'pq'] + extra_libs = ['xml2'] + extra_libs.extend('@EXTRA_LIBS@'.split(';') if '@EXTRA_LIBS@' else []) extra_macros = [('FATCLIENT', None)] else: uda_libs = [