From 1a3e71cfe603b331c23e68810712a65cad80edd9 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Tue, 19 Sep 2023 11:43:26 -0500 Subject: [PATCH 01/25] lcov update (#396) Update LCOV to v2.0 [ committed by @billschereriii ] [ reviewed by @ashao ] --- .github/workflows/run_tests.yml | 21 ++++++++++++++++++++- Makefile | 10 ++++++---- doc/changelog.rst | 15 +++++++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 04937744c..18d601e45 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -108,6 +108,25 @@ jobs: echo "CXX=icpx" >> $GITHUB_ENV && echo "FC=ifort" >> $GITHUB_ENV + # Set up perl environment for LCOV + - uses: actions/checkout@v3 + - name: Setup perl + uses: shogo82148/actions-setup-perl@v1 + with: + perl-version: '5.30' + install-modules: Memory::Process + + # Install additional perl Modules + - name: Add perl modules + run: | + sudo apt install libcapture-tiny-perl && \ + sudo apt install libdatetime-perl && \ + sudo apt install libdevel-cover-perl && \ + sudo apt install libdigest-md5-perl && \ + sudo apt install libfile-spec-perl && \ + sudo apt install libjson-xs-perl && \ + sudo apt install libtime-hires-perl + # Install additional dependencies - name: Install Cmake Linux if: contains(matrix.os, 'ubuntu') @@ -167,7 +186,7 @@ jobs: # Process and upload code coverage (Python was collected during pytest) - name: Collect coverage from C/C++/Fortran testers - run: third-party/lcov/install/usr/local/bin/lcov -c -d build/Coverage/CMakeFiles -o coverage.info + run: third-party/lcov/install/bin/lcov --ignore-errors gcov,mismatch --keep-going -c -d build/Coverage/CMakeFiles -o coverage.info - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 diff --git a/Makefile b/Makefile index a92489e2f..3c709379b 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ # General settings MAKEFLAGS += --no-print-directory SHELL:=/bin/bash +CWD := $(shell pwd) # Params for third-party software HIREDIS_URL := https://github.com/redis/hiredis.git @@ -42,7 +43,7 @@ REDISAI_URL := https://github.com/RedisAI/RedisAI.git CATCH2_URL := https://github.com/catchorg/Catch2.git CATCH2_VER := v2.13.6 LCOV_URL := https://github.com/linux-test-project/lcov.git -LCOV_VER := v1.15 +LCOV_VER := v2.0 # Build variables NPROC := $(shell nproc 2>/dev/null || python -c "import multiprocessing as mp; print (mp.cpu_count())" 2>/dev/null || echo 4) @@ -596,12 +597,13 @@ third-party/catch/single_include/catch2/catch.hpp: # LCOV (hidden test target) .PHONY: lcov -lcov: third-party/lcov/install/usr/local/bin/lcov -third-party/lcov/install/usr/local/bin/lcov: +lcov: third-party/lcov/install/bin/lcov +third-party/lcov/install/bin/lcov: + @echo Installing LCOV @mkdir -p third-party @cd third-party && \ git clone $(LCOV_URL) lcov --branch $(LCOV_VER) --depth=1 @cd third-party/lcov && \ mkdir -p install && \ - CC=gcc CXX=g++ DESTDIR="install/" make install && \ + CC=gcc CXX=g++ make PREFIX=$(CWD)/third-party/lcov/install/ install && \ echo "Finished installing LCOV" diff --git a/doc/changelog.rst b/doc/changelog.rst index c9663f40e..edbcae888 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,21 @@ Changelog ========= +Development branch +------------------ + +To be released at some future point in time + +Description + +- Updated the third-party lcov component + +Detailed Notes + +- Updated lcov from version 1.15 to 2.0 (PR395_) + +.. _PR395: https://github.com/CrayLabs/SmartRedis/pull/395 + 0.4.2 ----- From cfbb69aba3bdefcceb98458b32c6b6f11dc2639c Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Thu, 21 Sep 2023 14:25:13 -0500 Subject: [PATCH 02/25] Update RedisAI version to v1.2.7 (#402) Update RedisAI version [ committed by @billschereriii ] [ reviewed by @MattToast ] --- .github/workflows/run_tests.yml | 2 +- Makefile | 2 +- doc/changelog.rst | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 18d601e45..b4aea72a2 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -59,7 +59,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04] # cannot test on macOS as docker isn't supported on Mac - rai_v: [1.2.4, 1.2.5] # versions of RedisAI + rai_v: [1.2.7] # versions of RedisAI py_v: ['3.7.x', '3.8.x', '3.9.x', '3.10.x'] # versions of Python compiler: [intel, 8, 9, 10, 11] # intel compiler, and versions of GNU compiler env: diff --git a/Makefile b/Makefile index 3c709379b..4afa911ea 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ SR_TEST_REDIS_MODE := Clustered SR_TEST_UDS_FILE := /tmp/redis.sock SR_TEST_PORT := 6379 SR_TEST_NODES := 3 -SR_TEST_REDISAI_VER := v1.2.3 +SR_TEST_REDISAI_VER := v1.2.7 SR_TEST_DEVICE := cpu SR_TEST_PYTEST_FLAGS := -vv -s diff --git a/doc/changelog.rst b/doc/changelog.rst index edbcae888..4d47776c1 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,13 +8,16 @@ To be released at some future point in time Description +- Updated the third-party RedisAI component - Updated the third-party lcov component Detailed Notes -- Updated lcov from version 1.15 to 2.0 (PR395_) +- Updated from RedisAI v1.2.3 (test target)/v1.2.4 and v1.2.5 (CI/CD pipeline) to v1.2.7 (PR402_) +- Updated lcov from version 1.15 to 2.0 (PR396_) -.. _PR395: https://github.com/CrayLabs/SmartRedis/pull/395 +.. _PR402: https://github.com/CrayLabs/SmartRedis/pull/402 +.. _PR396: https://github.com/CrayLabs/SmartRedis/pull/396 0.4.2 ----- From faf58b328926275d76c622c46185c31664caf298 Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Thu, 28 Sep 2023 10:15:05 -0700 Subject: [PATCH 03/25] Add a contributing reference to SmartRedis (#395) The CONTRIBUTING.md file added to in this PR points to the documentation for SmartSim and SmartRedis that outlines a set of guidelines for contributors. [ committed by @ashao ] [ reviewed by @MattToast ] --- CONTRIBUTING.md | 3 +++ doc/changelog.rst | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..e4976aab8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,3 @@ +SmartRedis and SmartSim share the same contributor guidelines. Please refer to +[CONTRIBUTING.rst](https://github.com/CrayLabs/SmartSim/blob/develop/CONTRIBUTING.rst) +in the SmartSim repo or at CrayLabs[https://www.craylabs.org/docs/contributing.html] \ No newline at end of file diff --git a/doc/changelog.rst b/doc/changelog.rst index 4d47776c1..a18f9a5fd 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -10,14 +10,17 @@ Description - Updated the third-party RedisAI component - Updated the third-party lcov component +- Add link to contributing guidelines Detailed Notes - Updated from RedisAI v1.2.3 (test target)/v1.2.4 and v1.2.5 (CI/CD pipeline) to v1.2.7 (PR402_) - Updated lcov from version 1.15 to 2.0 (PR396_) +- Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) .. _PR402: https://github.com/CrayLabs/SmartRedis/pull/402 .. _PR396: https://github.com/CrayLabs/SmartRedis/pull/396 +.. _PR394: https://github.com/CrayLabs/SmartRedis/pull/395 0.4.2 ----- From 53def750f50b8920a4db103bb06807763c5f31c4 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:47:25 -0500 Subject: [PATCH 04/25] Model chunking support (#404) Add support for model chunking [ committed by @billschereriii ] [ reviewed by @ashao ] --- doc/changelog.rst | 5 +- include/client.h | 15 ++++ include/command.h | 15 ++++ include/commandreply.h | 8 ++ include/pyclient.h | 15 ++++ include/redis.h | 29 ++++++- include/rediscluster.h | 28 ++++++- include/redisserver.h | 48 ++++++++++- src/cpp/client.cpp | 80 +++++++++++++++++-- src/cpp/redis.cpp | 48 ++++++++++- src/cpp/rediscluster.cpp | 55 ++++++++++++- src/cpp/redisserver.cpp | 1 + src/python/bindings/bind.cpp | 1 + src/python/module/smartredis/client.py | 19 +++++ src/python/src/pyclient.cpp | 8 ++ tests/cpp/client_test_mnist.cpp | 1 + tests/cpp/unit-tests/test_client_ensemble.cpp | 1 + tests/python/test_errors.py | 10 ++- tests/python/test_model_methods_torch.py | 1 + 19 files changed, 363 insertions(+), 25 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index a18f9a5fd..b38b8eb3c 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,19 +8,22 @@ To be released at some future point in time Description +- Added support for model chunking - Updated the third-party RedisAI component - Updated the third-party lcov component - Add link to contributing guidelines Detailed Notes +- Models will now be automatically chunked when sent to/received from the backed database. This allows use of models greater than 511MB in size. (PR404_) - Updated from RedisAI v1.2.3 (test target)/v1.2.4 and v1.2.5 (CI/CD pipeline) to v1.2.7 (PR402_) - Updated lcov from version 1.15 to 2.0 (PR396_) - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) +.. _PR404: https://github.com/CrayLabs/SmartRedis/pull/404 .. _PR402: https://github.com/CrayLabs/SmartRedis/pull/402 .. _PR396: https://github.com/CrayLabs/SmartRedis/pull/396 -.. _PR394: https://github.com/CrayLabs/SmartRedis/pull/395 +.. _PR395: https://github.com/CrayLabs/SmartRedis/pull/395 0.4.2 ----- diff --git a/include/client.h b/include/client.h index 59747c33d..3444c707b 100644 --- a/include/client.h +++ b/include/client.h @@ -1269,6 +1269,21 @@ class Client : public SRObject const int start_index, const int end_index); + /*! + * \brief Reconfigure the chunking size that Redis uses for model + * serialization, replication, and the model_get command. + * \details This method triggers the AI.CONFIG method in the Redis + * database to change the model chunking size. + * + * NOTE: The default size of 511MB should be fine for most + * applications, so it is expected to be very rare that a + * client calls this method. It is not necessary to call + * this method a model to be chunked. + * \param chunk_size The new chunk size in bytes + * \throw SmartRedis::Exception if the command fails. + */ + void set_model_chunk_size(int chunk_size); + /*! * \brief Create a string representation of the client * \returns A string containing client details diff --git a/include/command.h b/include/command.h index 67a313114..4456f4212 100644 --- a/include/command.h +++ b/include/command.h @@ -147,6 +147,21 @@ class Command return *this; } + /*! + * \brief Add a vector of string_views to the command. + * \details The string values are copied to the command. + * To add a vector of keys, use the add_keys() + * method. + * \param fields The strings to add to the command + * \returns The command object, for chaining. + */ + virtual Command& operator<<(const std::vector& fields) { + for (size_t i = 0; i < fields.size(); i++) { + add_field_ptr(fields[i]); + } + return *this; + } + /*! * \brief Add a vector of strings to the command. * \details The string values are copied to the command. diff --git a/include/commandreply.h b/include/commandreply.h index a86671834..68e9bac94 100644 --- a/include/commandreply.h +++ b/include/commandreply.h @@ -267,6 +267,14 @@ class CommandReply { */ std::string redis_reply_type(); + /*! + * \brief Determine whether the response is an array + * \returns true iff the response is of type REDIS_REPLY_ARRAY + */ + bool is_array() { + return _reply->type == REDIS_REPLY_ARRAY; + } + /*! * \brief Print the reply structure of the CommandReply */ diff --git a/include/pyclient.h b/include/pyclient.h index 478c6e6a7..b1f82bcbb 100644 --- a/include/pyclient.h +++ b/include/pyclient.h @@ -925,6 +925,21 @@ class PyClient : public PySRObject const int start_index, const int end_index); + /*! + * \brief Reconfigure the chunking size that Redis uses for model + * serialization, replication, and the model_get command. + * \details This method triggers the AI.CONFIG method in the Redis + * database to change the model chunking size. + * + * NOTE: The default size of 511MB should be fine for most + * applications, so it is expected to be very rare that a + * client calls this method. It is not necessary to call + * this method a model to be chunked. + * \param chunk_size The new chunk size in bytes + * \throw SmartRedis::Exception if the command fails. + */ + void set_model_chunk_size(int chunk_size); + /*! * \brief Create a string representation of the Client * \returns A string representation of the Client diff --git a/include/redis.h b/include/redis.h index 2a8efc25a..4aa9f246c 100644 --- a/include/redis.h +++ b/include/redis.h @@ -276,7 +276,7 @@ class Redis : public RedisServer * \brief Set a model from std::string_view buffer in the * database for future execution * \param key The key to associate with the model - * \param model The model as a continuous buffer string_view + * \param model The model as a sequence of buffer string_view chunks * \param backend The name of the backend * (TF, TFLITE, TORCH, ONNX) * \param device The name of the device for execution @@ -292,7 +292,7 @@ class Redis : public RedisServer * \throw RuntimeException for all client errors */ virtual CommandReply set_model(const std::string& key, - std::string_view model, + const std::vector& model, const std::string& backend, const std::string& device, int batch_size = 0, @@ -307,7 +307,7 @@ class Redis : public RedisServer * \brief Set a model from std::string_view buffer in the * database for future execution in a multi-GPU system * \param name The name to associate with the model - * \param model The model as a continuous buffer string_view + * \param model The model as a sequence of buffer string_view chunks * \param backend The name of the backend * (TF, TFLITE, TORCH, ONNX) * \param first_gpu The first GPU to use with this model @@ -322,7 +322,7 @@ class Redis : public RedisServer * \throw RuntimeException for all client errors */ virtual void set_model_multigpu(const std::string& name, - const std::string_view& model, + const std::vector& model, const std::string& backend, int first_gpu, int num_gpus, @@ -505,6 +505,27 @@ class Redis : public RedisServer const std::string& key, const bool reset_stat); + /*! + * \brief Retrieve the current model chunk size + * \returns The size in bytes for model chunking + */ + virtual int get_model_chunk_size(); + + /*! + * \brief Reconfigure the chunking size that Redis uses for model + * serialization, replication, and the model_get command. + * \details This method triggers the AI.CONFIG method in the Redis + * database to change the model chunking size. + * + * NOTE: The default size of 511MB should be fine for most + * applications, so it is expected to be very rare that a + * client calls this method. It is not necessary to call + * this method a model to be chunked. + * \param chunk_size The new chunk size in bytes + * \throw SmartRedis::Exception if the command fails. + */ + virtual void set_model_chunk_size(int chunk_size); + /*! * \brief Run a CommandList via a Pipeline * \param cmdlist The list of commands to run diff --git a/include/rediscluster.h b/include/rediscluster.h index bf390472d..876c6b733 100644 --- a/include/rediscluster.h +++ b/include/rediscluster.h @@ -294,7 +294,7 @@ class RedisCluster : public RedisServer * \brief Set a model from std::string_view buffer in the * database for future execution * \param key The key to associate with the model - * \param model The model as a continuous buffer string_view + * \param model The model as a sequence of buffer string_view chunks * \param backend The name of the backend * (TF, TFLITE, TORCH, ONNX) * \param device The name of the device for execution @@ -312,7 +312,7 @@ class RedisCluster : public RedisServer * \throw RuntimeException for all client errors */ virtual CommandReply set_model(const std::string& key, - std::string_view model, + const std::vector& model, const std::string& backend, const std::string& device, int batch_size = 0, @@ -327,7 +327,7 @@ class RedisCluster : public RedisServer * \brief Set a model from std::string_view buffer in the * database for future execution in a multi-GPU system * \param name The name to associate with the model - * \param model The model as a continuous buffer string_view + * \param model The model as a sequence of buffer string_view chunks * \param backend The name of the backend * (TF, TFLITE, TORCH, ONNX) * \param first_gpu The first GPU to use with this model @@ -344,7 +344,7 @@ class RedisCluster : public RedisServer * \throw RuntimeException for all client errors */ virtual void set_model_multigpu(const std::string& name, - const std::string_view& model, + const std::vector& model, const std::string& backend, int first_gpu, int num_gpus, @@ -527,6 +527,11 @@ class RedisCluster : public RedisServer get_model_script_ai_info(const std::string& address, const std::string& key, const bool reset_stat); + /*! + * \brief Retrieve the current model chunk size + * \returns The size in bytes for model chunking + */ + virtual int get_model_chunk_size(); /*! * \brief Run a CommandList via a Pipeline. @@ -741,6 +746,21 @@ class RedisCluster : public RedisServer std::vector& inputs, std::vector& outputs); + /*! + * \brief Reconfigure the chunking size that Redis uses for model + * serialization, replication, and the model_get command. + * \details This method triggers the AI.CONFIG method in the Redis + * database to change the model chunking size. + * + * NOTE: The default size of 511MB should be fine for most + * applications, so it is expected to be very rare that a + * client calls this method. It is not necessary to call + * this method a model to be chunked. + * \param chunk_size The new chunk size in bytes + * \throw SmartRedis::Exception if the command fails. + */ + virtual void set_model_chunk_size(int chunk_size); + /*! * \brief Execute a pipeline for the provided commands. * The provided commands MUST be executable on a single diff --git a/include/redisserver.h b/include/redisserver.h index f3ab81d8c..b28d03a25 100644 --- a/include/redisserver.h +++ b/include/redisserver.h @@ -277,7 +277,7 @@ class RedisServer { * \brief Set a model from std::string_view buffer in the * database for future execution * \param key The key to associate with the model - * \param model The model as a continuous buffer string_view + * \param model The model as a sequence of buffer string_view chunks * \param backend The name of the backend * (TF, TFLITE, TORCH, ONNX) * \param device The name of the device for execution @@ -295,7 +295,7 @@ class RedisServer { * \throw RuntimeException for all client errors */ virtual CommandReply set_model(const std::string& key, - std::string_view model, + const std::vector& model, const std::string& backend, const std::string& device, int batch_size = 0, @@ -311,7 +311,7 @@ class RedisServer { * \brief Set a model from std::string_view buffer in the * database for future execution in a multi-GPU system * \param name The name to associate with the model - * \param model The model as a continuous buffer string_view + * \param model The model as a sequence of buffer string_view chunks * \param backend The name of the backend * (TF, TFLITE, TORCH, ONNX) * \param first_gpu The first GPU to use with this model @@ -328,7 +328,7 @@ class RedisServer { * \throw RuntimeException for all client errors */ virtual void set_model_multigpu(const std::string& name, - const std::string_view& model, + const std::vector& model, const std::string& backend, int first_gpu, int num_gpus, @@ -520,6 +520,35 @@ class RedisServer { const std::string& key, const bool reset_stat) = 0; + /*! + * \brief Retrieve the current model chunk size + * \returns The size in bytes for model chunking + */ + virtual int get_model_chunk_size() = 0; + + /*! + * \brief Reconfigure the chunking size that Redis uses for model + * serialization, replication, and the model_get command. + * \details This method triggers the AI.CONFIG method in the Redis + * database to change the model chunking size. + * + * NOTE: The default size of 511MB should be fine for most + * applications, so it is expected to be very rare that a + * client calls this method. It is not necessary to call + * this method a model to be chunked. + * \param chunk_size The new chunk size in bytes + * \throw SmartRedis::Exception if the command fails. + */ + virtual void set_model_chunk_size(int chunk_size) = 0; + + /*! + * \brief Store the current model chunk size + * \param chunk_size The updated model chunk size + */ + virtual void store_model_chunk_size(int chunk_size) { + _model_chunk_size = chunk_size; + } + /*! * \brief Run a CommandList via a Pipeline. For clustered databases * all commands must go to the same shard @@ -567,6 +596,12 @@ class RedisServer { */ int _command_attempts; + /*! + * \brief The chunk size into which models need to be broken for + * transfer to Redis + */ + int _model_chunk_size; + /*! * \brief Default value of connection timeout (seconds) */ @@ -630,6 +665,11 @@ class RedisServer { */ bool _is_domain_socket; + /*! + * \brief Default model chunk size + */ + static constexpr int _UNKNOWN_MODEL_CHUNK_SIZE = -1; + /*! * \brief Environment variable for connection timeout */ diff --git a/src/cpp/client.cpp b/src/cpp/client.cpp index 0f64895a7..cfb1867a5 100644 --- a/src/cpp/client.cpp +++ b/src/cpp/client.cpp @@ -28,6 +28,10 @@ #include #include +#include +#include +#include +#include #include "client.h" #include "srexception.h" #include "logger.h" @@ -602,9 +606,21 @@ void Client::set_model(const std::string& name, throw SRRuntimeException(device + " is not a valid device."); } + // Split model into chunks + size_t offset = 0; + std::vector model_segments; + size_t chunk_size = _redis_server->get_model_chunk_size(); + size_t remaining = model.length(); + for (offset = 0; offset < model.length(); offset += chunk_size) { + size_t this_chunk_size = remaining > chunk_size ? chunk_size : remaining; + std::string_view chunk(model.data() + offset, this_chunk_size); + model_segments.push_back(chunk); + remaining -= this_chunk_size; + } + std::string key = _build_model_key(name, false); auto response = _redis_server->set_model( - key, model, backend, device, + key, model_segments, backend, device, batch_size, min_batch_size, tag, inputs, outputs); if (response.has_error()) { @@ -661,9 +677,21 @@ void Client::set_model_multigpu(const std::string& name, throw SRParameterException(backend + " is not a valid backend."); } + // Split model into chunks + size_t offset = 0; + std::vector model_segments; + size_t chunk_size = _redis_server->get_model_chunk_size(); + size_t remaining = model.length(); + for (offset = 0; offset < model.length(); offset += chunk_size) { + size_t this_chunk_size = remaining > chunk_size ? chunk_size : remaining; + std::string_view chunk(model.data() + offset, this_chunk_size); + model_segments.push_back(chunk); + remaining -= this_chunk_size; + } + std::string key = _build_model_key(name, false); _redis_server->set_model_multigpu( - key, model, backend, first_gpu, num_gpus, + key, model_segments, backend, first_gpu, num_gpus, batch_size, min_batch_size, tag, inputs, outputs); } @@ -675,16 +703,35 @@ std::string_view Client::get_model(const std::string& name) // Track calls to this API function LOG_API_FUNCTION(); + // Get the model from the server std::string get_key = _build_model_key(name, true); CommandReply reply = _redis_server->get_model(get_key); if (reply.has_error()) throw SRRuntimeException("failed to get model from server"); - char* model = _model_queries.allocate(reply.str_len()); + // In most cases, the reply will be a single string + // consisting of the serialized model + if (!reply.is_array()) { + char* model = _model_queries.allocate(reply.str_len()); + if (model == NULL) + throw SRBadAllocException("model query"); + std::memcpy(model, reply.str(), reply.str_len()); + return std::string_view(model, reply.str_len()); + } + + // Otherwise, we need to concatenate the segments together + size_t model_length = 0; + size_t offset = 0; + for (size_t i = 0; i < reply.n_elements(); i++) { + model_length += reply[i].str_len(); + } + char* model = _model_queries.allocate(model_length); if (model == NULL) throw SRBadAllocException("model query"); - std::memcpy(model, reply.str(), reply.str_len()); - return std::string_view(model, reply.str_len()); + for (size_t i = 0; i < reply.n_elements(); i++) { + std::memcpy(model + offset, reply[i].str(), reply[i].str_len()); + } + return std::string_view(model, model_length); } // Set a script from file in the database for future execution @@ -2115,9 +2162,32 @@ bool Client::_poll_list_length(const std::string& name, int list_length, return false; } +// Reconfigure the model chunk size for the database +void Client::set_model_chunk_size(int chunk_size) +{ + // Track calls to this API function + LOG_API_FUNCTION(); + + // Build the command + AddressAnyCommand cmd; + cmd << "AI.CONFIG" << "MODEL_CHUNK_SIZE" << std::to_string(chunk_size); + std::cout << cmd.to_string() << std::endl; + + // Run it + CommandReply reply = _run(cmd); + if (reply.has_error() > 0) + throw SRRuntimeException("AI.CONFIG MODEL_CHUNK_SIZE command failed"); + + // Remember the new chunk size + _redis_server->store_model_chunk_size(chunk_size); +} + // Create a string representation of the client std::string Client::to_string() const { + // Track calls to this API function + LOG_API_FUNCTION(); + std::string result; result = "Client (" + _lname + "):\n"; result += _redis_server->to_string(); diff --git a/src/cpp/redis.cpp b/src/cpp/redis.cpp index b84cc77d1..0aa7560c6 100644 --- a/src/cpp/redis.cpp +++ b/src/cpp/redis.cpp @@ -294,7 +294,7 @@ CommandReply Redis::copy_tensors(const std::vector& src, // Set a model from std::string_view buffer in the database for future execution CommandReply Redis::set_model(const std::string& model_name, - std::string_view model, + const std::vector& model, const std::string& backend, const std::string& device, int batch_size, @@ -333,7 +333,7 @@ CommandReply Redis::set_model(const std::string& model_name, // Set a model from std::string_view buffer in the // database for future execution in a multi-GPU system void Redis::set_model_multigpu(const std::string& name, - const std::string_view& model, + const std::vector& model, const std::string& backend, int first_gpu, int num_gpus, @@ -581,7 +581,7 @@ CommandReply Redis::get_model_script_ai_info(const std::string& address, "non-cluster client connection."); } - //Build the Command + // Build the Command cmd.set_exec_address(db_address); cmd << "AI.INFO" << Keyfield(key); @@ -593,6 +593,48 @@ CommandReply Redis::get_model_script_ai_info(const std::string& address, return run(cmd); } +// Retrieve the current model chunk size +int Redis::get_model_chunk_size() +{ + // If we've already set a chunk size, just return it + if (_model_chunk_size != _UNKNOWN_MODEL_CHUNK_SIZE) + return _model_chunk_size; + + // Build the command + AddressAnyCommand cmd; + cmd << "AI.CONFIG" << "GET" << "MODEL_CHUNK_SIZE"; + + CommandReply reply = _run(cmd); + if (reply.has_error() > 0) + throw SRRuntimeException("AI.CONFIG GET MODEL_CHUNK_SIZE command failed"); + + if (reply.redis_reply_type() != "REDIS_REPLY_INTEGER") + throw SRRuntimeException("An unexpected type was returned for " + "for the model chunk size."); + + int chunk_size = reply.integer(); + + if (chunk_size < 0) + throw SRRuntimeException("An invalid, negative value was " + "returned for the model chunk size."); + + return chunk_size; +} + +// Reconfigure the model chunk size for the database +void Redis::set_model_chunk_size(int chunk_size) +{ + AddressAnyCommand cmd; + cmd << "AI.CONFIG" << "MODEL_CHUNK_SIZE" << std::to_string(chunk_size); + + CommandReply reply = _run(cmd); + if (reply.has_error() > 0) + throw SRRuntimeException("AI.CONFIG MODEL_CHUNK_SIZE command failed"); + + // Store the new model chunk size for later + _model_chunk_size = chunk_size; +} + inline CommandReply Redis::_run(const Command& cmd) { for (int i = 1; i <= _command_attempts; i++) { diff --git a/src/cpp/rediscluster.cpp b/src/cpp/rediscluster.cpp index bcdcf6c33..3c1ae259d 100644 --- a/src/cpp/rediscluster.cpp +++ b/src/cpp/rediscluster.cpp @@ -506,7 +506,7 @@ CommandReply RedisCluster::copy_tensors(const std::vector& src, // Set a model from a string buffer in the database for future execution CommandReply RedisCluster::set_model(const std::string& model_name, - std::string_view model, + const std::vector& model, const std::string& backend, const std::string& device, int batch_size, @@ -552,7 +552,7 @@ CommandReply RedisCluster::set_model(const std::string& model_name, // Set a model from std::string_view buffer in the // database for future execution in a multi-GPU system void RedisCluster::set_model_multigpu(const std::string& name, - const std::string_view& model, + const std::vector& model, const std::string& backend, int first_gpu, int num_gpus, @@ -941,6 +941,57 @@ CommandReply RedisCluster::get_model_script_ai_info(const std::string& address, return run(cmd); } +// Retrieve the current model chunk size +int RedisCluster::get_model_chunk_size() +{ + // If we've already set a chunk size, just return it + if (_model_chunk_size != _UNKNOWN_MODEL_CHUNK_SIZE) + return _model_chunk_size; + + // Build the command + AddressAnyCommand cmd; + cmd << "AI.CONFIG" << "GET" << "MODEL_CHUNK_SIZE"; + + CommandReply reply = run(cmd); + if (reply.has_error() > 0) + throw SRRuntimeException("AI.CONFIG GET MODEL_CHUNK_SIZE command failed"); + + if (reply.redis_reply_type() != "REDIS_REPLY_INTEGER") + throw SRRuntimeException("An unexpected type was returned for " + "for the model chunk size."); + + int chunk_size = reply.integer(); + + if (chunk_size < 0) + throw SRRuntimeException("An invalid, negative value was " + "returned for the model chunk size."); + + return chunk_size; +} + +// Reconfigure the model chunk size for the database +void RedisCluster::set_model_chunk_size(int chunk_size) +{ + // Repeat for each server node: + auto node = _db_nodes.cbegin(); + for ( ; node != _db_nodes.cend(); node++) { + // Pick a node for the command + AddressAtCommand cmd; + cmd.set_exec_address(node->address); + // Build the command + cmd << "AI.CONFIG" << "MODEL_CHUNK_SIZE" << std::to_string(chunk_size); + + // Run it + CommandReply reply = run(cmd); + if (reply.has_error() > 0) { + throw SRRuntimeException("set_model_chunk_size failed for node " + node->name); + } + } + + // Store the new model chunk size for later + _model_chunk_size = chunk_size; +} + inline CommandReply RedisCluster::_run(const Command& cmd, std::string db_prefix) { std::string_view sv_prefix(db_prefix.data(), db_prefix.size()); diff --git a/src/cpp/redisserver.cpp b/src/cpp/redisserver.cpp index e3a206e84..d4d04514c 100644 --- a/src/cpp/redisserver.cpp +++ b/src/cpp/redisserver.cpp @@ -59,6 +59,7 @@ RedisServer::RedisServer(const SRObject* context) _command_interval + 1; _tp = new ThreadPool(_context, _thread_count); + _model_chunk_size = _UNKNOWN_MODEL_CHUNK_SIZE; } // RedisServer destructor diff --git a/src/python/bindings/bind.cpp b/src/python/bindings/bind.cpp index b53c62515..0ab30fb57 100644 --- a/src/python/bindings/bind.cpp +++ b/src/python/bindings/bind.cpp @@ -117,6 +117,7 @@ PYBIND11_MODULE(smartredisPy, m) { .CLIENT_METHOD(poll_list_length_lte) .CLIENT_METHOD(get_datasets_from_list) .CLIENT_METHOD(get_dataset_list_range) + .CLIENT_METHOD(set_model_chunk_size) .CLIENT_METHOD(to_string) ; diff --git a/src/python/module/smartredis/client.py b/src/python/module/smartredis/client.py index b2223d06a..53270f966 100644 --- a/src/python/module/smartredis/client.py +++ b/src/python/module/smartredis/client.py @@ -1451,6 +1451,25 @@ def save(self, addresses: t.List[str]) -> None: typecheck(addresses, "addresses", list) self._client.save(addresses) + @exception_handler + def set_model_chunk_size(self, chunk_size: int) -> None: + """Reconfigures the chunking size that Redis uses for model + serialization, replication, and the model_get command. + This method triggers the AI.CONFIG method in the Redis + database to change the model chunking size. + + NOTE: The default size of 511MB should be fine for most + applications, so it is expected to be very rare that a + client calls this method. It is not necessary to call + this method a model to be chunked. + :param chunk_size: The new chunk size in bytes + :type addresses: int + :raises RedisReplyError: if there is an error + in command execution. + """ + typecheck(chunk_size, "chunk_size", int) + self._client.set_model_chunk_size(chunk_size) + @exception_handler def append_to_list(self, list_name: str, dataset: Dataset) -> None: """Appends a dataset to the aggregation list diff --git a/src/python/src/pyclient.cpp b/src/python/src/pyclient.cpp index 286a0a375..eb9b497d3 100644 --- a/src/python/src/pyclient.cpp +++ b/src/python/src/pyclient.cpp @@ -710,6 +710,14 @@ py::list PyClient::get_dataset_list_range( }); } +// Configure the Redis module chunk size +void PyClient::set_model_chunk_size(int chunk_size) +{ + return MAKE_CLIENT_API({ + return _client->set_model_chunk_size(chunk_size); + }); +} + // Create a string representation of the Client std::string PyClient::to_string() { diff --git a/tests/cpp/client_test_mnist.cpp b/tests/cpp/client_test_mnist.cpp index 7fd5f1d78..f07684495 100644 --- a/tests/cpp/client_test_mnist.cpp +++ b/tests/cpp/client_test_mnist.cpp @@ -81,6 +81,7 @@ int main(int argc, char* argv[]) { SmartRedis::Client client(use_cluster(), "client_test_mnist"); std::string model_key = "mnist_model"; std::string model_file = "mnist_data/mnist_cnn.pt"; + client.set_model_chunk_size(1024 * 1024); client.set_model_from_file(model_key, model_file, "TORCH", "CPU"); std::string script_key = "mnist_script"; diff --git a/tests/cpp/unit-tests/test_client_ensemble.cpp b/tests/cpp/unit-tests/test_client_ensemble.cpp index 6dc105e7c..9268a9b92 100644 --- a/tests/cpp/unit-tests/test_client_ensemble.cpp +++ b/tests/cpp/unit-tests/test_client_ensemble.cpp @@ -133,6 +133,7 @@ SCENARIO("Testing Client ensemble using a producer/consumer paradigm") Client producer_client(use_cluster(), "test_client_ensemble::producer"); producer_client.use_model_ensemble_prefix(true); + producer_client.set_model_chunk_size(1024 * 1024); // Tensors float* array = (float*)malloc(dims[0]*sizeof(float)); diff --git a/tests/python/test_errors.py b/tests/python/test_errors.py index 343a0e77c..bba7fde07 100644 --- a/tests/python/test_errors.py +++ b/tests/python/test_errors.py @@ -752,6 +752,14 @@ def test_bad_type_get_dataset_list_range(use_cluster, context): with pytest.raises(TypeError): c.get_dataset_list_range(listname, start_index, "not an integer") +def test_bad_type_set_model_chunk_size(use_cluster, context): + c = Client(None, use_cluster, logger_name=context) + with pytest.raises(TypeError): + c.set_model_chunk_size("not an integer") + +##### +# Test type errors from bad parameter types to logging calls + @pytest.mark.parametrize("log_fn", [ (log_data,), (log_warning,), (log_error,) ]) @@ -931,13 +939,11 @@ def test_get_integer_option_wrong_type(cfg_opts: ConfigOptions): def test_get_string_option_wrong_type(cfg_opts: ConfigOptions): """Ensure get_string_option raises an exception on an invalid key type""" - with pytest.raises(TypeError): _ = cfg_opts.get_string_option(42) def test_is_configured_wrong_type(cfg_opts: ConfigOptions): """Ensure is_configured raises an exception on an invalid key type""" - with pytest.raises(TypeError): _ = cfg_opts.is_configured(42) diff --git a/tests/python/test_model_methods_torch.py b/tests/python/test_model_methods_torch.py index 9220b6753..b1c7b078b 100644 --- a/tests/python/test_model_methods_torch.py +++ b/tests/python/test_model_methods_torch.py @@ -42,6 +42,7 @@ def test_set_model_from_file(mock_model, use_cluster, context): try: mock_model.create_torch_cnn(filepath="./torch_cnn.pt") c = Client(None, use_cluster, logger_name=context) + c.set_model_chunk_size(1024 * 1024) c.set_model_from_file("file_cnn", "./torch_cnn.pt", "TORCH", "CPU") assert c.model_exists("file_cnn") returned_model = c.get_model("file_cnn") From 2fe3228752b01b3ac58fdef173ebd53bd964df1b Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:41:01 -0500 Subject: [PATCH 05/25] Expose MINBATCHTIMEOUT parameter in set_model interface (#406) Expose MINBATCHTIMEOUT parameter to set_model() family calls Tweak CI/CD pipeline to clear up disk space needed for Intel compiler [ committed by @billschereriii ] [ reviewed by @ashao ] --- .github/workflows/run_tests.yml | 12 +++ doc/changelog.rst | 5 +- doc/data_structures.rst | 1 + include/c_client.h | 10 +- include/client.h | 8 ++ include/pyclient.h | 8 ++ include/redis.h | 4 + include/rediscluster.h | 4 + include/redisserver.h | 4 + src/c/c_client.cpp | 20 ++-- src/cpp/client.cpp | 50 ++++++++- src/cpp/redis.cpp | 11 +- src/cpp/rediscluster.cpp | 9 +- src/fortran/client.F90 | 48 +++++---- src/fortran/client/model_interfaces.inc | 24 +++-- src/python/module/smartredis/client.py | 20 ++++ src/python/src/pyclient.cpp | 18 ++-- tests/python/test_errors.py | 2 + tests/python/test_model_methods_torch.py | 125 +++++++++++++++++++++++ 19 files changed, 331 insertions(+), 52 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index b4aea72a2..bbcb343c2 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -75,6 +75,18 @@ jobs: with: python-version: ${{ matrix.py_v }} + # Free up some disk space + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet && + sudo rm -rf /opt/ghc && + sudo rm -rf "/usr/local/share/boost" + + # sudo rm -rf /usr/share/dotnet && + # sudo rm -rf /opt/ghc && + # sudo rm -rf "/usr/local/share/boost" && + # sudo rm -rf "$AGENT_TOOLSDIRECTORY" + # Install compilers (Intel or GCC) - name: Install GCC if: "!contains( matrix.compiler, 'intel' )" # if using GNU compiler diff --git a/doc/changelog.rst b/doc/changelog.rst index b38b8eb3c..1e12c96ad 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,18 +8,21 @@ To be released at some future point in time Description +- Improved support for model execution batching - Added support for model chunking - Updated the third-party RedisAI component - Updated the third-party lcov component -- Add link to contributing guidelines +- Added link to contributing guidelines Detailed Notes +- Exposed access to the Redis.AI MINBATCHTIMEOUT parameter, which limits the delay in model execution when trying to accumulate multiple executions in a batch (PR406_) - Models will now be automatically chunked when sent to/received from the backed database. This allows use of models greater than 511MB in size. (PR404_) - Updated from RedisAI v1.2.3 (test target)/v1.2.4 and v1.2.5 (CI/CD pipeline) to v1.2.7 (PR402_) - Updated lcov from version 1.15 to 2.0 (PR396_) - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) +.. _PR406: https://github.com/CrayLabs/SmartRedis/pull/406 .. _PR404: https://github.com/CrayLabs/SmartRedis/pull/404 .. _PR402: https://github.com/CrayLabs/SmartRedis/pull/402 .. _PR396: https://github.com/CrayLabs/SmartRedis/pull/396 diff --git a/doc/data_structures.rst b/doc/data_structures.rst index 2a1b86467..fb547c4bd 100644 --- a/doc/data_structures.rst +++ b/doc/data_structures.rst @@ -358,6 +358,7 @@ are uniform across all SmartRedis clients, and as an example, the C++ const std::string& device, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), diff --git a/include/c_client.h b/include/c_client.h index 3d0535d43..850b8ea9b 100644 --- a/include/c_client.h +++ b/include/c_client.h @@ -323,7 +323,7 @@ bool _isTensorFlow(const char* backend); /*! * \brief Check parameters for all parameters common to set_model methods * \details Make sure that all pointers are not void and that the size -* of the inputs and outputs is not zero +* of the inputs and outputs is not zero * \param c_client The client object to use for communication * \param name The name to associate with the model * \param backend The name of the backend (TF, TFLITE, TORCH, ONNX) @@ -372,6 +372,7 @@ void _check_params_set_model(void* c_client, * excluding null terminating character * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model execution +* \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for information purposes * \param tag_length The length of the tag string, * excluding null terminating character @@ -396,6 +397,7 @@ SRError set_model_from_file(void* c_client, const size_t device_length, const int batch_size, const int min_batch_size, + const int min_batch_timeout, const char* tag, const size_t tag_length, const char** inputs, @@ -428,6 +430,7 @@ SRError set_model_from_file(void* c_client, * \param num_gpus the number of gpus to use with the model * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model execution +* \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for information purposes * \param tag_length The length of the tag string, * excluding null terminating character @@ -452,6 +455,7 @@ SRError set_model_from_file_multigpu(void* c_client, const int num_gpus, const int batch_size, const int min_batch_size, + const int min_batch_timeout, const char* tag, const size_t tag_length, const char** inputs, @@ -486,6 +490,7 @@ SRError set_model_from_file_multigpu(void* c_client, * excluding null terminating character * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model execution +* \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for information purposes * \param tag_length The length of the tag string, * excluding null terminating character @@ -510,6 +515,7 @@ SRError set_model(void* c_client, const size_t device_length, const int batch_size, const int min_batch_size, + const int min_batch_timeout, const char* tag, const size_t tag_length, const char** inputs, @@ -542,6 +548,7 @@ SRError set_model(void* c_client, * \param num_gpus The number of GPUs to use with the model * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model execution +* \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for information purposes * \param tag_length The length of the tag string, * excluding null terminating character @@ -566,6 +573,7 @@ SRError set_model_multigpu(void* c_client, const int num_gpus, const int batch_size, const int min_batch_size, + const int min_batch_timeout, const char* tag, const size_t tag_length, const char** inputs, diff --git a/include/client.h b/include/client.h index 3444c707b..a9bb07614 100644 --- a/include/client.h +++ b/include/client.h @@ -344,6 +344,7 @@ class Client : public SRObject * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for information purposes * \param inputs One or more names of model input nodes * (TF models only). For other models, provide an @@ -359,6 +360,7 @@ class Client : public SRObject const std::string& device, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), @@ -383,6 +385,7 @@ class Client : public SRObject * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for * information purposes * \param inputs One or more names of model input nodes @@ -398,6 +401,7 @@ class Client : public SRObject int num_gpus, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), @@ -423,6 +427,7 @@ class Client : public SRObject * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for information purposes * \param inputs One or more names of model input nodes * (TF models only). For other models, provide an @@ -438,6 +443,7 @@ class Client : public SRObject const std::string& device, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), @@ -462,6 +468,7 @@ class Client : public SRObject * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for * information purposes * \param inputs One or more names of model input nodes @@ -477,6 +484,7 @@ class Client : public SRObject int num_gpus, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), diff --git a/include/pyclient.h b/include/pyclient.h index b1f82bcbb..5c407a3a3 100644 --- a/include/pyclient.h +++ b/include/pyclient.h @@ -285,6 +285,7 @@ class PyClient : public PySRObject * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for * information purposes * \param inputs One or more names of model input nodes @@ -299,6 +300,7 @@ class PyClient : public PySRObject const std::string& device, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), @@ -317,6 +319,7 @@ class PyClient : public PySRObject * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for * information purposes * \param inputs One or more names of model input nodes @@ -332,6 +335,7 @@ class PyClient : public PySRObject int num_gpus, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), @@ -350,6 +354,7 @@ class PyClient : public PySRObject * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for * information purposes * \param inputs One or more names of model input nodes @@ -364,6 +369,7 @@ class PyClient : public PySRObject const std::string& device, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), @@ -382,6 +388,7 @@ class PyClient : public PySRObject * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for * information purposes * \param inputs One or more names of model input nodes @@ -397,6 +404,7 @@ class PyClient : public PySRObject int num_gpus, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), diff --git a/include/redis.h b/include/redis.h index 4aa9f246c..b799577c1 100644 --- a/include/redis.h +++ b/include/redis.h @@ -283,6 +283,7 @@ class Redis : public RedisServer * (e.g. CPU or GPU) * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for information purposes * \param inputs One or more names of model input nodes * (TF models only) @@ -297,6 +298,7 @@ class Redis : public RedisServer const std::string& device, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), @@ -314,6 +316,7 @@ class Redis : public RedisServer * \param num_gpus The number of GPUs to use with this model * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for information purposes * \param inputs One or more names of model input nodes * (TF models only) @@ -328,6 +331,7 @@ class Redis : public RedisServer int num_gpus, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), diff --git a/include/rediscluster.h b/include/rediscluster.h index 876c6b733..7371ec343 100644 --- a/include/rediscluster.h +++ b/include/rediscluster.h @@ -302,6 +302,7 @@ class RedisCluster : public RedisServer * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for * information purposes * \param inputs One or more names of model input nodes @@ -317,6 +318,7 @@ class RedisCluster : public RedisServer const std::string& device, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), @@ -335,6 +337,7 @@ class RedisCluster : public RedisServer * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for * information purposes * \param inputs One or more names of model input nodes @@ -350,6 +353,7 @@ class RedisCluster : public RedisServer int num_gpus, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), diff --git a/include/redisserver.h b/include/redisserver.h index b28d03a25..39594c505 100644 --- a/include/redisserver.h +++ b/include/redisserver.h @@ -285,6 +285,7 @@ class RedisServer { * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for * information purposes * \param inputs One or more names of model input nodes @@ -300,6 +301,7 @@ class RedisServer { const std::string& device, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), @@ -319,6 +321,7 @@ class RedisServer { * \param batch_size The batch size for model execution * \param min_batch_size The minimum batch size for model * execution + * \param min_batch_timeout Max time (ms) to wait for min batch size * \param tag A tag to attach to the model for * information purposes * \param inputs One or more names of model input nodes @@ -334,6 +337,7 @@ class RedisServer { int num_gpus, int batch_size = 0, int min_batch_size = 0, + int min_batch_timeout = 0, const std::string& tag = "", const std::vector& inputs = std::vector(), diff --git a/src/c/c_client.cpp b/src/c/c_client.cpp index 99cd691da..6672184b5 100644 --- a/src/c/c_client.cpp +++ b/src/c/c_client.cpp @@ -392,6 +392,7 @@ extern "C" SRError set_model_from_file( const char* device, const size_t device_length, const int batch_size, const int min_batch_size, + const int min_batch_timeout, const char* tag, const size_t tag_length, const char** inputs, const size_t* input_lengths, const size_t n_inputs, const char** outputs, const size_t* output_lengths, const size_t n_outputs) @@ -430,8 +431,8 @@ extern "C" SRError set_model_from_file( } s->set_model_from_file(name_str, model_file_str, backend_str, device_str, - batch_size, min_batch_size, tag_str, input_vec, - output_vec); + batch_size, min_batch_size, min_batch_timeout, + tag_str, input_vec, output_vec); }); } @@ -443,6 +444,7 @@ extern "C" SRError set_model_from_file_multigpu( const char* backend, const size_t backend_length, const int first_gpu, const int num_gpus, const int batch_size, const int min_batch_size, + const int min_batch_timeout, const char* tag, const size_t tag_length, const char** inputs, const size_t* input_lengths, const size_t n_inputs, const char** outputs, @@ -481,8 +483,8 @@ extern "C" SRError set_model_from_file_multigpu( } s->set_model_from_file_multigpu(name_str, model_file_str, backend_str, first_gpu, - num_gpus, batch_size, min_batch_size, tag_str, - input_vec, output_vec); + num_gpus, batch_size, min_batch_size, min_batch_timeout, + tag_str, input_vec, output_vec); }); } @@ -494,6 +496,7 @@ extern "C" SRError set_model( const char* backend, const size_t backend_length, const char* device, const size_t device_length, const int batch_size, const int min_batch_size, + const int min_batch_timeout, const char* tag, const size_t tag_length, const char** inputs, const size_t* input_lengths, const size_t n_inputs, @@ -534,8 +537,8 @@ extern "C" SRError set_model( } s->set_model(name_str, model_str, backend_str, device_str, - batch_size, min_batch_size, tag_str, input_vec, - output_vec); + batch_size, min_batch_size, min_batch_timeout, + tag_str, input_vec, output_vec); }); } @@ -547,6 +550,7 @@ extern "C" SRError set_model_multigpu( const char* backend, const size_t backend_length, const int first_gpu, const int num_gpus, const int batch_size, const int min_batch_size, + const int min_batch_timeout, const char* tag, const size_t tag_length, const char** inputs, const size_t* input_lengths, const size_t n_inputs, @@ -586,8 +590,8 @@ extern "C" SRError set_model_multigpu( } s->set_model_multigpu(name_str, model_str, backend_str, first_gpu, num_gpus, - batch_size, min_batch_size, tag_str, input_vec, - output_vec); + batch_size, min_batch_size, min_batch_timeout, + tag_str, input_vec, output_vec); }); } diff --git a/src/cpp/client.cpp b/src/cpp/client.cpp index cfb1867a5..7e9e98c7d 100644 --- a/src/cpp/client.cpp +++ b/src/cpp/client.cpp @@ -502,6 +502,7 @@ void Client::set_model_from_file(const std::string& name, const std::string& device, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs) @@ -522,7 +523,7 @@ void Client::set_model_from_file(const std::string& name, std::string_view model(tmp.data(), tmp.length()); set_model(name, model, backend, device, batch_size, - min_batch_size, tag, inputs, outputs); + min_batch_size, min_batch_timeout, tag, inputs, outputs); } // Set a model from file in the database for future execution in a multi-GPU system @@ -533,6 +534,7 @@ void Client::set_model_from_file_multigpu(const std::string& name, int num_gpus, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs) @@ -553,8 +555,42 @@ void Client::set_model_from_file_multigpu(const std::string& name, std::string_view model(tmp.data(), tmp.length()); set_model_multigpu(name, model, backend, first_gpu, num_gpus, batch_size, - min_batch_size, tag, inputs, outputs); + min_batch_size, min_batch_timeout, tag, inputs, outputs); } + +// Validate batch settings for the set_model calls +inline void __check_batch_settings( + int batch_size, int min_batch_size, int min_batch_timeout) +{ + // Throw a usage exception if batch_size is zero but one of the other + // parameters is non-zero + if (batch_size == 0 && (min_batch_size > 0 || min_batch_timeout > 0)) { + throw SRRuntimeException( + "batch_size must be non-zero if min_batch_size or " + "min_batch_timeout is used; otherwise batching will " + "not be performed." + ); + } + + // Throw a usage exception if min_batch_timeout is nonzero and + // min_batch_size is zero. (batch_size also has to be non-zero, but + // this was caught in the previous clause.) + if (min_batch_timeout > 0 && min_batch_size == 0) { + throw SRRuntimeException( + "min_batch_size must be non-zero if min_batch_timeout " + "is used; otherwise the min_batch_timeout parameter is ignored." + ); + } + + // Issue a warning if min_batch_size is non-zero but min_batch_timeout is zero + if (min_batch_size > 0 && min_batch_timeout == 0) { + std::cerr << "WARNING: min_batch_timeout was not set when a non-zero " + << "min_batch_size was selected. " << std::endl + << "Setting a small value (~10ms) for min_batch_timeout " + << "may improve performance" << std::endl; + } +} + // Set a model from a string buffer in the database for future execution void Client::set_model(const std::string& name, const std::string_view& model, @@ -562,6 +598,7 @@ void Client::set_model(const std::string& name, const std::string& device, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs) @@ -606,6 +643,8 @@ void Client::set_model(const std::string& name, throw SRRuntimeException(device + " is not a valid device."); } + __check_batch_settings(batch_size, min_batch_size, min_batch_timeout); + // Split model into chunks size_t offset = 0; std::vector model_segments; @@ -621,7 +660,7 @@ void Client::set_model(const std::string& name, std::string key = _build_model_key(name, false); auto response = _redis_server->set_model( key, model_segments, backend, device, - batch_size, min_batch_size, + batch_size, min_batch_size, min_batch_timeout, tag, inputs, outputs); if (response.has_error()) { throw SRInternalException( @@ -636,6 +675,7 @@ void Client::set_model_multigpu(const std::string& name, int num_gpus, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs) @@ -677,6 +717,8 @@ void Client::set_model_multigpu(const std::string& name, throw SRParameterException(backend + " is not a valid backend."); } + __check_batch_settings(batch_size, min_batch_size, min_batch_timeout); + // Split model into chunks size_t offset = 0; std::vector model_segments; @@ -692,7 +734,7 @@ void Client::set_model_multigpu(const std::string& name, std::string key = _build_model_key(name, false); _redis_server->set_model_multigpu( key, model_segments, backend, first_gpu, num_gpus, - batch_size, min_batch_size, + batch_size, min_batch_size, min_batch_timeout, tag, inputs, outputs); } diff --git a/src/cpp/redis.cpp b/src/cpp/redis.cpp index 0aa7560c6..a00af813b 100644 --- a/src/cpp/redis.cpp +++ b/src/cpp/redis.cpp @@ -299,6 +299,7 @@ CommandReply Redis::set_model(const std::string& model_name, const std::string& device, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs @@ -318,6 +319,9 @@ CommandReply Redis::set_model(const std::string& model_name, if (min_batch_size > 0) { cmd << "MINBATCHSIZE" << std::to_string(min_batch_size); } + if (min_batch_timeout > 0) { + cmd << "MINBATCHTIMEOUT" << std::to_string(min_batch_timeout); + } if (inputs.size() > 0) { cmd << "INPUTS" << std::to_string(inputs.size()) << inputs; } @@ -339,6 +343,7 @@ void Redis::set_model_multigpu(const std::string& name, int num_gpus, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs) @@ -349,7 +354,8 @@ void Redis::set_model_multigpu(const std::string& name, std::string device = "GPU:" + std::to_string(i); std::string model_key = name + "." + device; result = set_model( - model_key, model, backend, device, batch_size, min_batch_size, tag, inputs, outputs); + model_key, model, backend, device, batch_size, min_batch_size, min_batch_timeout, + tag, inputs, outputs); if (result.has_error() > 0) { throw SRRuntimeException("Failed to set model for GPU " + std::to_string(i)); } @@ -357,7 +363,8 @@ void Redis::set_model_multigpu(const std::string& name, // Add a version for get_model to find result = set_model( - name, model, backend, "GPU", batch_size, min_batch_size, tag, inputs, outputs); + name, model, backend, "GPU", batch_size, min_batch_size, min_batch_timeout, + tag, inputs, outputs); if (result.has_error() > 0) { throw SRRuntimeException("Failed to set general model"); } diff --git a/src/cpp/rediscluster.cpp b/src/cpp/rediscluster.cpp index 3c1ae259d..6f847dc6e 100644 --- a/src/cpp/rediscluster.cpp +++ b/src/cpp/rediscluster.cpp @@ -511,6 +511,7 @@ CommandReply RedisCluster::set_model(const std::string& model_name, const std::string& device, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs) @@ -531,6 +532,9 @@ CommandReply RedisCluster::set_model(const std::string& model_name, if (min_batch_size > 0) { cmd << "MINBATCHSIZE" << std::to_string(min_batch_size); } + if (min_batch_timeout > 0) { + cmd << "MINBATCHTIMEOUT" << std::to_string(min_batch_timeout); + } if ( inputs.size() > 0) { cmd << "INPUTS" << std::to_string(inputs.size()) << inputs; } @@ -558,6 +562,7 @@ void RedisCluster::set_model_multigpu(const std::string& name, int num_gpus, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs) @@ -571,7 +576,7 @@ void RedisCluster::set_model_multigpu(const std::string& name, // Store it CommandReply result = set_model( model_key, model, backend, device, batch_size, min_batch_size, - tag, inputs, outputs); + min_batch_timeout, tag, inputs, outputs); if (result.has_error() > 0) { throw SRRuntimeException("Failed to set model for " + device); } @@ -580,7 +585,7 @@ void RedisCluster::set_model_multigpu(const std::string& name, // Add a version for get_model to find CommandReply result = set_model( name, model, backend, "GPU", batch_size, min_batch_size, - tag, inputs, outputs); + min_batch_timeout, tag, inputs, outputs); if (result.has_error() > 0) { throw SRRuntimeException("Failed to set general model"); } diff --git a/src/fortran/client.F90 b/src/fortran/client.F90 index 7c79f0148..c3acd35c7 100644 --- a/src/fortran/client.F90 +++ b/src/fortran/client.F90 @@ -746,8 +746,8 @@ function get_model(self, name, model) result(code) end function get_model !> Load the machine learning model from a file and set the configuration -function set_model_from_file(self, name, model_file, backend, device, batch_size, min_batch_size, tag, & - inputs, outputs) result(code) +function set_model_from_file(self, name, model_file, backend, device, batch_size, min_batch_size, & + min_batch_timeout, tag, inputs, outputs) result(code) class(client_type), intent(in) :: self !< An initialized SmartRedis client character(len=*), intent(in) :: name !< The name to use to place the model character(len=*), intent(in) :: model_file !< The file storing the model @@ -755,6 +755,7 @@ function set_model_from_file(self, name, model_file, backend, device, batch_size character(len=*), intent(in) :: device !< The name of the device (CPU, GPU, GPU:0, GPU:1...) integer, optional, intent(in) :: batch_size !< The batch size for model execution integer, optional, intent(in) :: min_batch_size !< The minimum batch size for model execution + integer, optional, intent(in) :: min_batch_timeout !< Max time (ms) to wait for min batch size character(len=*), optional, intent(in) :: tag !< A tag to attach to the model for !! information purposes character(len=*), dimension(:), optional, intent(in) :: inputs !< One or more names of model input nodes (TF @@ -775,7 +776,7 @@ function set_model_from_file(self, name, model_file, backend, device, batch_size integer(c_size_t), dimension(:), allocatable, target :: input_lengths, output_lengths integer(kind=c_size_t) :: name_length, model_file_length, backend_length, device_length, tag_length, n_inputs, & n_outputs - integer(kind=c_int) :: c_batch_size, c_min_batch_size + integer(kind=c_int) :: c_batch_size, c_min_batch_size, c_min_batch_timeout type(c_ptr) :: inputs_ptr, input_lengths_ptr, outputs_ptr, output_lengths_ptr type(c_ptr), dimension(:), allocatable :: ptrs_to_inputs, ptrs_to_outputs @@ -784,6 +785,8 @@ function set_model_from_file(self, name, model_file, backend, device, batch_size if (present(batch_size)) c_batch_size = batch_size c_min_batch_size = 0 if (present(min_batch_size)) c_min_batch_size = min_batch_size + c_min_batch_timeout = 0 + if (present(min_batch_timeout)) c_min_batch_timeout = min_batch_timeout if (present(tag)) then allocate(character(kind=c_char, len=len_trim(tag)) :: c_tag) c_tag = tag @@ -828,8 +831,8 @@ function set_model_from_file(self, name, model_file, backend, device, batch_size code = set_model_from_file_c(self%client_ptr, c_name, name_length, c_model_file, model_file_length, & c_backend, backend_length, c_device, device_length, c_batch_size, c_min_batch_size, & - c_tag, tag_length, inputs_ptr, input_lengths_ptr, n_inputs, outputs_ptr, & - output_lengths_ptr, n_outputs) + c_min_batch_timeout, c_tag, tag_length, inputs_ptr, input_lengths_ptr, n_inputs, & + outputs_ptr, output_lengths_ptr, n_outputs) if (allocated(c_inputs)) deallocate(c_inputs) if (allocated(input_lengths)) deallocate(input_lengths) if (allocated(ptrs_to_inputs)) deallocate(ptrs_to_inputs) @@ -840,7 +843,7 @@ end function set_model_from_file !> Load the machine learning model from a file and set the configuration for use in multi-GPU systems function set_model_from_file_multigpu(self, name, model_file, backend, first_gpu, num_gpus, batch_size, min_batch_size, & - tag, inputs, outputs) result(code) + min_batch_timeout, tag, inputs, outputs) result(code) class(client_type), intent(in) :: self !< An initialized SmartRedis client character(len=*), intent(in) :: name !< The name to use to place the model character(len=*), intent(in) :: model_file !< The file storing the model @@ -849,6 +852,7 @@ function set_model_from_file_multigpu(self, name, model_file, backend, first_gpu integer, intent(in) :: num_gpus !< The number of GPUs to use with the model integer, optional, intent(in) :: batch_size !< The batch size for model execution integer, optional, intent(in) :: min_batch_size !< The minimum batch size for model execution + integer, optional, intent(in) :: min_batch_timeout !< Max time (ms) to wait for min batch size character(len=*), optional, intent(in) :: tag !< A tag to attach to the model for !! information purposes character(len=*), dimension(:), optional, intent(in) :: inputs !< One or more names of model input nodes (TF @@ -868,7 +872,7 @@ function set_model_from_file_multigpu(self, name, model_file, backend, first_gpu integer(c_size_t), dimension(:), allocatable, target :: input_lengths, output_lengths integer(kind=c_size_t) :: name_length, model_file_length, backend_length, tag_length, n_inputs, & n_outputs - integer(kind=c_int) :: c_batch_size, c_min_batch_size, c_first_gpu, c_num_gpus + integer(kind=c_int) :: c_batch_size, c_min_batch_size, c_min_batch_timeout, c_first_gpu, c_num_gpus type(c_ptr) :: inputs_ptr, input_lengths_ptr, outputs_ptr, output_lengths_ptr type(c_ptr), dimension(:), allocatable :: ptrs_to_inputs, ptrs_to_outputs @@ -877,6 +881,8 @@ function set_model_from_file_multigpu(self, name, model_file, backend, first_gpu if (present(batch_size)) c_batch_size = batch_size c_min_batch_size = 0 if (present(min_batch_size)) c_min_batch_size = min_batch_size + c_min_batch_timeout = 0 + if (present(min_batch_timeout)) c_min_batch_timeout = min_batch_timeout if (present(tag)) then allocate(character(kind=c_char, len=len_trim(tag)) :: c_tag) c_tag = tag @@ -922,8 +928,8 @@ function set_model_from_file_multigpu(self, name, model_file, backend, first_gpu code = set_model_from_file_multigpu_c(self%client_ptr, c_name, name_length, c_model_file, model_file_length, & c_backend, backend_length, c_first_gpu, c_num_gpus, c_batch_size, c_min_batch_size, & - c_tag, tag_length, inputs_ptr, input_lengths_ptr, n_inputs, outputs_ptr, & - output_lengths_ptr, n_outputs) + c_min_batch_timeout, c_tag, tag_length, inputs_ptr, input_lengths_ptr, n_inputs, & + outputs_ptr, output_lengths_ptr, n_outputs) if (allocated(c_inputs)) deallocate(c_inputs) if (allocated(input_lengths)) deallocate(input_lengths) @@ -934,8 +940,8 @@ function set_model_from_file_multigpu(self, name, model_file, backend, first_gpu end function set_model_from_file_multigpu !> Establish a model to run -function set_model(self, name, model, backend, device, batch_size, min_batch_size, tag, & - inputs, outputs) result(code) +function set_model(self, name, model, backend, device, batch_size, min_batch_size, min_batch_timeout, & + tag, inputs, outputs) result(code) class(client_type), intent(in) :: self !< An initialized SmartRedis client character(len=*), intent(in) :: name !< The name to use to place the model character(len=*), intent(in) :: model !< The binary representation of the model @@ -943,6 +949,7 @@ function set_model(self, name, model, backend, device, batch_size, min_batch_siz character(len=*), intent(in) :: device !< The name of the device (CPU, GPU, GPU:0, GPU:1...) integer, intent(in) :: batch_size !< The batch size for model execution integer, intent(in) :: min_batch_size !< The minimum batch size for model execution + integer, intent(in) :: min_batch_timeout !< Max time (ms) to wait for min batch size character(len=*), intent(in) :: tag !< A tag to attach to the model for information purposes character(len=*), dimension(:), intent(in) :: inputs !< One or more names of model input nodes (TF models) character(len=*), dimension(:), intent(in) :: outputs !< One or more names of model output nodes (TF models) @@ -960,7 +967,7 @@ function set_model(self, name, model, backend, device, batch_size, min_batch_siz integer(c_size_t), dimension(:), allocatable, target :: input_lengths, output_lengths integer(kind=c_size_t) :: name_length, model_length, backend_length, device_length, tag_length, n_inputs, & n_outputs - integer(kind=c_int) :: c_batch_size, c_min_batch_size + integer(kind=c_int) :: c_batch_size, c_min_batch_size, c_min_batch_timeout type(c_ptr) :: inputs_ptr, input_lengths_ptr, outputs_ptr, output_lengths_ptr type(c_ptr), dimension(:), allocatable :: ptrs_to_inputs, ptrs_to_outputs @@ -984,12 +991,13 @@ function set_model(self, name, model, backend, device, batch_size, min_batch_siz output_lengths_ptr, n_outputs) if (code /= SRNoError) return - ! Cast the batch sizes to C integers + ! Cast the batch params to C integers c_batch_size = batch_size c_min_batch_size = min_batch_size + c_min_batch_timeout = min_batch_timeout code = set_model_c(self%client_ptr, c_name, name_length, c_model, model_length, c_backend, backend_length, & - c_device, device_length, batch_size, min_batch_size, c_tag, tag_length, & + c_device, device_length, batch_size, min_batch_size, c_min_batch_timeout, c_tag, tag_length, & inputs_ptr, input_lengths_ptr, n_inputs, outputs_ptr, output_lengths_ptr, n_outputs) if (allocated(c_inputs)) deallocate(c_inputs) @@ -1001,8 +1009,8 @@ function set_model(self, name, model, backend, device, batch_size, min_batch_siz end function set_model !> Set a model from a byte string to run on a system with multiple GPUs -function set_model_multigpu(self, name, model, backend, first_gpu, num_gpus, batch_size, min_batch_size, tag, & - inputs, outputs) result(code) +function set_model_multigpu(self, name, model, backend, first_gpu, num_gpus, batch_size, min_batch_size, & + min_batch_timeout, tag, inputs, outputs) result(code) class(client_type), intent(in) :: self !< An initialized SmartRedis client character(len=*), intent(in) :: name !< The name to use to place the model character(len=*), intent(in) :: model !< The binary representation of the model @@ -1011,6 +1019,7 @@ function set_model_multigpu(self, name, model, backend, first_gpu, num_gpus, bat integer, intent(in) :: num_gpus !< The number of GPUs to use with the model integer, intent(in) :: batch_size !< The batch size for model execution integer, intent(in) :: min_batch_size !< The minimum batch size for model execution + integer, intent(in) :: min_batch_timeout !< Max time (ms) to wait for min batch size character(len=*), intent(in) :: tag !< A tag to attach to the model for information purposes character(len=*), dimension(:), intent(in) :: inputs !< One or more names of model input nodes (TF models) character(len=*), dimension(:), intent(in) :: outputs !< One or more names of model output nodes (TF models) @@ -1026,7 +1035,7 @@ function set_model_multigpu(self, name, model, backend, first_gpu, num_gpus, bat integer(c_size_t), dimension(:), allocatable, target :: input_lengths, output_lengths integer(kind=c_size_t) :: name_length, model_length, backend_length, tag_length, n_inputs, n_outputs - integer(kind=c_int) :: c_batch_size, c_min_batch_size, c_first_gpu, c_num_gpus + integer(kind=c_int) :: c_batch_size, c_min_batch_size, c_min_batch_timeout, c_first_gpu, c_num_gpus type(c_ptr) :: inputs_ptr, input_lengths_ptr, outputs_ptr, output_lengths_ptr type(c_ptr), dimension(:), allocatable :: ptrs_to_inputs, ptrs_to_outputs @@ -1048,14 +1057,15 @@ function set_model_multigpu(self, name, model, backend, first_gpu, num_gpus, bat output_lengths_ptr, n_outputs) if (code /= SRNoError) return - ! Cast the batch sizes to C integers + ! Cast the batch params to C integers c_batch_size = batch_size c_min_batch_size = min_batch_size + c_min_batch_timeout = min_batch_timeout c_first_gpu = first_gpu c_num_gpus = num_gpus code = set_model_multigpu_c(self%client_ptr, c_name, name_length, c_model, model_length, c_backend, backend_length, & - c_first_gpu, c_num_gpus, c_batch_size, c_min_batch_size, c_tag, tag_length, & + c_first_gpu, c_num_gpus, c_batch_size, c_min_batch_size, c_min_batch_timeout, c_tag, tag_length, & inputs_ptr, input_lengths_ptr, n_inputs, outputs_ptr, output_lengths_ptr, n_outputs) if (allocated(c_inputs)) deallocate(c_inputs) diff --git a/src/fortran/client/model_interfaces.inc b/src/fortran/client/model_interfaces.inc index d3836bc1c..ef7d46661 100644 --- a/src/fortran/client/model_interfaces.inc +++ b/src/fortran/client/model_interfaces.inc @@ -40,8 +40,9 @@ end interface interface function set_model_from_file_c( c_client, key, key_length, model_file, model_file_length, & - backend, backend_length, device, device_length, batch_size, min_batch_size, tag, tag_length, & - inputs, input_lengths, n_inputs, outputs, output_lengths, n_outputs ) bind(c, name="set_model_from_file") + backend, backend_length, device, device_length, batch_size, min_batch_size, min_batch_timeout, & + tag, tag_length, inputs, input_lengths, n_inputs, outputs, output_lengths, n_outputs ) & + bind(c, name="set_model_from_file") use iso_c_binding, only : c_ptr, c_size_t, c_int, c_char import :: enum_kind integer(kind=enum_kind) :: set_model_from_file_c @@ -59,6 +60,7 @@ interface !! null terminating character integer(kind=c_int), value, intent(in) :: batch_size !< The batch size for model execution integer(kind=c_int), value, intent(in) :: min_batch_size !< The minimum batch size for model execution + integer(kind=c_int), value, intent(in) :: min_batch_timeout !< Max time (ms) to wait for min batch size character(kind=c_char), intent(in) :: tag(*) !< A tag to attach to the model for information !! purposes integer(kind=c_size_t), value, intent(in) :: tag_length !< The length of the tag c-string, excluding null @@ -77,8 +79,9 @@ end interface interface function set_model_from_file_multigpu_c( c_client, key, key_length, model_file, model_file_length, & - backend, backend_length, first_gpu, num_gpus, batch_size, min_batch_size, tag, tag_length, & - inputs, input_lengths, n_inputs, outputs, output_lengths, n_outputs ) bind(c, name="set_model_from_file_multigpu") + backend, backend_length, first_gpu, num_gpus, batch_size, min_batch_size, min_batch_timeout, & + tag, tag_length, inputs, input_lengths, n_inputs, outputs, output_lengths, n_outputs) & + bind(c, name="set_model_from_file_multigpu") use iso_c_binding, only : c_ptr, c_size_t, c_int, c_char import :: enum_kind integer(kind=enum_kind) :: set_model_from_file_multigpu_c @@ -96,6 +99,7 @@ interface !! null terminating character integer(kind=c_int), value, intent(in) :: batch_size !< The batch size for model execution integer(kind=c_int), value, intent(in) :: min_batch_size !< The minimum batch size for model execution + integer(kind=c_int), value, intent(in) :: min_batch_timeout !< Max time (ms) to wait for min batch size character(kind=c_char), intent(in) :: tag(*) !< A tag to attach to the model for information !! purposes integer(kind=c_size_t), value, intent(in) :: tag_length !< The length of the tag c-string, excluding null @@ -114,8 +118,9 @@ end interface interface function set_model_c( c_client, key, key_length, model, model_length, & - backend, backend_length, device, device_length, batch_size, min_batch_size, tag, tag_length, & - inputs, input_lengths, n_inputs, outputs, output_lengths, n_outputs ) bind(c, name="set_model") + backend, backend_length, device, device_length, batch_size, min_batch_size, min_batch_timeout, & + tag, tag_length, inputs, input_lengths, n_inputs, outputs, output_lengths, n_outputs) & + bind(c, name="set_model") use iso_c_binding, only : c_ptr, c_size_t, c_int, c_char import :: enum_kind integer(kind=enum_kind) :: set_model_c @@ -133,6 +138,7 @@ interface !! null terminating character integer(kind=c_int), value, intent(in) :: batch_size !< The batch size for model execution integer(kind=c_int), value, intent(in) :: min_batch_size !< The minimum batch size for model execution + integer(kind=c_int), value, intent(in) :: min_batch_timeout !< Max time (ms) to wait for min batch size character(kind=c_char), intent(in) :: tag(*) !< A tag to attach to the model for information !! purposes integer(kind=c_size_t), value, intent(in) :: tag_length !< The length of the tag c-string, excluding null @@ -151,8 +157,9 @@ end interface interface function set_model_multigpu_c( c_client, key, key_length, model, model_length, & - backend, backend_length, first_gpu, num_gpus, batch_size, min_batch_size, tag, tag_length, & - inputs, input_lengths, n_inputs, outputs, output_lengths, n_outputs ) bind(c, name="set_model_multigpu") + backend, backend_length, first_gpu, num_gpus, batch_size, min_batch_size, min_batch_timeout, & + tag, tag_length, inputs, input_lengths, n_inputs, outputs, output_lengths, n_outputs) & + bind(c, name="set_model_multigpu") use iso_c_binding, only : c_ptr, c_size_t, c_int, c_char import :: enum_kind integer(kind=enum_kind) :: set_model_multigpu_c @@ -170,6 +177,7 @@ interface !! null terminating character integer(kind=c_int), value, intent(in) :: batch_size !< The batch size for model execution integer(kind=c_int), value, intent(in) :: min_batch_size !< The minimum batch size for model execution + integer(kind=c_int), value, intent(in) :: min_batch_timeout !< Max time (ms) to wait for min batch size character(kind=c_char), intent(in) :: tag(*) !< A tag to attach to the model for information !! purposes integer(kind=c_size_t), value, intent(in) :: tag_length !< The length of the tag c-string, excluding null diff --git a/src/python/module/smartredis/client.py b/src/python/module/smartredis/client.py index 53270f966..30361d3ba 100644 --- a/src/python/module/smartredis/client.py +++ b/src/python/module/smartredis/client.py @@ -609,6 +609,7 @@ def set_model( device: str = "CPU", batch_size: int = 0, min_batch_size: int = 0, + min_batch_timeout: int = 0, tag: str = "", inputs: t.Optional[t.Union[str, t.List[str]]] = None, outputs: t.Optional[t.Union[str, t.List[str]]] = None, @@ -636,6 +637,8 @@ def set_model( :type batch_size: int, optional :param min_batch_size: minimum batch size for model execution, defaults to 0 :type min_batch_size: int, optional + :param min_batch_timeout: Max time (ms) to wait for min batch size + :type min_batch_timeout: int, optional :param tag: additional tag for model information, defaults to "" :type tag: str, optional :param inputs: model inputs (TF only), defaults to None @@ -649,6 +652,7 @@ def set_model( typecheck(device, "device", str) typecheck(batch_size, "batch_size", int) typecheck(min_batch_size, "min_batch_size", int) + typecheck(min_batch_timeout, "min_batch_timeout", int) typecheck(tag, "tag", str) device = self.__check_device(device) backend = self.__check_backend(backend) @@ -660,6 +664,7 @@ def set_model( device, batch_size, min_batch_size, + min_batch_timeout, tag, inputs, outputs, @@ -675,6 +680,7 @@ def set_model_multigpu( num_gpus: int, batch_size: int = 0, min_batch_size: int = 0, + min_batch_timeout: int = 0, tag: str = "", inputs: t.Optional[t.Union[str, t.List[str]]] = None, outputs: t.Optional[t.Union[str, t.List[str]]] = None, @@ -703,6 +709,8 @@ def set_model_multigpu( :type batch_size: int, optional :param min_batch_size: minimum batch size for model execution, defaults to 0 :type min_batch_size: int, optional + :param min_batch_timeout: Max time (ms) to wait for min batch size + :type min_batch_timeout: int, optional :param tag: additional tag for model information, defaults to "" :type tag: str, optional :param inputs: model inputs (TF only), defaults to None @@ -717,6 +725,7 @@ def set_model_multigpu( typecheck(num_gpus, "num_gpus", int) typecheck(batch_size, "batch_size", int) typecheck(min_batch_size, "min_batch_size", int) + typecheck(min_batch_timeout, "min_batch_timeout", int) typecheck(tag, "tag", str) backend = self.__check_backend(backend) inputs, outputs = self.__check_tensor_args(inputs, outputs) @@ -728,6 +737,7 @@ def set_model_multigpu( num_gpus, batch_size, min_batch_size, + min_batch_timeout, tag, inputs, outputs, @@ -742,6 +752,7 @@ def set_model_from_file( device: str = "CPU", batch_size: int = 0, min_batch_size: int = 0, + min_batch_timeout: int = 0, tag: str = "", inputs: t.Optional[t.Union[str, t.List[str]]] = None, outputs: t.Optional[t.Union[str, t.List[str]]] = None, @@ -769,6 +780,8 @@ def set_model_from_file( :type batch_size: int, optional :param min_batch_size: minimum batch size for model execution, defaults to 0 :type min_batch_size: int, optional + :param min_batch_timeout: Max time (ms) to wait for min batch size + :type min_batch_timeout: int, optional :param tag: additional tag for model information, defaults to "" :type tag: str, optional :param inputs: model inputs (TF only), defaults to None @@ -783,6 +796,7 @@ def set_model_from_file( typecheck(device, "device", str) typecheck(batch_size, "batch_size", int) typecheck(min_batch_size, "min_batch_size", int) + typecheck(min_batch_timeout, "min_batch_timeout", int) typecheck(tag, "tag", str) device = self.__check_device(device) backend = self.__check_backend(backend) @@ -795,6 +809,7 @@ def set_model_from_file( device, batch_size, min_batch_size, + min_batch_timeout, tag, inputs, outputs, @@ -810,6 +825,7 @@ def set_model_from_file_multigpu( num_gpus: int, batch_size: int = 0, min_batch_size: int = 0, + min_batch_timeout: int = 0, tag: str = "", inputs: t.Optional[t.Union[str, t.List[str]]] = None, outputs: t.Optional[t.Union[str, t.List[str]]] = None, @@ -838,6 +854,8 @@ def set_model_from_file_multigpu( :type batch_size: int, optional :param min_batch_size: minimum batch size for model execution, defaults to 0 :type min_batch_size: int, optional + :param min_batch_timeout: Max time (ms) to wait for min batch size + :type min_batch_timeout: int, optional :param tag: additional tag for model information, defaults to "" :type tag: str, optional :param inputs: model inputs (TF only), defaults to None @@ -853,6 +871,7 @@ def set_model_from_file_multigpu( typecheck(num_gpus, "num_gpus", int) typecheck(batch_size, "batch_size", int) typecheck(min_batch_size, "min_batch_size", int) + typecheck(min_batch_timeout, "min_batch_timeout", int) typecheck(tag, "tag", str) backend = self.__check_backend(backend) m_file = self.__check_file(model_file) @@ -865,6 +884,7 @@ def set_model_from_file_multigpu( num_gpus, batch_size, min_batch_size, + min_batch_timeout, tag, inputs, outputs, diff --git a/src/python/src/pyclient.cpp b/src/python/src/pyclient.cpp index eb9b497d3..f174fa253 100644 --- a/src/python/src/pyclient.cpp +++ b/src/python/src/pyclient.cpp @@ -326,14 +326,15 @@ void PyClient::set_model(const std::string& name, const std::string& device, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs) { MAKE_CLIENT_API({ _client->set_model(name, model, backend, device, - batch_size, min_batch_size, tag, - inputs, outputs); + batch_size, min_batch_size, min_batch_timeout, + tag, inputs, outputs); }); } @@ -344,14 +345,15 @@ void PyClient::set_model_multigpu(const std::string& name, int num_gpus, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs) { MAKE_CLIENT_API({ _client->set_model_multigpu(name, model, backend, first_gpu, num_gpus, - batch_size, min_batch_size, tag, - inputs, outputs); + batch_size, min_batch_size, min_batch_timeout, + tag, inputs, outputs); }); } @@ -361,14 +363,15 @@ void PyClient::set_model_from_file(const std::string& name, const std::string& device, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs) { MAKE_CLIENT_API({ _client->set_model_from_file(name, model_file, backend, device, - batch_size, min_batch_size, tag, - inputs, outputs); + batch_size, min_batch_size, min_batch_timeout, + tag, inputs, outputs); }); } @@ -379,6 +382,7 @@ void PyClient::set_model_from_file_multigpu(const std::string& name, int num_gpus, int batch_size, int min_batch_size, + int min_batch_timeout, const std::string& tag, const std::vector& inputs, const std::vector& outputs) @@ -386,7 +390,7 @@ void PyClient::set_model_from_file_multigpu(const std::string& name, MAKE_CLIENT_API({ _client->set_model_from_file_multigpu( name, model_file, backend, first_gpu, num_gpus, batch_size, - min_batch_size, tag, inputs, outputs); + min_batch_size, min_batch_timeout, tag, inputs, outputs); }); } diff --git a/tests/python/test_errors.py b/tests/python/test_errors.py index bba7fde07..c330dc996 100644 --- a/tests/python/test_errors.py +++ b/tests/python/test_errors.py @@ -444,6 +444,8 @@ def test_bad_type_set_model_from_file_multigpu(use_cluster, context): c.set_model_from_file_multigpu("simple_cnn", modelfile, "TORCH", 0, 1, batch_size="not_an_integer") with pytest.raises(TypeError): c.set_model_from_file_multigpu("simple_cnn", modelfile, "TORCH", 0, 1, min_batch_size="not_an_integer") + with pytest.raises(TypeError): + c.set_model_from_file_multigpu("simple_cnn", modelfile, "TORCH", 0, 1, min_batch_timeout="not_an_integer") with pytest.raises(TypeError): c.set_model_from_file_multigpu("simple_cnn", modelfile, "TORCH", 0, 1, tag=42) diff --git a/tests/python/test_model_methods_torch.py b/tests/python/test_model_methods_torch.py index b1c7b078b..d98c6bed7 100644 --- a/tests/python/test_model_methods_torch.py +++ b/tests/python/test_model_methods_torch.py @@ -27,8 +27,12 @@ import os import torch +import pytest +from os import environ from smartredis import Client +from smartredis.error import * +test_gpu = environ.get("SMARTREDIS_TEST_DEVICE","cpu").lower() == "gpu" def test_set_model(mock_model, use_cluster, context): model = mock_model.create_torch_cnn() @@ -69,3 +73,124 @@ def test_torch_inference(mock_model, use_cluster, context): c.run_model("torch_cnn", inputs=["torch_cnn_input"], outputs=["torch_cnn_output"]) out_data = c.get_tensor("torch_cnn_output") assert out_data.shape == (1, 1, 1, 1) + +def test_batch_exceptions(mock_model, use_cluster, context): + # get model and set into database + mock_model.create_torch_cnn(filepath="./torch_cnn.pt") + model = mock_model.create_torch_cnn() + c = Client(None, use_cluster, logger_name=context) + batch_size = 1 + min_batch_size = 1 + min_batch_timeout = 1 + with pytest.raises(RedisRuntimeError): + c.set_model_from_file( + "file_cnn", "./torch_cnn.pt", "TORCH", "CPU", + batch_size=0, min_batch_size=0, min_batch_timeout=min_batch_timeout + ) + with pytest.raises(RedisRuntimeError): + c.set_model_from_file( + "file_cnn", "./torch_cnn.pt", "TORCH", "CPU", + batch_size=0, min_batch_size=min_batch_size, min_batch_timeout=0 + ) + with pytest.raises(RedisRuntimeError): + c.set_model_from_file( + "file_cnn", "./torch_cnn.pt", "TORCH", "CPU", + batch_size=batch_size, min_batch_size=0, min_batch_timeout=min_batch_timeout + ) + with pytest.raises(RedisRuntimeError): + c.set_model_from_file_multigpu( + "file_cnn", "./torch_cnn.pt", "TORCH", 1, 1, + batch_size=0, min_batch_size=0, min_batch_timeout=min_batch_timeout + ) + with pytest.raises(RedisRuntimeError): + c.set_model_from_file_multigpu( + "file_cnn", "./torch_cnn.pt", "TORCH", 1, 1, + batch_size=0, min_batch_size=min_batch_size, min_batch_timeout=0 + ) + with pytest.raises(RedisRuntimeError): + c.set_model_from_file_multigpu( + "file_cnn", "./torch_cnn.pt", "TORCH", 1, 1, + batch_size=batch_size, min_batch_size=0, min_batch_timeout=min_batch_timeout + ) + with pytest.raises(RedisRuntimeError): + c.set_model( + "file_cnn", model, "TORCH", "CPU", + batch_size=0, min_batch_size=0, min_batch_timeout=min_batch_timeout + ) + with pytest.raises(RedisRuntimeError): + c.set_model( + "file_cnn", model, "TORCH", "CPU", + batch_size=0, min_batch_size=min_batch_size, min_batch_timeout=0 + ) + with pytest.raises(RedisRuntimeError): + c.set_model( + "file_cnn", model, "TORCH", "CPU", + batch_size=batch_size, min_batch_size=0, min_batch_timeout=min_batch_timeout + ) + with pytest.raises(RedisRuntimeError): + c.set_model_multigpu( + "file_cnn", model, "TORCH", 1, 1, + batch_size=0, min_batch_size=0, min_batch_timeout=min_batch_timeout + ) + with pytest.raises(RedisRuntimeError): + c.set_model_multigpu( + "file_cnn", model, "TORCH", 1, 1, + batch_size=0, min_batch_size=min_batch_size, min_batch_timeout=0 + ) + with pytest.raises(RedisRuntimeError): + c.set_model_multigpu( + "file_cnn", model, "TORCH", 1, 1, + batch_size=batch_size, min_batch_size=0, min_batch_timeout=min_batch_timeout + ) + +def test_batch_warning_set_model_from_file(mock_model, use_cluster, context, capfd): + # get model and set into database + mock_model.create_torch_cnn(filepath="./torch_cnn.pt") + c = Client(None, use_cluster, logger_name=context) + c.set_model_from_file( + "file_cnn", "./torch_cnn.pt", "TORCH", "CPU", + batch_size=1, min_batch_size=1, min_batch_timeout=0 + ) + captured = capfd.readouterr() + assert "WARNING" in captured.err + +@pytest.mark.skipif( + not test_gpu, + reason="SMARTREDIS_TEST_DEVICE does not specify 'gpu'" +) +def test_batch_warning_set_model_from_file_multigpu(mock_model, use_cluster, context, capfd): + # get model and set into database + mock_model.create_torch_cnn(filepath="./torch_cnn.pt") + c = Client(None, use_cluster, logger_name=context) + c.set_model_from_file_multigpu( + "file_cnn", "./torch_cnn.pt", "TORCH", 1, 1, + batch_size=1, min_batch_size=1, min_batch_timeout=0 + ) + captured = capfd.readouterr() + assert "WARNING" in captured.err + +def test_batch_warning_set_model(mock_model, use_cluster, context, capfd): + # get model and set into database + model = mock_model.create_torch_cnn() + c = Client(None, use_cluster, logger_name=context) + c.set_model( + "file_cnn", model, "TORCH", "CPU", + batch_size=1, min_batch_size=1, min_batch_timeout=0 + ) + captured = capfd.readouterr() + assert "WARNING" in captured.err + +@pytest.mark.skipif( + not test_gpu, + reason="SMARTREDIS_TEST_DEVICE does not specify 'gpu'" +) +def test_batch_warning_set_model_multigpu(mock_model, use_cluster, context, capfd): + # get model and set into database + model = mock_model.create_torch_cnn() + c = Client(None, use_cluster, logger_name=context) + c.set_model_multigpu( + "file_cnn", model, "TORCH", 1, 1, + batch_size=1, min_batch_size=1, min_batch_timeout=0 + ) + captured = capfd.readouterr() + assert "WARNING" in captured.err From 90f6c87caff69b1b8fdb669e200bf72bdc3cf9de Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Wed, 4 Oct 2023 17:08:17 -0500 Subject: [PATCH 06/25] Update RedisAI version in post commit checking (#408) Remove unsupported versions of RedisAI in post-commit testing [ committed by @billschereriii ] [ reviewed by @ashao ] --- .github/workflows/run_post_merge_tests.yml | 3 ++- .github/workflows/run_tests.yml | 5 ----- doc/changelog.rst | 3 +++ 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_post_merge_tests.yml b/.github/workflows/run_post_merge_tests.yml index 5660584fe..d61aaf62c 100644 --- a/.github/workflows/run_post_merge_tests.yml +++ b/.github/workflows/run_post_merge_tests.yml @@ -56,7 +56,7 @@ jobs: matrix: os: [ubuntu-20.04] # cannot test on macOS as docker isn't supported on Mac compiler: [intel, 8, 9, 10, 11] # intel compiler, and versions of GNU compiler - rai_v: [1.2.4, 1.2.5] # versions of RedisAI + rai_v: [1.2.7] # version(s) of RedisAI py_v: ['3.7.x', '3.8.x', '3.9.x', '3.10.x'] # versions of Python env: FC: gfortran-${{ matrix.compiler }} @@ -149,6 +149,7 @@ jobs: if: "!contains( matrix.compiler, 'intel' )" # if using GNU compiler run: make test-examples SR_FORTRAN=ON SR_PYTHON=ON \ SR_TEST_PORT=7000 SR_TEST_REDISAI_VER=v${{ matrix.rai_v }} + - name: Test static build -- intel compilers if: contains(matrix.compiler, 'intel') run: make build-example-serial SR_FORTRAN=ON diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index bbcb343c2..b527cde00 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -82,11 +82,6 @@ jobs: sudo rm -rf /opt/ghc && sudo rm -rf "/usr/local/share/boost" - # sudo rm -rf /usr/share/dotnet && - # sudo rm -rf /opt/ghc && - # sudo rm -rf "/usr/local/share/boost" && - # sudo rm -rf "$AGENT_TOOLSDIRECTORY" - # Install compilers (Intel or GCC) - name: Install GCC if: "!contains( matrix.compiler, 'intel' )" # if using GNU compiler diff --git a/doc/changelog.rst b/doc/changelog.rst index 1e12c96ad..4f3434710 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Updated RedisAI version used in post-commit check-in testing in Github pipeline - Improved support for model execution batching - Added support for model chunking - Updated the third-party RedisAI component @@ -16,12 +17,14 @@ Description Detailed Notes +- Updated RedisAI version used in post-commit check-in testing in Github pipeline to a version that supports fetch of model chunking size (PR408_) - Exposed access to the Redis.AI MINBATCHTIMEOUT parameter, which limits the delay in model execution when trying to accumulate multiple executions in a batch (PR406_) - Models will now be automatically chunked when sent to/received from the backed database. This allows use of models greater than 511MB in size. (PR404_) - Updated from RedisAI v1.2.3 (test target)/v1.2.4 and v1.2.5 (CI/CD pipeline) to v1.2.7 (PR402_) - Updated lcov from version 1.15 to 2.0 (PR396_) - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) +.. _PR408: https://github.com/CrayLabs/SmartRedis/pull/408 .. _PR406: https://github.com/CrayLabs/SmartRedis/pull/406 .. _PR404: https://github.com/CrayLabs/SmartRedis/pull/404 .. _PR402: https://github.com/CrayLabs/SmartRedis/pull/402 From dd5e8fec0a56ded2a3a72dbc32769610a37c796a Mon Sep 17 00:00:00 2001 From: amandarichardsonn <30413257+amandarichardsonn@users.noreply.github.com> Date: Fri, 6 Oct 2023 18:10:13 -0500 Subject: [PATCH 07/25] Allow strings in Python run_script methods (#407) Previously, users could only pass lists of strings into the run_script methods in Python. This was inconvenient because many users have scripts that only have single inputs and outputs. Users are now able to pass in a single string to these methods. [ committed by @amandarichardsonn ] [ reviewed by @billschereriii ] --- doc/changelog.rst | 4 ++ src/python/module/smartredis/client.py | 22 +++--- tests/python/test_errors.py | 95 ++++++++++++++++++++++++-- tests/python/test_script_methods.py | 52 ++++++++++++-- 4 files changed, 151 insertions(+), 22 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 4f3434710..11ca1a424 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -9,15 +9,18 @@ To be released at some future point in time Description - Updated RedisAI version used in post-commit check-in testing in Github pipeline +- Allow strings in Python interface for Client.run_script, Client.run_script_multiGPU - Improved support for model execution batching - Added support for model chunking - Updated the third-party RedisAI component - Updated the third-party lcov component +- Add link to contributing guidelines - Added link to contributing guidelines Detailed Notes - Updated RedisAI version used in post-commit check-in testing in Github pipeline to a version that supports fetch of model chunking size (PR408_) +- Allow users to pass single keys for the inputs and outputs parameters as a string for Python run_script and run_script_multigpu - Exposed access to the Redis.AI MINBATCHTIMEOUT parameter, which limits the delay in model execution when trying to accumulate multiple executions in a batch (PR406_) - Models will now be automatically chunked when sent to/received from the backed database. This allows use of models greater than 511MB in size. (PR404_) - Updated from RedisAI v1.2.3 (test target)/v1.2.4 and v1.2.5 (CI/CD pipeline) to v1.2.7 (PR402_) @@ -25,6 +28,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) .. _PR408: https://github.com/CrayLabs/SmartRedis/pull/408 +.. _PR407: https://github.com/CrayLabs/SmartRedis/pull/407 .. _PR406: https://github.com/CrayLabs/SmartRedis/pull/406 .. _PR404: https://github.com/CrayLabs/SmartRedis/pull/404 .. _PR402: https://github.com/CrayLabs/SmartRedis/pull/402 diff --git a/src/python/module/smartredis/client.py b/src/python/module/smartredis/client.py index 30361d3ba..bb02c587b 100644 --- a/src/python/module/smartredis/client.py +++ b/src/python/module/smartredis/client.py @@ -466,7 +466,11 @@ def get_script(self, name: str) -> str: @exception_handler def run_script( - self, name: str, fn_name: str, inputs: t.List[str], outputs: t.List[str] + self, + name: str, + fn_name: str, + inputs: t.Union[str, t.List[str]], + outputs: t.Union[str, t.List[str]] ) -> None: """Execute TorchScript stored inside the database @@ -482,15 +486,13 @@ def run_script( :param fn_name: name of a function within the script to execute :type fn_name: str :param inputs: database tensor names to use as script inputs - :type inputs: list[str] + :type inputs: str | list[str] :param outputs: database tensor names to receive script outputs - :type outputs: list[str] + :type outputs: str | list[str] :raises RedisReplyError: if script execution fails """ typecheck(name, "name", str) typecheck(fn_name, "fn_name", str) - typecheck(inputs, "inputs", list) - typecheck(outputs, "outputs", list) inputs, outputs = self.__check_tensor_args(inputs, outputs) self._client.run_script(name, fn_name, inputs, outputs) @@ -499,8 +501,8 @@ def run_script_multigpu( self, name: str, fn_name: str, - inputs: t.List[str], - outputs: t.List[str], + inputs: t.Union[str, t.List[str]], + outputs: t.Union[str, t.List[str]], offset: int, first_gpu: int, num_gpus: int, @@ -519,9 +521,9 @@ def run_script_multigpu( :param fn_name: name of a function within the script to execute :type fn_name: str :param inputs: database tensor names to use as script inputs - :type inputs: list[str] + :type inputs: str | list[str] :param outputs: database tensor names to receive script outputs - :type outputs: list[str] + :type outputs: str | list[str] :param offset: index of the current image, such as a processor ID or MPI rank :type offset: int @@ -533,8 +535,6 @@ def run_script_multigpu( """ typecheck(name, "name", str) typecheck(fn_name, "fn_name", str) - typecheck(inputs, "inputs", list) - typecheck(outputs, "outputs", list) typecheck(offset, "offset", int) typecheck(first_gpu, "first_gpu", int) typecheck(num_gpus, "num_gpus", int) diff --git a/tests/python/test_errors.py b/tests/python/test_errors.py index c330dc996..3a6b9afa4 100644 --- a/tests/python/test_errors.py +++ b/tests/python/test_errors.py @@ -28,10 +28,13 @@ import numpy as np import pytest +from os import environ from smartredis import * from smartredis.error import * +test_gpu = environ.get("SMARTREDIS_TEST_DEVICE","cpu").lower() == "gpu" + @pytest.fixture def cfg_opts() -> ConfigOptions: opts = ConfigOptions.create_from_environment("") @@ -88,6 +91,8 @@ def test_bad_function_execution(use_cluster, context): c.put_tensor("bad-func-tensor", data) with pytest.raises(RedisReplyError): c.run_script("bad-function", "bad_function", ["bad-func-tensor"], ["output"]) + with pytest.raises(RedisReplyError): + c.run_script("bad-function", "bad_function", "bad-func-tensor", "output") def test_missing_script_function(use_cluster, context): @@ -96,11 +101,42 @@ def test_missing_script_function(use_cluster, context): c = Client(None, use_cluster, logger_name=context) c.set_function("bad-function", bad_function) with pytest.raises(RedisReplyError): - c.run_script( - "bad-function", "not-a-function-in-script", ["bad-func-tensor"], ["output"] - ) + c.run_script("bad-function", "not-a-function-in-script", ["bad-func-tensor"], ["output"]) + with pytest.raises(RedisReplyError): + c.run_script("bad-function", "not-a-function-in-script", "bad-func-tensor", "output") +@pytest.mark.skipif( + not test_gpu, + reason="SMARTREDIS_TEST_DEVICE does not specify 'gpu'" +) +def test_bad_function_execution_multigpu(use_cluster, context): + """Error raised inside function""" + c = Client(None, use_cluster, logger_name=context) + c.set_function_multigpu("bad-function", bad_function, 0, 1) + data = np.array([1, 2, 3, 4]) + c.put_tensor("bad-func-tensor", data) + with pytest.raises(RedisReplyError): + c.run_script_multigpu("bad-function", "bad_function", ["bad-func-tensor"], ["output"], 0, 0, 2) + with pytest.raises(RedisReplyError): + c.run_script_multigpu("bad-function", "bad_function", "bad-func-tensor", "output", 0, 0, 2) + + +@pytest.mark.skipif( + not test_gpu, + reason="SMARTREDIS_TEST_DEVICE does not specify 'gpu'" +) +def test_missing_script_function_multigpu(use_cluster, context): + """User requests to run a function not in the script""" + + c = Client(None, use_cluster, logger_name=context) + c.set_function_multigpu("bad-function", bad_function, 0, 1) + with pytest.raises(RedisReplyError): + c.run_script_multigpu("bad-function", "not-a-function-in-script", ["bad-func-tensor"], ["output"], 0, 0, 2) + with pytest.raises(RedisReplyError): + c.run_script_multigpu("bad-function", "not-a-function-in-script", "bad-func-tensor", "output", 0, 0, 2) + + def test_wrong_model_name(mock_data, mock_model, use_cluster, context): """User requests to run a model that is not there""" @@ -306,12 +342,12 @@ def test_bad_type_get_script(use_cluster, context): c.get_script(42) -def test_bad_type_run_script(use_cluster, context): +def test_bad_type_run_script_str(use_cluster, context): c = Client(None, use_cluster, logger_name=context) key = "my_script" fn_name = "phred" - inputs = ["list", "of", "strings"] - outputs = ["another", "string", "list"] + inputs = "a string" + outputs = "another string" with pytest.raises(TypeError): c.run_script(42, fn_name, inputs, outputs) with pytest.raises(TypeError): @@ -322,12 +358,28 @@ def test_bad_type_run_script(use_cluster, context): c.run_script(key, fn_name, inputs, 42) -def test_bad_type_run_script_multigpu(use_cluster, context): +def test_bad_type_run_script_list(use_cluster, context): c = Client(None, use_cluster, logger_name=context) key = "my_script" fn_name = "phred" inputs = ["list", "of", "strings"] outputs = ["another", "string", "list"] + with pytest.raises(TypeError): + c.run_script(42, fn_name, inputs, outputs) + with pytest.raises(TypeError): + c.run_script(key, 42, inputs, outputs) + with pytest.raises(TypeError): + c.run_script(key, fn_name, 42, outputs) + with pytest.raises(TypeError): + c.run_script(key, fn_name, inputs, 42) + + +def test_bad_type_run_script_multigpu_str(use_cluster, context): + c = Client(None, use_cluster, logger_name=context) + key = "my_script" + fn_name = "phred" + inputs = "a string" + outputs = "another string" offset = 0 first_gpu = 0 num_gpus = 1 @@ -351,6 +403,35 @@ def test_bad_type_run_script_multigpu(use_cluster, context): c.run_script_multigpu(key, fn_name, inputs, outputs, offset, first_gpu, 0) +def test_bad_type_run_script_multigpu_list(use_cluster, context): + c = Client(None, use_cluster, logger_name=context) + key = "my_script" + fn_name = "phred" + inputs = ["list", "of", "strings"] + outputs = ["another", "string", "list"] + offset = 0 + first_gpu = 0 + num_gpus = 1 + with pytest.raises(TypeError): + c.run_script_multigpu(42, fn_name, inputs, outputs, offset, first_gpu, num_gpus) + with pytest.raises(TypeError): + c.run_script_multigpu(key, 42, inputs, outputs, offset, first_gpu, num_gpus) + with pytest.raises(TypeError): + c.run_script_multigpu(key, fn_name, 42, outputs, offset, first_gpu, num_gpus) + with pytest.raises(TypeError): + c.run_script_multigpu(key, fn_name, inputs, 42, offset, first_gpu, num_gpus) + with pytest.raises(TypeError): + c.run_script_multigpu(key, fn_name, inputs, outputs, "not an integer", first_gpu, num_gpus) + with pytest.raises(TypeError): + c.run_script_multigpu(key, fn_name, inputs, outputs, offset, "not an integer", num_gpus) + with pytest.raises(TypeError): + c.run_script_multigpu(key, fn_name, inputs, outputs, offset, first_gpu, "not an integer") + with pytest.raises(ValueError): + c.run_script_multigpu(key, fn_name, inputs, outputs, offset, -1, num_gpus) + with pytest.raises(ValueError): + c.run_script_multigpu(key, fn_name, inputs, outputs, offset, first_gpu, 0) + + def test_bad_type_get_model(use_cluster, context): c = Client(None, use_cluster, logger_name=context) with pytest.raises(TypeError): diff --git a/tests/python/test_script_methods.py b/tests/python/test_script_methods.py index 4e9295661..00655c938 100644 --- a/tests/python/test_script_methods.py +++ b/tests/python/test_script_methods.py @@ -24,9 +24,10 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import pytest import inspect import os.path as osp - +from os import environ import numpy as np import torch from smartredis import Client @@ -34,6 +35,8 @@ file_path = osp.dirname(osp.abspath(__file__)) +test_gpu = environ.get("SMARTREDIS_TEST_DEVICE","cpu").lower() == "gpu" + def test_set_get_function(use_cluster, context): c = Client(None, use_cluster, logger_name=context) c.set_function("test-set-function", one_to_one) @@ -63,18 +66,18 @@ def test_set_script_from_file(use_cluster, context): assert not c.model_exists("test-script-file") -def test_run_script(use_cluster, context): +def test_run_script_str(use_cluster, context): data = np.array([[1, 2, 3, 4, 5]]) c = Client(None, use_cluster, logger_name=context) c.put_tensor("script-test-data", data) c.set_function("one-to-one", one_to_one) - c.run_script("one-to-one", "one_to_one", ["script-test-data"], ["script-test-out"]) + c.run_script("one-to-one", "one_to_one", "script-test-data", "script-test-out") out = c.get_tensor("script-test-out") assert out == 5 -def test_run_script_multi(use_cluster, context): +def test_run_script_list(use_cluster, context): data = np.array([[1, 2, 3, 4]]) data_2 = np.array([[5, 6, 7, 8]]) @@ -94,6 +97,47 @@ def test_run_script_multi(use_cluster, context): out, expected, "Returned array from script not equal to expected result" ) +@pytest.mark.skipif( + not test_gpu, + reason="SMARTREDIS_TEST_DEVICE does not specify 'gpu'" +) +def test_run_script_multigpu_str(use_cluster, context): + data = np.array([[1, 2, 3, 4, 5]]) + + c = Client(None, use_cluster, logger_name=context) + c.put_tensor("script-test-data", data) + c.set_function_multigpu("one-to-one", one_to_one, 0, 2) + c.run_script_multigpu("one-to-one", "one_to_one", "script-test-data", "script-test-out", 0, 0, 2) + out = c.get_tensor("script-test-out") + assert out == 5 + +@pytest.mark.skipif( + not test_gpu, + reason="SMARTREDIS_TEST_DEVICE does not specify 'gpu'" +) +def test_run_script_multigpu_list(use_cluster, context): + data = np.array([[1, 2, 3, 4]]) + data_2 = np.array([[5, 6, 7, 8]]) + + c = Client(None, use_cluster, logger_name=context) + c.put_tensor("srpt-multi-out-data-1", data) + c.put_tensor("srpt-multi-out-data-2", data_2) + c.set_function_multigpu("two-to-one", two_to_one, 0, 2) + c.run_script_multigpu( + "two-to-one", + "two_to_one", + ["srpt-multi-out-data-1", "srpt-multi-out-data-2"], + ["srpt-multi-out-output"], + 0, + 0, + 2 + ) + out = c.get_tensor("srpt-multi-out-output") + expected = np.array([4, 8]) + np.testing.assert_array_equal( + out, expected, "Returned array from script not equal to expected result" + ) + def one_to_one(data): """Sample torchscript script that returns the From 8e82e01aba49238bc83ce6da51f9b48e05573163 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Wed, 11 Oct 2023 16:25:56 -0500 Subject: [PATCH 08/25] Multidb support for SmartRedis (#353) Add support for Multiple Databases in SmartRedis. This involves a Client class constructor update -- the type of database is now stored in an environment variable rather than being passed in to the Client constructor. The legacy constructor will remain supported for the short term but will be phased out in favor of the new constructor. Updated all test cases and examples with the new Client class constructor. Updated documentation. [ committed by @billschereriii ] [ reviewed by @ashao @MattToast ] --- .github/workflows/run_post_merge_tests.yml | 2 - Makefile | 6 +- conftest.py | 4 - doc/advanced_topics.rst | 71 +++- doc/changelog.rst | 5 +- doc/developer/testing.rst | 2 +- doc/install/docker.rst | 43 +-- doc/runtime.rst | 26 +- examples/parallel/cpp/smartredis_mnist.cpp | 3 +- .../parallel/cpp/smartredis_put_get_3D.cpp | 3 +- examples/parallel/fortran/example_utils.F90 | 14 +- .../parallel/fortran/smartredis_dataset.F90 | 2 +- .../parallel/fortran/smartredis_mnist.F90 | 2 +- .../fortran/smartredis_put_get_3D.F90 | 2 +- examples/serial/c/example_put_get_3D.c | 3 +- examples/serial/c/example_put_unpack_1D.c | 3 +- examples/serial/cpp/smartredis_dataset.cpp | 3 +- examples/serial/cpp/smartredis_mnist.cpp | 7 +- examples/serial/cpp/smartredis_model.cpp | 3 +- examples/serial/cpp/smartredis_put_get_3D.cpp | 3 +- .../serial/fortran/smartredis_dataset.F90 | 2 +- .../serial/fortran/smartredis_put_get_3D.F90 | 2 +- .../serial/python/example_model_file_torch.py | 3 +- examples/serial/python/example_model_torch.py | 3 +- .../serial/python/example_put_get_dataset.py | 3 +- .../serial/python/example_put_get_tensor.py | 2 +- examples/serial/python/example_script_file.py | 2 +- examples/test_examples.py | 2 +- images/Dockerfile | 2 +- include/c_client.h | 44 +++ include/c_configoptions.h | 14 +- include/client.h | 46 ++- include/configoptions.h | 60 +++- include/pyclient.h | 20 +- include/pyconfigoptions.h | 12 +- include/redis.h | 10 +- include/rediscluster.h | 8 +- include/redisserver.h | 13 +- src/c/c_client.cpp | 49 ++- src/c/c_configoptions.cpp | 10 +- src/cpp/client.cpp | 100 +++++- src/cpp/configoptions.cpp | 50 ++- src/cpp/redis.cpp | 19 +- src/cpp/rediscluster.cpp | 20 +- src/cpp/redisserver.cpp | 41 ++- src/fortran/client.F90 | 78 ++++- src/fortran/client/client_interfaces.inc | 31 +- src/fortran/configoptions.F90 | 14 +- .../configoptions_interfaces.inc | 6 +- src/python/bindings/bind.cpp | 2 + src/python/module/smartredis/client.py | 90 ++++- src/python/module/smartredis/configoptions.py | 27 +- src/python/src/pyclient.cpp | 18 + src/python/src/pyconfigoptions.cpp | 4 +- tests/c/c_client_test_utils.h | 14 +- tests/c/client_test_dataset_aggregation.c | 2 +- tests/c/client_test_dataset_exists.c | 4 +- tests/c/client_test_logging.c | 5 +- tests/c/client_test_put_get_1D.c | 25 +- tests/c/client_test_put_get_2D.c | 24 +- tests/c/client_test_put_get_3D.c | 23 +- tests/c/client_test_put_unpack_1D.c | 9 +- tests/c/test_c_client.py | 6 +- tests/cpp/client_test_copy_dataset.cpp | 2 +- tests/cpp/client_test_dataset.cpp | 2 +- tests/cpp/client_test_dataset_aggregation.cpp | 4 +- .../client_test_dataset_copy_assignment.cpp | 2 +- .../client_test_dataset_copy_constructor.cpp | 4 +- tests/cpp/client_test_dataset_empty.cpp | 2 +- .../client_test_dataset_move_assignment.cpp | 2 +- .../client_test_dataset_move_constructor.cpp | 2 +- ...lient_test_dataset_multiple_get_tensor.cpp | 2 +- tests/cpp/client_test_dataset_no_meta.cpp | 2 +- tests/cpp/client_test_dataset_no_tensors.cpp | 2 +- tests/cpp/client_test_delete_dataset.cpp | 2 +- tests/cpp/client_test_ensemble.cpp | 10 +- tests/cpp/client_test_ensemble_dataset.cpp | 8 +- tests/cpp/client_test_mnist.cpp | 4 +- tests/cpp/client_test_mnist_dataset.cpp | 4 +- tests/cpp/client_test_put_get_1D.cpp | 2 +- tests/cpp/client_test_put_get_2D.cpp | 2 +- tests/cpp/client_test_put_get_3D.cpp | 2 +- .../client_test_put_get_3D_static_values.cpp | 2 +- .../cpp/client_test_put_get_contiguous_3D.cpp | 2 +- .../cpp/client_test_put_get_transpose_3D.cpp | 2 +- tests/cpp/client_test_rename_dataset.cpp | 2 +- tests/cpp/client_test_utils.h | 95 +++--- tests/cpp/dataset_test_utils.h | 8 +- tests/cpp/test_cpp_client.py | 6 +- .../cpp/unit-tests/test_aggregation_list.cpp | 2 +- tests/cpp/unit-tests/test_client.cpp | 31 +- tests/cpp/unit-tests/test_client_ensemble.cpp | 4 +- .../cpp/unit-tests/test_client_prefixing.cpp | 4 +- tests/cpp/unit-tests/test_configopts.cpp | 28 +- tests/cpp/unit-tests/test_logger.cpp | 2 +- tests/cpp/unit-tests/test_redisserver.cpp | 24 +- tests/cpp/unit-tests/test_ssdb.cpp | 16 +- tests/cpp/unit-tests/test_unit_cpp_client.py | 6 +- tests/docker/c/test_docker.c | 2 +- tests/docker/cpp/docker_test.cpp | 2 +- tests/docker/fortran/CMakeLists.txt | 23 +- tests/docker/fortran/test_docker.F90 | 2 +- tests/docker/python/test_docker.py | 2 +- tests/fortran/client_test_configoptions.F90 | 24 +- tests/fortran/client_test_dataset.F90 | 2 +- .../client_test_dataset_aggregation.F90 | 3 +- tests/fortran/client_test_ensemble.F90 | 4 +- tests/fortran/client_test_errors.F90 | 6 +- tests/fortran/client_test_initialized.F90 | 2 +- tests/fortran/client_test_logging.F90 | 2 +- tests/fortran/client_test_misc_tensor.F90 | 2 +- tests/fortran/client_test_mnist.F90 | 2 +- tests/fortran/client_test_mnist_multigpu.F90 | 3 +- tests/fortran/client_test_prefixing.F90 | 2 +- tests/fortran/client_test_put_get_1D.F90 | 2 +- tests/fortran/client_test_put_get_2D.F90 | 2 +- tests/fortran/client_test_put_get_3D.F90 | 2 +- .../client_test_put_get_unpack_dataset.F90 | 3 +- tests/fortran/test_utils.F90 | 20 +- tests/python/test_address.py | 13 +- tests/python/test_configoptions.py | 8 +- tests/python/test_dataset_aggregation.py | 6 +- tests/python/test_dataset_ops.py | 24 +- tests/python/test_errors.py | 314 +++++++++--------- tests/python/test_logging.py | 6 +- tests/python/test_model_methods_torch.py | 32 +- tests/python/test_nonkeyed_cmd.py | 93 ++---- tests/python/test_prefixing.py | 5 +- tests/python/test_put_get_dataset.py | 8 +- tests/python/test_put_get_tensor.py | 12 +- tests/python/test_script_methods.py | 20 +- tests/python/test_tensor_ops.py | 24 +- 132 files changed, 1302 insertions(+), 795 deletions(-) diff --git a/.github/workflows/run_post_merge_tests.yml b/.github/workflows/run_post_merge_tests.yml index d61aaf62c..37995c0af 100644 --- a/.github/workflows/run_post_merge_tests.yml +++ b/.github/workflows/run_post_merge_tests.yml @@ -43,8 +43,6 @@ env: HOMEBREW_NO_GITHUB_API: "ON" HOMEBREW_NO_INSTALL_CLEANUP: "ON" DEBIAN_FRONTEND: "noninteractive" # disable interactive apt installs - SSDB: "127.0.0.1:6379" - SMARTREDIS_TEST_CLUSTER: False jobs: diff --git a/Makefile b/Makefile index 4afa911ea..b15d14ad9 100644 --- a/Makefile +++ b/Makefile @@ -319,7 +319,7 @@ SSDB_STRING := $(shell echo $(SSDB_STRING) | tr -d " ") # 1: the test directory in which to run tests define run_smartredis_tests_with_standalone_server echo "Launching standalone Redis server" && \ - export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_SERVER_MODE=Standalone && \ + export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_DB_TYPE=Standalone && \ export SMARTREDIS_TEST_CLUSTER=False SMARTREDIS_TEST_DEVICE=$(SR_TEST_DEVICE) && \ export SSDB=127.0.0.1:$(SR_TEST_PORT) && \ python utils/launch_redis.py --port $(SR_TEST_PORT) --nodes 1 \ @@ -341,7 +341,7 @@ endef # 1: the test directory in which to run tests define run_smartredis_tests_with_clustered_server echo "Launching clustered Redis server" && \ - export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_SERVER_MODE=Clustered && \ + export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_DB_TYPE=Clustered && \ export SMARTREDIS_TEST_CLUSTER=True SMARTREDIS_TEST_DEVICE=$(SR_TEST_DEVICE) && \ export SSDB=$(SSDB_STRING) && \ python utils/launch_redis.py --port $(SR_TEST_PORT) --nodes $(SR_TEST_NODES) \ @@ -365,7 +365,7 @@ endef # 1: the test directory in which to run tests define run_smartredis_tests_with_uds_server echo "Launching standalone Redis server with Unix Domain Socket support" - export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_SERVER_MODE=Standalone && \ + export SR_TEST_DEVICE=$(SR_TEST_DEVICE) SR_DB_TYPE=Standalone && \ export SMARTREDIS_TEST_CLUSTER=False SMARTREDIS_TEST_DEVICE=$(SR_TEST_DEVICE) && \ export SSDB=unix://$(SR_TEST_UDS_FILE) && \ python utils/launch_redis.py --port $(SR_TEST_PORT) --nodes 1 \ diff --git a/conftest.py b/conftest.py index 1cb4c924e..a379dbdf2 100644 --- a/conftest.py +++ b/conftest.py @@ -53,10 +53,6 @@ np.uint64, ] -@pytest.fixture -def use_cluster(): - return os.getenv('SMARTREDIS_TEST_CLUSTER', "").lower() == 'true' - @pytest.fixture def mock_data(): return MockTestData diff --git a/doc/advanced_topics.rst b/doc/advanced_topics.rst index d21d828a9..50be73c7e 100644 --- a/doc/advanced_topics.rst +++ b/doc/advanced_topics.rst @@ -92,15 +92,15 @@ predicate is matched on the length of the list: .. code-block:: cpp - # Block until the list reaches a specific length + // Block until the list reaches a specific length bool poll_list_length(const std::string& name, int list_length, int poll_frequency_ms, int num_tries); - # Block until the list reaches or exceeds a specific length + // Block until the list reaches or exceeds a specific length bool poll_list_length_gte(const std::string& name, int list_length, int poll_frequency_ms, int num_tries); - # Block until the list is no longer than a specific length + // Block until the list is no longer than a specific length bool poll_list_length_lte(const std::string& name, int list_length, int poll_frequency_ms, int num_tries); @@ -114,13 +114,72 @@ lead to race conditions: .. code-block:: cpp - # Copy an aggregation list + // Copy an aggregation list void copy_list(const std::string& src_name, const std::string& dest_name); - # Rename an aggregation list + // Rename an aggregation list void rename_list(const std::string& src_name, const std::string& dest_name); - # Delete an aggregation list + // Delete an aggregation list void delete_list(const std::string& list_name); + +.. _advanced_topics_dataset_aggregation: + +Multiple Database Support +========================= + +SmartRedis offers clients the ability to interact with multiple databases +concurrently. Each Client represents a connection to a specific database, +but an application with multiple clients can have each one connected to a +different database. + +Differentiating databases via environment variables +--------------------------------------------------- + +In order to differentiate the databases that clients connect to, SmartRedis +relies on differentiation in the environment variables that the client uses +to initialize itself. Of primary importance here are the SSDB and SR_DB_TYPE +variables, but all environment variables (other than SR_LOG_LEVEL and +SR_LOG_FILE, which are shared for all databases) are duplicated in order to +represent additional databases. + +This duplication is done via suffixing: an underscore and the identifier for +the database are suffixed to the base variable names to derive a set of +environment variables specific to each database. For example, SSDB_INPUT +and SR_DB_TYPE_INPUT reflect a database named ``INPUT``. + +For backward compatibility, the default database is anonymous and thus its +environment variables use neither an underscore nor a database name. This +behavior exactly matches earlier releases of SmartRedis. + +Instantiating Clients for named databases +----------------------------------------- + +Beginning with version 0.5.0 of SmartRedis, users can initialize Clients +using a new construction method that accepts a ConfigOptions object as +an input parameter. In turn, the ConfigOptions object can be constructed +via the ConfigOptions create_from_environment() factory method, which +accepts the suffix to be applied to environment variables when looking +them up (or an empty string, to indicate that the default names should be +used, as for an anonymous database). Depending on the programming language +for the SmartRedis client, variously None, NULL, or skipping the +ConfigOptions parameter altogether also implicitly requests an anonymous +database. + +For example, to create a Client for a database named ``INPUT`` in C++, +one would write the following code: + +.. code-block:: cpp + + // Create a ConfigOptions object + auto co = ConfigOptions::create_from_environment("INPUT"); + + // Pass it to the Client constructor along with an identifier for logging + Client* input_client = new Client(co, "input_client"); + +Note that with the Client constructor that accepts a ConfigOptions object, +there is no parameter for whether the database is clustered or not. This is +because the type of database is now read in from the SR_DB_TYPE environment +variable (with optional {_suffix}). \ No newline at end of file diff --git a/doc/changelog.rst b/doc/changelog.rst index 11ca1a424..5dcfbd1d1 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -16,6 +16,7 @@ Description - Updated the third-party lcov component - Add link to contributing guidelines - Added link to contributing guidelines +- Added support for multiple backend databases via a new Client constructor that accepts a ConfigOptions object Detailed Notes @@ -26,6 +27,7 @@ Detailed Notes - Updated from RedisAI v1.2.3 (test target)/v1.2.4 and v1.2.5 (CI/CD pipeline) to v1.2.7 (PR402_) - Updated lcov from version 1.15 to 2.0 (PR396_) - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) +- Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) .. _PR408: https://github.com/CrayLabs/SmartRedis/pull/408 .. _PR407: https://github.com/CrayLabs/SmartRedis/pull/407 @@ -34,6 +36,7 @@ Detailed Notes .. _PR402: https://github.com/CrayLabs/SmartRedis/pull/402 .. _PR396: https://github.com/CrayLabs/SmartRedis/pull/396 .. _PR395: https://github.com/CrayLabs/SmartRedis/pull/395 +.. _PR353: https://github.com/CrayLabs/SmartRedis/pull/353 0.4.2 ----- @@ -131,7 +134,7 @@ Detailed Notes - Enable parallel build for the SmartRedis examples by moving utility Fortran code into a small static library (PR349_) - For the NVidia toolchain only: Replaces the assumed rank feature of F2018 used in the Fortran client with assumed shape arrays, making it possible to compile SmartRedis with the Nvidia toolchain. (PR346_) - Rework the build and test system to improve maintainability of the library. There have been several significant changes, including that Python and Fortran clients are no longer built by defaults and that there are Make variables that customize the build process. Please review the build documentation and ``make help`` to see all that has changed. (PR341_) -- Many Fortran routines were returning logical kind = c_bool which turns out not to be the same default kind of most Fortran compilers. These have now been refactored so that users need not import `iso_c_binding` in their own applications (PR340_) +- Many Fortran routines were returning logical kind = c_bool which turns out not to be the same default kind of most Fortran compilers. These have now been refactored so that users need not import `iso_c_binding` in their own applications (PR340_) - Update MacOS version in CI/CD tests from 10.15 to 12.0 (PR339_) - Correct the spelling of the C DataSet destruction interface from DeallocateeDataSet to DeallocateDataSet (PR338_) - Updated the version of Redis++ to v1.3.8 to pull in a change that ensures the redis++.pc file properly points to the generated libraries (PR334_) diff --git a/doc/developer/testing.rst b/doc/developer/testing.rst index 11b806dd5..d0b899a0b 100644 --- a/doc/developer/testing.rst +++ b/doc/developer/testing.rst @@ -8,7 +8,7 @@ Quick instructions To run the tests, assuming that all requirements have been installed 1. Activate your environment with SmartSim and SmartRedis installed -2. Modify `SMARTREDIS_TEST_CLUSTER` (`True` or `False` and +2. Modify `SR_DB_TYPE` (`Clustered` or `Standalone`) and `SMARTREDIS_TEST_DEVICE` (`gpu` or `cpu`) as necessary in `setup_test_env.sh`. 3. `source setup_test_env.sh` diff --git a/doc/install/docker.rst b/doc/install/docker.rst index dbca0d616..11113a93c 100644 --- a/doc/install/docker.rst +++ b/doc/install/docker.rst @@ -149,15 +149,14 @@ for a containerized application. **Fortran** -In addition to the SmartRedis dynamic -library installed in ``/usr/local/lib/`` and library -header files installed in ``/usr/local/include/smartredis/``, -the Fortran source files needed to compile a Fortran application -with SmartRedis are installed in -``/usr/local/src/SmartRedis/src/fortran/``. +The SmartRedis and SmartRedis-Fortran dynamic +library needed to compile a Fortran application +with SmartRedis are installed in ``/usr/local/lib/`` +and the library header files are installed in +``/usr/local/include/smartredis/``. An example CMake file that builds a Fortran application -using the ``smartredis`` image can be found +using the ``smartredis`` images can be found at ``./tests/docker/fortran/CMakeLists.txt``. This CMake file is also shown below along with an example Docker file that invokes the CMake files @@ -168,31 +167,37 @@ for a containerized application. # Fortran containerized CMake file cmake_minimum_required(VERSION 3.13) - project(DockerTester) + project(DockerTesterFortran) enable_language(Fortran) + # Configure the build set(CMAKE_CXX_STANDARD 17) - set(CMAKE_C_STANDARD 99) - - set(ftn_client_src - /usr/local/src/SmartRedis/src/fortran/fortran_c_interop.F90 - /usr/local/src/SmartRedis/src/fortran/dataset.F90 - /usr/local/src/SmartRedis/src/fortran/client.F90 + SET(CMAKE_C_STANDARD 99) + set(CMAKE_BUILD_TYPE Debug) + + # Locate dependencies + find_library(SR_LIB smartredis REQUIRED) + find_library(SR_FTN_LIB smartredis-fortran REQUIRED) + set(SMARTREDIS_LIBRARIES + ${SR_LIB} + ${SR_FTN_LIB} ) - find_library(SR_LIB smartredis) - + # Define include directories for header files include_directories(SYSTEM /usr/local/include/smartredis ) - add_executable(docker_test + # Build the test + add_executable(docker_test_fortran test_docker.F90 - ${ftn_client_src} ) + set_target_properties(docker_test_fortran PROPERTIES + OUTPUT_NAME docker_test + ) + target_link_libraries(docker_test_fortran ${SMARTREDIS_LIBRARIES} pthread) - target_link_libraries(docker_test ${SR_LIB} pthread) .. code-block:: docker diff --git a/doc/runtime.rst b/doc/runtime.rst index 844bd1c7d..8cc668950 100644 --- a/doc/runtime.rst +++ b/doc/runtime.rst @@ -10,8 +10,8 @@ and to retrieve the correct information from the database. In the following sections, these requirements will be described. -Setting Redis Database Location -=============================== +Setting Redis Database Location and Type +======================================== The C++, C, and Fortran clients retrieve the Redis database location from the @@ -37,10 +37,22 @@ at three different addresses, each using port ``6379``: export SSDB="10.128.0.153:6379,10.128.0.154:6379,10.128.0.155:6379" -The Python client relies on ``SSDB`` to determine database -location. However, the Python ``Client`` constructor also allows -for the database location to be set as an input parameter. In -this case, it sets SSDB from the input parameter. + +There are two types of Redis databases that can be used by the +SmartRedis library. A ``Clustered`` database, such as the one in +the previous example, is replicated across multiple shards. +By way of comparison, a ``Standalone`` database only has a single +shard that services all traffic; this is the form used when a +colocated database or a standard deployment with a non-sharded +database is requested. + +The ``SR_DB_TYPE`` environment variable informs the SmartRedis +library which form is in use. Below is an example of setting +``SR_DB_TYPE`` for a Redis cluster: + +.. code-block:: bash + + export SR_DB_TYPE="Clustered" Logging Environment Variables ============================= @@ -69,7 +81,7 @@ though this can happen only if the variables to set up logging are in place. If this parameter is not set, a default logging level of ``INFO`` will be adopted. The runtime impact of log levels NONE or INFO should be minimal on -client performance; however, seting the log level to DEBUG may cause some +client performance; however, setting the log level to DEBUG may cause some degradation. Ensemble Environment Variables diff --git a/examples/parallel/cpp/smartredis_mnist.cpp b/examples/parallel/cpp/smartredis_mnist.cpp index ea4bb1fca..97a8d24f5 100644 --- a/examples/parallel/cpp/smartredis_mnist.cpp +++ b/examples/parallel/cpp/smartredis_mnist.cpp @@ -104,8 +104,7 @@ int main(int argc, char* argv[]) { logger_name += std::to_string(rank); // Initialize a Client object - bool cluster_mode = true; // Set to false if not using a clustered database - SmartRedis::Client client(cluster_mode, logger_name); + SmartRedis::Client client(logger_name); // Set the model and script that will be used by all ranks // from MPI rank 0. diff --git a/examples/parallel/cpp/smartredis_put_get_3D.cpp b/examples/parallel/cpp/smartredis_put_get_3D.cpp index 370a3a245..fbdc86ee1 100644 --- a/examples/parallel/cpp/smartredis_put_get_3D.cpp +++ b/examples/parallel/cpp/smartredis_put_get_3D.cpp @@ -57,8 +57,7 @@ int main(int argc, char* argv[]) { logger_name += std::to_string(rank); // Initialize a SmartRedis client - bool cluster_mode = true; // Set to false if not using a clustered database - SmartRedis::Client client(cluster_mode, logger_name); + SmartRedis::Client client(logger_name); // Put the tensor in the database std::string key = "3d_tensor_" + std::to_string(rank); diff --git a/examples/parallel/fortran/example_utils.F90 b/examples/parallel/fortran/example_utils.F90 index 23e5397e4..4e999ebac 100644 --- a/examples/parallel/fortran/example_utils.F90 +++ b/examples/parallel/fortran/example_utils.F90 @@ -44,15 +44,15 @@ end function irand logical function use_cluster() - character(len=16) :: smartredis_test_cluster + character(len=16) :: server_type - call get_environment_variable('SMARTREDIS_TEST_CLUSTER', smartredis_test_cluster) - smartredis_test_cluster = to_lower(smartredis_test_cluster) - if (len_trim(smartredis_test_cluster)>0) then - select case (smartredis_test_cluster) - case ('true') + call get_environment_variable('SR_DB_TYPE', server_type) + server_type = to_lower(server_type) + if (len_trim(server_type)>0) then + select case (server_type) + case ('clustered') use_cluster = .true. - case ('false') + case ('standalone') use_cluster = .false. case default use_cluster = .false. diff --git a/examples/parallel/fortran/smartredis_dataset.F90 b/examples/parallel/fortran/smartredis_dataset.F90 index b16534f4b..7b8da6a87 100644 --- a/examples/parallel/fortran/smartredis_dataset.F90 +++ b/examples/parallel/fortran/smartredis_dataset.F90 @@ -85,7 +85,7 @@ program main if (result .ne. SRNoError) error stop 'dataset%get_meta_scalars failed' ! Initialize a client - result = client%initialize(.true., "smartredis_dataset") ! Change .false. to .true. if not using a clustered database + result = client%initialize("smartredis_dataset") if (result .ne. SRNoError) error stop 'client%initialize failed' ! Send the dataset to the database via the client diff --git a/examples/parallel/fortran/smartredis_mnist.F90 b/examples/parallel/fortran/smartredis_mnist.F90 index 912744e62..90771a988 100644 --- a/examples/parallel/fortran/smartredis_mnist.F90 +++ b/examples/parallel/fortran/smartredis_mnist.F90 @@ -51,7 +51,7 @@ program mnist_example write(key_suffix, "(A,I1.1)") "_",pe_id ! Initialize a client - result = client%initialize(.true., "smartredis_mnist") ! Change .false. to .true. if not using a clustered database + result = client%initialize("smartredis_mnist") if (result .ne. SRNoError) error stop 'client%initialize failed' ! Set up model and script for the computation diff --git a/examples/parallel/fortran/smartredis_put_get_3D.F90 b/examples/parallel/fortran/smartredis_put_get_3D.F90 index e1f308984..cc26dd2b4 100644 --- a/examples/parallel/fortran/smartredis_put_get_3D.F90 +++ b/examples/parallel/fortran/smartredis_put_get_3D.F90 @@ -57,7 +57,7 @@ program main call random_number(recv_array_real_64) ! Initialize a client - result = client%initialize(.true., "smartredis_put_get_3D") ! Change .false. to .true. if not using a clustered database + result = client%initialize("smartredis_put_get_3D") if (result .ne. SRNoError) error stop 'client%initialize failed' ! Add a tensor to the database and verify that we can retrieve it diff --git a/examples/serial/c/example_put_get_3D.c b/examples/serial/c/example_put_get_3D.c index 02c497320..ea66f0d35 100644 --- a/examples/serial/c/example_put_get_3D.c +++ b/examples/serial/c/example_put_get_3D.c @@ -47,8 +47,7 @@ int main(int argc, char* argv[]) { dims[2] = 3; void* client = NULL; - bool cluster_mode = true; // Set to false if not using a clustered database - if (SRNoError != SmartRedisCClient(cluster_mode, logger_name, cid_len, &client)) { + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) { printf("Client initialization failed!\n"); exit(-1); } diff --git a/examples/serial/c/example_put_unpack_1D.c b/examples/serial/c/example_put_unpack_1D.c index df0d28278..7b67efdbf 100644 --- a/examples/serial/c/example_put_unpack_1D.c +++ b/examples/serial/c/example_put_unpack_1D.c @@ -47,8 +47,7 @@ int main(int argc, char* argv[]) { tensor[i] = ((float)rand())/(float)RAND_MAX; void* client = NULL; - bool cluster_mode = true; // Set to false if not using a clustered database - if (SRNoError != SmartRedisCClient(cluster_mode, logger_name, cid_len, &client)) { + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) { printf("Client initialization failed!\n"); exit(-1); } diff --git a/examples/serial/cpp/smartredis_dataset.cpp b/examples/serial/cpp/smartredis_dataset.cpp index 563a7dd0b..b51c58942 100644 --- a/examples/serial/cpp/smartredis_dataset.cpp +++ b/examples/serial/cpp/smartredis_dataset.cpp @@ -55,8 +55,7 @@ int main(int argc, char* argv[]) { int64_t meta_scalar_3 = 3; // Initialize a SmartRedis client - bool cluster_mode = true; // Set to false if not using a clustered database - SmartRedis::Client client(cluster_mode, __FILE__); + SmartRedis::Client client(__FILE__); // Create a DataSet SmartRedis::DataSet dataset("example_dataset"); diff --git a/examples/serial/cpp/smartredis_mnist.cpp b/examples/serial/cpp/smartredis_mnist.cpp index 002acfa28..3dba01e1a 100644 --- a/examples/serial/cpp/smartredis_mnist.cpp +++ b/examples/serial/cpp/smartredis_mnist.cpp @@ -74,11 +74,8 @@ void run_mnist(const std::string& model_name, int main(int argc, char* argv[]) { - // Initialize a client for setting the model and script. - // In general, Client objects should be reused, but for this - // small example, Client objects are not reused. - bool cluster_mode = true; // Set to false if not using a clustered database - SmartRedis::Client client(cluster_mode, __FILE__); + // Initialize a client for setting the model and script + SmartRedis::Client client(__FILE__); // Build model key, file name, and then set model // from file using client API diff --git a/examples/serial/cpp/smartredis_model.cpp b/examples/serial/cpp/smartredis_model.cpp index 93e20d2f9..a76925e5e 100644 --- a/examples/serial/cpp/smartredis_model.cpp +++ b/examples/serial/cpp/smartredis_model.cpp @@ -47,8 +47,7 @@ int main(int argc, char* argv[]) { std::memcpy(img.data(), tmp.data(), img.size()*sizeof(float)); // Initialize a SmartRedis client to connect to the Redis database - bool cluster_mode = true; // Set to false if not using a clustered database - SmartRedis::Client client(cluster_mode, __FILE__); + SmartRedis::Client client(__FILE__); // Use the client to set a model in the database from a file std::string model_key = "mnist_model"; diff --git a/examples/serial/cpp/smartredis_put_get_3D.cpp b/examples/serial/cpp/smartredis_put_get_3D.cpp index 20a04035d..cc7cbfe8b 100644 --- a/examples/serial/cpp/smartredis_put_get_3D.cpp +++ b/examples/serial/cpp/smartredis_put_get_3D.cpp @@ -47,8 +47,7 @@ int main(int argc, char* argv[]) { input_tensor[i] = 2.0*rand()/(double)RAND_MAX - 1.0; // Initialize a SmartRedis client - bool cluster_mode = true; // Set to false if not using a clustered database - SmartRedis::Client client(cluster_mode, __FILE__); + SmartRedis::Client client(__FILE__); // Put the tensor in the database std::string key = "3d_tensor"; diff --git a/examples/serial/fortran/smartredis_dataset.F90 b/examples/serial/fortran/smartredis_dataset.F90 index e3836f457..76d9a4b21 100644 --- a/examples/serial/fortran/smartredis_dataset.F90 +++ b/examples/serial/fortran/smartredis_dataset.F90 @@ -75,7 +75,7 @@ program main if (result .ne. SRNoError) error stop 'dataset%get_meta_scalars failed' ! Initialize a client - result = client%initialize(.true., "smartredis_dataset") ! Change .false. to .true. if not using a clustered database + result = client%initialize("smartredis_dataset") if (result .ne. SRNoError) error stop 'client%initialize failed' ! Send the dataset to the database via the client diff --git a/examples/serial/fortran/smartredis_put_get_3D.F90 b/examples/serial/fortran/smartredis_put_get_3D.F90 index 1b075aada..543affa56 100644 --- a/examples/serial/fortran/smartredis_put_get_3D.F90 +++ b/examples/serial/fortran/smartredis_put_get_3D.F90 @@ -46,7 +46,7 @@ program main call random_number(send_array_real_64) ! Initialize a client - result = client%initialize(.true., "smartredis_put_get_3D") ! Change .false. to .true. if not using a clustered database + result = client%initialize("smartredis_put_get_3D") if (result .ne. SRNoError) error stop 'client%initialize failed' ! Send a tensor to the database via the client and verify that we can retrieve it diff --git a/examples/serial/python/example_model_file_torch.py b/examples/serial/python/example_model_file_torch.py index 826ef1ce2..cd54c0a32 100644 --- a/examples/serial/python/example_model_file_torch.py +++ b/examples/serial/python/example_model_file_torch.py @@ -42,8 +42,7 @@ def forward(self, x): return self.conv(x) # Connect a SmartRedis client -db_address = "127.0.0.1:6379" -client = Client(address=db_address, cluster=True, logger_name="example_model_file_torch.py") +client = Client(logger_name="example_model_file_torch.py") try: net = Net() diff --git a/examples/serial/python/example_model_torch.py b/examples/serial/python/example_model_torch.py index 41e5a5d84..cbfd3fef3 100644 --- a/examples/serial/python/example_model_torch.py +++ b/examples/serial/python/example_model_torch.py @@ -53,8 +53,7 @@ def forward(self, x): model = buffer.getvalue() # Connect a SmartRedis client and set the model in the database -db_address = "127.0.0.1:6379" -client = Client(address=db_address, cluster=True, logger_name="example_model_torch.py") +client = Client(logger_name="example_model_torch.py") client.set_model("torch_cnn", model, "TORCH", "CPU") # Retrieve the model and verify that the retrieved diff --git a/examples/serial/python/example_put_get_dataset.py b/examples/serial/python/example_put_get_dataset.py index 38df029e0..4c041d999 100644 --- a/examples/serial/python/example_put_get_dataset.py +++ b/examples/serial/python/example_put_get_dataset.py @@ -38,8 +38,7 @@ dataset.add_tensor("tensor_2", data_2) # Connect SmartRedis client to Redis database -db_address = "127.0.0.1:6379" -client = Client(address=db_address, cluster=True, logger_name="example_put_get_dataset.py") +client = Client(logger_name="example_put_get_dataset.py") # Place the DataSet into the database client.put_dataset(dataset) diff --git a/examples/serial/python/example_put_get_tensor.py b/examples/serial/python/example_put_get_tensor.py index 30fc941d2..cd5178ac0 100644 --- a/examples/serial/python/example_put_get_tensor.py +++ b/examples/serial/python/example_put_get_tensor.py @@ -29,7 +29,7 @@ # Connect a SmartRedis client to Redis database db_address = "127.0.0.1:6379" -client = Client(address=db_address, cluster=True, logger_name="example_put_get_tensor.py") +client = Client(logger_name="example_put_get_tensor.py") # Send a 2D tensor to the database key = "2D_array" diff --git a/examples/serial/python/example_script_file.py b/examples/serial/python/example_script_file.py index 8d6abd889..8f2a50c0f 100644 --- a/examples/serial/python/example_script_file.py +++ b/examples/serial/python/example_script_file.py @@ -33,7 +33,7 @@ # Connect to the Redis database db_address = "127.0.0.1:6379" -client = Client(address=db_address, cluster=True, logger_name="example_script_file.py") +client = Client(logger_name="example_script_file.py") # Place the script in the database client.set_script_from_file( diff --git a/examples/test_examples.py b/examples/test_examples.py index 966af8a30..46abddc04 100644 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -48,7 +48,7 @@ def get_test_names(): return test_names @pytest.mark.parametrize("test", get_test_names()) -def test_example(test, use_cluster, build, sr_fortran, link): +def test_example(test, build, sr_fortran, link): if (sr_fortran == "ON" or ".F90" not in test): # Build the path to the test executable from the source file name # . keep only the last three parts of the path: (parallel/serial, language, basename) diff --git a/images/Dockerfile b/images/Dockerfile index fde64428e..2a278b613 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -50,7 +50,7 @@ COPY . /usr/local/src/SmartRedis # Compile and install WORKDIR /usr/local/src/SmartRedis RUN pip3 install --upgrade pip -RUN make clobber && make lib && pip install . && rm -rf ~/.cache/pip \ +RUN make clobber && make lib SR_FORTRAN=ON SR_PYTHON=ON && pip install . && rm -rf ~/.cache/pip \ && rm -rf build tests examples images utils third-party doc images # Copy install files and directories to /usr/local/lib and diff --git a/include/c_client.h b/include/c_client.h index 850b8ea9b..9afa554df 100644 --- a/include/c_client.h +++ b/include/c_client.h @@ -42,6 +42,34 @@ extern "C" { #endif +/*! +* \brief C-client simple constructor that uses default environment variables +* to locate configuration settings +* \param logger_name Identifier for the current client +* \param logger_name_length Length in characters of the logger_name string +* \param new_client Receives the new client +* \return Returns SRNoError on success or an error code on failure +*/ +SRError SimpleCreateClient( + const char* logger_name, + const size_t logger_name_length, + void** new_client); + +/*! +* \brief C-client constructor that uses a ConfigOptions object +* to locate configuration settings +* \param config_options The ConfigOptions object to use +* \param logger_name Identifier for the current client +* \param logger_name_length Length in characters of the logger_name string +* \param new_client Receives the new client +* \return Returns SRNoError on success or an error code on failure +*/ +SRError CreateClient( + void* config_options, + const char* logger_name, + const size_t logger_name_length, + void** new_client); + /*! * \brief C-client constructor * \param cluster Flag to indicate if a database cluster is being used @@ -56,6 +84,22 @@ SRError SmartRedisCClient( const size_t logger_name_length, void **new_client); + + +/*! +* \brief C-client constructor (deprecated) +* \param cluster Flag to indicate if a database cluster is being used +* \param logger_name Identifier for the current client +* \param logger_name_length Length in characters of the logger_name string +* \param new_client Receives the new client +* \return Returns SRNoError on success or an error code on failure +*/ +SRError SmartRedisCClient( + bool cluster, + const char* logger_name, + const size_t logger_name_length, + void **new_client); + /*! * \brief C-client destructor * \param c_client A pointer to a pointer to the c-client to destroy. diff --git a/include/c_configoptions.h b/include/c_configoptions.h index f3f342339..642a638a3 100644 --- a/include/c_configoptions.h +++ b/include/c_configoptions.h @@ -44,19 +44,19 @@ extern "C" { /*! * \brief Instantiate ConfigOptions, getting selections from -* environment variables. If \p db_prefix is non-empty, -* then "{db_prefix}_" will be prepended to the name of +* environment variables. If \p db_suffix is non-empty, +* then "_{db_suffix}" will be appended to the name of * each environment variable that is read. -* \param db_prefix The prefix to use with environment variables, -* or an empty string to disable prefixing -* \param db_prefix_length The length of the db_prefix string, +* \param db_suffix The suffix to use with environment variables, +* or an empty string to disable suffixing +* \param db_suffix_length The length of the db_suffix string, * excluding null terminating character * \param new_configoptions Receives the new configoptions object * \returns Returns SRNoError on success or an error code on failure */ SRError create_configoptions_from_environment( - const char* db_prefix, - const size_t db_prefix_length, + const char* db_suffix, + const size_t db_suffix_length, void** new_configoptions); ///////////////////////////////////////////////////////////// diff --git a/include/client.h b/include/client.h index a9bb07614..78c0b40e0 100644 --- a/include/client.h +++ b/include/client.h @@ -56,6 +56,9 @@ namespace SmartRedis { +// class declarations +class ConfigOptions; + /*! * \brief The database response to a command */ @@ -71,8 +74,36 @@ class Client : public SRObject public: + /*! + * \brief Simple Client constructor with default configuration: + * environment variables, no suffix + * \param logger_name Name to use for this client when logging + * \throw SmartRedis::Exception if client connection or + * object initialization fails + */ + Client(const char* logger_name); + + /*! + * \brief Simple Client constructor with default configuration: + * environment variables, no suffix + * \param logger_name Name to use for this client when logging + * \throw SmartRedis::Exception if client connection or + * object initialization fails + */ + Client(const std::string& logger_name = "default") + : Client(logger_name.c_str()) {} + /*! * \brief Client constructor + * \param cfgopts source from which to access runtime settings + * \param logger_name Name to use for this client when logging + * \throw SmartRedis::Exception if client connection or + * object initialization fails + */ + Client(ConfigOptions* cfgopts, const std::string& logger_name = "default"); + + /*! + * \brief Client constructor (deprecated) * \param cluster Flag for if a database cluster is being used * \param logger_name Name to use for this client when logging * \throw SmartRedis::Exception if client connection or @@ -1382,9 +1413,9 @@ class Client : public SRObject /*! * \brief Set the prefixes that are used for set and get methods - * using SSKEYIN and SSKEYOUT environment variables. + * using SSKEYIN and SSKEYOUT config settings */ - void _set_prefixes_from_env(); + void _get_prefix_settings(); /*! * \brief Get the key prefix for placement methods @@ -1535,6 +1566,11 @@ class Client : public SRObject */ bool _use_list_prefix; + /*! + * \brief Our configuration options, used to access runtime settings + */ + ConfigOptions* _cfgopts; + /*! * \brief Build full formatted key of a tensor, based on * current prefix settings. @@ -1706,6 +1742,12 @@ class Client : public SRObject int poll_frequency_ms, int num_tries, std::function comp_func); + /*! + * \brief Initialize a connection to the back-end database + * \throw SmartRedis::Exception if the connection fails + */ + void _establish_server_connection(); + }; /*! diff --git a/include/configoptions.h b/include/configoptions.h index 0ac1d9c91..175359b1e 100644 --- a/include/configoptions.h +++ b/include/configoptions.h @@ -39,6 +39,7 @@ #include #include "srobject.h" #include "sr_enums.h" +#include "srexception.h" ///@file @@ -98,22 +99,43 @@ class ConfigOptions */ virtual ~ConfigOptions(); + /*! + * \brief Deep copy a ConfigOptions object + * \returns The cloned object + * \throw std::bad_alloc on allocation failure + */ + ConfigOptions* clone(); + ///////////////////////////////////////////////////////////// // Factory construction methods /*! * \brief Instantiate ConfigOptions, getting selections from - * environment variables. If \p db_prefix is non-empty, - * then "{db_prefix}_" will be prepended to the name of + * environment variables. If \p db_suffix is non-empty, + * then "{db_suffix}_" will be prepended to the name of * each environment variable that is read. - * \param db_prefix The prefix to use with environment variables, - * or an empty string to disable prefixing + * \param db_suffix The suffix to use with environment variables, + * or an empty string to disable suffixing * \returns The constructed ConfigOptions object - * \throw SmartRedis::Exception if db_prefix contains invalid + * \throw SmartRedis::Exception if db_suffix contains invalid * characters */ static std::unique_ptr create_from_environment( - const std::string& db_prefix); + const std::string& db_suffix); + + /*! + * \brief Instantiate ConfigOptions, getting selections from + * environment variables. If \p db_suffix is non-empty, + * then "{db_suffix}_" will be prepended to the name of + * each environment variable that is read. + * \param db_suffix The suffix to use with environment variables, + * or an empty string to disable suffixing + * \returns The constructed ConfigOptions object + * \throw SmartRedis::Exception if db_suffix contains invalid + * characters + */ + static std::unique_ptr create_from_environment( + const char* db_suffix); ///////////////////////////////////////////////////////////// // Option access @@ -181,16 +203,28 @@ class ConfigOptions * \brief Retrieve the logging context * \returns The log context associated with this object */ - std::string _get_log_context() { return _log_context; } + SRObject* _get_log_context() { + if (_log_context == NULL) { + throw SRRuntimeException( + "Attempt to _get_log_context() before context was set!"); + } + return _log_context; + } /*! * \brief Store the logging context * \param log_context The context to associate with logging */ - void _set_log_context(const std::string& log_context) { + void _set_log_context(SRObject* log_context) { _log_context = log_context; } + /*! + * \brief Clear a configuration option from the cache + * \param option_name The name of the option to clear + */ + void _clear_option_from_cache(const std::string& option_name); + /*! * \brief Stash a string buffer so we can delete it on cleanup * \param buf The buffer to store @@ -238,11 +272,11 @@ class ConfigOptions void _populate_options(); /*! - * \brief Apply a prefix to an option name if the source is environment - * variables and the prefix is nonempty - * \param option_name The name of the option to prefix + * \brief Apply a suffix to an option name if the source is environment + * variables and the suffix is nonempty + * \param option_name The name of the option to suffix */ - std::string _prefixed(const std::string& option_name); + std::string _suffixed(const std::string& option_name); /*! * \brief Integer option map @@ -273,7 +307,7 @@ class ConfigOptions /*! * \brief Logging context */ - std::string _log_context; + SRObject* _log_context; /*! * \brief Stash of string buffers to free at cleanup time diff --git a/include/pyclient.h b/include/pyclient.h index 5c407a3a3..deb01f57a 100644 --- a/include/pyclient.h +++ b/include/pyclient.h @@ -39,6 +39,7 @@ #include "client.h" #include "pydataset.h" #include "pysrobject.h" +#include "pyconfigoptions.h" ///@file @@ -56,7 +57,24 @@ class PyClient : public PySRObject public: /*! - * \brief PyClient constructor + * \brief Simple constructor that uses default environment variables + * to locate configuration settings + * \param logger_name Identifier for the current client + */ + PyClient(const std::string& logger_name); + + /*! + * \brief Constructor that uses a ConfigOptions object + * to locate configuration settings + * \param config_options The ConfigOptions object to use + * \param logger_name Identifier for the current client + */ + PyClient( + PyConfigOptions& config_options, + const std::string& logger_name); + + /*! + * \brief PyClient constructor (deprecated) * \param cluster Flag to indicate if a database cluster * is being used * \param logger_name Identifier for the current client diff --git a/include/pyconfigoptions.h b/include/pyconfigoptions.h index bfbeacb59..0eb887a3f 100644 --- a/include/pyconfigoptions.h +++ b/include/pyconfigoptions.h @@ -85,17 +85,17 @@ class PyConfigOptions /*! * \brief Instantiate ConfigOptions, getting selections from - * environment variables. If \p db_prefix is non-empty, - * then "{db_prefix}_" will be prepended to the name of + * environment variables. If \p db_suffix is non-empty, + * then "_{db_suffix}" will be appended to the name of * each environment variable that is read. - * \param db_prefix The prefix to use with environment variables, - * or an empty string to disable prefixing + * \param db_suffix The suffix to use with environment variables, + * or an empty string to disable suffixing * \returns The constructed ConfigOptions object - * \throw SmartRedis::Exception if db_prefix contains invalid + * \throw SmartRedis::Exception if db_suffix contains invalid * characters */ static PyConfigOptions* create_from_environment( - const std::string& db_prefix); + const std::string& db_suffix); ///////////////////////////////////////////////////////////// // Option access diff --git a/include/redis.h b/include/redis.h index b799577c1..8233c45f5 100644 --- a/include/redis.h +++ b/include/redis.h @@ -35,7 +35,7 @@ namespace SmartRedis { -class SRObject; +class ConfigOptions; /*! * \brief The Redis class executes RedisServer @@ -46,19 +46,19 @@ class Redis : public RedisServer public: /*! * \brief Redis constructor. - * \param context The owning context + * \param cfgopts Our source for configuration options */ - Redis(const SRObject* context); + Redis(ConfigOptions* cfgopts); /*! * \brief Redis constructor. * Uses address provided to constructor instead * of environment variables. - * \param context The owning context + * \param cfgopts Our source for configuration options * \param addr_spec The TCP or UDS server address * \throw SmartRedis::Exception if connection fails */ - Redis(const SRObject* context, std::string addr_spec); + Redis(ConfigOptions* cfgopts, std::string addr_spec); /*! * \brief Redis copy constructor is not allowed diff --git a/include/rediscluster.h b/include/rediscluster.h index 7371ec343..1d818ea26 100644 --- a/include/rediscluster.h +++ b/include/rediscluster.h @@ -55,18 +55,18 @@ class RedisCluster : public RedisServer /*! * \brief RedisCluster constructor. - * \param context The owning context + * \param cfgopts Our source for configuration options */ - RedisCluster(const SRObject* context); + RedisCluster(ConfigOptions* cfgopts); /*! * \brief RedisCluster constructor. Uses address provided to * constructor instead of environment variables. - * \param context The owning context + * \param cfgopts Our source for configuration options * \param address_spec The TCP or UDS address of the server * \throw SmartRedis::Exception if connection fails */ - RedisCluster(const SRObject* context, std::string address_spec); + RedisCluster(ConfigOptions* cfgopts, std::string address_spec); /*! * \brief RedisCluster copy constructor is not allowed diff --git a/include/redisserver.h b/include/redisserver.h index 39594c505..e3faeeb6a 100644 --- a/include/redisserver.h +++ b/include/redisserver.h @@ -58,7 +58,7 @@ namespace SmartRedis { -class SRObject; +class ConfigOptions; /*! * \brief Abstract class that defines interface for @@ -70,10 +70,10 @@ class RedisServer { /*! * \brief Default constructor - * \param context The owning context + * \param cfgopts Our source for configuration options * \throw SmartRedis::Exception if connection fails */ - RedisServer(const SRObject* context); + RedisServer(ConfigOptions* cfgopts); /*! * \brief Destructor @@ -638,7 +638,12 @@ class RedisServer { static constexpr int _DEFAULT_THREAD_COUNT = 4; /*! - * \brief The owning context + * \brief Our source for configuration options + */ + ConfigOptions* _cfgopts; + + /*! + * \brief Our logging context */ const SRObject* _context; diff --git a/src/c/c_client.cpp b/src/c/c_client.cpp index 6672184b5..38e226659 100644 --- a/src/c/c_client.cpp +++ b/src/c/c_client.cpp @@ -67,8 +67,55 @@ auto c_client_api(T&& client_api_func, const char* name) #define MAKE_CLIENT_API(stuff)\ c_client_api([&] { stuff }, __func__)() +// Create a simple Client +SRError SimpleCreateClient( + const char* logger_name, + const size_t logger_name_length, + void** new_client) +{ + return MAKE_CLIENT_API({ + // Sanity check params + SR_CHECK_PARAMS(new_client != NULL && logger_name != NULL); + + std::string _logger_name(logger_name, logger_name_length); + try { + *new_client = NULL; + Client* s = new Client(_logger_name); + *new_client = reinterpret_cast(s); + } + catch (const std::bad_alloc& e) { + throw SRBadAllocException("client allocation"); + } + }); +} + +// Create a Client that uses a ConfigOptions object +SRError CreateClient( + void* config_options, + const char* logger_name, + const size_t logger_name_length, + void** new_client) +{ + return MAKE_CLIENT_API({ + // Sanity check params + SR_CHECK_PARAMS( + config_options != NULL && new_client != NULL && logger_name != NULL); + + ConfigOptions* cfgopts = reinterpret_cast(config_options); + + std::string _logger_name(logger_name, logger_name_length); + try { + *new_client = NULL; + Client* s = new Client(cfgopts, _logger_name); + *new_client = reinterpret_cast(s); + } + catch (const std::bad_alloc& e) { + throw SRBadAllocException("client allocation"); + } + }); +} -// Return a pointer to a new Client +// Return a pointer to a new Client (deprecated) extern "C" SRError SmartRedisCClient( bool cluster, const char* logger_name, diff --git a/src/c/c_configoptions.cpp b/src/c/c_configoptions.cpp index f7c0f8ad7..e7b3083d2 100644 --- a/src/c/c_configoptions.cpp +++ b/src/c/c_configoptions.cpp @@ -69,18 +69,18 @@ auto c_cfgopt_api(T&& cfgopt_api_func, const char* name) // Instantiate ConfigOptions from environment variables extern "C" SRError create_configoptions_from_environment( - const char* db_prefix, - const size_t db_prefix_length, + const char* db_suffix, + const size_t db_suffix_length, void** new_configoptions) { return MAKE_CFGOPT_API({ try { // Sanity check params - SR_CHECK_PARAMS(db_prefix != NULL && new_configoptions != NULL); + SR_CHECK_PARAMS(db_suffix != NULL && new_configoptions != NULL); - std::string db_prefix_str(db_prefix, db_prefix_length); + std::string db_suffix_str(db_suffix, db_suffix_length); - auto cfgOpts = ConfigOptions::create_from_environment(db_prefix_str); + auto cfgOpts = ConfigOptions::create_from_environment(db_suffix_str); ConfigOptions* pCfgOpts = cfgOpts.release(); *new_configoptions = reinterpret_cast(pCfgOpts); } diff --git a/src/cpp/client.cpp b/src/cpp/client.cpp index 7e9e98c7d..2e4581762 100644 --- a/src/cpp/client.cpp +++ b/src/cpp/client.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -36,28 +37,105 @@ #include "srexception.h" #include "logger.h" #include "utility.h" +#include "configoptions.h" using namespace SmartRedis; -// Constructor +// Simple Client constructor +Client::Client(const char* logger_name) + : SRObject(logger_name) +{ + // Create our ConfigOptions object (default: no suffixing) + auto cfgopts = ConfigOptions::create_from_environment(""); + _cfgopts = cfgopts.release(); + _cfgopts->_set_log_context(this); + + // Log that a new client has been instantiated + log_data(LLDebug, "New client created"); + + // Establish our server connection + _establish_server_connection(); +} + +// Constructor with config options +Client::Client(ConfigOptions* cfgopts, const std::string& logger_name) + : SRObject(logger_name), _cfgopts(cfgopts->clone()) +{ + // Log that a new client has been instantiated + _cfgopts->_set_log_context(this); + log_data(LLDebug, "New client created"); + + // Establish our server connection + _establish_server_connection(); +} + +// Initialize a connection to the back-end database +void Client::_establish_server_connection() +{ + // See what type of connection the user wants + std::string server_type = _cfgopts->_resolve_string_option( + "SR_DB_TYPE", "Clustered"); + std::transform(server_type.begin(), server_type.end(), server_type.begin(), + [](unsigned char c){ return std::tolower(c); }); + + // Set up Redis server connection + // A std::bad_alloc exception on the initializer will be caught + // by the call to new for the client + if (server_type == "clustered") { + log_data(LLDeveloper, "Instantiating clustered Redis connection"); + _redis_cluster = new RedisCluster(_cfgopts); + _redis = NULL; + _redis_server = _redis_cluster; + } + else { // Standalone or Colocated + log_data(LLDeveloper, "Instantiating standalone Redis connection"); + _redis_cluster = NULL; + _redis = new Redis(_cfgopts); + _redis_server = _redis; + } + log_data(LLDeveloper, "Redis connection established"); + + // Initialize key prefixing + _get_prefix_settings(); + _use_tensor_prefix = true; + _use_dataset_prefix = true; + _use_model_prefix = false; + _use_list_prefix = true; +} + +// Constructor (deprecated) Client::Client(bool cluster, const std::string& logger_name) : SRObject(logger_name) { + std::cout << "In deprecated constructor" << std::endl; // Log that a new client has been instantiated log_data(LLDebug, "New client created"); + // Log deprecation warning for this method + log_data( + LLInfo, + "Deprecation Notice: Client::Client(bool, std::string) constructor " + "should not be used. Please migrate your code to use the " + "Client::Client(ConfigOptions*) constructor and set the server type " + "in the SR_DB_TYPE environment variable."); + + // Create our ConfigOptions object (default = no suffixing) + auto cfgopts = ConfigOptions::create_from_environment(""); + _cfgopts = cfgopts.release(); + _cfgopts->_set_log_context(this); + // Set up Redis server connection // A std::bad_alloc exception on the initializer will be caught // by the call to new for the client - _redis_cluster = (cluster ? new RedisCluster(this) : NULL); - _redis = (cluster ? NULL : new Redis(this)); + _redis_cluster = (cluster ? new RedisCluster(_cfgopts) : NULL); + _redis = (cluster ? NULL : new Redis(_cfgopts)); if (cluster) _redis_server = _redis_cluster; else _redis_server = _redis; // Initialize key prefixing - _set_prefixes_from_env(); + _get_prefix_settings(); _use_tensor_prefix = true; _use_dataset_prefix = true; _use_model_prefix = false; @@ -78,6 +156,8 @@ Client::~Client() _redis = NULL; } _redis_server = NULL; + delete _cfgopts; + _cfgopts = NULL; // Log Client destruction log_data(LLDebug, "Client destroyed"); @@ -1672,20 +1752,20 @@ std::vector Client::get_dataset_list_range(const std::string& list_name } // Set the prefixes that are used for set and get methods using SSKEYIN -// and SSKEYOUT environment variables. -void Client::_set_prefixes_from_env() +// and SSKEYOUT configuration settings +void Client::_get_prefix_settings() { // Establish set prefix - std::string put_key_prefix; - get_config_string(put_key_prefix, "SSKEYOUT", ""); + std::string put_key_prefix = _cfgopts->_resolve_string_option( + "SSKEYOUT", ""); if (put_key_prefix.length() > 0) _put_key_prefix = put_key_prefix; else _put_key_prefix.clear(); // Establish get prefix(es) - std::string get_key_prefixes; - get_config_string(get_key_prefixes, "SSKEYIN", ""); + std::string get_key_prefixes = _cfgopts->_resolve_string_option( + "SSKEYIN", ""); if (get_key_prefixes.length() > 0) { const char* a = get_key_prefixes.c_str(); const char* b = a; diff --git a/src/cpp/configoptions.cpp b/src/cpp/configoptions.cpp index f5061ddf8..753248ddb 100644 --- a/src/cpp/configoptions.cpp +++ b/src/cpp/configoptions.cpp @@ -40,7 +40,7 @@ ConfigOptions::ConfigOptions( cfgSrc source, const std::string& string) : _source(source), _string(string), _lazy(source == cs_envt), - _log_context("") + _log_context(NULL) { // Read in options if needed if (!_lazy) { @@ -48,6 +48,16 @@ ConfigOptions::ConfigOptions( } } +// Deep copy a ConfigOptions object +ConfigOptions* ConfigOptions::clone() +{ + ConfigOptions* result = new ConfigOptions(_source, _string); + result->_log_context = _log_context; + result->_int_options = _int_options; + result->_string_options = _string_options; + return result; +} + // ConfigOptions destructor ConfigOptions::~ConfigOptions() { @@ -59,12 +69,20 @@ ConfigOptions::~ConfigOptions() // Instantiate ConfigOptions, getting selections from environment variables std::unique_ptr ConfigOptions::create_from_environment( - const std::string& db_prefix) + const std::string& db_suffix) { // NOTE: We can't use std::make_unique<> here because our constructor // is private return std::unique_ptr( - new ConfigOptions(cs_envt, db_prefix)); + new ConfigOptions(cs_envt, db_suffix)); +} + +// Instantiate ConfigOptions, getting selections from environment variables +std::unique_ptr ConfigOptions::create_from_environment( + const char* db_suffix) +{ + std::string str_suffix(db_suffix != NULL ? db_suffix : ""); + return create_from_environment(str_suffix); } // Retrieve the value of a numeric configuration option @@ -81,7 +99,7 @@ int64_t ConfigOptions::get_integer_option(const std::string& option_name) if (_lazy) { int temp = 0; get_config_integer( - temp, _prefixed(option_name), default_value, throw_on_absent); + temp, _suffixed(option_name), default_value, throw_on_absent); result = (int64_t)temp; } @@ -103,7 +121,7 @@ std::string ConfigOptions::get_string_option(const std::string& option_name) std::string result(default_value); if (_lazy) { get_config_string( - result, _prefixed(option_name), default_value, throw_on_absent); + result, _suffixed(option_name), default_value, throw_on_absent); } // Store the final value before we exit @@ -124,7 +142,7 @@ int64_t ConfigOptions::_resolve_integer_option( int64_t result = default_value; if (_lazy) { int temp = 0; - get_config_integer(temp, _prefixed(option_name), default_value); + get_config_integer(temp, _suffixed(option_name), default_value); result = (int64_t)temp; } @@ -145,7 +163,7 @@ std::string ConfigOptions::_resolve_string_option( // If we're doing lazy evaluation of option names, fetch the value std::string result(default_value); if (_lazy) { - get_config_string(result, _prefixed(option_name), default_value); + get_config_string(result, _suffixed(option_name), default_value); } // Store the final value before we exit @@ -165,8 +183,8 @@ bool ConfigOptions::is_configured(const std::string& option_name) // Check to see if the value is available and we just haven't // seen it yet if (_lazy) { - std::string prefixed = _prefixed(option_name); - char* environment_string = std::getenv(prefixed.c_str()); + std::string suffixed = _suffixed(option_name); + char* environment_string = std::getenv(suffixed.c_str()); return NULL != environment_string; } @@ -197,9 +215,9 @@ void ConfigOptions::_populate_options() ); } -// Apply a prefix to a option_name if the source is environment -// variables and the prefix is nonempty -std::string ConfigOptions::_prefixed(const std::string& option_name) +// Apply a suffix to a option_name if the source is environment +// variables and the suffix is nonempty +std::string ConfigOptions::_suffixed(const std::string& option_name) { // Sanity check if ("" == option_name) { @@ -208,7 +226,13 @@ std::string ConfigOptions::_prefixed(const std::string& option_name) } std::string result(option_name); if (_source == cs_envt && _string != "") - result = _string + + "_" + option_name; + result = option_name + + "_" + _string; return result; } +// Clear a configuration option from the cache +void ConfigOptions::_clear_option_from_cache(const std::string& option_name) +{ + _int_options.erase(option_name); + _string_options.erase(option_name); +} diff --git a/src/cpp/redis.cpp b/src/cpp/redis.cpp index a00af813b..533852b57 100644 --- a/src/cpp/redis.cpp +++ b/src/cpp/redis.cpp @@ -35,8 +35,8 @@ using namespace SmartRedis; // Redis constructor. -Redis::Redis(const SRObject* context) - : RedisServer(context) +Redis::Redis(ConfigOptions* cfgopts) + : RedisServer(cfgopts) { SRAddress db_address(_get_ssdb()); // Remember whether it's a unix domain socket for later @@ -46,8 +46,8 @@ Redis::Redis(const SRObject* context) } // Redis constructor. Uses address provided to constructor instead of environment variables -Redis::Redis(const SRObject* context, std::string addr_spec) - : RedisServer(context) +Redis::Redis(ConfigOptions* cfgopts, std::string addr_spec) + : RedisServer(cfgopts) { SRAddress db_address(addr_spec); _add_to_address_map(db_address); @@ -80,10 +80,13 @@ CommandReply Redis::run(CompoundCommand& cmd){ // Run an address-at Command on the server CommandReply Redis::run(AddressAtCommand& cmd){ - if (!is_addressable(cmd.get_address())) - throw SRRuntimeException("The provided address does not match "\ - "the address used to initialize the "\ - "non-cluster client connection."); + if (!is_addressable(cmd.get_address())) { + std::string msg("The provided address does not match "\ + "the address used to initialize the "\ + "non-cluster client connection."); + msg += " Received: " + cmd.get_address().to_string(); + throw SRRuntimeException(msg); + } return this->_run(cmd); } diff --git a/src/cpp/rediscluster.cpp b/src/cpp/rediscluster.cpp index 6f847dc6e..4165cee47 100644 --- a/src/cpp/rediscluster.cpp +++ b/src/cpp/rediscluster.cpp @@ -33,12 +33,13 @@ #include "srexception.h" #include "utility.h" #include "srobject.h" +#include "configoptions.h" using namespace SmartRedis; // RedisCluster constructor -RedisCluster::RedisCluster(const SRObject* context) - : RedisServer(context) +RedisCluster::RedisCluster(ConfigOptions* cfgopts) + : RedisServer(cfgopts) { SRAddress db_address(_get_ssdb()); if (!db_address._is_tcp) { @@ -57,8 +58,8 @@ RedisCluster::RedisCluster(const SRObject* context) // RedisCluster constructor. Uses address provided to constructor instead of // environment variables -RedisCluster::RedisCluster(const SRObject* context, std::string address_spec) - : RedisServer(context) +RedisCluster::RedisCluster(ConfigOptions* cfgopts, std::string address_spec) + : RedisServer(cfgopts) { SRAddress db_address(address_spec); _connect(db_address); @@ -1074,7 +1075,12 @@ inline CommandReply RedisCluster::_run(const Command& cmd, std::string db_prefix // Connect to the cluster at the address and port inline void RedisCluster::_connect(SRAddress& db_address) { + std::string msg; for (int i = 1; i <= _connection_attempts; i++) { + msg = "Connection attempt " + std::to_string(i) + " of " + + std::to_string(_connection_attempts); + _cfgopts->_get_log_context()->log_data(LLDeveloper, msg); + try { // Attempt the connection _redis_cluster = new sw::redis::RedisCluster(db_address.to_string(true)); @@ -1083,10 +1089,13 @@ inline void RedisCluster::_connect(SRAddress& db_address) catch (std::bad_alloc& e) { // On a memory error, bail immediately _redis_cluster = NULL; + _cfgopts->_get_log_context()->log_data(LLDeveloper, "Memory error"); throw SRBadAllocException("RedisCluster connection"); } catch (sw::redis::Error& e) { // For an error from Redis, retry unless we're out of chances + msg = "redis error: "; msg += e.what(); + _cfgopts->_get_log_context()->log_data(LLDeveloper, msg); _redis_cluster = NULL; std::string message("Unable to connect to backend database: "); message += e.what(); @@ -1100,6 +1109,8 @@ inline void RedisCluster::_connect(SRAddress& db_address) } catch (std::exception& e) { // Should never hit this, so bail immediately if we do + msg = "std::exception: "; msg += e.what(); + _cfgopts->_get_log_context()->log_data(LLDeveloper, msg); _redis_cluster = NULL; throw SRInternalException( std::string("Unexpected exception while connecting: ") + @@ -1107,6 +1118,7 @@ inline void RedisCluster::_connect(SRAddress& db_address) } catch (...) { // Should never hit this, so bail immediately if we do + _cfgopts->_get_log_context()->log_data(LLDeveloper, "unknown exception"); _redis_cluster = NULL; throw SRInternalException( "A non-standard exception was encountered during client "\ diff --git a/src/cpp/redisserver.cpp b/src/cpp/redisserver.cpp index d4d04514c..3e799c108 100644 --- a/src/cpp/redisserver.cpp +++ b/src/cpp/redisserver.cpp @@ -32,23 +32,25 @@ #include "srexception.h" #include "utility.h" #include "srobject.h" +#include "configoptions.h" using namespace SmartRedis; // RedisServer constructor -RedisServer::RedisServer(const SRObject* context) - : _context(context), _gen(_rd()) +RedisServer::RedisServer(ConfigOptions* cfgopts) + : _cfgopts(cfgopts), _context(cfgopts->_get_log_context()), + _gen(_rd()) { - get_config_integer(_connection_timeout, _CONN_TIMEOUT_ENV_VAR, - _DEFAULT_CONN_TIMEOUT); - get_config_integer(_connection_interval, _CONN_INTERVAL_ENV_VAR, - _DEFAULT_CONN_INTERVAL); - get_config_integer(_command_timeout, _CMD_TIMEOUT_ENV_VAR, - _DEFAULT_CMD_TIMEOUT); - get_config_integer(_command_interval, _CMD_INTERVAL_ENV_VAR, - _DEFAULT_CMD_INTERVAL); - get_config_integer(_thread_count, _TP_THREAD_COUNT, - _DEFAULT_THREAD_COUNT); + _connection_timeout = _cfgopts->_resolve_integer_option( + _CONN_TIMEOUT_ENV_VAR, _DEFAULT_CONN_TIMEOUT); + _connection_interval = _cfgopts->_resolve_integer_option( + _CONN_INTERVAL_ENV_VAR, _DEFAULT_CONN_INTERVAL); + _command_timeout = _cfgopts->_resolve_integer_option( + _CMD_TIMEOUT_ENV_VAR, _DEFAULT_CMD_TIMEOUT); + _command_interval = _cfgopts->_resolve_integer_option( + _CMD_INTERVAL_ENV_VAR, _DEFAULT_CMD_INTERVAL); + _thread_count = _cfgopts->_resolve_integer_option( + _TP_THREAD_COUNT, _DEFAULT_THREAD_COUNT); _check_runtime_variables(); @@ -76,8 +78,7 @@ RedisServer::~RedisServer() SRAddress RedisServer::_get_ssdb() { // Retrieve the environment variable - std::string db_spec; - get_config_string(db_spec, "SSDB", ""); + std::string db_spec = _cfgopts->_resolve_string_option("SSDB", ""); if (db_spec.length() == 0) throw SRRuntimeException("The environment variable SSDB "\ "must be set to use the client."); @@ -103,9 +104,19 @@ SRAddress RedisServer::_get_ssdb() address_choices.push_back(addr_spec); } + std::string msg = "Found " + std::to_string(address_choices.size()) + " addresses:"; + _cfgopts->_get_log_context()->log_data(LLDeveloper, msg); + for (size_t i = 0; i < address_choices.size(); i++) { + _cfgopts->_get_log_context()->log_data( + LLDeveloper, "\t" + address_choices[i].to_string()); + } + // Pick an entry from the list at random std::uniform_int_distribution<> distrib(0, address_choices.size() - 1); - return address_choices[distrib(_gen)]; + auto choice = address_choices[distrib(_gen)]; + _cfgopts->_get_log_context()->log_data( + LLDeveloper, "Picked: " + choice.to_string()); + return choice; } // Check that the SSDB environment variable value does not have any errors diff --git a/src/fortran/client.F90 b/src/fortran/client.F90 index c3acd35c7..a7b16f081 100644 --- a/src/fortran/client.F90 +++ b/src/fortran/client.F90 @@ -42,6 +42,7 @@ module smartredis_client use, intrinsic :: iso_fortran_env, only: stderr => error_unit use smartredis_dataset, only : dataset_type +use smartredis_configoptions, only : configoptions_type use fortran_c_interop, only : convert_char_array_to_c, enum_kind, C_MAX_STRING @@ -69,12 +70,13 @@ module smartredis_client type, public :: client_type private - logical(kind=c_bool) :: cluster = .false. !< True if a database cluster is being used - type(c_ptr) :: client_ptr = c_null_ptr !< Pointer to the initialized SmartRedisClient - logical :: is_initialized = .false. !< True if client is initialized + type(c_ptr) :: client_ptr = c_null_ptr !< Pointer to the initialized SmartRedisClient + logical :: is_initialized = .false. !< True if client is initialized contains ! Public procedures + !> Initializes a new instance of the SmartRedis client + generic :: initialize => initialize_client_deprecated, initialize_client_simple, initialize_client_cfgopts !> Puts a tensor into the database (overloaded) generic :: put_tensor => put_tensor_i8, put_tensor_i16, put_tensor_i32, put_tensor_i64, & put_tensor_float, put_tensor_double @@ -84,8 +86,6 @@ module smartredis_client !> Decode a response code from an API function procedure :: SR_error_parser - !> Initializes a new instance of the SmartRedis client - procedure :: initialize => initialize_client !> Check if a SmartRedis client has been initialized procedure :: isinitialized !> Destructs a new instance of the SmartRedis client @@ -198,6 +198,9 @@ module smartredis_client procedure :: print_client ! Private procedures + procedure, private :: initialize_client_deprecated + procedure, private :: initialize_client_simple + procedure, private :: initialize_client_cfgopts procedure, private :: put_tensor_i8 procedure, private :: put_tensor_i16 procedure, private :: put_tensor_i32 @@ -251,15 +254,63 @@ function SR_error_parser(self, response_code) result(is_error) end function SR_error_parser !> Initializes a new instance of a SmartRedis client -function initialize_client(self, cluster, logger_name) - integer(kind=enum_kind) :: initialize_client - class(client_type), intent(inout) :: self !< Receives the initialized client - logical, optional, intent(in ) :: cluster !< If true, client uses a database cluster (Default: .false.) +function initialize_client_simple(self, logger_name) + integer(kind=enum_kind) :: initialize_client_simple + class(client_type), intent(inout) :: self !< Receives the initialized client + character(len=*), optional, intent(in ) :: logger_name !< Identifier for the current client + + ! Local variables + character(kind=c_char, len=:), allocatable :: c_logger_name + integer(kind=c_size_t) :: logger_name_length + + if (present(logger_name)) then + c_logger_name = logger_name + else + c_logger_name = 'default' + endif + logger_name_length = len_trim(c_logger_name) + + initialize_client_simple = c_simple_constructor( & + c_logger_name, logger_name_length, self%client_ptr) + self%is_initialized = initialize_client_simple .eq. SRNoError + if (allocated(c_logger_name)) deallocate(c_logger_name) +end function initialize_client_simple + +!> Initializes a new instance of a SmartRedis client +function initialize_client_cfgopts(self, cfgopts, logger_name) + integer(kind=enum_kind) :: initialize_client_cfgopts + class(client_type), intent(inout) :: self !< Receives the initialized client + type(configoptions_type), intent(in ) :: cfgopts !< Source for configuration settings + character(len=*), optional, intent(in ) :: logger_name !< Identifier for the current client + + ! Local variables + character(kind=c_char, len=:), allocatable :: c_logger_name + integer(kind=c_size_t) :: logger_name_length + + if (present(logger_name)) then + c_logger_name = logger_name + else + c_logger_name = 'default' + endif + logger_name_length = len_trim(c_logger_name) + + initialize_client_cfgopts = c_constructor( & + cfgopts%get_c_pointer(), c_logger_name, logger_name_length, self%client_ptr) + self%is_initialized = initialize_client_cfgopts .eq. SRNoError + if (allocated(c_logger_name)) deallocate(c_logger_name) +end function initialize_client_cfgopts + +!> Initializes a new instance of a SmartRedis client (deprecated) +function initialize_client_deprecated(self, cluster, logger_name) + integer(kind=enum_kind) :: initialize_client_deprecated + class(client_type), intent(inout) :: self !< Receives the initialized client + logical, intent(in ) :: cluster !< If true, client uses a database cluster (Default: .false.) character(len=*), optional, intent(in ) :: logger_name !< Identifier for the current client ! Local variables character(kind=c_char, len=:), allocatable :: c_logger_name integer(kind=c_size_t) :: logger_name_length + logical(kind=c_bool) :: c_cluster if (present(logger_name)) then c_logger_name = logger_name @@ -267,12 +318,13 @@ function initialize_client(self, cluster, logger_name) c_logger_name = 'default' endif logger_name_length = len_trim(c_logger_name) + c_cluster = cluster - if (present(cluster)) self%cluster = cluster - initialize_client = c_constructor(self%cluster, c_logger_name, logger_name_length, self%client_ptr) - self%is_initialized = initialize_client .eq. SRNoError + initialize_client_deprecated = c_deprecated_constructor( & + c_cluster, c_logger_name, logger_name_length, self%client_ptr) + self%is_initialized = initialize_client_deprecated .eq. SRNoError if (allocated(c_logger_name)) deallocate(c_logger_name) -end function initialize_client +end function initialize_client_deprecated !> Check whether the client has been initialized logical function isinitialized(this) diff --git a/src/fortran/client/client_interfaces.inc b/src/fortran/client/client_interfaces.inc index fd3b50ca2..6e707dfff 100644 --- a/src/fortran/client/client_interfaces.inc +++ b/src/fortran/client/client_interfaces.inc @@ -25,18 +25,43 @@ ! OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. interface - function c_constructor(cluster, logger_name, logger_name_length, new_client) bind(c, name="SmartRedisCClient") + function c_simple_constructor(logger_name, logger_name_length, new_client) & + bind(c, name="SimpleCreateClient") use iso_c_binding, only : c_ptr, c_bool, c_char, c_size_t import :: enum_kind - integer(kind=enum_kind) :: c_constructor - logical(kind=c_bool), value :: cluster !< True if a database cluster is being used + integer(kind=enum_kind) :: c_simple_constructor character(kind=c_char) :: logger_name(*) integer(kind=c_size_t), value :: logger_name_length + type(c_ptr) :: new_client !< Receives the newly constructed client + end function c_simple_constructor +end interface +interface + function c_constructor(cfgopts, logger_name, logger_name_length, new_client) & + bind(c, name="CreateClient") + use iso_c_binding, only : c_ptr, c_bool, c_char, c_size_t + import :: enum_kind + integer(kind=enum_kind) :: c_constructor + type(c_ptr), value :: cfgopts + character(kind=c_char) :: logger_name(*) + integer(kind=c_size_t), value :: logger_name_length type(c_ptr) :: new_client !< Receives the newly constructed client end function c_constructor end interface +interface + function c_deprecated_constructor(cluster, logger_name, logger_name_length, new_client) & + bind(c, name="SmartRedisCClient") + use iso_c_binding, only : c_ptr, c_bool, c_char, c_size_t + import :: enum_kind + integer(kind=enum_kind) :: c_deprecated_constructor + logical(kind=c_bool), value :: cluster !< True if a database cluster is being used + character(kind=c_char) :: logger_name(*) + integer(kind=c_size_t), value :: logger_name_length + type(c_ptr) :: new_client !< Receives the newly constructed client + end function c_deprecated_constructor +end interface + interface function c_destructor(client) bind(c, name="DeleteCClient") use iso_c_binding, only : c_ptr diff --git a/src/fortran/configoptions.F90 b/src/fortran/configoptions.F90 index c04fc19e8..7907fc658 100644 --- a/src/fortran/configoptions.F90 +++ b/src/fortran/configoptions.F90 @@ -84,21 +84,21 @@ function get_c_pointer(self) end function get_c_pointer !> Instantiate ConfigOptions, getting selections from environment variables -function create_configoptions_from_environment(self, db_prefix) result(code) +function create_configoptions_from_environment(self, db_suffix) result(code) class(configoptions_type), intent(inout) :: self !< Receives the configoptions - character(len=*), intent(in) :: db_prefix !< Prefix to apply to environment + character(len=*), intent(in) :: db_suffix !< Suffix to apply to environment !! variables; empty string for none integer(kind=enum_kind) :: code !< Result of the operation ! Local variables - integer(kind=c_size_t) :: db_prefix_length - character(kind=c_char, len=len_trim(db_prefix)) :: c_db_prefix + integer(kind=c_size_t) :: db_suffix_length + character(kind=c_char, len=len_trim(db_suffix)) :: c_db_suffix - db_prefix_length = len_trim(db_prefix) - c_db_prefix = trim(db_prefix) + db_suffix_length = len_trim(db_suffix) + c_db_suffix = trim(db_suffix) code = create_configoptions_from_environment_c( & - c_db_prefix, db_prefix_length, self%configoptions_ptr) + c_db_suffix, db_suffix_length, self%configoptions_ptr) end function create_configoptions_from_environment !> Retrieve the value of a numeric configuration option diff --git a/src/fortran/configoptions/configoptions_interfaces.inc b/src/fortran/configoptions/configoptions_interfaces.inc index d4ea290a7..87b87e1ed 100644 --- a/src/fortran/configoptions/configoptions_interfaces.inc +++ b/src/fortran/configoptions/configoptions_interfaces.inc @@ -27,13 +27,13 @@ interface function create_configoptions_from_environment_c( & - db_prefix, db_prefix_len, configoptions) & + db_suffix, db_suffix_len, configoptions) & bind(c, name="create_configoptions_from_environment") use iso_c_binding, only : c_ptr, c_char, c_size_t import :: enum_kind integer(kind=enum_kind) :: create_configoptions_from_environment_c - character(kind=c_char) :: db_prefix(*) !< Prefix to add to environment vars - integer(kind=c_size_t), value :: db_prefix_len !< How many characters in db_prefix + character(kind=c_char) :: db_suffix(*) !< Suffix to add to environment vars + integer(kind=c_size_t), value :: db_suffix_len !< How many characters in db_suffix type(c_ptr) :: configoptions !< Receives the constructed configoptions end function create_configoptions_from_environment_c diff --git a/src/python/bindings/bind.cpp b/src/python/bindings/bind.cpp index 0ab30fb57..f5d1304ca 100644 --- a/src/python/bindings/bind.cpp +++ b/src/python/bindings/bind.cpp @@ -59,6 +59,8 @@ PYBIND11_MODULE(smartredisPy, m) { #define CLIENT_METHOD(name) CLASS_METHOD(PyClient, name) py::class_(m, "PyClient") .def(py::init()) + .def(py::init()) + .def(py::init()) .CLIENT_METHOD(put_tensor) .CLIENT_METHOD(get_tensor) .CLIENT_METHOD(delete_tensor) diff --git a/src/python/module/smartredis/client.py b/src/python/module/smartredis/client.py index bb02c587b..c0338940e 100644 --- a/src/python/module/smartredis/client.py +++ b/src/python/module/smartredis/client.py @@ -29,10 +29,11 @@ import os import os.path as osp import typing as t - +import warnings as w import numpy as np from .dataset import Dataset +from .configoptions import ConfigOptions from .error import RedisConnectionError from .smartredisPy import PyClient from .smartredisPy import RedisReplyError as PybindRedisReplyError @@ -41,14 +42,52 @@ class Client(SRObject): - def __init__( - self, - address: t.Optional[str] = None, - cluster: bool = False, - logger_name: str = "default", - ) -> None: + def __init__(self, *a: t.Any, **kw: t.Any): """Initialize a RedisAI client + At this time, the Client can be initialized with one of two + signatures. The first version is preferred, though the second is + still supported. Note that the order was swapped for first two + parameters in the second signature relative to previous releases + of SmartRedis; this was necessary to remove ambiguity. Support for + the second signature will be removed in a future version of the + SmartRedis library. + + Client(config_options: ConfigOptions=None, + logger_name: str="Default") + Client(cluster: bool, address: optional(str)=None, + logger_name: str="Default") <= Deprecated! + + For detailed information on the first signature, please refer + to the __new_construction() method below. + + For detailed information on the second signature, please refer + to the __deprecated_construction() method below. + + :param a: The positional arguments supplied to this method; see above for + valid options + :type a: tuple[any]; see above for valid options + :param kw: Keyword arguments supplied to this method; see above for + valid options + :type kw: dict[string, any]; see above for valid options + :raises RedisConnectionError: if connection initialization fails + """ + if a: + if isinstance(a[0], bool): + pyclient = self.__deprecated_construction(*a, **kw) + elif isinstance(a[0], ConfigOptions) or a[0] is None: + pyclient = self.__new_construction(*a, **kw) + else: + raise TypeError(f"Invalid type for argument 0: {type(a[0])}") + else: + config_object = kw.get("config_object", None) + logger_name = kw.get("logger_name", "default") + pyclient = self.__new_construction(config_object, logger_name) + super().__init__(pyclient) + + def __deprecated_construction(self, cluster, address=None, logger_name="Default"): + """Initialize a RedisAI client (Deprecated) + For clusters, the address can be a single tcp/ip address and port of a database node. The rest of the cluster will be discovered by the client itself. (e.g. address="127.0.0.1:6379") @@ -56,19 +95,50 @@ def __init__( If an address is not set, the client will look for the environment variable ``SSDB`` (e.g. SSDB="127.0.0.1:6379;") - :param address: Address of the database + DEPRECATION NOTICE: This construction method is deprecated and will + be removed in the next release of the SmartRedis client. + :param cluster: True if connecting to a redis cluster, defaults to False - :type cluster: bool, optional + :type cluster: bool + :param address: Address of the database + :type address: str, optional :param logger_name: Identifier for the current client :type logger_name: str :raises RedisConnectionError: if connection initialization fails """ + w.warn( + 'This construction method is deprecated and will be removed in the next ' + + 'release of the SmartRedis client.', + DeprecationWarning, + stacklevel=3 + ) if address: self.__set_address(address) if "SSDB" not in os.environ: raise RedisConnectionError("Could not connect to database. $SSDB not set") try: - super().__init__(PyClient(cluster, logger_name)) + return PyClient(cluster, logger_name) + except (PybindRedisReplyError, RuntimeError) as e: + raise RedisConnectionError(str(e)) from None + + def __new_construction(self, config_options=None, logger_name="Default"): # pylint: disable=no-self-use + """Initialize a RedisAI client + + The address of the Redis database is expected to be found in the + SSDB environment variable (or a suffixed variable if a suffix was + used when building the config_options object). + + :param config_options: Source for configuration data + :type config_options: ConfigOptions, optional + :param logger_name: Identifier for the current client + :type logger_name: str + :raises RedisConnectionError: if connection initialization fails + """ + try: + if config_options: + pybind_config_options = config_options.get_data() + return PyClient(pybind_config_options, logger_name) + return PyClient(logger_name) except PybindRedisReplyError as e: raise RedisConnectionError(str(e)) from None except RuntimeError as e: diff --git a/src/python/module/smartredis/configoptions.py b/src/python/module/smartredis/configoptions.py index 8422416e4..ad60542ac 100644 --- a/src/python/module/smartredis/configoptions.py +++ b/src/python/module/smartredis/configoptions.py @@ -92,6 +92,17 @@ def from_pybind(configoptions: PyConfigOptions) -> "ConfigOptions": opts.set_configoptions(configoptions) return opts + @exception_handler + @managed + def get_data(self): + """Return the PyConfigOptions attribute + + :return: The PyConfigOptions attribute containing + the ConfigOptions information + :rtype: PyConfigOptions + """ + return self._config_opts + @exception_handler def set_configoptions(self, configoptions: PyConfigOptions) -> None: """Set the PyConfigOptions attribute @@ -104,22 +115,22 @@ def set_configoptions(self, configoptions: PyConfigOptions) -> None: @classmethod @exception_handler - def create_from_environment(cls, db_prefix: str) -> "ConfigOptions": + def create_from_environment(cls, db_suffix: str) -> "ConfigOptions": """Instantiate ConfigOptions, getting selections from - environment variables. If db_prefix is non-empty, - then "{db_prefix}_" will be prepended to the name of + environment variables. If db_suffix is non-empty, + then "_{db_suffix}" will be appended to the name of each environment variable that is read :param cls: The ConfigOptions class :type cls: type - :param db_prefix: Prefix to prepend to environment variables - or an empty string to eschew prepending - :type db_prefix: str + :param db_suffix: Prefix to append to environment variables + or an empty string to eschew appending + :type db_suffix: str :return: An instantiated ConfigOptions object :rtype: ConfigOptions """ - typecheck(db_prefix, "db_prefix", str) - configoptions = PyConfigOptions.create_from_environment(db_prefix) + typecheck(db_suffix, "db_suffix", str) + configoptions = PyConfigOptions.create_from_environment(db_suffix) opts: ConfigOptions = create_managed_instance(ConfigOptions) opts.set_configoptions(configoptions) return opts diff --git a/src/python/src/pyclient.cpp b/src/python/src/pyclient.cpp index f174fa253..7577932c2 100644 --- a/src/python/src/pyclient.cpp +++ b/src/python/src/pyclient.cpp @@ -69,6 +69,24 @@ auto pb_client_api(T&& client_api_func, const char* name) #define MAKE_CLIENT_API(stuff)\ pb_client_api([&] { stuff }, __func__)() +PyClient::PyClient(const std::string& logger_name) + : PySRObject(logger_name) +{ + MAKE_CLIENT_API({ + _client = new Client(logger_name); + }); +} + +PyClient::PyClient( + PyConfigOptions& config_options, + const std::string& logger_name) + : PySRObject(logger_name) +{ + MAKE_CLIENT_API({ + ConfigOptions* co = config_options.get(); + _client = new Client(co, logger_name); + }); +} PyClient::PyClient(bool cluster, const std::string& logger_name) : PySRObject(logger_name) diff --git a/src/python/src/pyconfigoptions.cpp b/src/python/src/pyconfigoptions.cpp index 60efb628b..8f0488df9 100644 --- a/src/python/src/pyconfigoptions.cpp +++ b/src/python/src/pyconfigoptions.cpp @@ -96,10 +96,10 @@ ConfigOptions* PyConfigOptions::get() { // Instantiate ConfigOptions from environment variables PyConfigOptions* PyConfigOptions::create_from_environment( - const std::string& db_prefix) + const std::string& db_suffix) { return MAKE_CFGOPT_API({ - auto cfgOpts = ConfigOptions::create_from_environment(db_prefix); + auto cfgOpts = ConfigOptions::create_from_environment(db_suffix); ConfigOptions* pCfgOpts = cfgOpts.release(); return new PyConfigOptions(pCfgOpts); }); diff --git a/tests/c/c_client_test_utils.h b/tests/c/c_client_test_utils.h index fa35a3942..56a3e49ac 100644 --- a/tests/c/c_client_test_utils.h +++ b/tests/c/c_client_test_utils.h @@ -53,15 +53,15 @@ void to_lower(char* s, int maxchars) { bool use_cluster() { - /* This function determines if a cluster - configuration should be used in the test - when creating a Client. + /* This function determines if a cluster configuration should be used in + the test when creating a Client. + (deprecated) */ - char* smartredis_test_cluster = getenv("SMARTREDIS_TEST_CLUSTER"); - to_lower(smartredis_test_cluster, 256); + char* server_type = getenv("SR_DB_TYPE"); + to_lower(server_type, 256); - if(smartredis_test_cluster) { - if(strcmp(smartredis_test_cluster, "true")==0) + if(server_type) { + if(strcmp(server_type, "clustered")==0) return true; } return false; diff --git a/tests/c/client_test_dataset_aggregation.c b/tests/c/client_test_dataset_aggregation.c index a87806610..d9f817ea8 100644 --- a/tests/c/client_test_dataset_aggregation.c +++ b/tests/c/client_test_dataset_aggregation.c @@ -135,7 +135,7 @@ int main(int argc, char* argv[]) // Initialize client void *client = NULL; - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client) || + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client) || NULL == client) { printf("Failed to initialize client!\n"); printf("Test passed: NO\n"); diff --git a/tests/c/client_test_dataset_exists.c b/tests/c/client_test_dataset_exists.c index a2174b164..c08b97078 100644 --- a/tests/c/client_test_dataset_exists.c +++ b/tests/c/client_test_dataset_exists.c @@ -43,7 +43,7 @@ int missing_dataset(char *dataset_name, size_t dataset_name_len) void *client = NULL; const char* logger_name = "missing_dataset"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; bool exists = false; @@ -80,7 +80,7 @@ int present_dataset(char *dataset_name, size_t dataset_name_len) size_t cid_len = strlen(logger_name); // Initialize client and dataset - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client) || NULL == client) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client) || NULL == client) return -1; if (SRNoError != CDataSet(dataset_name, dataset_name_len, &dataset) || NULL == dataset) return -1; diff --git a/tests/c/client_test_logging.c b/tests/c/client_test_logging.c index 7ea097d43..96f391b42 100644 --- a/tests/c/client_test_logging.c +++ b/tests/c/client_test_logging.c @@ -61,9 +61,8 @@ int main(int argc, char* argv[]) size_t ctx_logcontext_len = strlen(ctx_logcontext); // Initialize client, dataset, logcontext - if (SRNoError != SmartRedisCClient( - use_cluster(), ctx_client, ctx_client_len, - &client) || NULL == client) { + if (SRNoError != SimpleCreateClient( + ctx_client, ctx_client_len, &client) || NULL == client) { return -1; } if (SRNoError != CDataSet( diff --git a/tests/c/client_test_put_get_1D.c b/tests/c/client_test_put_get_1D.c index 913449d18..0b36a7fbb 100644 --- a/tests/c/client_test_put_get_1D.c +++ b/tests/c/client_test_put_get_1D.c @@ -119,7 +119,7 @@ int put_get_1D_tensor_double(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_1D_tensor_double"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; double* tensor = (double*)malloc(dims[0]*sizeof(double)); @@ -156,7 +156,7 @@ int put_get_1D_tensor_float(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_1D_tensor_float"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; float* tensor = (float*)malloc(dims[0]*sizeof(float)); @@ -166,7 +166,7 @@ int put_get_1D_tensor_float(size_t* dims, size_t n_dims, tensor[i] = ((float)rand())/(float)RAND_MAX; int r_value = 0; - r_value = put_get_1D_tensor(client,(void*)tensor, + r_value = put_get_1D_tensor(client, (void*)tensor, dims, n_dims, (void**)(&result), key_suffix, key_suffix_length, SRTensorTypeFloat); @@ -188,7 +188,7 @@ int put_get_1D_tensor_i8(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_1D_tensor_i8"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int8_t* tensor = (int8_t*)malloc(dims[0]*sizeof(int8_t)); @@ -228,7 +228,7 @@ int put_get_1D_tensor_i16(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_1D_tensor_i16"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int16_t* tensor = (int16_t*)malloc(dims[0]*sizeof(int16_t)); @@ -268,7 +268,7 @@ int put_get_1D_tensor_i32(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_1D_tensor_i32"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int32_t* tensor = (int32_t*)malloc(dims[0]*sizeof(int32_t)); @@ -308,7 +308,7 @@ int put_get_1D_tensor_i64(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_1D_tensor_i64"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int64_t* tensor = (int64_t*)malloc(dims[0]*sizeof(int64_t)); @@ -348,7 +348,7 @@ int put_get_1D_tensor_ui8(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_1D_tensor_ui8"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; uint8_t* tensor = (uint8_t*)malloc(dims[0]*sizeof(uint8_t)); @@ -385,7 +385,7 @@ int put_get_1D_tensor_ui16(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_1D_tensor_ui16"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; uint16_t* tensor = (uint16_t*)malloc(dims[0]*sizeof(uint16_t)); @@ -459,11 +459,6 @@ int main(int argc, char* argv[]) { ui16_suffix, strlen(ui16_suffix)); free(dims); - printf("%s","Test passed: "); - if(result==0) - printf("%s", "YES\n"); - else - printf("%s", "NO\n"); - + printf("Test passed: %s\n", result == 0 ? "YES" : "NO"); return result; } diff --git a/tests/c/client_test_put_get_2D.c b/tests/c/client_test_put_get_2D.c index a8b6bf122..51117c956 100644 --- a/tests/c/client_test_put_get_2D.c +++ b/tests/c/client_test_put_get_2D.c @@ -76,6 +76,7 @@ int put_get_2D_tensor(void* client, (void*)tensor, dims, n_dims, type, layout)) { return -1; } + if (SRNoError != get_tensor(client, key, key_length, result, &g_dims, &g_n_dims, &g_type, layout)) { @@ -120,7 +121,7 @@ int put_get_2D_tensor_double(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_2D_tensor_double"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; double** tensor = (double**)malloc(dims[0]*sizeof(double*)); @@ -169,7 +170,7 @@ int put_get_2D_tensor_float(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_2D_tensor_float"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; float** tensor = (float**)malloc(dims[0]*sizeof(float*)); @@ -218,7 +219,7 @@ int put_get_2D_tensor_i8(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_2D_tensor_i8"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int8_t** tensor = (int8_t**)malloc(dims[0]*sizeof(int8_t*)); @@ -271,7 +272,7 @@ int put_get_2D_tensor_i16(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_2D_tensor_i16"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int16_t** tensor = (int16_t**)malloc(dims[0]*sizeof(int16_t*)); @@ -324,7 +325,7 @@ int put_get_2D_tensor_i32(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_2D_tensor_i32"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int32_t** tensor = (int32_t**)malloc(dims[0]*sizeof(int32_t*)); @@ -377,7 +378,7 @@ int put_get_2D_tensor_i64(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_2D_tensor_i64"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int64_t** tensor = (int64_t**)malloc(dims[0]*sizeof(int64_t*)); @@ -430,7 +431,7 @@ int put_get_2D_tensor_ui8(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_2D_tensor_ui8"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; uint8_t** tensor = (uint8_t**)malloc(dims[0]*sizeof(uint8_t*)); @@ -481,7 +482,7 @@ int put_get_2D_tensor_ui16(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_2D_tensor_ui16"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; uint16_t** tensor = (uint16_t**)malloc(dims[0]*sizeof(uint16_t*)); @@ -569,11 +570,6 @@ int main(int argc, char* argv[]) { ui16_suffix, strlen(ui16_suffix)); free(dims); - printf("%s","Test passed: "); - if(result==0) - printf("%s", "YES\n"); - else - printf("%s", "NO\n"); - + printf("Test passed: %s\n", result == 0 ? "YES" : "NO"); return result; } diff --git a/tests/c/client_test_put_get_3D.c b/tests/c/client_test_put_get_3D.c index 7e9a42b3d..358e276dc 100644 --- a/tests/c/client_test_put_get_3D.c +++ b/tests/c/client_test_put_get_3D.c @@ -120,7 +120,7 @@ int put_get_3D_tensor_double(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_3D_tensor_double"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; double*** tensor = (double***)malloc(dims[0]*sizeof(double**)); @@ -179,7 +179,7 @@ int put_get_3D_tensor_float(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_3D_tensor_float"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; float*** tensor = (float***)malloc(dims[0]*sizeof(float**)); @@ -237,7 +237,7 @@ int put_get_3D_tensor_i8(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_3D_tensor_i8"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int8_t*** tensor = (int8_t***)malloc(dims[0]*sizeof(int8_t**)); @@ -299,7 +299,7 @@ int put_get_3D_tensor_i16(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_3D_tensor_i16"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int16_t*** tensor = (int16_t***)malloc(dims[0]*sizeof(int16_t**)); @@ -362,7 +362,7 @@ int put_get_3D_tensor_i32(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_3D_tensor_i32"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int32_t*** tensor = (int32_t***)malloc(dims[0]*sizeof(int32_t**)); @@ -425,7 +425,7 @@ int put_get_3D_tensor_i64(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_3D_tensor_i64"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; int64_t*** tensor = (int64_t***)malloc(dims[0]*sizeof(int64_t**)); @@ -488,7 +488,7 @@ int put_get_3D_tensor_ui8(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_3D_tensor_ui8"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; uint8_t*** tensor = (uint8_t***)malloc(dims[0]*sizeof(uint8_t**)); @@ -551,7 +551,7 @@ int put_get_3D_tensor_ui16(size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_get_3D_tensor_ui16"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; uint16_t*** tensor = (uint16_t***)malloc(dims[0]*sizeof(uint16_t**)); @@ -654,11 +654,6 @@ int main(int argc, char* argv[]) { ui16_suffix, strlen(ui16_suffix)); free(dims); - printf("%s","Test passed: "); - if(result==0) - printf("%s", "YES\n"); - else - printf("%s", "NO\n"); - + printf("Test passed: %s\n", result == 0 ? "YES" : "NO"); return result; } diff --git a/tests/c/client_test_put_unpack_1D.c b/tests/c/client_test_put_unpack_1D.c index 310e2b1f8..85250f61d 100644 --- a/tests/c/client_test_put_unpack_1D.c +++ b/tests/c/client_test_put_unpack_1D.c @@ -49,7 +49,7 @@ int put_unpack_1D_tensor(void* tensor, size_t* dims, size_t n_dims, void* client = NULL; const char* logger_name = "put_unpack_1D_tensor"; size_t cid_len = strlen(logger_name); - if (SRNoError != SmartRedisCClient(use_cluster(), logger_name, cid_len, &client)) + if (SRNoError != SimpleCreateClient(logger_name, cid_len, &client)) return -1; char* prefix_str = "1D_tensor_test"; @@ -364,11 +364,6 @@ int main(int argc, char* argv[]) { ui16_suffix, strlen(ui16_suffix)); free(dims); - printf("%s","Test passed: "); - if(result==0) - printf("%s", "YES\n"); - else - printf("%s", "NO\n"); - + printf("Test passed: %s\n", result == 0 ? "YES" : "NO"); return result; } diff --git a/tests/c/test_c_client.py b/tests/c/test_c_client.py index f461ba8df..170833f21 100644 --- a/tests/c/test_c_client.py +++ b/tests/c/test_c_client.py @@ -46,7 +46,7 @@ def get_test_names(): @pytest.mark.parametrize("test", get_test_names()) -def test_c_client(test, use_cluster, build, link): +def test_c_client(test, build, link): """This function actually runs the tests using the parameterization function provided in Pytest @@ -61,9 +61,7 @@ def test_c_client(test, use_cluster, build, link): # . prepend the path to the built test executable test = f"{getcwd()}/build/{build}/tests/{link}/{test}" cmd = [test] - print(f"Running test: {osp.basename(test)}") - print(f"Test command {' '.join(cmd)}") - print(f"Using cluster: {use_cluster}") + print(f"\nRunning test: {osp.basename(test)}") execute_cmd(cmd) time.sleep(1) diff --git a/tests/cpp/client_test_copy_dataset.cpp b/tests/cpp/client_test_copy_dataset.cpp index cc0e5a07b..69d3538dc 100644 --- a/tests/cpp/client_test_copy_dataset.cpp +++ b/tests/cpp/client_test_copy_dataset.cpp @@ -52,7 +52,7 @@ void put_and_copy_dataset( fill_array(t_send_3, dims[0], dims[1], dims[2]); //Create Client and DataSet - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_copy_dataset"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_copy_dataset"); SmartRedis::DataSet source_dataset(dataset_name); //Add metadata to the DataSet diff --git a/tests/cpp/client_test_dataset.cpp b/tests/cpp/client_test_dataset.cpp index a879e0dc2..fedd51501 100644 --- a/tests/cpp/client_test_dataset.cpp +++ b/tests/cpp/client_test_dataset.cpp @@ -52,7 +52,7 @@ void put_get_dataset( fill_array(t_send_3, dims[0], dims[1], dims[2]); //Create Client and DataSet - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_dataset"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_dataset"); SmartRedis::DataSet sent_dataset(dataset_name); //Add metadata to the DataSet diff --git a/tests/cpp/client_test_dataset_aggregation.cpp b/tests/cpp/client_test_dataset_aggregation.cpp index 392d5ec4e..4beba37e2 100644 --- a/tests/cpp/client_test_dataset_aggregation.cpp +++ b/tests/cpp/client_test_dataset_aggregation.cpp @@ -105,14 +105,12 @@ void check_dataset(SmartRedis::DataSet& dataset_1, //Check that the metadata values are correct for the metadata DATASET_TEST_UTILS::check_dataset_metadata(dataset_1); DATASET_TEST_UTILS::check_dataset_metadata(dataset_2); - - return; } int main(int argc, char* argv[]) { // Create client for dataset and aggregation list actions - SmartRedis::Client client(use_cluster(), "client_test_dataset_aggregation"); + SmartRedis::Client client("client_test_dataset_aggregation"); // Set a fill function for dataset creation void (*fill_function)(double***, int, int, int) = diff --git a/tests/cpp/client_test_dataset_copy_assignment.cpp b/tests/cpp/client_test_dataset_copy_assignment.cpp index 7fa444e81..2e0c62443 100644 --- a/tests/cpp/client_test_dataset_copy_assignment.cpp +++ b/tests/cpp/client_test_dataset_copy_assignment.cpp @@ -52,7 +52,7 @@ void copy_assignment( fill_array(t_send_3, dims[0], dims[1], dims[2]); //Create Client and DataSets - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_dataset_copy_assignment"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_dataset_copy_assignment"); SmartRedis::DataSet* dataset = new SmartRedis::DataSet(dataset_name); //Add tensors to the DataSet diff --git a/tests/cpp/client_test_dataset_copy_constructor.cpp b/tests/cpp/client_test_dataset_copy_constructor.cpp index 8d7380fa4..22e8dc90f 100644 --- a/tests/cpp/client_test_dataset_copy_constructor.cpp +++ b/tests/cpp/client_test_dataset_copy_constructor.cpp @@ -52,7 +52,7 @@ void copy_constructor( fill_array(t_send_3, dims[0], dims[1], dims[2]); //Create Client and DataSet - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_dataset_copy_constructor"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_dataset_copy_constructor"); SmartRedis::DataSet* dataset = new SmartRedis::DataSet(dataset_name); //Add tensors to the DataSet @@ -189,8 +189,6 @@ void copy_constructor( //Check that the metadata values are correct for the metadata DATASET_TEST_UTILS::check_dataset_metadata(full_dataset); - - return; } int main(int argc, char* argv[]) { diff --git a/tests/cpp/client_test_dataset_empty.cpp b/tests/cpp/client_test_dataset_empty.cpp index 4ca0c84dc..afbbc3d2b 100644 --- a/tests/cpp/client_test_dataset_empty.cpp +++ b/tests/cpp/client_test_dataset_empty.cpp @@ -35,7 +35,7 @@ void put_get_empty_dataset(std::string dataset_name) { //Create Client and DataSet - SmartRedis::Client client(use_cluster(), "client_test_dataset_empty"); + SmartRedis::Client client("client_test_dataset_empty"); SmartRedis::DataSet sent_dataset(dataset_name); //Put the DataSet into the database diff --git a/tests/cpp/client_test_dataset_move_assignment.cpp b/tests/cpp/client_test_dataset_move_assignment.cpp index 775897c21..e143f15b7 100644 --- a/tests/cpp/client_test_dataset_move_assignment.cpp +++ b/tests/cpp/client_test_dataset_move_assignment.cpp @@ -52,7 +52,7 @@ void put_get_3D_array( fill_array(t_send_3, dims[0], dims[1], dims[2]); //Create Client and DataSets - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_dataset_move_assignment"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_dataset_move_assignment"); SmartRedis::DataSet* dataset = new SmartRedis::DataSet(dataset_name); //Add tensors to the DataSet diff --git a/tests/cpp/client_test_dataset_move_constructor.cpp b/tests/cpp/client_test_dataset_move_constructor.cpp index 8e7b594f8..6a2476dbe 100644 --- a/tests/cpp/client_test_dataset_move_constructor.cpp +++ b/tests/cpp/client_test_dataset_move_constructor.cpp @@ -52,7 +52,7 @@ void move_constructor( fill_array(t_send_3, dims[0], dims[1], dims[2]); //Create Client and DataSets - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_dataet_move_constructor"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_dataet_move_constructor"); SmartRedis::DataSet* dataset = new SmartRedis::DataSet(dataset_name); //Add tensors to the DataSet diff --git a/tests/cpp/client_test_dataset_multiple_get_tensor.cpp b/tests/cpp/client_test_dataset_multiple_get_tensor.cpp index 10a838d1b..c5abc464b 100644 --- a/tests/cpp/client_test_dataset_multiple_get_tensor.cpp +++ b/tests/cpp/client_test_dataset_multiple_get_tensor.cpp @@ -52,7 +52,7 @@ void get_multiple_tensors( fill_array(t_send_3, dims[0], dims[1], dims[2]); //Create Client and DataSet - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_dataset_multiple_get_tensor"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_dataset_multiple_get_tensor"); SmartRedis::DataSet sent_dataset(dataset_name); //Add metadata to the DataSet diff --git a/tests/cpp/client_test_dataset_no_meta.cpp b/tests/cpp/client_test_dataset_no_meta.cpp index 04f84fd7b..bd4388d41 100644 --- a/tests/cpp/client_test_dataset_no_meta.cpp +++ b/tests/cpp/client_test_dataset_no_meta.cpp @@ -52,7 +52,7 @@ void put_get_dataset( fill_array(t_send_3, dims[0], dims[1], dims[2]); //Create Client and DataSet - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_dataset_no_meta"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_dataset_no_meta"); SmartRedis::DataSet sent_dataset(dataset_name); //Add tensors to the DataSet diff --git a/tests/cpp/client_test_dataset_no_tensors.cpp b/tests/cpp/client_test_dataset_no_tensors.cpp index 2fc52c795..a537b2bc3 100644 --- a/tests/cpp/client_test_dataset_no_tensors.cpp +++ b/tests/cpp/client_test_dataset_no_tensors.cpp @@ -34,7 +34,7 @@ void put_dataset_no_tensors(std::string dataset_name) { //Create Client and DataSet - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_dataset_no_tensors"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_dataset_no_tensors"); SmartRedis::DataSet sent_dataset(dataset_name); //Add metadata to the DataSet diff --git a/tests/cpp/client_test_delete_dataset.cpp b/tests/cpp/client_test_delete_dataset.cpp index 86201729b..5ba19b65b 100644 --- a/tests/cpp/client_test_delete_dataset.cpp +++ b/tests/cpp/client_test_delete_dataset.cpp @@ -52,7 +52,7 @@ void dataset_delete( fill_array(t_send_3, dims[0], dims[1], dims[2]); //Create Client and DataSets - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_delete_dataset"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_delete_dataset"); SmartRedis::DataSet* dataset = new SmartRedis::DataSet(dataset_name); //Add tensors to the DataSet diff --git a/tests/cpp/client_test_ensemble.cpp b/tests/cpp/client_test_ensemble.cpp index d7b978b02..6d76987bf 100644 --- a/tests/cpp/client_test_ensemble.cpp +++ b/tests/cpp/client_test_ensemble.cpp @@ -54,11 +54,11 @@ void load_mnist_image_to_array(float**** img) } void produce( - std::vector dims, - std::string keyout="", - std::string keyin="") + std::vector dims, + std::string keyout="", + std::string keyin="") { - SmartRedis::Client client(use_cluster(), "client_test_ensemble::producer"); + SmartRedis::Client client("client_test_ensemble::producer"); client.use_model_ensemble_prefix(true); // Tensors @@ -128,7 +128,7 @@ void consume(std::vector dims, std::string keyout="", std::string keyin="") { - SmartRedis::Client client(use_cluster(), "client_test_ensemble::consumer"); + SmartRedis::Client client("client_test_ensemble::consumer"); client.use_model_ensemble_prefix(true); // Tensors diff --git a/tests/cpp/client_test_ensemble_dataset.cpp b/tests/cpp/client_test_ensemble_dataset.cpp index 2447d0b40..12ba8f265 100644 --- a/tests/cpp/client_test_ensemble_dataset.cpp +++ b/tests/cpp/client_test_ensemble_dataset.cpp @@ -35,7 +35,7 @@ void rename_dataset(std::string keyout) { std::vector dims({10,10,2}); - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_ensemble_dataset"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_ensemble_dataset"); client.use_tensor_ensemble_prefix(true); client.use_dataset_ensemble_prefix(true); @@ -130,15 +130,13 @@ void rename_dataset(std::string keyout) //Check that the metadata values are correct for the metadata DATASET_TEST_UTILS::check_dataset_metadata(retrieved_dataset); - - return; } void add_to_aggregation_list(std::string keyout) { std::vector dims({10,10,2}); - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_ensemble_dataset"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_ensemble_dataset"); client.use_tensor_ensemble_prefix(true); client.use_dataset_ensemble_prefix(true); client.use_list_ensemble_prefix(true); @@ -237,8 +235,6 @@ void add_to_aggregation_list(std::string keyout) //Check that the metadata values are correct for the metadata DATASET_TEST_UTILS::check_dataset_metadata(retrieved_dataset); - - return; } int main(int argc, char* argv[]) { diff --git a/tests/cpp/client_test_mnist.cpp b/tests/cpp/client_test_mnist.cpp index f07684495..4ec4ecef9 100644 --- a/tests/cpp/client_test_mnist.cpp +++ b/tests/cpp/client_test_mnist.cpp @@ -52,7 +52,7 @@ void load_mnist_image_to_array(float**** img) void run_mnist(const std::string& model_name, const std::string& script_name) { - SmartRedis::Client client(use_cluster(), "client_test_mnist"); + SmartRedis::Client client("client_test_mnist"); float**** array = allocate_4D_array(1,1,28,28); float** result = allocate_2D_array(1, 10); @@ -78,7 +78,7 @@ void run_mnist(const std::string& model_name, int main(int argc, char* argv[]) { - SmartRedis::Client client(use_cluster(), "client_test_mnist"); + SmartRedis::Client client("client_test_mnist"); std::string model_key = "mnist_model"; std::string model_file = "mnist_data/mnist_cnn.pt"; client.set_model_chunk_size(1024 * 1024); diff --git a/tests/cpp/client_test_mnist_dataset.cpp b/tests/cpp/client_test_mnist_dataset.cpp index d732b27ea..e96aa4530 100644 --- a/tests/cpp/client_test_mnist_dataset.cpp +++ b/tests/cpp/client_test_mnist_dataset.cpp @@ -52,7 +52,7 @@ void load_mnist_image_to_array(float**** img) void run_mnist(const std::string& model_name, const std::string& script_name) { - SmartRedis::Client client(use_cluster(), "client_test_mnist_dataset"); + SmartRedis::Client client("client_test_mnist_dataset"); float**** array = allocate_4D_array(1,1,28,28); float** result = allocate_2D_array(1, 10); @@ -82,7 +82,7 @@ void run_mnist(const std::string& model_name, int main(int argc, char* argv[]) { - SmartRedis::Client client(use_cluster(), "client_test_mnist_dataset"); + SmartRedis::Client client("client_test_mnist_dataset"); std::string model_key = "mnist_model"; std::string model_file = "mnist_data/mnist_cnn.pt"; client.set_model_from_file(model_key, model_file, "TORCH", "CPU"); diff --git a/tests/cpp/client_test_put_get_1D.cpp b/tests/cpp/client_test_put_get_1D.cpp index 4b2f98c75..9a9b6d5cd 100644 --- a/tests/cpp/client_test_put_get_1D.cpp +++ b/tests/cpp/client_test_put_get_1D.cpp @@ -38,7 +38,7 @@ void put_get_1D_array( SRTensorType type, std::string key_suffix="") { - SmartRedis::Client client(use_cluster(), "client_test_put_get_1D"); + SmartRedis::Client client("client_test_put_get_1D"); //Allocate and fill arrays T_send* array = (T_send*)malloc(dims[0]*sizeof(T_send)); diff --git a/tests/cpp/client_test_put_get_2D.cpp b/tests/cpp/client_test_put_get_2D.cpp index 54ca6b18a..025ffce80 100644 --- a/tests/cpp/client_test_put_get_2D.cpp +++ b/tests/cpp/client_test_put_get_2D.cpp @@ -39,7 +39,7 @@ void put_get_2D_array( std::string key_suffix="") { - SmartRedis::Client client(use_cluster(), "client_test_put_get_2D"); + SmartRedis::Client client("client_test_put_get_2D"); //Allocate and fill arrays T_send** array = allocate_2D_array(dims[0], dims[1]); diff --git a/tests/cpp/client_test_put_get_3D.cpp b/tests/cpp/client_test_put_get_3D.cpp index 8b5b4c6ee..54fda99eb 100644 --- a/tests/cpp/client_test_put_get_3D.cpp +++ b/tests/cpp/client_test_put_get_3D.cpp @@ -38,7 +38,7 @@ void put_get_3D_array( SRTensorType type, std::string key_suffix="") { - SmartRedis::Client client(use_cluster(), "client_test_put_get_3D"); + SmartRedis::Client client("client_test_put_get_3D"); //Allocate and fill arrays T_send*** array = allocate_3D_array(dims[0], dims[1], dims[2]); diff --git a/tests/cpp/client_test_put_get_3D_static_values.cpp b/tests/cpp/client_test_put_get_3D_static_values.cpp index 01d5184eb..b8ce24421 100644 --- a/tests/cpp/client_test_put_get_3D_static_values.cpp +++ b/tests/cpp/client_test_put_get_3D_static_values.cpp @@ -37,7 +37,7 @@ void put_get_3D_array( SRTensorType type, std::string key_suffix="") { - SmartRedis::Client client(use_cluster(), "client_test_put_get_3D_static_values"); + SmartRedis::Client client("client_test_put_get_3D_static_values"); //Allocate and fill arrays T_send*** array = allocate_3D_array(dims[0], dims[1], dims[2]); diff --git a/tests/cpp/client_test_put_get_contiguous_3D.cpp b/tests/cpp/client_test_put_get_contiguous_3D.cpp index 634ce96fa..138d1afc9 100644 --- a/tests/cpp/client_test_put_get_contiguous_3D.cpp +++ b/tests/cpp/client_test_put_get_contiguous_3D.cpp @@ -38,7 +38,7 @@ void put_get_3D_array( SRTensorType type, std::string key_suffix="") { - SmartRedis::Client client(use_cluster(), "client_test_put_get_contiguous_3D"); + SmartRedis::Client client("client_test_put_get_contiguous_3D"); //Allocate and fill arrays T_send* array = diff --git a/tests/cpp/client_test_put_get_transpose_3D.cpp b/tests/cpp/client_test_put_get_transpose_3D.cpp index 0f5fff81f..d0184a72d 100644 --- a/tests/cpp/client_test_put_get_transpose_3D.cpp +++ b/tests/cpp/client_test_put_get_transpose_3D.cpp @@ -60,7 +60,7 @@ void put_get_3D_array( SRMemoryLayout send_direction = SRMemLayoutContiguous, SRMemoryLayout recv_direction = SRMemLayoutContiguous) { - SmartRedis::Client client(use_cluster(), "client_test_put_get_transpose_3D"); + SmartRedis::Client client("client_test_put_get_transpose_3D"); //Allocate and fill arrays T_send* array = (T_send*)malloc(dims[0]*dims[1]*dims[2]*sizeof(T_send)); diff --git a/tests/cpp/client_test_rename_dataset.cpp b/tests/cpp/client_test_rename_dataset.cpp index 148e583d8..5d14719bb 100644 --- a/tests/cpp/client_test_rename_dataset.cpp +++ b/tests/cpp/client_test_rename_dataset.cpp @@ -52,7 +52,7 @@ void rename_dataset( fill_array(t_send_3, dims[0], dims[1], dims[2]); //Create Client and DataSet - DATASET_TEST_UTILS::DatasetTestClient client(use_cluster(), "client_test_rename_dataset"); + DATASET_TEST_UTILS::DatasetTestClient client("client_test_rename_dataset"); SmartRedis::DataSet sent_dataset(dataset_name); //Add metadata to the DataSet diff --git a/tests/cpp/client_test_utils.h b/tests/cpp/client_test_utils.h index 1965bbb13..eb4df51e4 100644 --- a/tests/cpp/client_test_utils.h +++ b/tests/cpp/client_test_utils.h @@ -30,18 +30,20 @@ #define SMARTREDIS_TEST_UTILS_H #include +#include #include #include "rediscluster.h" #include "srobject.h" +#include "configoptions.h" using namespace SmartRedis; class RedisClusterTestObject : public RedisCluster { public: - RedisClusterTestObject(const SRObject* context) - : RedisCluster(context) {}; + RedisClusterTestObject(ConfigOptions* cfgopts) + : RedisCluster(cfgopts) {}; std::string get_crc16_prefix(uint64_t hash_slot) { return _get_crc16_prefix(hash_slot); @@ -53,31 +55,34 @@ inline void to_lower(char* s) { c-str into the lowercase value. This assumes the c-str is null terminated. */ - if(!s) + if (s == NULL) return; - while((*s)!=0) { - if( *s>='A' && *s<='Z') + while (*s != '\0') { + if (*s >= 'A' && *s <= 'Z') *s = *s - 'A' + 'a'; s++; } - return; } inline bool use_cluster() { /* This function determines if a cluster - configuration should be used in the test - when creating a Client. + configuration is being used */ - char* smartredis_test_cluster = std::getenv("SMARTREDIS_TEST_CLUSTER"); - to_lower(smartredis_test_cluster); - - if(smartredis_test_cluster) { - if(std::strcmp(smartredis_test_cluster, "true")==0) - return true; + char* server_type = std::getenv("SR_DB_TYPE"); + if (server_type == NULL) + return false; + + // lower-case a copy of the environment variable + // not the value in memory + char copy[256]; + for (int i = 0; i < 256; i++) { + copy[i] = std::tolower(server_type[i]); + if (copy[i] == '\0') + break; } - return false; + return std::strcmp(copy, "clustered") == 0; } template @@ -86,9 +91,9 @@ T** allocate_2D_array(int dim_1, int dim_2) /* This function allocates a 2D array and and returns a pointer to that 2D array. */ - T **array = (T **)malloc(dim_1*sizeof(T *)); - for (int i=0; i void free_4D_array(T**** array, int dim_1, int dim_2, int dim_3) { - for(int i=0; i @@ -176,9 +180,10 @@ bool is_equal_1D_array(T* a, U* b, int dim_1) /* This function compares two arrays to make sure their values are identical. */ - for(int i=0; i distribution; - for(int i=0; i @@ -253,7 +260,7 @@ void set_1D_array_integral_values(T* a, int dim_1) T t_min = std::numeric_limits::min(); T t_max = std::numeric_limits::max(); std::uniform_int_distribution distribution(t_min, t_max); - for(int i=0; i datasets; diff --git a/tests/cpp/unit-tests/test_client.cpp b/tests/cpp/unit-tests/test_client.cpp index dd637b383..bc551491d 100644 --- a/tests/cpp/unit-tests/test_client.cpp +++ b/tests/cpp/unit-tests/test_client.cpp @@ -101,7 +101,7 @@ SCENARIO("Testing Dataset Functions on Client Object", "[Client]") log_data(context, LLDebug, "***Beginning Client testing***"); GIVEN("A Client object") { - Client client(use_cluster(), "test_client"); + Client client("test_client"); THEN("get, rename, and copy DataSet called on " "a nonexistent DataSet throws errors") @@ -111,7 +111,7 @@ SCENARIO("Testing Dataset Functions on Client Object", "[Client]") KeyException); CHECK_THROWS_AS( client.rename_dataset("DNE", "rename_DNE"), - KeyException); + KeyException); CHECK_THROWS_AS( client.copy_dataset("src_DNE", "dest_DNE"), KeyException); @@ -200,7 +200,7 @@ SCENARIO("Testing Tensor Functions on Client Object", "[Client]") log_data(context, LLDebug, "***Beginning Client tensor testing***"); GIVEN("A Client object") { - Client client(use_cluster(), "test_client"); + Client client("test_client"); AND_WHEN("Tensors of each type are created and put into the Client") { @@ -483,7 +483,7 @@ SCENARIO("Testing INFO Functions on Client Object", "[Client]") log_data(context, LLDebug, "***Beginning Client INFO testing***"); GIVEN("A Client object") { - Client client(use_cluster(), "test_client"); + Client client("test_client"); THEN("The client can be serialized") { @@ -491,7 +491,6 @@ SCENARIO("Testing INFO Functions on Client Object", "[Client]") CHECK(serial.length() > 0); std::cout << client; } - WHEN("INFO or CLUSTER INFO is called on database with " "an invalid address") { @@ -510,7 +509,7 @@ SCENARIO("Testing INFO Functions on Client Object", "[Client]") { THEN("No errors with be thrown for both cluster and " - "non-cluster environemnts") + "non-cluster environments") { std::string db_address = parse_SSDB(std::getenv("SSDB")); @@ -542,7 +541,7 @@ SCENARIO("Testing AI.INFO Functions on Client Object", "[Client]") log_data(context, LLDebug, "***Beginning Client AI.INFO testing***"); GIVEN("A Client object") { - Client client(use_cluster(), "test_client"); + Client client("test_client"); WHEN("AI.INFO called on database with an invalid address") { @@ -595,7 +594,7 @@ SCENARIO("Testing FLUSHDB on empty Client Object", "[Client][FLUSHDB]") GIVEN("An empty non-cluster Client object") { - Client client(use_cluster(), "test_client"); + Client client("test_client"); WHEN("FLUSHDB is called on database with " "an invalid address") @@ -639,7 +638,7 @@ SCENARIO("Testing FLUSHDB on Client Object", "[Client][FLUSHDB]") if (use_cluster()) return; - Client client(use_cluster(), "test_client"); + Client client("test_client"); std::string dataset_name = "test_dataset_name"; DataSet dataset(dataset_name); dataset.add_meta_string("meta_string_name", "meta_string_val"); @@ -679,7 +678,7 @@ SCENARIO("Testing CONFIG GET and CONFIG SET on Client Object", "[Client]") GIVEN("A Client object") { - Client client(use_cluster(), "test_client"); + Client client("test_client"); WHEN("CONFIG GET or CONFIG SET are called on databases with " "invalid addresses ") @@ -728,7 +727,7 @@ SCENARIO("Test CONFIG GET on an unsupported command", "[Client]") GIVEN("A client object") { - Client client(use_cluster(), "test_client"); + Client client("test_client"); std::string address = parse_SSDB(std::getenv("SSDB")); WHEN("CONFIG GET is called with an unsupported command") @@ -753,7 +752,7 @@ SCENARIO("Test CONFIG SET on an unsupported command", "[Client]") GIVEN("A client object") { - Client client(use_cluster(), "test_client"); + Client client("test_client"); std::string address = parse_SSDB(std::getenv("SSDB")); WHEN("CONFIG SET is called with an unsupported command") @@ -778,7 +777,7 @@ SCENARIO("Testing SAVE command on Client Object", "[!mayfail][Client][SAVE]") GIVEN("A client object and some data") { - Client client(use_cluster(), "test_client"); + Client client("test_client"); std::string dataset_name = "test_save_dataset"; DataSet dataset(dataset_name); dataset.add_meta_string("meta_string_save_name", "meta_string_val"); @@ -825,8 +824,10 @@ SCENARIO("Test that prefixing covers all hash slots of a cluster", "[Client]") GIVEN("A test RedisCluster test object") { + ConfigOptions* cfgopts = ConfigOptions::create_from_environment("").release(); LogContext context("test_client"); - RedisClusterTestObject redis_cluster(&context); + cfgopts->_set_log_context(&context); + RedisClusterTestObject redis_cluster(cfgopts); WHEN("A prefix is requested for a hash slot between 0 and 16384") { @@ -865,7 +866,7 @@ SCENARIO("Testing Multi-GPU Function error cases", "[Client]") GIVEN("A Client object, a script, and a model") { - Client client(use_cluster(), "test_client"); + Client client("test_client"); std::string model_key = "a_model"; std::string model_file = "../mnist_data/mnist_cnn.pt"; std::string script_key = "a_script"; diff --git a/tests/cpp/unit-tests/test_client_ensemble.cpp b/tests/cpp/unit-tests/test_client_ensemble.cpp index 9268a9b92..2ce69a894 100644 --- a/tests/cpp/unit-tests/test_client_ensemble.cpp +++ b/tests/cpp/unit-tests/test_client_ensemble.cpp @@ -131,7 +131,7 @@ SCENARIO("Testing Client ensemble using a producer/consumer paradigm") setenv("SSKEYIN", keyin_env_put, (old_keyin != NULL)); setenv("SSKEYOUT", keyout_env_put, (old_keyout != NULL)); - Client producer_client(use_cluster(), "test_client_ensemble::producer"); + Client producer_client("test_client_ensemble::producer"); producer_client.use_model_ensemble_prefix(true); producer_client.set_model_chunk_size(1024 * 1024); @@ -185,7 +185,7 @@ SCENARIO("Testing Client ensemble using a producer/consumer paradigm") setenv("SSKEYIN", keyin_env_get, 1); setenv("SSKEYOUT", keyout_env_get, 1); - Client consumer_client(use_cluster(), "test_client_ensemble::consumer"); + Client consumer_client("test_client_ensemble::consumer"); consumer_client.use_model_ensemble_prefix(true); // Tensors diff --git a/tests/cpp/unit-tests/test_client_prefixing.cpp b/tests/cpp/unit-tests/test_client_prefixing.cpp index 57922ce49..4b6138be8 100644 --- a/tests/cpp/unit-tests/test_client_prefixing.cpp +++ b/tests/cpp/unit-tests/test_client_prefixing.cpp @@ -65,11 +65,11 @@ SCENARIO("Testing Client prefixing") std::vector g_dims; void* g_result; - THEN(" Client prefixing can be tested") + THEN("Client prefixing can be tested") { setenv("SSKEYIN", keyin_env_put, (old_keyin != NULL)); setenv("SSKEYOUT", keyout_env_put, (old_keyout != NULL)); - Client client(use_cluster(), context); + Client client(context); client.use_dataset_ensemble_prefix(true); client.use_tensor_ensemble_prefix(true); diff --git a/tests/cpp/unit-tests/test_configopts.cpp b/tests/cpp/unit-tests/test_configopts.cpp index c373c8612..5469d8d01 100644 --- a/tests/cpp/unit-tests/test_configopts.cpp +++ b/tests/cpp/unit-tests/test_configopts.cpp @@ -111,20 +111,20 @@ SCENARIO("Testing for ConfigOptions", "[CfgOpts]") log_data(context, LLDebug, "***End ConfigOptions testing***"); } -SCENARIO("Prefix Testing for ConfigOptions", "[CfgOpts]") +SCENARIO("Suffix Testing for ConfigOptions", "[CfgOpts]") { - std::cout << std::to_string(get_time_offset()) << ": Prefix Testing for ConfigOptions" << std::endl; + std::cout << std::to_string(get_time_offset()) << ": Suffix Testing for ConfigOptions" << std::endl; std::string context("test_configopts"); - log_data(context, LLDebug, "***Beginning ConfigOptions prefix testing***"); + log_data(context, LLDebug, "***Beginning ConfigOptions suffix testing***"); GIVEN("A ConfigOptions object") { // Make sure keys aren't set before we start const char* keys[] = { - "prefixtest_integer_key_that_is_not_really_present", - "prefixtest_string_key_that_is_not_really_present", - "prefixtest_integer_key", - "prefixtest_string_key" + "integer_key_that_is_not_really_present_suffixtest", + "string_key_that_is_not_really_present_suffixtest", + "integer_key_suffixtest", + "string_key_suffixtest" }; INFO("Reserved keys must not be set before running this test."); for (size_t i = 0; i < sizeof(keys)/sizeof(keys[0]); i++) { @@ -132,12 +132,12 @@ SCENARIO("Prefix Testing for ConfigOptions", "[CfgOpts]") } // Set up keys for testing - setenv("prefixtest_integer_key", "42", true); - setenv("prefixtest_string_key", "charizard", true); + setenv("integer_key_suffixtest", "42", true); + setenv("string_key_suffixtest", "charizard", true); - auto co = ConfigOptions::create_from_environment("prefixtest"); + auto co = ConfigOptions::create_from_environment("suffixtest"); - THEN("Prefixed options should be configurable") + THEN("Suffixed options should be configurable") { // integer option tests CHECK(co->get_integer_option("integer_key") == 42); @@ -180,9 +180,9 @@ SCENARIO("Prefix Testing for ConfigOptions", "[CfgOpts]") } // Clean up test keys - unsetenv("prefixtest_integer_key"); - unsetenv("prefixtest_string_key"); + unsetenv("integer_key_suffixtest"); + unsetenv("string_key_suffixtest"); - log_data(context, LLDebug, "***End ConfigOptions prefix testing***"); + log_data(context, LLDebug, "***End ConfigOptions suffix testing***"); } diff --git a/tests/cpp/unit-tests/test_logger.cpp b/tests/cpp/unit-tests/test_logger.cpp index 6ffe5c911..d9b1bdcdb 100644 --- a/tests/cpp/unit-tests/test_logger.cpp +++ b/tests/cpp/unit-tests/test_logger.cpp @@ -45,7 +45,7 @@ SCENARIO("Additional Testing for logging", "[LOG]") GIVEN("A Logger object") { - Client client(use_cluster(), "test_logger"); + Client client("test_logger"); THEN("Logging should be able to be done") { diff --git a/tests/cpp/unit-tests/test_redisserver.cpp b/tests/cpp/unit-tests/test_redisserver.cpp index 79626609e..60875b0c9 100644 --- a/tests/cpp/unit-tests/test_redisserver.cpp +++ b/tests/cpp/unit-tests/test_redisserver.cpp @@ -57,7 +57,7 @@ using namespace SmartRedis; class RedisTest : public Redis { public: - RedisTest(SRObject* c) : Redis(c) {} + RedisTest(ConfigOptions* c) : Redis(c) {} int get_connection_timeout() {return _connection_timeout;} int get_connection_interval() {return _connection_interval;} int get_command_timeout() {return _command_timeout;} @@ -73,7 +73,7 @@ class RedisTest : public Redis class RedisClusterTest : public RedisCluster { public: - RedisClusterTest(SRObject* c) : RedisCluster(c) {} + RedisClusterTest(ConfigOptions* c) : RedisCluster(c) {} int get_connection_timeout() {return _connection_timeout;} int get_connection_interval() {return _connection_interval;} int get_command_timeout() {return _command_timeout;} @@ -97,12 +97,14 @@ const char* CMD_INTERVAL_ENV_VAR = "SR_CMD_INTERVAL"; // error to be thrown void invoke_constructor() { + ConfigOptions* cfgopts = ConfigOptions::create_from_environment("").release(); LogContext context("test_redisserver"); + cfgopts->_set_log_context(&context); if (use_cluster()) { - RedisClusterTest cluster_obj(&context); + RedisClusterTest cluster_obj(cfgopts); } else { - RedisTest non_cluster_obj(&context); + RedisTest non_cluster_obj(cfgopts); } } @@ -177,6 +179,8 @@ SCENARIO("Test runtime settings are initialized correctly", "[RedisServer]") std::string context("test_redisserver"); log_data(context, LLDebug, "***Beginning RedisServer testing***"); LogContext lc("test_redisserver"); + ConfigOptions* cfgopts = ConfigOptions::create_from_environment("").release(); + cfgopts->_set_log_context(&lc); char* __conn_timeout; char* __conn_interval; @@ -188,14 +192,14 @@ SCENARIO("Test runtime settings are initialized correctly", "[RedisServer]") { unset_all_env_vars(); if (use_cluster()) { - RedisClusterTest redis_server(&lc); + RedisClusterTest redis_server(cfgopts); THEN("Default member variable values are used") { check_all_defaults(redis_server); } } else { - RedisTest redis_server(&lc); + RedisTest redis_server(cfgopts); THEN("Default member variable values are used") { check_all_defaults(redis_server); @@ -211,14 +215,14 @@ SCENARIO("Test runtime settings are initialized correctly", "[RedisServer]") setenv(CMD_INTERVAL_ENV_VAR, "", true); if (use_cluster()) { - RedisClusterTest redis_server(&lc); + RedisClusterTest redis_server(cfgopts); THEN("Default member variable values are used") { check_all_defaults(redis_server); } } else { - RedisTest redis_server(&lc); + RedisTest redis_server(cfgopts); THEN("Default member variable values are used") { check_all_defaults(redis_server); @@ -241,7 +245,7 @@ SCENARIO("Test runtime settings are initialized correctly", "[RedisServer]") setenv(CMD_INTERVAL_ENV_VAR, std::to_string(cmd_interval).c_str(), true); if (use_cluster()) { - RedisClusterTest redis_server(&lc); + RedisClusterTest redis_server(cfgopts); THEN("Environment variables are used for member variables") { CHECK(redis_server.get_connection_timeout() == @@ -260,7 +264,7 @@ SCENARIO("Test runtime settings are initialized correctly", "[RedisServer]") } } else { - RedisTest redis_server(&lc); + RedisTest redis_server(cfgopts); THEN("Environment variables are used for member variables") { CHECK(redis_server.get_connection_timeout() == diff --git a/tests/cpp/unit-tests/test_ssdb.cpp b/tests/cpp/unit-tests/test_ssdb.cpp index a66d4a3cc..3e6238708 100644 --- a/tests/cpp/unit-tests/test_ssdb.cpp +++ b/tests/cpp/unit-tests/test_ssdb.cpp @@ -33,6 +33,7 @@ #include "logger.h" #include "logcontext.h" #include "srobject.h" +#include "configoptions.h" unsigned long get_time_offset(); @@ -42,12 +43,17 @@ using namespace SmartRedis; class TestSSDB : public Redis { public: - TestSSDB(const SRObject* c) : Redis(c) {} + TestSSDB(ConfigOptions* c) : Redis(c) {} SRAddress get_ssdb() { return _get_ssdb(); } + + void clear_cached_SSDB() + { + _cfgopts->_clear_option_from_cache("SSDB"); + } }; // helper function for putting the SSDB environment @@ -64,7 +70,9 @@ SCENARIO("Additional Testing for various SSDBs", "[SSDB]") std::cout << std::to_string(get_time_offset()) << ": Additional Testing for various SSDBs" << std::endl; std::string context("test_ssdb"); log_data(context, LLDebug, "***Beginning SSDB testing***"); + ConfigOptions* cfgopts = ConfigOptions::create_from_environment("").release(); LogContext lc("test_ssdb"); + cfgopts->_set_log_context(&lc); GIVEN("A TestSSDB object") { @@ -74,7 +82,7 @@ SCENARIO("Additional Testing for various SSDBs", "[SSDB]") "port before running this test."); REQUIRE(old_ssdb != NULL); - TestSSDB test_ssdb(&lc); + TestSSDB test_ssdb(cfgopts); Client* c = NULL; THEN("SSDB environment variable must exist " @@ -82,17 +90,21 @@ SCENARIO("Additional Testing for various SSDBs", "[SSDB]") { // SSDB is nullptr unsetenv("SSDB"); + test_ssdb.clear_cached_SSDB(); CHECK_THROWS_AS(test_ssdb.get_ssdb(), SmartRedis::RuntimeException); // SSDB contains invalid characters setenv_ssdb ("127.0.0.1:*&^9"); + test_ssdb.clear_cached_SSDB(); CHECK_THROWS_AS(test_ssdb.get_ssdb(), SmartRedis::RuntimeException); // Valid SSDB. Ensure one of 127 or 128 is chosen setenv_ssdb("127,128"); + test_ssdb.clear_cached_SSDB(); CHECK_THROWS_AS(test_ssdb.get_ssdb(), SmartRedis::RuntimeException); // SSDB points to a unix domain socket and we're using clustered Redis + // FINDME: This test uses a deprecated constructor and will need to be rewritten setenv_ssdb ("unix://127.0.0.1:6349"); CHECK_THROWS_AS(c = new Client(true, "test_ssdb"), SmartRedis::RuntimeException); diff --git a/tests/cpp/unit-tests/test_unit_cpp_client.py b/tests/cpp/unit-tests/test_unit_cpp_client.py index 71956a8fd..c4b6ddd0a 100644 --- a/tests/cpp/unit-tests/test_unit_cpp_client.py +++ b/tests/cpp/unit-tests/test_unit_cpp_client.py @@ -46,16 +46,14 @@ def get_test_names(): @pytest.mark.parametrize("test", get_test_names()) -def test_unit_cpp_client(test, use_cluster, build, link): +def test_unit_cpp_client(test, build, link): # Build the path to the test executable from the source file name # . keep only the last three parts of the path: (language, unit-tests, basename) test = "/".join(test.split("/")[-3:]) # . prepend the path to the built test executable test = f"{getcwd()}/build/{build}/tests/{link}/{test}" cmd = [test] - print(f"Running test: {osp.basename(test)}") - print(f"Test command {' '.join(cmd)}") - print(f"Using cluster: {use_cluster}") + print(f"\nRunning test: {osp.basename(test)}") execute_cmd(cmd) time.sleep(1) diff --git a/tests/docker/c/test_docker.c b/tests/docker/c/test_docker.c index 8f24cef94..56f3581e3 100644 --- a/tests/docker/c/test_docker.c +++ b/tests/docker/c/test_docker.c @@ -40,7 +40,7 @@ int main(int argc, char* argv[]) { size_t cid_len = strlen(logger_name); SRError return_code = SRNoError; - return_code = SmartRedisCClient(false, logger_name, cid_len, &client); + return_code = SimpleCreateClient(logger_name, cid_len, &client); if (return_code != SRNoError) { return -1; diff --git a/tests/docker/cpp/docker_test.cpp b/tests/docker/cpp/docker_test.cpp index 0df4786a5..215c3670a 100644 --- a/tests/docker/cpp/docker_test.cpp +++ b/tests/docker/cpp/docker_test.cpp @@ -30,7 +30,7 @@ int main(int argc, char* argv[]) { - SmartRedis::Client client(false, __FILE__); + SmartRedis::Client client(__FILE__); std::vector data = {1.0, 2.0, 3.0}; std::vector dims = {3}; diff --git a/tests/docker/fortran/CMakeLists.txt b/tests/docker/fortran/CMakeLists.txt index 5e3217a3a..cf989bbd6 100644 --- a/tests/docker/fortran/CMakeLists.txt +++ b/tests/docker/fortran/CMakeLists.txt @@ -30,28 +30,29 @@ cmake_minimum_required(VERSION 3.13) project(DockerTesterFortran) enable_language(Fortran) -set(CMAKE_VERBOSE_MAKEFILE ON) -set(CMAKE_BUILD_TYPE Debug) +# Configure the build set(CMAKE_CXX_STANDARD 17) -set(CMAKE_C_STANDARD 99) +SET(CMAKE_C_STANDARD 99) +set(CMAKE_BUILD_TYPE Debug) -set(ftn_client_src - /usr/local/src/SmartRedis/src/fortran/fortran_c_interop.F90 - /usr/local/src/SmartRedis/src/fortran/dataset.F90 - /usr/local/src/SmartRedis/src/fortran/client.F90 +# Locate dependencies +find_library(SR_LIB smartredis REQUIRED) +find_library(SR_FTN_LIB smartredis-fortran REQUIRED) +set(SMARTREDIS_LIBRARIES + ${SR_LIB} + ${SR_FTN_LIB} ) -find_library(SR_LIB smartredis) - +# Define include directories for header files include_directories(SYSTEM /usr/local/include/smartredis ) +# Build the test add_executable(docker_test_fortran test_docker.F90 - ${ftn_client_src} ) set_target_properties(docker_test_fortran PROPERTIES OUTPUT_NAME docker_test ) -target_link_libraries(docker_test_fortran ${SR_LIB} pthread) +target_link_libraries(docker_test_fortran ${SMARTREDIS_LIBRARIES} pthread) diff --git a/tests/docker/fortran/test_docker.F90 b/tests/docker/fortran/test_docker.F90 index 0bc705fc2..d889eebea 100644 --- a/tests/docker/fortran/test_docker.F90 +++ b/tests/docker/fortran/test_docker.F90 @@ -38,7 +38,7 @@ program main real(kind=8), dimension(dim1) :: tensor real(kind=8), dimension(dim1) :: returned - result = client%initialize(.FALSE., "test_docker.F90") + result = client%initialize("test_docker.F90") if (result .ne. SRNoError) stop call random_number(tensor) diff --git a/tests/docker/python/test_docker.py b/tests/docker/python/test_docker.py index 313a5aec4..c66d4188e 100644 --- a/tests/docker/python/test_docker.py +++ b/tests/docker/python/test_docker.py @@ -29,7 +29,7 @@ from smartredis import Client import numpy as np -client = Client(None, False) +client = Client(None) tensor = np.random.randint(-10, 10, size=(2,4)) diff --git a/tests/fortran/client_test_configoptions.F90 b/tests/fortran/client_test_configoptions.F90 index f160bf91e..9111b81cf 100644 --- a/tests/fortran/client_test_configoptions.F90 +++ b/tests/fortran/client_test_configoptions.F90 @@ -43,15 +43,15 @@ program main !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !! Establish test keys - ! non-prefixed testing keys + ! non-suffixed testing keys call setenv("test_integer_key", "42") call setenv("test_string_key", "charizard") - ! prefixed testing keys - call setenv("prefixtest_integer_key", "42") - call setenv("prefixtest_string_key", "charizard") + ! suffixed testing keys + call setenv("integer_key_suffixtest", "42") + call setenv("string_key_suffixtest", "charizard") !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! non-prefixed option testing + !! non-suffixed option testing result = co%create_configoptions_from_environment(""); if (result .ne. SRNoError) error stop @@ -118,12 +118,12 @@ program main !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! Prefixed testing - result = co%create_configoptions_from_environment("prefixtest"); + ! suffixtest testing + result = co%create_configoptions_from_environment("suffixtest"); if (result .ne. SRNoError) error stop ! integer option tests - write(*,*) "ConfigOption testing: prefixed integer option tests" + write(*,*) "ConfigOption testing: suffixed integer option tests" result = co%get_integer_option("integer_key", iresult) if (result .ne. SRNoError) error stop @@ -155,7 +155,7 @@ program main ! string option tests - write(*,*) "ConfigOption testing: prefixed string option tests" + write(*,*) "ConfigOption testing: suffixed string option tests" result = co%get_string_option("string_key", sresult) if (result .ne. SRNoError) error stop @@ -189,9 +189,9 @@ program main ! non-prefixed testing keys call unsetenv("test_integer_key") call unsetenv("test_string_key") - ! prefixed testing keys - call unsetenv("prefixtest_integer_key") - call unsetenv("prefixtest_string_key") + ! suffixed testing keys + call unsetenv("integer_key_suffixtest") + call unsetenv("string_key_suffixtest") ! Done write(*,*) "ConfigOption testing: passed" diff --git a/tests/fortran/client_test_dataset.F90 b/tests/fortran/client_test_dataset.F90 index 388e435f6..b4e82e5e2 100644 --- a/tests/fortran/client_test_dataset.F90 +++ b/tests/fortran/client_test_dataset.F90 @@ -222,7 +222,7 @@ program main call dataset%print_dataset(stdout) ! test dataset_existence - result = client%initialize(use_cluster(), "client_test_dataset") + result = client%initialize("client_test_dataset") if (result .ne. SRNoError) error stop result = client%dataset_exists("nonexistent", exists) if (result .ne. SRNoError) error stop diff --git a/tests/fortran/client_test_dataset_aggregation.F90 b/tests/fortran/client_test_dataset_aggregation.F90 index bc7b24689..67420b2ea 100644 --- a/tests/fortran/client_test_dataset_aggregation.F90 +++ b/tests/fortran/client_test_dataset_aggregation.F90 @@ -51,8 +51,7 @@ program main character(len=12) :: dataset_name integer :: result - result = client%initialize(use_cluster(), & - "client_test_dataset_aggregation") + result = client%initialize("client_test_dataset_aggregation") if (result .ne. SRNoError) error stop call random_number(true_vectors) diff --git a/tests/fortran/client_test_ensemble.F90 b/tests/fortran/client_test_ensemble.F90 index 262a6fd99..d0522fadb 100644 --- a/tests/fortran/client_test_ensemble.F90 +++ b/tests/fortran/client_test_ensemble.F90 @@ -49,7 +49,7 @@ program main call setenv("SSKEYIN", "producer_0,producer_1") call setenv("SSKEYOUT", ensemble_keyout) -result = client%initialize(use_cluster(), "client_test_ensemble") +result = client%initialize("client_test_ensemble") if (result .ne. SRNoError) error stop result = client%use_model_ensemble_prefix(.true.) if (result .ne. SRNoError) error stop @@ -103,7 +103,7 @@ program main call setenv("SSKEYIN", "producer_1,producer_0") call setenv("SSKEYOUT", ensemble_keyout) -result = client%initialize(use_cluster(), "client_test_ensemble") +result = client%initialize("client_test_ensemble") if (result .ne. SRNoError) error stop result = client%use_model_ensemble_prefix(.true.) if (result .ne. SRNoError) error stop diff --git a/tests/fortran/client_test_errors.F90 b/tests/fortran/client_test_errors.F90 index a515b1ff9..9dbefe029 100644 --- a/tests/fortran/client_test_errors.F90 +++ b/tests/fortran/client_test_errors.F90 @@ -39,11 +39,11 @@ program main type(client_type) :: client integer :: result - result = client%initialize(use_cluster()) - if (result .ne. SRNoError) error stop + result = client%initialize() + if (result .ne. SRNoError) error stop "Initialization failed" result = client%rename_tensor("vanilla", "chocolate") - if (result .eq. SRNoError) error stop + if (result .eq. SRNoError) error stop "rename didn't fail" write(*,*) "Printing last error retrieved as string" write(*,*) get_last_error() diff --git a/tests/fortran/client_test_initialized.F90 b/tests/fortran/client_test_initialized.F90 index f4eac43e9..2a30027a5 100644 --- a/tests/fortran/client_test_initialized.F90 +++ b/tests/fortran/client_test_initialized.F90 @@ -42,7 +42,7 @@ program main if (client%isinitialized()) error stop 'client not initialized' - result = client%initialize(use_cluster(), "client_test_initialized") + result = client%initialize("client_test_initialized") if (result .ne. SRNoError) error stop if (.not. client%isinitialized()) error stop 'client is initialized' diff --git a/tests/fortran/client_test_logging.F90 b/tests/fortran/client_test_logging.F90 index 2f6023e0c..c14993821 100644 --- a/tests/fortran/client_test_logging.F90 +++ b/tests/fortran/client_test_logging.F90 @@ -46,7 +46,7 @@ program main result = logcontext%initialize("client_test_logging (logcontext)") if (result .ne. SRNoError) error stop - result = client%initialize(use_cluster(), "client_test_logging (client)") + result = client%initialize("client_test_logging (client)") if (result .ne. SRNoError) error stop result = dataset%initialize("client_test_logging (dataset)") if (result .ne. SRNoError) error stop diff --git a/tests/fortran/client_test_misc_tensor.F90 b/tests/fortran/client_test_misc_tensor.F90 index 30372d558..2c20b74f6 100644 --- a/tests/fortran/client_test_misc_tensor.F90 +++ b/tests/fortran/client_test_misc_tensor.F90 @@ -46,7 +46,7 @@ program main integer :: result logical :: exists - result = client%initialize(use_cluster(), "client_test_misc_tensor") + result = client%initialize("client_test_misc_tensor") if (result .ne. SRNoError) error stop print *, "Putting tensor" diff --git a/tests/fortran/client_test_mnist.F90 b/tests/fortran/client_test_mnist.F90 index 52e126dba..9cf43a1e1 100644 --- a/tests/fortran/client_test_mnist.F90 +++ b/tests/fortran/client_test_mnist.F90 @@ -44,7 +44,7 @@ program mnist_test character(len=2) :: key_suffix integer :: result - result = client%initialize(use_cluster(), "client_test_mnist") + result = client%initialize("client_test_mnist") if (result .ne. SRNoError) error stop result = client%set_model_from_file(model_key, model_file, "TORCH", "CPU") diff --git a/tests/fortran/client_test_mnist_multigpu.F90 b/tests/fortran/client_test_mnist_multigpu.F90 index e8d5b7c47..e59fdae21 100644 --- a/tests/fortran/client_test_mnist_multigpu.F90 +++ b/tests/fortran/client_test_mnist_multigpu.F90 @@ -47,8 +47,7 @@ program mnist_test character(len=2) :: key_suffix integer :: sr_return_code - sr_return_code = client%initialize(use_cluster(), & - "client_test_mnist_multigpu") + sr_return_code = client%initialize("client_test_mnist_multigpu") if (sr_return_code .ne. SRNoError) error stop sr_return_code = client%set_model_from_file_multigpu(model_key, model_file, "TORCH", first_gpu, num_gpus) diff --git a/tests/fortran/client_test_prefixing.F90 b/tests/fortran/client_test_prefixing.F90 index e95c6a1f8..e322f3309 100644 --- a/tests/fortran/client_test_prefixing.F90 +++ b/tests/fortran/client_test_prefixing.F90 @@ -50,7 +50,7 @@ program main call setenv("SSKEYIN", prefix) call setenv("SSKEYOUT", prefix) -result = client%initialize(use_cluster(), "client_test_prefixing") +result = client%initialize("client_test_prefixing") if (result .ne. SRNoError) error stop result = client%use_tensor_ensemble_prefix(.true.) if (result .ne. SRNoError) error stop diff --git a/tests/fortran/client_test_put_get_1D.F90 b/tests/fortran/client_test_put_get_1D.F90 index 97a2a4939..9a064c389 100644 --- a/tests/fortran/client_test_put_get_1D.F90 +++ b/tests/fortran/client_test_put_get_1D.F90 @@ -73,7 +73,7 @@ program main recv_array_integer_64(i) = irand() enddo - result = client%initialize(use_cluster(), "client_test_put_get_1D") + result = client%initialize("client_test_put_get_1D") if (result .ne. SRNoError) error stop result = client%put_tensor("true_array_real_32", true_array_real_32, shape(true_array_real_32)) diff --git a/tests/fortran/client_test_put_get_2D.F90 b/tests/fortran/client_test_put_get_2D.F90 index a7485ef95..a18b6b200 100644 --- a/tests/fortran/client_test_put_get_2D.F90 +++ b/tests/fortran/client_test_put_get_2D.F90 @@ -74,7 +74,7 @@ program main recv_array_integer_64(i,j) = irand() enddo; enddo - result = client%initialize(use_cluster(), "client_test_put_get_2D") + result = client%initialize("client_test_put_get_2D") if (result .ne. SRNoError) error stop result = client%put_tensor("true_array_real_32", true_array_real_32, shape(true_array_real_32)) diff --git a/tests/fortran/client_test_put_get_3D.F90 b/tests/fortran/client_test_put_get_3D.F90 index 87510546b..5a0c935f0 100644 --- a/tests/fortran/client_test_put_get_3D.F90 +++ b/tests/fortran/client_test_put_get_3D.F90 @@ -76,7 +76,7 @@ program main recv_array_integer_64(i,j,k) = irand() enddo; enddo; enddo - result = client%initialize(use_cluster(), "client_test_put_get_3D") + result = client%initialize("client_test_put_get_3D") if (result .ne. SRNoError) error stop result = client%put_tensor("true_array_real_32", true_array_real_32, shape(true_array_real_32)) diff --git a/tests/fortran/client_test_put_get_unpack_dataset.F90 b/tests/fortran/client_test_put_get_unpack_dataset.F90 index 6f1c70ecf..5dc21e977 100644 --- a/tests/fortran/client_test_put_get_unpack_dataset.F90 +++ b/tests/fortran/client_test_put_get_unpack_dataset.F90 @@ -61,8 +61,7 @@ program main integer :: err_code - result = client%initialize(use_cluster(), & - "client_test_put_get_unpack_dataset") + result = client%initialize("client_test_put_get_unpack_dataset") if (result .ne. SRNoError) error stop call random_number(true_array_real_32) diff --git a/tests/fortran/test_utils.F90 b/tests/fortran/test_utils.F90 index 67b6b095e..d0ff69ab2 100644 --- a/tests/fortran/test_utils.F90 +++ b/tests/fortran/test_utils.F90 @@ -65,23 +65,21 @@ integer function irand() end function irand logical function use_cluster() + character(len=16) :: server_type - character(len=16) :: smartredis_test_cluster - - call get_environment_variable('SMARTREDIS_TEST_CLUSTER', smartredis_test_cluster) - smartredis_test_cluster = to_lower(smartredis_test_cluster) + call get_environment_variable('SR_DB_TYPE', server_type) + server_type = to_lower(server_type) use_cluster = .false. - if (len_trim(smartredis_test_cluster)>0) then - select case (smartredis_test_cluster) - case ('true') + if (len_trim(server_type)>0) then + select case (server_type) + case ('clustered') use_cluster = .true. - case ('false') + case ('standalone') use_cluster = .false. case default use_cluster = .false. end select endif - end function use_cluster !> Returns a lower case version of the string. Only supports a-z @@ -99,7 +97,6 @@ function to_lower(str) result(lower_str) i_low = index(caps,str(i:i)) if (i_low > 0) lower_str(i:i) = lows(i_low:i_low) enddo - end function to_lower !> Convert a Fortran string to a C-string (i.e. append a null character) @@ -108,7 +105,6 @@ function c_str(f_str) character(kind=c_char,len=len_trim(f_str)+1) :: c_str !< The resultant C-style string c_str = trim(f_str)//C_NULL_CHAR - end function c_str !> Set an environment variable to a given value @@ -134,7 +130,6 @@ subroutine setenv(env_var, env_val, replace) write(STDERR,*) "Error setting", c_env_var, c_env_val error stop endif - end subroutine setenv !> Clear an environment variable @@ -150,7 +145,6 @@ subroutine unsetenv(env_var) write(STDERR,*) "Error clearing", c_env_var error stop endif - end subroutine unsetenv end module test_utils diff --git a/tests/python/test_address.py b/tests/python/test_address.py index e5bed1482..9b8ed4cf6 100644 --- a/tests/python/test_address.py +++ b/tests/python/test_address.py @@ -28,20 +28,21 @@ from smartredis import Client -def test_serialization(use_cluster, context): - # get env var to set through client init - ssdb = os.environ["SSDB"] - c = Client(address=ssdb, cluster=use_cluster, logger_name=context) +def test_serialization(context): + c = Client(None, logger_name=context) assert str(c) != repr(c) -def test_address(use_cluster, context): +def test_address(context): + # This test evaluates deprecated behavior and will be removed + # in the next release when the deprecated Client construction + # is removed. # get env var to set through client init ssdb = os.environ["SSDB"] del os.environ["SSDB"] # client init should fail if SSDB not set - c = Client(address=ssdb, cluster=use_cluster, logger_name=context) + _ = Client(False, address=ssdb, logger_name=context) # check if SSDB was set anyway assert os.environ["SSDB"] == ssdb diff --git a/tests/python/test_configoptions.py b/tests/python/test_configoptions.py index a042867c8..0990a0b6b 100644 --- a/tests/python/test_configoptions.py +++ b/tests/python/test_configoptions.py @@ -71,10 +71,10 @@ def test_options(monkeypatch): assert co.get_string_option( "test_string_key_that_is_not_really_present") == "meowth" -def test_options_with_prefix(monkeypatch): - monkeypatch.setenv("prefixtest_integer_key", "42") - monkeypatch.setenv("prefixtest_string_key", "charizard") - co = ConfigOptions.create_from_environment("prefixtest") +def test_options_with_suffix(monkeypatch): + monkeypatch.setenv("integer_key_suffixtest", "42") + monkeypatch.setenv("string_key_suffixtest", "charizard") + co = ConfigOptions.create_from_environment("suffixtest") # integer option tests assert co.get_integer_option("integer_key") == 42 diff --git a/tests/python/test_dataset_aggregation.py b/tests/python/test_dataset_aggregation.py index 1c826dade..b28111c65 100644 --- a/tests/python/test_dataset_aggregation.py +++ b/tests/python/test_dataset_aggregation.py @@ -24,17 +24,15 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os import numpy as np -import pytest from smartredis import Client, Dataset from smartredis.error import * from smartredis import * -def test_aggregation(use_cluster, context): +def test_aggregation(context): num_datasets = 4 - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) log_data(context, LLDebug, "Initialization complete") # Build datasets diff --git a/tests/python/test_dataset_ops.py b/tests/python/test_dataset_ops.py index a516465b5..32f2b471f 100644 --- a/tests/python/test_dataset_ops.py +++ b/tests/python/test_dataset_ops.py @@ -31,12 +31,12 @@ from smartredis.error import * -def test_copy_dataset(use_cluster, context): +def test_copy_dataset(context): # test copying dataset from one key to another dataset = create_dataset("test_dataset_copy") - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) client.put_dataset(dataset) client.copy_dataset("test_dataset_copy", "test_dataset_copied") @@ -67,12 +67,12 @@ def test_copy_dataset(use_cluster, context): ) -def test_rename_dataset(use_cluster, context): +def test_rename_dataset(context): # test renaming a dataset in the database dataset = create_dataset("dataset_rename") - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) client.put_dataset(dataset) client.rename_dataset("dataset_rename", "dataset_renamed") @@ -105,12 +105,12 @@ def test_rename_dataset(use_cluster, context): ) -def test_delete_dataset(use_cluster, context): +def test_delete_dataset(context): # test renaming a dataset in the database dataset = create_dataset("dataset_delete") - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) client.put_dataset(dataset) client.delete_dataset( @@ -123,25 +123,25 @@ def test_delete_dataset(use_cluster, context): # ----------- Error handling ------------------------------------ -def test_rename_nonexisting_dataset(use_cluster, context): +def test_rename_nonexisting_dataset(context): - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) with pytest.raises(RedisReplyError): client.rename_dataset("not-a-tensor", "still-not-a-tensor") -def test_copy_nonexistant_dataset(use_cluster, context): +def test_copy_nonexistant_dataset(context): - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) with pytest.raises(RedisReplyError): client.copy_dataset("not-a-tensor", "still-not-a-tensor") -def test_copy_not_dataset(use_cluster, context): +def test_copy_not_dataset(context): def test_func(param): print(param) - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) client.set_function("test_func_dataset", test_func) with pytest.raises(RedisReplyError): client.copy_dataset("test_func_dataset", "test_fork_dataset") diff --git a/tests/python/test_errors.py b/tests/python/test_errors.py index 3a6b9afa4..519d6e281 100644 --- a/tests/python/test_errors.py +++ b/tests/python/test_errors.py @@ -41,51 +41,51 @@ def cfg_opts() -> ConfigOptions: return opts -def test_SSDB_not_set(use_cluster, context): +def test_SSDB_not_set(context): ssdb = os.environ["SSDB"] del os.environ["SSDB"] with pytest.raises(RedisConnectionError): - c = Client(None, use_cluster, logger_name=context) + _ = Client(None, logger_name=context) os.environ["SSDB"] = ssdb -def test_bad_SSDB(use_cluster, context): +def test_bad_SSDB(context): ssdb = os.environ["SSDB"] del os.environ["SSDB"] os.environ["SSDB"] = "not-an-address:6379;" with pytest.raises(RedisConnectionError): - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) os.environ["SSDB"] = ssdb -def test_bad_get_tensor(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_get_tensor(context): + c = Client(None, logger_name=context) with pytest.raises(RedisReplyError): c.get_tensor("not-a-key") -def test_bad_get_dataset(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_get_dataset(context): + c = Client(None, logger_name=context) with pytest.raises(RedisKeyError): c.get_dataset("not-a-key") -def test_bad_script_file(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_script_file(context): + c = Client(None, logger_name=context) with pytest.raises(FileNotFoundError): c.set_script_from_file("key", "not-a-file") -def test_get_non_existant_script(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_get_non_existant_script(context): + c = Client(None, logger_name=context) with pytest.raises(RedisReplyError): script = c.get_script("not-a-script") -def test_bad_function_execution(use_cluster, context): +def test_bad_function_execution(context): """Error raised inside function""" - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_function("bad-function", bad_function) data = np.array([1, 2, 3, 4]) c.put_tensor("bad-func-tensor", data) @@ -95,10 +95,10 @@ def test_bad_function_execution(use_cluster, context): c.run_script("bad-function", "bad_function", "bad-func-tensor", "output") -def test_missing_script_function(use_cluster, context): +def test_missing_script_function(context): """User requests to run a function not in the script""" - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_function("bad-function", bad_function) with pytest.raises(RedisReplyError): c.run_script("bad-function", "not-a-function-in-script", ["bad-func-tensor"], ["output"]) @@ -126,38 +126,38 @@ def test_bad_function_execution_multigpu(use_cluster, context): not test_gpu, reason="SMARTREDIS_TEST_DEVICE does not specify 'gpu'" ) -def test_missing_script_function_multigpu(use_cluster, context): +def test_missing_script_function_multigpu(context): """User requests to run a function not in the script""" - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_function_multigpu("bad-function", bad_function, 0, 1) with pytest.raises(RedisReplyError): c.run_script_multigpu("bad-function", "not-a-function-in-script", ["bad-func-tensor"], ["output"], 0, 0, 2) with pytest.raises(RedisReplyError): c.run_script_multigpu("bad-function", "not-a-function-in-script", "bad-func-tensor", "output", 0, 0, 2) - - -def test_wrong_model_name(mock_data, mock_model, use_cluster, context): + + +def test_wrong_model_name(mock_data, mock_model, context): """User requests to run a model that is not there""" data = mock_data.create_data(1) model = mock_model.create_torch_cnn() - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_model("simple_cnn", model, "TORCH", "CPU") c.put_tensor("input", data[0]) with pytest.raises(RedisReplyError): c.run_model("wrong_cnn", ["input"], ["output"]) -def test_wrong_model_name_from_file(mock_data, mock_model, use_cluster, context): +def test_wrong_model_name_from_file(mock_data, mock_model, context): """User requests to run a model that is not there that was loaded from file.""" try: data = mock_data.create_data(1) mock_model.create_torch_cnn(filepath="./torch_cnn.pt") - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_model_from_file("simple_cnn_from_file", "./torch_cnn.pt", "TORCH", "CPU") c.put_tensor("input", data[0]) with pytest.raises(RedisReplyError): @@ -166,16 +166,16 @@ def test_wrong_model_name_from_file(mock_data, mock_model, use_cluster, context) os.remove("torch_cnn.pt") -def test_bad_device(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_device(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.set_script("key", "some_script", device="not-a-gpu") ##### # Test type errors from bad parameter types to Client API calls -def test_bad_type_put_tensor(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_put_tensor(context): + c = Client(None, logger_name=context) array = np.array([1, 2, 3, 4]) with pytest.raises(TypeError): c.put_tensor(42, array) @@ -183,79 +183,79 @@ def test_bad_type_put_tensor(use_cluster, context): c.put_tensor("key", [1, 2, 3, 4]) -def test_unsupported_type_put_tensor(use_cluster, context): +def test_unsupported_type_put_tensor(context): """test an unsupported numpy type""" - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) data = np.array([1, 2, 3, 4]).astype(np.uint64) with pytest.raises(TypeError): c.put_tensor("key", data) -def test_bad_type_get_tensor(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_get_tensor(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.get_tensor(42) -def test_bad_type_delete_tensor(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_delete_tensor(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.delete_tensor(42) -def test_bad_type_copy_tensor(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_copy_tensor(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.copy_tensor(42, "newname") with pytest.raises(TypeError): c.copy_tensor("oldname", 42) -def test_bad_type_rename_tensor(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_rename_tensor(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.rename_tensor(42, "newname") with pytest.raises(TypeError): c.rename_tensor("oldname", 42) -def test_bad_type_put_dataset(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_put_dataset(context): + c = Client(None, logger_name=context) array = np.array([1, 2, 3, 4]) with pytest.raises(TypeError): c.put_dataset(array) -def test_bad_type_get_dataset(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_get_dataset(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.get_dataset(42) -def test_bad_type_delete_dataset(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_delete_dataset(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.delete_dataset(42) -def test_bad_type_copy_dataset(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_copy_dataset(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.copy_dataset(42, "dest") with pytest.raises(TypeError): c.copy_dataset("src", 42) -def test_bad_type_rename_dataset(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_rename_dataset(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.rename_dataset(42, "oldkey") with pytest.raises(TypeError): c.rename_dataset("newkey", 42) -def test_bad_type_set_function(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_set_function(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.set_function(42, bad_function) with pytest.raises(TypeError): @@ -263,8 +263,8 @@ def test_bad_type_set_function(use_cluster, context): with pytest.raises(TypeError): c.set_function("key", bad_function, 42) -def test_bad_type_set_function_multigpu(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_set_function_multigpu(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.set_function_multigpu(42, bad_function, 0, 1) with pytest.raises(TypeError): @@ -278,8 +278,8 @@ def test_bad_type_set_function_multigpu(use_cluster, context): with pytest.raises(ValueError): c.set_function_multigpu("key", bad_function, 0, 0) # invalid num GPUs -def test_bad_type_set_script(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_set_script(context): + c = Client(None, logger_name=context) key = "key_for_script" script = "bad script but correct parameter type" device = "CPU" @@ -290,8 +290,8 @@ def test_bad_type_set_script(use_cluster, context): with pytest.raises(TypeError): c.set_script(key, script, 42) -def test_bad_type_set_script_multigpu(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_set_script_multigpu(context): + c = Client(None, logger_name=context) key = "key_for_script" script = "bad script but correct parameter type" first_gpu = 0 @@ -309,8 +309,8 @@ def test_bad_type_set_script_multigpu(use_cluster, context): with pytest.raises(ValueError): c.set_script_multigpu(key, script, first_gpu, 0) -def test_bad_type_set_script_from_file(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_set_script_from_file(context): + c = Client(None, logger_name=context) key = "key_for_script" scriptfile = "bad filename but correct parameter type" device = "CPU" @@ -321,8 +321,8 @@ def test_bad_type_set_script_from_file(use_cluster, context): with pytest.raises(TypeError): c.set_script_from_file(key, scriptfile, 42) -def test_bad_type_set_script_from_file_multigpu(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_set_script_from_file_multigpu(context): + c = Client(None, logger_name=context) key = "key_for_script" scriptfile = "bad filename but correct parameter type" first_gpu = 0 @@ -336,14 +336,14 @@ def test_bad_type_set_script_from_file_multigpu(use_cluster, context): with pytest.raises(TypeError): c.set_script_from_file_multigpu(key, scriptfile, first_gpu, "not an integer") -def test_bad_type_get_script(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_get_script(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.get_script(42) -def test_bad_type_run_script_str(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_run_script_str(context): + c = Client(None, logger_name=context) key = "my_script" fn_name = "phred" inputs = "a string" @@ -358,8 +358,8 @@ def test_bad_type_run_script_str(use_cluster, context): c.run_script(key, fn_name, inputs, 42) -def test_bad_type_run_script_list(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_run_script_list(context): + c = Client(None, logger_name=context) key = "my_script" fn_name = "phred" inputs = ["list", "of", "strings"] @@ -374,8 +374,8 @@ def test_bad_type_run_script_list(use_cluster, context): c.run_script(key, fn_name, inputs, 42) -def test_bad_type_run_script_multigpu_str(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_run_script_multigpu_str(context): + c = Client(None, logger_name=context) key = "my_script" fn_name = "phred" inputs = "a string" @@ -403,8 +403,8 @@ def test_bad_type_run_script_multigpu_str(use_cluster, context): c.run_script_multigpu(key, fn_name, inputs, outputs, offset, first_gpu, 0) -def test_bad_type_run_script_multigpu_list(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_run_script_multigpu_list(context): + c = Client(None, logger_name=context) key = "my_script" fn_name = "phred" inputs = ["list", "of", "strings"] @@ -430,17 +430,17 @@ def test_bad_type_run_script_multigpu_list(use_cluster, context): c.run_script_multigpu(key, fn_name, inputs, outputs, offset, -1, num_gpus) with pytest.raises(ValueError): c.run_script_multigpu(key, fn_name, inputs, outputs, offset, first_gpu, 0) - - -def test_bad_type_get_model(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) + + +def test_bad_type_get_model(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.get_model(42) -def test_bad_type_set_model(mock_model, use_cluster, context): +def test_bad_type_set_model(mock_model, context): model = mock_model.create_torch_cnn() - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.set_model(42, model, "TORCH", "CPU") with pytest.raises(TypeError): @@ -458,10 +458,10 @@ def test_bad_type_set_model(mock_model, use_cluster, context): with pytest.raises(TypeError): c.set_model("simple_cnn", model, "TORCH", "CPU", tag=42) -def test_bad_type_set_model_multigpu(mock_model, use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_set_model_multigpu(mock_model, context): + c = Client(None, logger_name=context) model = mock_model.create_torch_cnn() - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.set_model_multigpu(42, model, "TORCH", 0, 1) with pytest.raises(TypeError): @@ -484,9 +484,9 @@ def test_bad_type_set_model_multigpu(mock_model, use_cluster, context): c.set_model_multigpu("simple_cnn", model, "TORCH", 0, 1, tag=42) -def test_bad_type_set_model_from_file(use_cluster, context): +def test_bad_type_set_model_from_file(context): modelfile = "bad filename but right parameter type" - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.set_model_from_file(42, modelfile, "TORCH", "CPU") with pytest.raises(TypeError): @@ -506,9 +506,9 @@ def test_bad_type_set_model_from_file(use_cluster, context): with pytest.raises(TypeError): c.set_model_from_file("simple_cnn", modelfile, "TORCH", "CPU", tag=42) -def test_bad_type_set_model_from_file_multigpu(use_cluster, context): +def test_bad_type_set_model_from_file_multigpu(context): modelfile = "bad filename but right parameter type" - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.set_model_from_file_multigpu(42, modelfile, "TORCH", 0, 1) with pytest.raises(TypeError): @@ -530,14 +530,14 @@ def test_bad_type_set_model_from_file_multigpu(use_cluster, context): with pytest.raises(TypeError): c.set_model_from_file_multigpu("simple_cnn", modelfile, "TORCH", 0, 1, tag=42) -def test_bad_type_run_model(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_run_model(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.run_model(42) -def test_bad_type_run_model_multigpu(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_run_model_multigpu(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.run_model_multigpu(42, 0, 0, 1) with pytest.raises(TypeError): @@ -551,8 +551,8 @@ def test_bad_type_run_model_multigpu(use_cluster, context): with pytest.raises(ValueError): c.run_model_multigpu("simple_cnn", 0, 0, 0) -def test_bad_type_delete_model_multigpu(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_delete_model_multigpu(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.delete_model_multigpu(42, 0, 1) with pytest.raises(TypeError): @@ -564,8 +564,8 @@ def test_bad_type_delete_model_multigpu(use_cluster, context): with pytest.raises(ValueError): c.delete_model_multigpu("simple_cnn", 0, 0) -def test_bad_type_delete_script_multigpu(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_delete_script_multigpu(context): + c = Client(None, logger_name=context) script_name = "my_script" with pytest.raises(TypeError): c.delete_script_multigpu(42, 0, 1) @@ -578,32 +578,32 @@ def test_bad_type_delete_script_multigpu(use_cluster, context): with pytest.raises(ValueError): c.delete_script_multigpu(script_name, 0, 0) -def test_bad_type_tensor_exists(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_tensor_exists(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.tensor_exists(42) -def test_bad_type_dataset_exists(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_dataset_exists(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.dataset_exists(42) -def test_bad_type_model_exists(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_model_exists(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.model_exists(42) -def test_bad_type_key_exists(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_key_exists(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.key_exists(42) -def test_bad_type_poll_key(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_poll_key(context): + c = Client(None, logger_name=context) name = "some_key" freq = 42 num_tries = 42 @@ -616,8 +616,8 @@ def test_bad_type_poll_key(use_cluster, context): c.poll_key(name, freq, bogus) -def test_bad_type_poll_tensor(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_poll_tensor(context): + c = Client(None, logger_name=context) name = "some_key" freq = 42 num_tries = 42 @@ -630,8 +630,8 @@ def test_bad_type_poll_tensor(use_cluster, context): c.poll_tensor(name, freq, bogus) -def test_bad_type_poll_dataset(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_poll_dataset(context): + c = Client(None, logger_name=context) name = "some_key" freq = 42 num_tries = 42 @@ -644,8 +644,8 @@ def test_bad_type_poll_dataset(use_cluster, context): c.poll_dataset(name, freq, bogus) -def test_bad_type_poll_model(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_poll_model(context): + c = Client(None, logger_name=context) name = "some_key" freq = 42 num_tries = 42 @@ -658,50 +658,50 @@ def test_bad_type_poll_model(use_cluster, context): c.poll_model(name, freq, bogus) -def test_bad_type_set_data_source(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_set_data_source(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.set_data_source(42) -def test_bad_type_use_model_ensemble_prefix(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_use_model_ensemble_prefix(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.use_model_ensemble_prefix("not a boolean") -def test_bad_type_use_list_ensemble_prefix(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_use_list_ensemble_prefix(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.use_list_ensemble_prefix("not a boolean") -def test_bad_type_use_tensor_ensemble_prefix(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_use_tensor_ensemble_prefix(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.use_tensor_ensemble_prefix("not a boolean") -def test_bad_type_use_dataset_ensemble_prefix(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_use_dataset_ensemble_prefix(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.use_dataset_ensemble_prefix("not a boolean") -def test_bad_type_get_db_node_info(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_get_db_node_info(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.get_db_node_info("not a list") -def test_bad_type_get_db_cluster_info(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_get_db_cluster_info(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.get_db_cluster_info("not a list") -def test_bad_type_get_ai_info(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_get_ai_info(context): + c = Client(None, logger_name=context) address = ["list", "of", "str"] key = "ai.info.key" with pytest.raises(TypeError): @@ -712,22 +712,22 @@ def test_bad_type_get_ai_info(use_cluster, context): c.get_ai_info(address, key, "not a boolean") -def test_bad_type_flush_db(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_flush_db(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.flush_db("not a list") -def test_bad_type_config_get(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_config_get(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.config_get("timeout", 42) with pytest.raises(TypeError): c.config_get(42, "address") -def test_bad_type_config_set(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_config_set(context): + c = Client(None, logger_name=context) param = "timeout" value = "never" address = "127.0.0.1:6379" @@ -739,42 +739,42 @@ def test_bad_type_config_set(use_cluster, context): c.config_set(param, value, 42) -def test_bad_type_save(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_save(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.save("not a list") -def test_bad_type_append_to_list(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_append_to_list(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.append_to_list(42, 42) -def test_bad_type_delete_list(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_delete_list(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.delete_list(42) -def test_bad_type_copy_list(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_copy_list(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.copy_list(42, "dest") with pytest.raises(TypeError): c.copy_list("src", 42) -def test_bad_type_rename_list(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_rename_list(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.rename_list(42, "dest") with pytest.raises(TypeError): c.rename_list("src", 42) -def test_bad_type_get_list_length(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_get_list_length(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.get_list_length(42) -def test_bad_type_poll_list_length(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_poll_list_length(context): + c = Client(None, logger_name=context) name = "mylist" len = 42 pollfreq = 42 @@ -788,8 +788,8 @@ def test_bad_type_poll_list_length(use_cluster, context): with pytest.raises(TypeError): c.poll_list_length(name, len, pollfreq, "not an integer") -def test_bad_type_poll_list_length_gte(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_poll_list_length_gte(context): + c = Client(None, logger_name=context) name = "mylist" len = 42 pollfreq = 42 @@ -803,8 +803,8 @@ def test_bad_type_poll_list_length_gte(use_cluster, context): with pytest.raises(TypeError): c.poll_list_length_gte(name, len, pollfreq, "not an integer") -def test_bad_type_poll_list_length_lte(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_poll_list_length_lte(context): + c = Client(None, logger_name=context) name = "mylist" len = 42 pollfreq = 42 @@ -818,13 +818,13 @@ def test_bad_type_poll_list_length_lte(use_cluster, context): with pytest.raises(TypeError): c.poll_list_length_lte(name, len, pollfreq, "not an integer") -def test_bad_type_get_datasets_from_list(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_get_datasets_from_list(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.get_datasets_from_list(42) -def test_bad_type_get_dataset_list_range(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_get_dataset_list_range(context): + c = Client(None, logger_name=context) listname = "my_list" start_index = 0 end_index = 42 @@ -835,8 +835,8 @@ def test_bad_type_get_dataset_list_range(use_cluster, context): with pytest.raises(TypeError): c.get_dataset_list_range(listname, start_index, "not an integer") -def test_bad_type_set_model_chunk_size(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_set_model_chunk_size(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.set_model_chunk_size("not an integer") @@ -846,8 +846,8 @@ def test_bad_type_set_model_chunk_size(use_cluster, context): @pytest.mark.parametrize("log_fn", [ (log_data,), (log_warning,), (log_error,) ]) -def test_bad_type_log_function(use_cluster, context, log_fn): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_log_function(context, log_fn): + c = Client(None, logger_name=context) with pytest.raises(TypeError): log_fn(42, LLInfo, "Data to be logged") with pytest.raises(TypeError): @@ -855,8 +855,8 @@ def test_bad_type_log_function(use_cluster, context, log_fn): with pytest.raises(TypeError): log_fn("test_bad_type_log_function", LLInfo, 42) -def test_bad_type_client_log(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_bad_type_client_log(context): + c = Client(None, logger_name=context) with pytest.raises(TypeError): c.log_data("Not a logging level", "Data to be logged") with pytest.raises(TypeError): diff --git a/tests/python/test_logging.py b/tests/python/test_logging.py index 100016cc2..c9245d9dc 100644 --- a/tests/python/test_logging.py +++ b/tests/python/test_logging.py @@ -31,7 +31,7 @@ @pytest.mark.parametrize("log_level", [ LLQuiet, LLInfo, LLDebug, LLDeveloper ]) -def test_logging_string(use_cluster, context, log_level): +def test_logging_string(context, log_level): log_data(context, log_level, f"This is data logged from a string ({log_level.name})") log_warning(context, log_level, f"This is a warning logged from a string ({log_level.name})") log_error(context, log_level, f"This is an error logged from a string ({log_level.name})") @@ -39,8 +39,8 @@ def test_logging_string(use_cluster, context, log_level): @pytest.mark.parametrize("log_level", [ LLQuiet, LLInfo, LLDebug, LLDeveloper ]) -def test_logging_client(use_cluster, context, log_level): - c = Client(None, use_cluster, logger_name=context) +def test_logging_client(context, log_level): + c = Client(None, logger_name=context) c.log_data(log_level, f"This is data logged from a client ({log_level.name})") c.log_warning(log_level, f"This is a warning logged from a client ({log_level.name})") c.log_error(log_level, f"This is an error logged from a client ({log_level.name})") diff --git a/tests/python/test_model_methods_torch.py b/tests/python/test_model_methods_torch.py index d98c6bed7..e1a9bec9a 100644 --- a/tests/python/test_model_methods_torch.py +++ b/tests/python/test_model_methods_torch.py @@ -34,18 +34,18 @@ test_gpu = environ.get("SMARTREDIS_TEST_DEVICE","cpu").lower() == "gpu" -def test_set_model(mock_model, use_cluster, context): +def test_set_model(mock_model, context): model = mock_model.create_torch_cnn() - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_model("simple_cnn", model, "TORCH", "CPU") returned_model = c.get_model("simple_cnn") assert model == returned_model -def test_set_model_from_file(mock_model, use_cluster, context): +def test_set_model_from_file(mock_model, context): try: mock_model.create_torch_cnn(filepath="./torch_cnn.pt") - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_model_chunk_size(1024 * 1024) c.set_model_from_file("file_cnn", "./torch_cnn.pt", "TORCH", "CPU") assert c.model_exists("file_cnn") @@ -59,10 +59,10 @@ def test_set_model_from_file(mock_model, use_cluster, context): os.remove("torch_cnn.pt") -def test_torch_inference(mock_model, use_cluster, context): +def test_torch_inference(mock_model, context): # get model and set into database model = mock_model.create_torch_cnn() - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_model("torch_cnn", model, "TORCH") # setup input tensor @@ -74,11 +74,11 @@ def test_torch_inference(mock_model, use_cluster, context): out_data = c.get_tensor("torch_cnn_output") assert out_data.shape == (1, 1, 1, 1) -def test_batch_exceptions(mock_model, use_cluster, context): +def test_batch_exceptions(mock_model, context): # get model and set into database mock_model.create_torch_cnn(filepath="./torch_cnn.pt") model = mock_model.create_torch_cnn() - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) batch_size = 1 min_batch_size = 1 min_batch_timeout = 1 @@ -143,10 +143,10 @@ def test_batch_exceptions(mock_model, use_cluster, context): batch_size=batch_size, min_batch_size=0, min_batch_timeout=min_batch_timeout ) -def test_batch_warning_set_model_from_file(mock_model, use_cluster, context, capfd): +def test_batch_warning_set_model_from_file(mock_model, context, capfd): # get model and set into database mock_model.create_torch_cnn(filepath="./torch_cnn.pt") - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_model_from_file( "file_cnn", "./torch_cnn.pt", "TORCH", "CPU", batch_size=1, min_batch_size=1, min_batch_timeout=0 @@ -158,10 +158,10 @@ def test_batch_warning_set_model_from_file(mock_model, use_cluster, context, cap not test_gpu, reason="SMARTREDIS_TEST_DEVICE does not specify 'gpu'" ) -def test_batch_warning_set_model_from_file_multigpu(mock_model, use_cluster, context, capfd): +def test_batch_warning_set_model_from_file_multigpu(mock_model, context, capfd): # get model and set into database mock_model.create_torch_cnn(filepath="./torch_cnn.pt") - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_model_from_file_multigpu( "file_cnn", "./torch_cnn.pt", "TORCH", 1, 1, batch_size=1, min_batch_size=1, min_batch_timeout=0 @@ -169,10 +169,10 @@ def test_batch_warning_set_model_from_file_multigpu(mock_model, use_cluster, con captured = capfd.readouterr() assert "WARNING" in captured.err -def test_batch_warning_set_model(mock_model, use_cluster, context, capfd): +def test_batch_warning_set_model(mock_model, context, capfd): # get model and set into database model = mock_model.create_torch_cnn() - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_model( "file_cnn", model, "TORCH", "CPU", batch_size=1, min_batch_size=1, min_batch_timeout=0 @@ -184,10 +184,10 @@ def test_batch_warning_set_model(mock_model, use_cluster, context, capfd): not test_gpu, reason="SMARTREDIS_TEST_DEVICE does not specify 'gpu'" ) -def test_batch_warning_set_model_multigpu(mock_model, use_cluster, context, capfd): +def test_batch_warning_set_model_multigpu(mock_model, context, capfd): # get model and set into database model = mock_model.create_torch_cnn() - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_model_multigpu( "file_cnn", model, "TORCH", 1, 1, batch_size=1, min_batch_size=1, min_batch_timeout=0 diff --git a/tests/python/test_nonkeyed_cmd.py b/tests/python/test_nonkeyed_cmd.py index 79a280df0..9a0fb93ba 100644 --- a/tests/python/test_nonkeyed_cmd.py +++ b/tests/python/test_nonkeyed_cmd.py @@ -29,45 +29,30 @@ import numpy as np import pytest from smartredis import Client +from smartredis import ConfigOptions from smartredis.error import * -def test_dbnode_info_command(use_cluster, context): - # get env var to set through client init +def test_dbnode_info_command(context): ssdb = os.environ["SSDB"] - db_info_addr = [ssdb] - del os.environ["SSDB"] - - client = Client(address=ssdb, cluster=use_cluster, logger_name=context) - - info = client.get_db_node_info(db_info_addr) - + addresses = ssdb.split(',') + client = Client(None, logger_name=context) + info = client.get_db_node_info(addresses) assert len(info) > 0 - -def test_dbcluster_info_command(mock_model, use_cluster, context): - # get env var to set through client init +def test_dbcluster_info_command(mock_model, context): ssdb = os.environ["SSDB"] - address = [ssdb] - del os.environ["SSDB"] - - client = Client(address=ssdb, cluster=use_cluster, logger_name=context) + addresses = ssdb.split(',') + co = ConfigOptions().create_from_environment("") + client = Client(co, logger_name=context) - if use_cluster: - info = client.get_db_cluster_info(address) + if os.environ["SR_DB_TYPE"] == "Clustered": + info = client.get_db_cluster_info(addresses) assert len(info) > 0 else: # cannot call get_db_cluster_info in non-cluster environment with pytest.raises(RedisReplyError): - client.get_db_cluster_info(address) - - # get env var to set through client init - ssdb = os.environ["SSDB"] - address = [ssdb] - del os.environ["SSDB"] - - # Init client - client = Client(address=ssdb, cluster=use_cluster, logger_name=context) + client.get_db_cluster_info(addresses) # Get a mock model model = mock_model.create_torch_cnn() @@ -76,7 +61,7 @@ def test_dbcluster_info_command(mock_model, use_cluster, context): client.set_model("ai_info_cnn", model, "TORCH", "CPU") # Check with valid address and model key - ai_info = client.get_ai_info(address, "ai_info_cnn") + ai_info = client.get_ai_info(addresses, "ai_info_cnn") assert len(ai_info) != 0 # Check that invalid address throws error @@ -85,83 +70,63 @@ def test_dbcluster_info_command(mock_model, use_cluster, context): # Check that invalid model name throws error with pytest.raises(RedisRuntimeError): - client.get_ai_info(address, "bad_key") + client.get_ai_info(addresses, "bad_key") -def test_flushdb_command(use_cluster, context): +def test_flushdb_command(context): # from within the testing framework, there is no way # of knowing each db node that is being used, so skip # if on cluster - if use_cluster: - return - # get env var to set through client init ssdb = os.environ["SSDB"] - address = [ssdb] - del os.environ["SSDB"] + addresses = ssdb.split(',') + if os.environ["SR_DB_TYPE"] == "Clustered": + return - client = Client(address=ssdb, cluster=use_cluster, logger_name=context) + client = Client(None, logger_name=context) # add key to client via put_tensor tensor = np.array([1, 2]) client.put_tensor("test_copy", tensor) assert client.tensor_exists("test_copy") - client.flush_db(address) + client.flush_db(addresses) assert not client.tensor_exists("test_copy") -def test_config_set_get_command(use_cluster, context): +def test_config_set_get_command(context): # get env var to set through client init ssdb = os.environ["SSDB"] - - del os.environ["SSDB"] - - client = Client(address=ssdb, cluster=use_cluster, logger_name=context) + client = Client(None, logger_name=context) value = "6000" client.config_set("lua-time-limit", value, ssdb) - get_reply = client.config_get("lua-time-limit", ssdb) assert len(get_reply) > 0 assert get_reply["lua-time-limit"] == value -def test_config_set_command_DNE(use_cluster, context): - # get env var to set through client init +def test_config_set_command_DNE(context): ssdb = os.environ["SSDB"] - - del os.environ["SSDB"] - - client = Client(address=ssdb, cluster=use_cluster, logger_name=context) + client = Client(None, logger_name=context) # The CONFIG parameter "config_param_DNE" is unsupported with pytest.raises(RedisReplyError): client.config_set("config_param_DNE", "10", ssdb) -def test_config_get_command_DNE(use_cluster, context): - # get env var to set through client init +def test_config_get_command_DNE(context): ssdb = os.environ["SSDB"] - - del os.environ["SSDB"] - - client = Client(address=ssdb, cluster=use_cluster, logger_name=context) + client = Client(None, logger_name=context) # CONFIG GET returns an empty dictionary if the config_param is unsupported get_reply = client.config_get("config_param_DNE", ssdb) assert get_reply == dict() -def test_save_command(use_cluster, mock_data, context): - # get env var to set through client init +def test_save_command(context): ssdb = os.environ["SSDB"] - if use_cluster: - addresses = ssdb.split(",") - else: - addresses = [ssdb] - del os.environ["SSDB"] + client = Client(None, logger_name=context) - # client init should fail if SSDB not set - client = Client(address=ssdb, cluster=use_cluster, logger_name=context) + addresses = ssdb.split(",") # for each address, check that the timestamp of the last SAVE increases after calling Client::save for address in addresses: diff --git a/tests/python/test_prefixing.py b/tests/python/test_prefixing.py index 68388e912..565ba9668 100644 --- a/tests/python/test_prefixing.py +++ b/tests/python/test_prefixing.py @@ -24,18 +24,17 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os import numpy as np from smartredis import Client, Dataset -def test_prefixing(use_cluster, context, monkeypatch): +def test_prefixing(context, monkeypatch): # configure prefix variables monkeypatch.setenv("SSKEYOUT", "prefix_test") monkeypatch.setenv("SSKEYIN", "prefix_test") # Set up client - c = Client(address=None, cluster=use_cluster, logger_name=context) + c = Client(address=None, logger_name=context) c.use_dataset_ensemble_prefix(True) c.use_tensor_ensemble_prefix(True) d = Dataset("test_dataset") diff --git a/tests/python/test_put_get_dataset.py b/tests/python/test_put_get_dataset.py index 453563050..007c73300 100644 --- a/tests/python/test_put_get_dataset.py +++ b/tests/python/test_put_get_dataset.py @@ -30,7 +30,7 @@ from smartredis import Client, Dataset -def test_put_get_dataset(mock_data, use_cluster, context): +def test_put_get_dataset(mock_data, context): """test sending and receiving a dataset with 2D tensors of every datatype """ @@ -43,7 +43,7 @@ def test_put_get_dataset(mock_data, use_cluster, context): key = f"tensor_{str(index)}" dataset.add_tensor(key, tensor) - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) assert not client.dataset_exists( "nonexistent-dataset" @@ -64,7 +64,7 @@ def test_put_get_dataset(mock_data, use_cluster, context): ) -def test_augment_dataset(mock_data, use_cluster, context): +def test_augment_dataset(mock_data, context): """Test sending, receiving, altering, and sending a Dataset. """ @@ -75,7 +75,7 @@ def test_augment_dataset(mock_data, use_cluster, context): dataset_name = "augment-dataset" # Initialize a client - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) # Create a dataset to put into the database dataset = Dataset(dataset_name) diff --git a/tests/python/test_put_get_tensor.py b/tests/python/test_put_get_tensor.py index f067e17f0..ea40628d8 100644 --- a/tests/python/test_put_get_tensor.py +++ b/tests/python/test_put_get_tensor.py @@ -31,28 +31,28 @@ # ----- Tests ----------------------------------------------------------- -def test_1D_put_get(mock_data, use_cluster, context): +def test_1D_put_get(mock_data, context): """Test put/get_tensor for 1D numpy arrays""" - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) data = mock_data.create_data(10) send_get_arrays(client, data) -def test_2D_put_get(mock_data, use_cluster, context): +def test_2D_put_get(mock_data, context): """Test put/get_tensor for 2D numpy arrays""" - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) data = mock_data.create_data((10, 10)) send_get_arrays(client, data) -def test_3D_put_get(mock_data, use_cluster, context): +def test_3D_put_get(mock_data, context): """Test put/get_tensor for 3D numpy arrays""" - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) data = mock_data.create_data((10, 10, 10)) send_get_arrays(client, data) diff --git a/tests/python/test_script_methods.py b/tests/python/test_script_methods.py index 00655c938..83c0dff7e 100644 --- a/tests/python/test_script_methods.py +++ b/tests/python/test_script_methods.py @@ -37,25 +37,25 @@ test_gpu = environ.get("SMARTREDIS_TEST_DEVICE","cpu").lower() == "gpu" -def test_set_get_function(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_set_get_function(context): + c = Client(None, logger_name=context) c.set_function("test-set-function", one_to_one) script = c.get_script("test-set-function") sent_script = inspect.getsource(one_to_one) assert script == sent_script -def test_set_get_script(use_cluster, context): - c = Client(None, use_cluster, logger_name=context) +def test_set_get_script(context): + c = Client(None, logger_name=context) sent_script = read_script_from_file() c.set_script("test-set-script", sent_script) script = c.get_script("test-set-script") assert sent_script == script -def test_set_script_from_file(use_cluster, context): +def test_set_script_from_file(context): sent_script = read_script_from_file() - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.set_script_from_file( "test-script-file", osp.join(file_path, "./data_processing_script.txt") ) @@ -66,10 +66,10 @@ def test_set_script_from_file(use_cluster, context): assert not c.model_exists("test-script-file") -def test_run_script_str(use_cluster, context): +def test_run_script_str(context): data = np.array([[1, 2, 3, 4, 5]]) - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.put_tensor("script-test-data", data) c.set_function("one-to-one", one_to_one) c.run_script("one-to-one", "one_to_one", "script-test-data", "script-test-out") @@ -77,11 +77,11 @@ def test_run_script_str(use_cluster, context): assert out == 5 -def test_run_script_list(use_cluster, context): +def test_run_script_list(context): data = np.array([[1, 2, 3, 4]]) data_2 = np.array([[5, 6, 7, 8]]) - c = Client(None, use_cluster, logger_name=context) + c = Client(None, logger_name=context) c.put_tensor("srpt-multi-out-data-1", data) c.put_tensor("srpt-multi-out-data-2", data_2) c.set_function("two-to-one", two_to_one) diff --git a/tests/python/test_tensor_ops.py b/tests/python/test_tensor_ops.py index f25dde36b..833d66ff9 100644 --- a/tests/python/test_tensor_ops.py +++ b/tests/python/test_tensor_ops.py @@ -31,10 +31,10 @@ from smartredis.error import RedisReplyError -def test_copy_tensor(use_cluster, context): +def test_copy_tensor(context): # test copying tensor - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) tensor = np.array([1, 2]) client.put_tensor("test_copy", tensor) @@ -46,10 +46,10 @@ def test_copy_tensor(use_cluster, context): assert np.array_equal(tensor, returned) -def test_rename_tensor(use_cluster, context): +def test_rename_tensor(context): # test renaming tensor - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) tensor = np.array([1, 2]) client.put_tensor("test_rename", tensor) @@ -61,10 +61,10 @@ def test_rename_tensor(use_cluster, context): assert np.array_equal(tensor, returned) -def test_delete_tensor(use_cluster, context): +def test_delete_tensor(context): # test renaming tensor - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) tensor = np.array([1, 2]) client.put_tensor("test_delete", tensor) @@ -76,25 +76,25 @@ def test_delete_tensor(use_cluster, context): # --------------- Error handling ---------------------- -def test_rename_nonexisting_key(use_cluster, context): +def test_rename_nonexisting_key(context): - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) with pytest.raises(RedisReplyError): client.rename_tensor("not-a-tensor", "still-not-a-tensor") -def test_copy_nonexistant_key(use_cluster, context): +def test_copy_nonexistant_key(context): - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) with pytest.raises(RedisReplyError): client.copy_tensor("not-a-tensor", "still-not-a-tensor") -def test_copy_not_tensor(use_cluster, context): +def test_copy_not_tensor(context): def test_func(param): print(param) - client = Client(None, use_cluster, logger_name=context) + client = Client(None, logger_name=context) client.set_function("test_func", test_func) with pytest.raises(RedisReplyError): client.copy_tensor("test_func", "test_fork") From 3551b157b3e9509c77dc224a3b1f3ce9997e5a74 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Thu, 12 Oct 2023 12:05:52 -0500 Subject: [PATCH 09/25] Run examples on commit (#412) Run examples on commit to keep them fresh [ committed by @billschereriii ] [ reviewed by @ashao ] --- ...ost_merge_tests.yml => run_static_and_examples.yml} | 10 +++++----- doc/changelog.rst | 3 +++ examples/test_examples.py | 1 - 3 files changed, 8 insertions(+), 6 deletions(-) rename .github/workflows/{run_post_merge_tests.yml => run_static_and_examples.yml} (98%) diff --git a/.github/workflows/run_post_merge_tests.yml b/.github/workflows/run_static_and_examples.yml similarity index 98% rename from .github/workflows/run_post_merge_tests.yml rename to .github/workflows/run_static_and_examples.yml index 37995c0af..5687f7733 100644 --- a/.github/workflows/run_post_merge_tests.yml +++ b/.github/workflows/run_static_and_examples.yml @@ -26,16 +26,17 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -name: run_post_merge_tests - -# This file is for tests that are to be run rarely, once on each merge. +name: run_static_and_examples on: push: branches: - master - develop - + pull_request: + branches: + - master + - develop env: HOMEBREW_NO_ANALYTICS: "ON" # Make Homebrew installation a little quicker HOMEBREW_NO_AUTO_UPDATE: "ON" @@ -147,7 +148,6 @@ jobs: if: "!contains( matrix.compiler, 'intel' )" # if using GNU compiler run: make test-examples SR_FORTRAN=ON SR_PYTHON=ON \ SR_TEST_PORT=7000 SR_TEST_REDISAI_VER=v${{ matrix.rai_v }} - - name: Test static build -- intel compilers if: contains(matrix.compiler, 'intel') run: make build-example-serial SR_FORTRAN=ON diff --git a/doc/changelog.rst b/doc/changelog.rst index 5dcfbd1d1..fee3a63db 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Moved testing of examples to on-commit testing in CI/CD pipeline - Updated RedisAI version used in post-commit check-in testing in Github pipeline - Allow strings in Python interface for Client.run_script, Client.run_script_multiGPU - Improved support for model execution batching @@ -20,6 +21,7 @@ Description Detailed Notes +- Moved testing of examples to on-commit testing in CI/CD pipeline (PR412_) - Updated RedisAI version used in post-commit check-in testing in Github pipeline to a version that supports fetch of model chunking size (PR408_) - Allow users to pass single keys for the inputs and outputs parameters as a string for Python run_script and run_script_multigpu - Exposed access to the Redis.AI MINBATCHTIMEOUT parameter, which limits the delay in model execution when trying to accumulate multiple executions in a batch (PR406_) @@ -29,6 +31,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR412: https://github.com/CrayLabs/SmartRedis/pull/412 .. _PR408: https://github.com/CrayLabs/SmartRedis/pull/408 .. _PR407: https://github.com/CrayLabs/SmartRedis/pull/407 .. _PR406: https://github.com/CrayLabs/SmartRedis/pull/406 diff --git a/examples/test_examples.py b/examples/test_examples.py index 46abddc04..ff9b50d94 100644 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -61,7 +61,6 @@ def test_example(test, build, sr_fortran, link): cmd = [test] print(f"Running test: {osp.basename(test)}") print(f"Test command {' '.join(cmd)}") - print(f"Using cluster: {use_cluster}") execute_cmd(cmd, test_subdir) time.sleep(1) else: From 6a25426c5d6d48e5ad788c9983c41615a67b27f0 Mon Sep 17 00:00:00 2001 From: amandarichardsonn <30413257+amandarichardsonn@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:55:58 -0500 Subject: [PATCH 10/25] Retrieve the name of a dataset added to python API (#411) Allow users to query the name of a dataset after it has been created and/or retrieved from the database. [ committed by @amandarichardsonn ] [ reviewed by @billschereriii ] --- doc/changelog.rst | 3 +++ doc/clients/python.rst | 1 + src/python/module/smartredis/dataset.py | 9 +++++++++ tests/python/test_dataset_methods.py | 1 + tests/python/test_dataset_ops.py | 8 ++++++++ 5 files changed, 22 insertions(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index fee3a63db..17398bd72 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Added name retrieval function to the DataSet object - Moved testing of examples to on-commit testing in CI/CD pipeline - Updated RedisAI version used in post-commit check-in testing in Github pipeline - Allow strings in Python interface for Client.run_script, Client.run_script_multiGPU @@ -21,6 +22,7 @@ Description Detailed Notes +- Added a function to the DataSet class and added a test - Moved testing of examples to on-commit testing in CI/CD pipeline (PR412_) - Updated RedisAI version used in post-commit check-in testing in Github pipeline to a version that supports fetch of model chunking size (PR408_) - Allow users to pass single keys for the inputs and outputs parameters as a string for Python run_script and run_script_multigpu @@ -31,6 +33,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR411: https://github.com/CrayLabs/SmartRedis/pull/411 .. _PR412: https://github.com/CrayLabs/SmartRedis/pull/412 .. _PR408: https://github.com/CrayLabs/SmartRedis/pull/408 .. _PR407: https://github.com/CrayLabs/SmartRedis/pull/407 diff --git a/doc/clients/python.rst b/doc/clients/python.rst index 2160a7b15..b82fdf2f2 100644 --- a/doc/clients/python.rst +++ b/doc/clients/python.rst @@ -103,6 +103,7 @@ Dataset Class Method Overview Dataset.get_meta_strings Dataset.get_metadata_field_names Dataset.get_metadata_field_type + Dataset.get_name Dataset.get_tensor_type Dataset.get_tensor_names Dataset.get_tensor_dims diff --git a/src/python/module/smartredis/dataset.py b/src/python/module/smartredis/dataset.py index 60c6f39be..a9be73b81 100644 --- a/src/python/module/smartredis/dataset.py +++ b/src/python/module/smartredis/dataset.py @@ -120,6 +120,15 @@ def get_tensor(self, name: str) -> np.ndarray: typecheck(name, "name", str) return self._data.get_tensor(name) + @exception_handler + def get_name(self) -> str: + """Get the name of a Dataset + + :return: the name of the in-memory dataset + :rtype: str + """ + return self._data.get_name() + @exception_handler def add_meta_scalar(self, name: str, data: t.Union[int, float]) -> None: """Add metadata scalar field (non-string) with value to the DataSet diff --git a/tests/python/test_dataset_methods.py b/tests/python/test_dataset_methods.py index 457dcb0c9..debd5f6fe 100644 --- a/tests/python/test_dataset_methods.py +++ b/tests/python/test_dataset_methods.py @@ -95,6 +95,7 @@ def test_add_get_strings(mock_data): data = mock_data.create_metadata_strings(10) add_get_strings(dataset, data) + def test_dataset_inspection(context): d = Dataset(context) data = np.uint8([[2, 4, 6, 8], [1, 3, 5, 7]]) diff --git a/tests/python/test_dataset_ops.py b/tests/python/test_dataset_ops.py index 32f2b471f..2f52609ca 100644 --- a/tests/python/test_dataset_ops.py +++ b/tests/python/test_dataset_ops.py @@ -137,6 +137,14 @@ def test_copy_nonexistant_dataset(context): client.copy_dataset("not-a-tensor", "still-not-a-tensor") +def test_dataset_get_name(): + """Test getting a dataset name + """ + dataset = Dataset("test-dataset") + name = dataset.get_name() + assert name == "test-dataset" + + def test_copy_not_dataset(context): def test_func(param): print(param) From 83d413bd9898748afdfd038b480e46a7af9fd0c4 Mon Sep 17 00:00:00 2001 From: amandarichardsonn <30413257+amandarichardsonn@users.noreply.github.com> Date: Tue, 17 Oct 2023 13:26:59 -0500 Subject: [PATCH 11/25] Python Code coverage (#414) Added coverage to missing SmartRedis python API functions. Wrote tests in SmartRedis test directory. [ committed by @amandarichardsonn ] [ reviewed by @billschereriii ] --- doc/changelog.rst | 5 +- src/python/module/smartredis/srobject.py | 19 ------- tests/python/test_dataset_aggregation.py | 34 ++++++++++++- tests/python/test_errors.py | 14 +++++ tests/python/test_prefixing.py | 65 +++++++++++++++++++++++- tests/python/test_tensor_ops.py | 3 +- 6 files changed, 117 insertions(+), 23 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 17398bd72..444bf3c8e 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Added coverage to SmartRedis Python API functions - Added name retrieval function to the DataSet object - Moved testing of examples to on-commit testing in CI/CD pipeline - Updated RedisAI version used in post-commit check-in testing in Github pipeline @@ -22,6 +23,7 @@ Description Detailed Notes +- Added tests to increase Python code coverage - Added a function to the DataSet class and added a test - Moved testing of examples to on-commit testing in CI/CD pipeline (PR412_) - Updated RedisAI version used in post-commit check-in testing in Github pipeline to a version that supports fetch of model chunking size (PR408_) @@ -33,6 +35,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR414: https://github.com/CrayLabs/SmartRedis/pull/414 .. _PR411: https://github.com/CrayLabs/SmartRedis/pull/411 .. _PR412: https://github.com/CrayLabs/SmartRedis/pull/412 .. _PR408: https://github.com/CrayLabs/SmartRedis/pull/408 @@ -503,4 +506,4 @@ Released on April 1, 2021 Description -- Initial 0.1.0 release of SmartRedis +- Initial 0.1.0 release of SmartRedis \ No newline at end of file diff --git a/src/python/module/smartredis/srobject.py b/src/python/module/smartredis/srobject.py index 4ca4fa052..ccb642e5a 100644 --- a/src/python/module/smartredis/srobject.py +++ b/src/python/module/smartredis/srobject.py @@ -42,25 +42,6 @@ def __init__(self, context: str) -> None: else: self._srobject = context - @exception_handler - def get_srobject(self) -> PySRObject: - """Return the PySRObject attribute - - :return: The PySRObject attribute containing the srobject information - :rtype: PySRObject - """ - return self._srobject - - @exception_handler - def set_srobject(self, srobject: PySRObject) -> None: - """Set the PySRObject attribute - - :param srobject: The PySRObject object - :type srobject: PySRObject - """ - typecheck(srobject, "srobject", PySRObject) - self._srobject = srobject - @exception_handler def log_data(self, level: SRLoggingLevel, data: str) -> None: """Conditionally log data if the logging level is high enough diff --git a/tests/python/test_dataset_aggregation.py b/tests/python/test_dataset_aggregation.py index b28111c65..ba806e509 100644 --- a/tests/python/test_dataset_aggregation.py +++ b/tests/python/test_dataset_aggregation.py @@ -117,6 +117,14 @@ def test_aggregation(context): f"The list length of {list_length} does not match expected " f"value of {actual_length}.") log_data(context, LLDebug, "List length check") + + # Check the return of a range of datasets from the aggregated list + num_datasets = client.get_dataset_list_range(list_name, 0, 1) + if (len(num_datasets) != 2): + raise RuntimeError( + f"The length is {len(num_datasets)}, which does not " + f"match expected value of 2.") + log_data(context, LLDebug, "Retrieve datasets from list checked") # Retrieve datasets via the aggregation list datasets = client.get_datasets_from_list(list_name) @@ -126,7 +134,31 @@ def test_aggregation(context): f"does not match expected value of {list_length}.") for ds in datasets: check_dataset(ds) - log_data(context, LLDebug, "DataSet retrieval") + log_data(context, LLDebug, "DataSet list retrieval") + + # Rename a list of datasets + client.rename_list(list_name, "new_list_name") + renamed_list_datasets = client.get_datasets_from_list("new_list_name") + if len(renamed_list_datasets) != list_length: + raise RuntimeError( + f"The number of datasets received {len(datasets)} " + f"does not match expected value of {list_length}.") + for ds in renamed_list_datasets: + check_dataset(ds) + log_data(context, LLDebug, "DataSet list rename complete") + + # Copy a list of datasets + client.copy_list("new_list_name", "copied_list_name") + copied_list_datasets = client.get_datasets_from_list("copied_list_name") + if len(copied_list_datasets) != list_length: + raise RuntimeError( + f"The number of datasets received {len(datasets)} " + f"does not match expected value of {list_length}.") + for ds in copied_list_datasets: + check_dataset(ds) + log_data(context, LLDebug, "DataSet list copied") + + # ------------ helper functions --------------------------------- diff --git a/tests/python/test_errors.py b/tests/python/test_errors.py index 519d6e281..67f9289ad 100644 --- a/tests/python/test_errors.py +++ b/tests/python/test_errors.py @@ -31,6 +31,7 @@ from os import environ from smartredis import * from smartredis.error import * +from smartredis.util import Dtypes test_gpu = environ.get("SMARTREDIS_TEST_DEVICE","cpu").lower() == "gpu" @@ -999,6 +1000,19 @@ def test_get_metadata_field_type_wrong_type(): with pytest.raises(TypeError): d.get_metadata_field_type(42) +def test_from_string_wrong_type(): + """A call to Dataset.get_metadata_field_type is made with the wrong type + """ + with pytest.raises(TypeError): + Dtypes.from_string("Incorrect input") + +def test_metadata_from_numpy_wrong_type(): + """A call to Dataset.add_meta_scalar is made with the wrong type + """ + array = np.array(["Incorrect Input"]) + with pytest.raises(TypeError): + Dtypes.metadata_from_numpy(array) + def test_get_tensor_names_wrong_type(): """A call to Dataset.get_tensor_names is made with the wrong type """ diff --git a/tests/python/test_prefixing.py b/tests/python/test_prefixing.py index 565ba9668..503ab7b8e 100644 --- a/tests/python/test_prefixing.py +++ b/tests/python/test_prefixing.py @@ -31,12 +31,15 @@ def test_prefixing(context, monkeypatch): # configure prefix variables monkeypatch.setenv("SSKEYOUT", "prefix_test") - monkeypatch.setenv("SSKEYIN", "prefix_test") + monkeypatch.setenv("SSKEYIN", "prefix_test,prefix_ignore") # Set up client c = Client(address=None, logger_name=context) c.use_dataset_ensemble_prefix(True) c.use_tensor_ensemble_prefix(True) + c.set_data_source("prefix_test") + + # Create Dataset d = Dataset("test_dataset") data = np.uint16([1, 2, 3, 4]) d.add_tensor("dataset_tensor", data) @@ -50,3 +53,63 @@ def test_prefixing(context, monkeypatch): assert c.tensor_exists("test_tensor") assert c.key_exists("prefix_test.test_tensor") assert not c.key_exists("test_tensor") + + +def test_model_prefixing(mock_model, context, monkeypatch): + # configure prefix variables + monkeypatch.setenv("SSKEYOUT", "prefix_test") + monkeypatch.setenv("SSKEYIN", "prefix_test,prefix_ignore") + + # Set up client + c = Client(address=None, logger_name=context) + c.use_model_ensemble_prefix(True) + c.set_data_source("prefix_test") + + # Create model + model = mock_model.create_torch_cnn() + c.set_model("simple_cnn", model, "TORCH", "CPU") + + # Validate keys to see whether prefixing was applied properly + assert c.model_exists("simple_cnn") + assert not c.key_exists("simple_cnn") + + +def test_list_prefixing(context, monkeypatch): + # configure prefix variables + monkeypatch.setenv("SSKEYOUT", "prefix_test") + monkeypatch.setenv("SSKEYIN", "prefix_test,prefix_ignore") + + # Set up client + c = Client(address=None, logger_name=context) + c.use_list_ensemble_prefix(True) + c.set_data_source("prefix_test") + + # Build datasets + num_datasets = 4 + original_datasets = [create_dataset(f"dataset_{i}") for i in range(num_datasets)] + + # Make sure the list is cleared + list_name = "dataset_test_list" + c.delete_list(list_name) + + # Put datasets into the list + for i in range(num_datasets): + c.put_dataset(original_datasets[i]) + c.append_to_list(list_name, original_datasets[i]) + + # Validate keys to see whether prefixing was applied properly + assert c.key_exists("prefix_test.dataset_test_list") + assert not c.key_exists("dataset_test_list") + +# ------------ helper functions --------------------------------- + +def create_dataset(name): + array = np.array([1, 2, 3, 4]) + string = "test_meta_strings" + scalar = 7 + + dataset = Dataset(name) + dataset.add_tensor("test_array", array) + dataset.add_meta_string("test_string", string) + dataset.add_meta_scalar("test_scalar", scalar) + return dataset \ No newline at end of file diff --git a/tests/python/test_tensor_ops.py b/tests/python/test_tensor_ops.py index 833d66ff9..89ddd3f66 100644 --- a/tests/python/test_tensor_ops.py +++ b/tests/python/test_tensor_ops.py @@ -39,7 +39,8 @@ def test_copy_tensor(context): client.put_tensor("test_copy", tensor) client.copy_tensor("test_copy", "test_copied") - + bool_poll_key = client.poll_key(get_prefix() + "test_copy", 100, 100) + assert bool_poll_key == True assert client.key_exists(get_prefix() + "test_copy") assert client.key_exists(get_prefix() + "test_copied") returned = client.get_tensor("test_copied") From 33b7fd959d6025303f4a87d38c469000f6caf858 Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Tue, 17 Oct 2023 16:04:49 -0700 Subject: [PATCH 12/25] Update allowed special characters in SSDB name (#415) SmartSim allows users to declare their own name for a UDS socket file, however the _check_ssdb_string function is overly strict and throws an exception for two commonly used special characters. The modification here allows users to name their UDS socket file with _ and -. [ committed by @ashao ] [ reviewed by @billschereriii ] --- doc/changelog.rst | 9 ++++++--- src/cpp/redisserver.cpp | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 444bf3c8e..ec8f186c6 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Expanded list of allowed characters in the SSDB address - Added coverage to SmartRedis Python API functions - Added name retrieval function to the DataSet object - Moved testing of examples to on-commit testing in CI/CD pipeline @@ -23,9 +24,10 @@ Description Detailed Notes -- Added tests to increase Python code coverage -- Added a function to the DataSet class and added a test +- The SSDB address can now include '-' and '_' as special characters in the name. This gives users more options for naming the UDS socket file (PR415_) +- Added tests to increase Python code coverage (PR414_) - Moved testing of examples to on-commit testing in CI/CD pipeline (PR412_) +- Added a function to the DataSet class and added a test (PR411_) - Updated RedisAI version used in post-commit check-in testing in Github pipeline to a version that supports fetch of model chunking size (PR408_) - Allow users to pass single keys for the inputs and outputs parameters as a string for Python run_script and run_script_multigpu - Exposed access to the Redis.AI MINBATCHTIMEOUT parameter, which limits the delay in model execution when trying to accumulate multiple executions in a batch (PR406_) @@ -35,9 +37,10 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR415: https://github.com/CrayLabs/SmartRedis/pull/415 .. _PR414: https://github.com/CrayLabs/SmartRedis/pull/414 -.. _PR411: https://github.com/CrayLabs/SmartRedis/pull/411 .. _PR412: https://github.com/CrayLabs/SmartRedis/pull/412 +.. _PR411: https://github.com/CrayLabs/SmartRedis/pull/411 .. _PR408: https://github.com/CrayLabs/SmartRedis/pull/408 .. _PR407: https://github.com/CrayLabs/SmartRedis/pull/407 .. _PR406: https://github.com/CrayLabs/SmartRedis/pull/406 diff --git a/src/cpp/redisserver.cpp b/src/cpp/redisserver.cpp index 3e799c108..a08dfeaea 100644 --- a/src/cpp/redisserver.cpp +++ b/src/cpp/redisserver.cpp @@ -121,9 +121,10 @@ SRAddress RedisServer::_get_ssdb() // Check that the SSDB environment variable value does not have any errors void RedisServer::_check_ssdb_string(const std::string& env_str) { + std::string allowed_specials = ".:,/_-"; for (size_t i = 0; i < env_str.size(); i++) { char c = env_str[i]; - if (!isalnum(c) && c != '.' && c != ':' && c != ',' && c != '/') { + if (!isalnum(c) && (allowed_specials.find(c) == std::string::npos)) { throw SRRuntimeException("The provided SSDB value, " + env_str + " is invalid because of character " + c); } From c355ed2000271ee5cbb41b66271c38535f4e4a59 Mon Sep 17 00:00:00 2001 From: amandarichardsonn <30413257+amandarichardsonn@users.noreply.github.com> Date: Thu, 19 Oct 2023 14:45:25 -0500 Subject: [PATCH 13/25] Clarifying and Updating DataSet and Client API docs (#416) Updated the Client and Dataset docs to specify differences between the two APIs. [ committed by @amandarichardsonn ] [ reviewed by @billschereriii ] --- doc/advanced_topics.rst | 4 +- doc/changelog.rst | 3 + doc/clients/c-plus.rst | 29 +++- doc/clients/c.rst | 28 +++- doc/clients/fortran.rst | 28 +++- doc/clients/python.rst | 29 +++- doc/data_structures.rst | 193 +++++++++++++----------- doc/examples/cpp_api_examples.rst | 11 +- doc/examples/fortran_api_examples.rst | 24 +-- doc/examples/python_api_examples.rst | 13 +- src/python/module/smartredis/dataset.py | 57 ++++--- 11 files changed, 268 insertions(+), 151 deletions(-) diff --git a/doc/advanced_topics.rst b/doc/advanced_topics.rst index 50be73c7e..a84d090af 100644 --- a/doc/advanced_topics.rst +++ b/doc/advanced_topics.rst @@ -5,7 +5,7 @@ Advanced Topics This page of documentation is reserved for advanced topics that may not be needed for all users. -.. _advanced_topics_dataset_aggregation: +.. _advanced-topics-dataset-aggregation: Dataset Aggregation =================== @@ -125,7 +125,7 @@ lead to race conditions: // Delete an aggregation list void delete_list(const std::string& list_name); -.. _advanced_topics_dataset_aggregation: +.. _advanced-topics-dataset-aggregation: Multiple Database Support ========================= diff --git a/doc/changelog.rst b/doc/changelog.rst index ec8f186c6..25ffc9e45 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Updated Client and Dataset documentation - Expanded list of allowed characters in the SSDB address - Added coverage to SmartRedis Python API functions - Added name retrieval function to the DataSet object @@ -24,6 +25,7 @@ Description Detailed Notes +- Updated the Client and Dataset API documentation to clarify which interacts with the backend db (PR416_) - The SSDB address can now include '-' and '_' as special characters in the name. This gives users more options for naming the UDS socket file (PR415_) - Added tests to increase Python code coverage (PR414_) - Moved testing of examples to on-commit testing in CI/CD pipeline (PR412_) @@ -37,6 +39,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR416: https://github.com/CrayLabs/SmartRedis/pull/416 .. _PR415: https://github.com/CrayLabs/SmartRedis/pull/415 .. _PR414: https://github.com/CrayLabs/SmartRedis/pull/414 .. _PR412: https://github.com/CrayLabs/SmartRedis/pull/412 diff --git a/doc/clients/c-plus.rst b/doc/clients/c-plus.rst index 4ed831470..67844c8db 100644 --- a/doc/clients/c-plus.rst +++ b/doc/clients/c-plus.rst @@ -1,10 +1,26 @@ -*** -C++ -*** +******** +C++ APIs +******** + +The following page provides a comprehensive overview of the SmartRedis C++ +Client and Dataset APIs. +Further explanation and details of each are presented below. Client API ========== +The Client API is purpose-built for interaction with the backend database, +which extends the capabilities of the Redis in-memory data store. +It's important to note that the SmartRedis Client API is the exclusive +means for altering, transmitting, and receiving data within the backend +database. More specifically, the Client API is responsible for both +creating and modifying data structures, which encompass :ref:`Models `, +:ref:`Scripts `, and :ref:`Tensors `. +It also handles the transmission and reception of +the aforementioned data structures in addition to :ref:`Dataset ` +data structure. Creating and modifying the ``DataSet`` object +is confined to local operation by the DataSet API. + .. doxygenclass:: SmartRedis::Client :project: cpp_client :members: @@ -14,6 +30,13 @@ Client API Dataset API =========== +The C++ DataSet API enables a user to manage a group of tensors +and associated metadata within a datastructure called a ``DataSet`` object. +The DataSet API operates independently of the database and solely +maintains the dataset object in-memory. The actual interaction with the Redis database, +where a snapshot of the DataSet object is sent, is handled by the Client API. For more +information on the ``DataSet`` object, click :ref:`here `. + .. doxygenclass:: SmartRedis::DataSet :project: cpp_client :members: diff --git a/doc/clients/c.rst b/doc/clients/c.rst index 354e8d6c0..91b3ebb29 100644 --- a/doc/clients/c.rst +++ b/doc/clients/c.rst @@ -1,11 +1,26 @@ +******* +C APIs +******* -*** - C -*** +The following page provides a comprehensive overview of the SmartRedis C +Client and Dataset APIs. +Further explanation and details of each are presented below. Client API ========== +The Client API is purpose-built for interaction with the backend database, +which extends the capabilities of the Redis in-memory data store. +It's important to note that the SmartRedis Client API is the exclusive +means for altering, transmitting, and receiving data within the backend +database. More specifically, the Client API is responsible for both +creating and modifying data structures, which encompass :ref:`Models `, +:ref:`Scripts `, and :ref:`Tensors `. +It also handles the transmission and reception of +the aforementioned data structures in addition to :ref:`Dataset ` +data structure. Creating and modifying the ``DataSet`` object +is confined to local operation by the DataSet API. + .. doxygenfile:: c_client.h :project: c_client @@ -13,6 +28,13 @@ Client API Dataset API =========== +The C DataSet API enables a user to manage a group of tensors +and associated metadata within a datastructure called a ``DataSet`` object. +The DataSet API operates independently of the database and solely +maintains the dataset object in-memory. The actual interaction with the Redis database, +where a snapshot of the DataSet object is sent, is handled by the Client API. For more +information on the ``DataSet`` object, click :ref:`here `. + .. doxygenfile:: c_dataset.h :project: c_client diff --git a/doc/clients/fortran.rst b/doc/clients/fortran.rst index fdde448fe..0a31c2a3f 100644 --- a/doc/clients/fortran.rst +++ b/doc/clients/fortran.rst @@ -1,11 +1,26 @@ +************ +Fortran APIs +************ -******* -Fortran -******* +The following page provides a comprehensive overview of the SmartRedis Fortran +Client and Dataset APIs. +Further explanation and details of each are presented below. Client API ========== +The Client API is purpose-built for interaction with the backend database, +which extends the capabilities of the Redis in-memory data store. +It's important to note that the SmartRedis Client API is the exclusive +means for altering, transmitting, and receiving data within the backend +database. More specifically, the Client API is responsible for both +creating and modifying data structures, which encompass :ref:`Models `, +:ref:`Scripts `, and :ref:`Tensors `. +It also handles the transmission and reception of +the aforementioned data structures in addition to :ref:`Dataset ` +data structure. Creating and modifying the ``DataSet`` object +is confined to local operation by the DataSet API. + The following are overloaded interfaces which support 32/64-bit ``real`` and 8, 16, 32, and 64-bit ``integer`` tensors @@ -17,6 +32,13 @@ The following are overloaded interfaces which support Dataset API =========== +The Fortran DataSet API enables a user to manage a group of tensors +and associated metadata within a datastructure called a ``DataSet`` object. +The DataSet API operates independently of the database and solely +maintains the dataset object in-memory. The actual interaction with the Redis database, +where a snapshot of the DataSet object is sent, is handled by the Client API. For more +information on the ``DataSet`` object, click :ref:`here `. + The following are overloaded interfaces which support 32/64-bit ``real`` and 8, 16, 32, and 64-bit ``integer`` tensors diff --git a/doc/clients/python.rst b/doc/clients/python.rst index b82fdf2f2..442037535 100644 --- a/doc/clients/python.rst +++ b/doc/clients/python.rst @@ -1,10 +1,26 @@ -****** -Python -****** +*********** +Python APIs +*********** + +The following page provides a comprehensive overview of the SmartRedis Python +Client, DataSet and Logging APIs. +Further explanation and details of each are presented below. Client API ========== +The Client API is purpose-built for interaction with the backend database, +which extends the capabilities of the Redis in-memory data store. +It's important to note that the SmartRedis Client API is the exclusive +means for altering, transmitting, and receiving data within the backend +database. More specifically, the Client API is responsible for both +creating and modifying data structures, which encompass :ref:`Models `, +:ref:`Scripts `, and :ref:`Tensors `. +It also handles the transmission and reception of +the aforementioned data structures in addition to :ref:`Dataset ` +data structure. Creating and modifying the ``DataSet`` object +is confined to local operation by the DataSet API. + Client Class Method Overview ---------------------------- @@ -86,6 +102,13 @@ Client Class Method Detailed View DataSet API =========== +The Python DataSet API enables a user to manage a group of tensors +and associated metadata within a datastructure called a ``DataSet`` object. +The DataSet API operates independently of the database and solely +maintains the dataset object **in-memory**. The actual interaction with the Redis database, +where a snapshot of the DataSet object is sent, is handled by the Client API. For more +information on the ``DataSet`` object, click :ref:`here `. + Dataset Class Method Overview ----------------------------- diff --git a/doc/data_structures.rst b/doc/data_structures.rst index fb547c4bd..bd4441e86 100644 --- a/doc/data_structures.rst +++ b/doc/data_structures.rst @@ -2,32 +2,33 @@ Data Structures *************** -RedisAI defines three new data structures to be -used in redis databases: tensor, model, and script. -In addition, SmartRedis defines an additional data -structure ``DataSet``. In this section, the SmartRedis -API for interacting with these data structures -will be described, and when applicable, -comments on performance and best practices will be made. - -In general, concepts and capabilities will be -demonstrated for the Python and C++ API. -The C and Fortran function signatures closely -resemble the C++ API, and as a result, -they are not discussed in detail in the interest -of brevity. For more detailed explanations of the C -and Fortran API, refer to the documentation pages for those -clients. - - -.. _data_structures_tensor: +SmartSim defines primary three data structures designed for use within backend databases: + +* ``Tensor`` : represents an n-dimensional array of values. +* ``Model`` : represents a computational ML model for one of the supported backend frameworks. +* ``Script`` : represents a TorchScript program. + +In addition, SmartRedis defines a data +structure named ``DataSet`` that enables a user to manage a group of tensors +and associated metadata in-memory. In this section, we will provide an explanation +of the SmartRedis API used to interact with these four data structures, +along with relevant insights on performance and best practices. + +We illustrate concepts and capabilities of the Python +and C++ SmartRedis APIs. The C and Fortran function signatures closely +mirror the C++ API, and for brevity, we won't delve +into them. For full discussion of the C and Fortran APIs, +please refer to their respective documentation pages. + + +.. _data-structures-tensor: Tensor ====== An n-dimensional tensor is used by RedisAI to store and manipulate numerical data. SmartRedis provides functions to -put a key and tensor pair into the Redis database and retrieve +put a key and tensor pair into the backend database and retrieve a tensor associated with a key from the database. .. note:: @@ -88,7 +89,7 @@ Retrieving ---------- The C++, C, and Fortran clients provide two methods for retrieving -tensors from the Redis database. The first method is referred to +tensors from the backend database. The first method is referred to as *unpacking* a tensor. When a tensor is retrieved via ``unpack_tensor()``, the memory space to store the retrieved tensor data is provided by the user. This has the advantage @@ -172,21 +173,22 @@ Note that all of the client ``get_tensor()`` functions will internally modify the provided tensor name if the client is being used with SmartSim ensemble capabilities. -.. _data_structures_dataset: +.. _data-structures-dataset: Dataset ======= -In many situations, a ``Client`` might be tasked with sending a -group of tensors and metadata which are closely related and -naturally grouped into a collection for future retrieval. -The ``DataSet`` object stages these items so that they can be -more efficiently placed in the redis database and can later be -retrieved with the name given to the ``DataSet``. +When dealing with multi-modal data or complex data sets, +one may have different types of tensors (e.g., images, text embeddings, +numerical data) and metadata for each data point. Grouping them into a +collection represents each data point as a cohesive unit. +The ``DataSet`` data structure provides this functionality to stage tensors and metadata +in-memory via the ``DataSet API``. After the creation of a +``DataSet`` object, the grouped data can be efficiently stored in the backend database +by the ``Client API`` and subsequently retrieved using the assigned ``DataSet`` name. +In the upcoming sections, we outline the process of building, sending, and retrieving a ``DataSet``. Listed below are the supported tensor and metadata types. -In the following sections, building, sending, and retrieving -a ``DataSet`` will be described. .. list-table:: Supported Data Types :widths: 25 25 25 @@ -230,36 +232,37 @@ a ``DataSet`` will be described. - - X -Sending -------- +Build and Send a DataSet +------------------------ -When building a ``DataSet`` to be stored in the database, -a user can add any combination of tensors and metadata. -To add a tensor to the ``DataSet``, the user simply uses -the ``DataSet.add_tensor()`` function defined in -each language. The ``DataSet.add_tensor()`` parameters are the same -as ``Client.put_tensor()``, and as a result, details of the function -signatures will not be reiterated here. +When building a ``DataSet`` object in-memory, +a user can group various combinations of tensors and metadata that +constrain to the supported data types in the table above. To illustrate, +tensors can be inserted into a ``dataset`` object via the ``Dataset.add_tensor()`` method. +The SmartRedis DataSet API functions +are available in C, C++, Python, and Fortran. The ``DataSet.add_tensor()`` function, +operates independently of the database and solely +maintains the dataset object. Storing the dataset in the backend +database is done via the Client API ``put_dataset()`` method. .. note:: - ``DataSet.add_tensor()`` copies the tensor data - provided by the user to eliminate errors from user-provided - data being cleared or deallocated. This additional memory - will be freed when the DataSet - object is destroyed. + The ``DataSet.add_tensor()`` function copies user-provided + tensor data; this prevents potential issues arising from the user's + data being cleared or deallocated. Any additional memory allocated + for this purpose will be released when the DataSet object is destroyed. -Metadata can be added to the ``DataSet`` with the +Metadata can be added to an in-memory ``DataSet`` object with the ``DataSet.add_meta_scalar()`` and ``DataSet.add_meta_string()`` -functions. As the aforementioned function names suggest, -there are separate functions to add metadata that is a scalar -(e.g. double) and a string. For both functions, the first -function input is the name of the metadata field. This field -name is an internal ``DataSet`` identifier for the metadata -value(s) that is used for future retrieval, and because it -is an internal identifier, the user does not have to worry -about any key conflicts in the database (i.e. multiple ``DataSet`` -can have the same metadata field names). To clarify these -and future descriptions, the C++ interface for adding +functions. Methods exist for adding scalar metadata (e.g., double) and string metadata. +For both functions, the first input +parameter is the name of the metadata field. +The field name serves as an internal identifier within the ``DataSet`` +for grouped metadata values. It's used to retrieve metadata in the future. +Since it's an internal identifier, users don't need to be concerned +about conflicts with keys in the database. In other words, multiple +``DataSet`` objects can use the same metadata field names without causing +issues because these names are managed within the ``DataSet`` and won't +interfere with external database keys. The C++ interface for adding metadata is shown below: .. code-block:: cpp @@ -277,50 +280,58 @@ metadata is shown below: When adding a scalar or string metadata value, the value is copied by the ``DataSet``, and as a result, the user does not need to ensure that the metadata values provided -are still in memory after they have been added. +are still in-memory. In other words, +the ``DataSet`` handles the memory management of these metadata values, +and you don't need to retain or manage the original copies separately +once they have been included in the ``DataSet`` object. Additionally, multiple metadata values can be added to a -single field, and the default behavior is to append the value to the -existing field. In this way, the ``DataSet`` metadata supports -one-dimensional arrays, but the entries in the array must be added -iteratively by the user. Also, note that in the above C++ example, +single field name, and the default behavior is to append the value to the +field name (creating the field if not already present). This behavior allows the ``DataSet`` metadata +to function like one-dimensional arrays. + +Also, note that in the above C++ example, the metadata scalar type must be specified with a -``SRMetaDataType`` enum value, and similar +``SRMetaDataType`` enum value; similar requirements exist for C and Fortran ``DataSet`` implementations. Finally, the ``DataSet`` object is sent to the database using the ``Client.put_dataset()`` function, which is uniform across all clients. +To emphasize once more, all interactions with the backend database are handle by +the Client API, not the DataSet API. -Retrieving ----------- +Retrieving a DataSet +-------------------- In all clients, the ``DataSet`` is retrieved with a single function call to ``Client.get_dataset()``, which requires only the name of the ``DataSet`` (i.e. the name used in the constructor of the ``DataSet`` when it was -built and placed in the database). ``Client.get_dataset()`` -returns to the user a DataSet object or a pointer to a -DataSet object that can be used to access all of the +built and placed in the database by the Client API). ``Client.get_dataset()`` +returns to the user a ``DataSet`` object (in C, a pointer to a +``DataSet`` object) from the database that is used to access all of the dataset tensors and metadata. -The functions for retrieving tensors from ``DataSet`` +The functions for retrieving tensors from an in-memory ``DataSet`` object are identical to the functions provided by ``Client``, and the same return values and memory management -paradigm is followed. As a result, please refer to +paradigm is followed. As a result, please refer to the previous section for details on tensor retrieve function calls. -There are two functions for retrieving metadata: -``get_meta_scalars()`` and ``get_meta_strings()``. -As the names suggest, the first function -is used for retrieving numerical metadata values, -and the second is for retrieving metadata string -values. The metadata retrieval function prototypes +There are four functions for retrieving metadata information from a ``DataSet`` object in-memory: +``get_meta_scalars()``, ``get_meta_strings()``, ``get_metadata_field_names()`` +and ``get_metadata_field_type()``. As the names suggest, the ``get_meta_scalars()`` function +is used for retrieving numerical metadata values, while the ``get_meta_strings()`` function +is for retrieving metadata string values. The ``get_metadata_field_names()`` function +retrieves a list of all metadata field names in the ``DataSet`` object. Lastly, +the ``get_metadata_field_type()`` function returns the type (scalar or string) of the metadata +attached to the specified field name. The metadata retrieval function prototypes vary across the clients based on programming language constraints, and as a result, please refer to the ``DataSet`` API documentation -for a description of input parameters and memory management. It is +for a description of input parameters and memory management. It is important to note, however, that all functions require the name of the -metadata field to be retrieved, and this name is the same name that +metadata field to be retrieved. This name is the same name that was used when constructing the metadata field with ``add_meta_scalar()`` and ``add_meta_string()`` functions. @@ -329,7 +340,9 @@ Aggregating SmartRedis also supports an advanced API for working with aggregate lists of DataSets; details may be found -:ref:`here <_advanced_topics_dataset_aggregation>`. +:ref:`here `. + +.. _data-structures-model: Model ===== @@ -341,8 +354,8 @@ RedisAI supports PyTorch, TensorFlow, TensorFlow Lite, and ONNX backends, and specifying the backend to be used is done through the ``Client`` function calls. -Sending -------- +Build and Send a Model +---------------------- A model is placed in the database through the ``Client.set_model()`` function. While data types may differ, the function parameters @@ -372,14 +385,14 @@ documentation or the RedisAI documentation for a description of each parameter. .. note:: - With a Redis cluster configuration, ``Client.set_model()`` + With a clustered Redis backend configuration, ``Client.set_model()`` will distribute a copy of the model to each database node in the cluster. As a result, the model that has been placed in the cluster with ``Client.set_model()`` will not be addressable directly with the Redis CLI because of key manipulation that is required to accomplish this distribution. Despite the internal key - manipulation, models in a Redis cluster that have been + manipulation, models in a clustered Redis backend that have been set through the SmartRedis ``Client`` can be accessed and run through the SmartRedis ``Client`` API using the name provided to ``set_model()``. The user @@ -401,7 +414,7 @@ A model can be retrieved from the database using the type varies between languages, only the model name that was used with ``Client.set_model()`` is needed to reference the model in the database. Note that -in a Redis cluster configuration, only one copy of the +in a clustered Redis backend configuration, only one copy of the model is returned to the user. .. note:: @@ -416,7 +429,7 @@ Executing A model can be executed using the ``Client.run_model()`` function. The only required inputs to execute a model are the model name, a list of input tensor names, and a list of output tensor names. -If using a Redis cluster configuration, a copy of the model +If using a clustered Redis backend configuration, a copy of the model referenced by the provided name will be chosen based on data locality. It is worth noting that the names of input and output tensors will be altered with ensemble member identifications if the SmartSim @@ -464,13 +477,15 @@ via ``Client.set_model_multigpu()``. it must have been set via ``Client.set_model_multigpu()``. The ``first_gpu`` and ``num_gpus`` parameters must be constant across both calls. +.. _data-structures-script: + Script ====== Data processing is an essential step in most machine learning workflows. For this reason, RedisAI provides the ability to evaluate PyTorch programs using the hardware -co-located with the Redis database (either CPU or GPU). +co-located with the backend database (either CPU or GPU). The SmartRedis ``Client`` provides functions for users to place a script in the database, retrieve a script from the database, and run a script. @@ -493,14 +508,14 @@ need to be provided by the user. const std::string_view& script); .. note:: - With a Redis cluster configuration, ``Client.set_script()`` + With a clustered Redis backend configuration, ``Client.set_script()`` will distribute a copy of the script to each database node in the cluster. As a result, the script that has been placed in the cluster with ``Client.set_script()`` will not be addressable directly with the Redis CLI because of key manipulation that is required to accomplish this distribution. Despite the internal key - manipulation, scripts in a Redis cluster that have been + manipulation, scripts in a clustered Redis backend that have been set through the SmartRedis ``Client`` can be accessed and run through the SmartRedis ``Client`` API using the name provided to ``set_script()``. The user @@ -522,7 +537,7 @@ A script can be retrieved from the database using the type varies between languages, only the script name that was used with ``Client.set_script()`` is needed to reference the script in the database. Note that -in a Redis cluster configuration, only one copy of the +in a clustered Redis backend configuration, only one copy of the script is returned to the user. .. note:: @@ -538,7 +553,7 @@ A script can be executed using the ``Client.run_script()`` function. The only required inputs to execute a script are the script name, the name of the function in the script to execute, a list of input tensor names, and a list of output tensor names. -If using a Redis cluster configuration, a copy of the script +If using a clustered Redis backend configuration, a copy of the script referenced by the provided name will be chosen based on data locality. It is worth noting that the names of input and output tensors will be altered with ensemble member identifications if the SmartSim @@ -583,4 +598,4 @@ via ``Client.set_script_multigpu()``. In order for a script to be executed via ``Client.run_script_multigpu()``, or deleted via ``Client.delete_script_multigpu()``, it must have been set via ``Client.set_script_multigpu()``. The - ``first_gpu`` and ``num_gpus`` parameters must be constant across both calls. + ``first_gpu`` and ``num_gpus`` parameters must be constant across both calls. \ No newline at end of file diff --git a/doc/examples/cpp_api_examples.rst b/doc/examples/cpp_api_examples.rst index cdaa8fd78..a24efa003 100644 --- a/doc/examples/cpp_api_examples.rst +++ b/doc/examples/cpp_api_examples.rst @@ -17,8 +17,8 @@ SmartRedis ``DataSet`` API is also provided. .. note:: The C++ API examples are written - to connect to a Redis cluster database. Update the - ``Client`` constructor call to connect to a Redis non-cluster database. + to connect to a clustered backend database. Update the + ``Client`` constructor call to connect to a non-clustered backend database. Tensors ======= @@ -33,7 +33,10 @@ SmartRedis C++ client API. DataSets ======== -The C++ client can store and retrieve tensors and metadata in datasets. +The C++ ``Client`` API stores and retrieve datasets from the backend database. The C++ +``DataSet`` API can store and retrieve tensors and metadata from an in-memory ``DataSet`` object. +To reiterate, the actual interaction with the backend database, +where a snapshot of the ``DataSet`` object is sent, is handled by the Client API. For further information about datasets, please refer to the :ref:`Dataset section of the Data Structures documentation page `. @@ -97,4 +100,4 @@ source code is also shown. .. literalinclude:: ../../examples/common/mnist_data/data_processing_script.txt :linenos: :language: Python - :lines: 15-20 + :lines: 15-20 \ No newline at end of file diff --git a/doc/examples/fortran_api_examples.rst b/doc/examples/fortran_api_examples.rst index e25537547..9011db3c0 100644 --- a/doc/examples/fortran_api_examples.rst +++ b/doc/examples/fortran_api_examples.rst @@ -13,19 +13,19 @@ SmartRedis ``DataSet`` API is also provided. .. note:: The Fortran API examples rely on the ``SSDB`` environment - variable being set to the address and port of the Redis database. + variable being set to the address and port of the backend database. .. note:: The Fortran API examples are written - to connect to a Redis cluster database. Update the - ``Client`` constructor call to connect to a non-cluster Redis instance. + to connect to a clustered backend database. Update the + ``Client`` constructor call to connect to a non-cluster backend instance. Tensors ======= The SmartRedis Fortran client is used to communicate between -a Fortran client and the Redis database. In this example, +a Fortran client and the backend database. In this example, the client will be used to send an array to the database and then unpack the data into another Fortran array. @@ -107,9 +107,13 @@ into a different array. Datasets ======== -The following code snippet shows how to use the Fortran -Client to store and retrieve dataset tensors and -dataset metadata scalars. +The Fortran ``Client`` API stores and retrieve datasets from the backend database. The Fortran +``DataSet`` API can store and retrieve tensors and metadata from an in-memory ``DataSet`` object. +To reiterate, the actual interaction with the backend database, +where a snapshot of the ``DataSet`` object is sent, is handled by the Client API. + +The code below shows how to store and retrieve tensors and metadata +which belong to a ``DataSet``. .. literalinclude:: ../../examples/serial/fortran/smartredis_dataset.F90 :linenos: @@ -300,14 +304,14 @@ constructed by including a suffix based on MPI tasks. The subroutine, in place of an actual simulation, next generates an array of random numbers and puts this array -into the Redis database. +into the backend database. .. code-block:: Fortran call random_number(array) call client%put_tensor(in_key, array, shape(array)) -The Redis database can now be called to run preprocessing +The backend database can now be called to run preprocessing scripts on these data. .. code-block:: Fortran @@ -382,4 +386,4 @@ Python Pre-Processing: .. literalinclude:: ../../examples/common/mnist_data/data_processing_script.txt :linenos: :language: Python - :lines: 15-20 + :lines: 15-20 \ No newline at end of file diff --git a/doc/examples/python_api_examples.rst b/doc/examples/python_api_examples.rst index c1c154ed4..1adf575ab 100644 --- a/doc/examples/python_api_examples.rst +++ b/doc/examples/python_api_examples.rst @@ -18,13 +18,13 @@ SmartRedis ``DataSet`` API is also provided. .. note:: The Python API examples are written - to connect to a Redis cluster database. Update the - ``Client`` constructor call to connect to a Redis non-cluster database. + to connect to a clustered backend database. Update the + ``Client`` constructor call to connect to a non-clustered backend database. Tensors ======= The Python client has the ability to send and receive tensors from -the Redis database. The tensors are stored in the Redis database +the backend database. The tensors are stored in the backend database as RedisAI data structures. Additionally, Python client API functions involving tensor data are compatible with Numpy arrays and do not require any other data types. @@ -37,7 +37,10 @@ and do not require any other data types. Datasets ======== -The Python client can store and retrieve tensors and metadata in datasets. +The Python ``Client`` API stores and retrieve datasets from the backend database. The Python +``DataSet`` API can store and retrieve tensors and metadata from an in-memory ``DataSet`` object. +To reiterate, the actual interaction with the backend database, +where a snapshot of the ``DataSet`` object is sent, is handled by the Client API. For further information about datasets, please refer to the :ref:`Dataset section of the Data Structures documentation page `. @@ -92,4 +95,4 @@ looks like this: .. literalinclude:: ../../examples/serial/python/data_processing_script.txt :language: python - :linenos: + :linenos: \ No newline at end of file diff --git a/src/python/module/smartredis/dataset.py b/src/python/module/smartredis/dataset.py index a9be73b81..0816d11c8 100644 --- a/src/python/module/smartredis/dataset.py +++ b/src/python/module/smartredis/dataset.py @@ -59,15 +59,16 @@ def _data(self) -> PyDataset: @staticmethod def from_pybind(dataset: PyDataset) -> "Dataset": - """Initialize a Dataset object from - a PyDataset object + """Initialize a Dataset object from a PyDataset object + + Create a new Dataset object using the data and properties + of a PyDataset object as the initial values. :param dataset: The pybind PyDataset object to use for construction :type dataset: PyDataset - :return: The newly constructed Dataset from - the PyDataset - :rtype: Dataset + :return: The newly constructed Dataset object + :rtype: Dataset object """ typecheck(dataset, "dataset", PyDataset) new_dataset = Dataset(dataset.get_name()) @@ -96,9 +97,9 @@ def set_data(self, dataset: PyDataset) -> None: @exception_handler def add_tensor(self, name: str, data: np.ndarray) -> None: - """Add a named tensor to this dataset + """Add a named multi-dimensional data array (tensor) to this dataset - :param name: tensor name + :param name: name associated to the tensor data :type name: str :param data: tensor data :type data: np.ndarray @@ -131,16 +132,15 @@ def get_name(self) -> str: @exception_handler def add_meta_scalar(self, name: str, data: t.Union[int, float]) -> None: - """Add metadata scalar field (non-string) with value to the DataSet + """Add scalar (non-string) metadata to a field name if it exists; + otherwise, create and add - If the field does not exist, it will be created. - If the field exists, the value - will be appended to existing field. + If the field name exists, append the scalar metadata; otherwise, + create the field within the DataSet object and add the scalar metadata. - :param name: The name used to reference the metadata - field + :param name: The name used to reference the scalar metadata field :type name: str - :param data: a scalar + :param data: scalar metadata input :type data: int | float """ typecheck(name, "name", str) @@ -155,15 +155,14 @@ def add_meta_scalar(self, name: str, data: t.Union[int, float]) -> None: @exception_handler def add_meta_string(self, name: str, data: str) -> None: - """Add metadata string field with value to the DataSet + """Add string metadata to a field name if it exists; otherwise, create and add - If the field does not exist, it will be created - If the field exists the value will - be appended to existing field. + If the field name exists, append the string metadata; otherwise, + create the field within the DataSet object and add the string metadata. - :param name: The name used to reference the metadata field + :param name: The name used to reference the string metadata field :type name: str - :param data: The string to add to the field + :param data: string metadata input :type data: str """ typecheck(name, "name", str) @@ -172,10 +171,9 @@ def add_meta_string(self, name: str, data: str) -> None: @exception_handler def get_meta_scalars(self, name: str) -> t.Union[t.List[int], t.List[float]]: - """Get the metadata scalar field values from the DataSet + """Get the scalar values from the DataSet assigned to a field name - :param name: The name used to reference the metadata - field in the DataSet + :param name: The field name to retrieve from :type name: str :rtype: list[int] | list[float] """ @@ -184,10 +182,9 @@ def get_meta_scalars(self, name: str) -> t.Union[t.List[int], t.List[float]]: @exception_handler def get_meta_strings(self, name: str) -> t.List[str]: - """Get the metadata scalar field values from the DataSet + """Get the string values from the DataSet assigned to a field name - :param name: The name used to reference the metadata - field in the DataSet + :param name: The field name to retrieve from :type name: str :rtype: list[str] """ @@ -196,16 +193,16 @@ def get_meta_strings(self, name: str) -> t.List[str]: @exception_handler def get_metadata_field_names(self) -> t.List[str]: - """Get the names of all metadata scalars and strings from the DataSet + """Get all field names from the DataSet - :return: a list of metadata field names + :return: a list of all metadata field names :rtype: list[str] """ return self._data.get_metadata_field_names() @exception_handler def get_metadata_field_type(self, name: str) -> t.Type: - """Get the names of all metadata scalars and strings from the DataSet + """Get the type of metadata for a field name (scalar or string) :param name: The name used to reference the metadata field in the DataSet @@ -243,6 +240,8 @@ def get_tensor_names(self) -> t.List[str]: def get_tensor_dims(self, name: str) -> t.List[int]: """Get the dimensions of a tensor in the DataSet + :param name: name associated to the tensor data + :type name: str :return: a list of the tensor dimensions :rtype: list[int] """ From 251e34e14c74e98f678ee3e79273b4c5102afda3 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Thu, 19 Oct 2023 17:06:52 -0500 Subject: [PATCH 14/25] Use Redis++ ConnectionObject to set TCP socket timeout (#413) By default, the TCP connection timeout is 15 minutes and the UDP connection timeout 30 seconds. This reduces the timeout to 1/4 second in order to have our connections be more responsive. [ committed by @billschereriii ] [ reviewed by @ashao ] --- doc/changelog.rst | 9 ++++++--- include/redisserver.h | 5 +++++ src/cpp/redis.cpp | 19 ++++++++++++++++++- src/cpp/rediscluster.cpp | 20 ++++++++++++++++++-- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 25ffc9e45..065403fc3 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -11,8 +11,9 @@ Description - Updated Client and Dataset documentation - Expanded list of allowed characters in the SSDB address - Added coverage to SmartRedis Python API functions -- Added name retrieval function to the DataSet object +- Improved responsiveness of library when attempting connection to missing backend database - Moved testing of examples to on-commit testing in CI/CD pipeline +- Added name retrieval function to the DataSet object - Updated RedisAI version used in post-commit check-in testing in Github pipeline - Allow strings in Python interface for Client.run_script, Client.run_script_multiGPU - Improved support for model execution batching @@ -27,9 +28,10 @@ Detailed Notes - Updated the Client and Dataset API documentation to clarify which interacts with the backend db (PR416_) - The SSDB address can now include '-' and '_' as special characters in the name. This gives users more options for naming the UDS socket file (PR415_) -- Added tests to increase Python code coverage (PR414_) +- Added tests to increase Python code coverage +- Employed a Redis++ ConnectionsObject in the connection process to establish a TCP timeout of 100ms during connection attempts (PR413_) - Moved testing of examples to on-commit testing in CI/CD pipeline (PR412_) -- Added a function to the DataSet class and added a test (PR411_) +- Added a function to the DataSet class and added a test - Updated RedisAI version used in post-commit check-in testing in Github pipeline to a version that supports fetch of model chunking size (PR408_) - Allow users to pass single keys for the inputs and outputs parameters as a string for Python run_script and run_script_multigpu - Exposed access to the Redis.AI MINBATCHTIMEOUT parameter, which limits the delay in model execution when trying to accumulate multiple executions in a batch (PR406_) @@ -42,6 +44,7 @@ Detailed Notes .. _PR416: https://github.com/CrayLabs/SmartRedis/pull/416 .. _PR415: https://github.com/CrayLabs/SmartRedis/pull/415 .. _PR414: https://github.com/CrayLabs/SmartRedis/pull/414 +.. _PR413: https://github.com/CrayLabs/SmartRedis/pull/413 .. _PR412: https://github.com/CrayLabs/SmartRedis/pull/412 .. _PR411: https://github.com/CrayLabs/SmartRedis/pull/411 .. _PR408: https://github.com/CrayLabs/SmartRedis/pull/408 diff --git a/include/redisserver.h b/include/redisserver.h index e3faeeb6a..76a7b7c56 100644 --- a/include/redisserver.h +++ b/include/redisserver.h @@ -606,6 +606,11 @@ class RedisServer { */ int _model_chunk_size; + /*! + * \brief Default socket timeout (milliseconds) + */ + static constexpr int _DEFAULT_SOCKET_TIMEOUT = 250; + /*! * \brief Default value of connection timeout (seconds) */ diff --git a/src/cpp/redis.cpp b/src/cpp/redis.cpp index 533852b57..9b493406a 100644 --- a/src/cpp/redis.cpp +++ b/src/cpp/redis.cpp @@ -722,10 +722,27 @@ inline void Redis::_add_to_address_map(SRAddress& db_address) inline void Redis::_connect(SRAddress& db_address) { + // Build a connections object for this connection + // No need to repeat the build on each connection attempt + // so we do it outside the loop + sw::redis::ConnectionOptions connectOpts; + if (db_address._is_tcp) { + connectOpts.host = db_address._tcp_host; + connectOpts.port = db_address._tcp_port; + connectOpts.type = sw::redis::ConnectionType::TCP; + } + else { + connectOpts.path = db_address._uds_file; + connectOpts.type = sw::redis::ConnectionType::UNIX; + } + connectOpts.socket_timeout = std::chrono::milliseconds( + _DEFAULT_SOCKET_TIMEOUT); + + // Connect for (int i = 1; i <= _connection_attempts; i++) { try { // Try to create the sw::redis::Redis object - _redis = new sw::redis::Redis(db_address.to_string(true)); + _redis = new sw::redis::Redis(connectOpts); // Attempt to have the sw::redis::Redis object // make a connection using the PING command diff --git a/src/cpp/rediscluster.cpp b/src/cpp/rediscluster.cpp index 4165cee47..f68ecbe3b 100644 --- a/src/cpp/rediscluster.cpp +++ b/src/cpp/rediscluster.cpp @@ -1075,6 +1075,23 @@ inline CommandReply RedisCluster::_run(const Command& cmd, std::string db_prefix // Connect to the cluster at the address and port inline void RedisCluster::_connect(SRAddress& db_address) { + // Build a connections object for this connection + // No need to repeat the build on each connection attempt + // so we do it outside the loop + sw::redis::ConnectionOptions connectOpts; + if (db_address._is_tcp) { + connectOpts.host = db_address._tcp_host; + connectOpts.port = db_address._tcp_port; + connectOpts.type = sw::redis::ConnectionType::TCP; + } + else { + throw SRInternalException( + "RedisCluster encountered a UDS request in _connect()"); + } + connectOpts.socket_timeout = std::chrono::milliseconds( + _DEFAULT_SOCKET_TIMEOUT); + + // Connect std::string msg; for (int i = 1; i <= _connection_attempts; i++) { msg = "Connection attempt " + std::to_string(i) + " of " + @@ -1083,8 +1100,7 @@ inline void RedisCluster::_connect(SRAddress& db_address) try { // Attempt the connection - _redis_cluster = new sw::redis::RedisCluster(db_address.to_string(true)); - return; + _redis_cluster = new sw::redis::RedisCluster(connectOpts); return; } catch (std::bad_alloc& e) { // On a memory error, bail immediately From 56701f2ee361dd1b338bb3447e641d20ffac84d2 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:13:44 -0500 Subject: [PATCH 15/25] Python Client constructor update (#419) Clean up Python Client constructor [ committed by @billschereriii ] [ reviewed by @MattToast ] --- doc/changelog.rst | 3 ++ doc/clients/python.rst | 35 +++++++------ src/python/module/smartredis/client.py | 71 +++++++++++++++----------- tests/python/test_prefixing.py | 20 ++++---- 4 files changed, 71 insertions(+), 58 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 065403fc3..a280040d2 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Improved robustness of Python client construction - Updated Client and Dataset documentation - Expanded list of allowed characters in the SSDB address - Added coverage to SmartRedis Python API functions @@ -26,6 +27,7 @@ Description Detailed Notes +- Improved robustness of Python client construction by adding detection of invalid kwargs (PR419_) - Updated the Client and Dataset API documentation to clarify which interacts with the backend db (PR416_) - The SSDB address can now include '-' and '_' as special characters in the name. This gives users more options for naming the UDS socket file (PR415_) - Added tests to increase Python code coverage @@ -41,6 +43,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR419: https://github.com/CrayLabs/SmartRedis/pull/419 .. _PR416: https://github.com/CrayLabs/SmartRedis/pull/416 .. _PR415: https://github.com/CrayLabs/SmartRedis/pull/415 .. _PR414: https://github.com/CrayLabs/SmartRedis/pull/414 diff --git a/doc/clients/python.rst b/doc/clients/python.rst index 442037535..7b446664f 100644 --- a/doc/clients/python.rst +++ b/doc/clients/python.rst @@ -3,23 +3,24 @@ Python APIs *********** The following page provides a comprehensive overview of the SmartRedis Python -Client, DataSet and Logging APIs. +Client, DataSet and Logging APIs. Further explanation and details of each are presented below. Client API ========== -The Client API is purpose-built for interaction with the backend database, -which extends the capabilities of the Redis in-memory data store. -It's important to note that the SmartRedis Client API is the exclusive -means for altering, transmitting, and receiving data within the backend -database. More specifically, the Client API is responsible for both -creating and modifying data structures, which encompass :ref:`Models `, -:ref:`Scripts `, and :ref:`Tensors `. -It also handles the transmission and reception of -the aforementioned data structures in addition to :ref:`Dataset ` -data structure. Creating and modifying the ``DataSet`` object -is confined to local operation by the DataSet API. +The Client API is purpose-built for interaction with the backend database, +which extends the capabilities of the Redis in-memory data store. +It's important to note that the SmartRedis Client API is the exclusive +means for altering, transmitting, and receiving data within the backend +database. More specifically, the Client API is responsible for both +creating and modifying data structures, which encompass +:ref:`Models `, :ref:`Scripts `, +and :ref:`Tensors `. +It also handles the transmission and reception of the aforementioned data +structures in addition to :ref:`Dataset ` data +structure. Creating and modifying the ``DataSet`` object is confined to local +operation by the DataSet API. Client Class Method Overview ---------------------------- @@ -102,11 +103,11 @@ Client Class Method Detailed View DataSet API =========== -The Python DataSet API enables a user to manage a group of tensors -and associated metadata within a datastructure called a ``DataSet`` object. -The DataSet API operates independently of the database and solely -maintains the dataset object **in-memory**. The actual interaction with the Redis database, -where a snapshot of the DataSet object is sent, is handled by the Client API. For more +The Python DataSet API enables a user to manage a group of tensors +and associated metadata within a datastructure called a ``DataSet`` object. +The DataSet API operates independently of the database and solely +maintains the dataset object **in-memory**. The actual interaction with the Redis database, +where a snapshot of the DataSet object is sent, is handled by the Client API. For more information on the ``DataSet`` object, click :ref:`here `. Dataset Class Method Overview diff --git a/src/python/module/smartredis/client.py b/src/python/module/smartredis/client.py index c0338940e..985045ea5 100644 --- a/src/python/module/smartredis/client.py +++ b/src/python/module/smartredis/client.py @@ -29,7 +29,6 @@ import os import os.path as osp import typing as t -import warnings as w import numpy as np from .dataset import Dataset @@ -43,50 +42,65 @@ class Client(SRObject): def __init__(self, *a: t.Any, **kw: t.Any): - """Initialize a RedisAI client + """Initialize a SmartRedis client At this time, the Client can be initialized with one of two signatures. The first version is preferred, though the second is - still supported. Note that the order was swapped for first two - parameters in the second signature relative to previous releases - of SmartRedis; this was necessary to remove ambiguity. Support for - the second signature will be removed in a future version of the - SmartRedis library. + supported (primarily for use in driver scripts). Note that the + order was swapped for first two parameters in the second signature + relative to previous releases of SmartRedis; this was necessary to + remove ambiguity. Client(config_options: ConfigOptions=None, logger_name: str="Default") Client(cluster: bool, address: optional(str)=None, - logger_name: str="Default") <= Deprecated! + logger_name: str="Default") For detailed information on the first signature, please refer - to the __new_construction() method below. + to the __standard_construction() method below. For detailed information on the second signature, please refer - to the __deprecated_construction() method below. + to the __address_construction() method below. - :param a: The positional arguments supplied to this method; see above for - valid options + :param a: The positional arguments supplied to this method; + see above for valid options :type a: tuple[any]; see above for valid options - :param kw: Keyword arguments supplied to this method; see above for - valid options + :param kw: Keyword arguments supplied to this method; + see above for valid options :type kw: dict[string, any]; see above for valid options :raises RedisConnectionError: if connection initialization fails """ if a: if isinstance(a[0], bool): - pyclient = self.__deprecated_construction(*a, **kw) + for arg in kw: + if arg not in ["cluster", "address", "logger_name"]: + raise TypeError( + f"__init__() got an unexpected keyword argument '{arg}'" + ) + pyclient = self.__address_construction(*a, **kw) elif isinstance(a[0], ConfigOptions) or a[0] is None: - pyclient = self.__new_construction(*a, **kw) + pyclient = self.__standard_construction(*a, **kw) else: raise TypeError(f"Invalid type for argument 0: {type(a[0])}") else: - config_object = kw.get("config_object", None) - logger_name = kw.get("logger_name", "default") - pyclient = self.__new_construction(config_object, logger_name) + # Only kwargs in the call + if "address" in kw: + pyclient = self.__address_construction(*a, **kw) + else: + pyclient = self.__standard_construction(*a, **kw) super().__init__(pyclient) - def __deprecated_construction(self, cluster, address=None, logger_name="Default"): - """Initialize a RedisAI client (Deprecated) + def __address_construction( + self, + cluster: bool, + address: t.Optional[str] = None, + logger_name: str = "Default" + ) -> PyClient: + """Initialize a SmartRedis client + + This construction method is primarily intended for use by driver + scripts. It is preferred to set up configuration via environment + variables. For clusters, the address can be a single tcp/ip address and port of a database node. The rest of the cluster will be discovered @@ -95,9 +109,6 @@ def __deprecated_construction(self, cluster, address=None, logger_name="Default" If an address is not set, the client will look for the environment variable ``SSDB`` (e.g. SSDB="127.0.0.1:6379;") - DEPRECATION NOTICE: This construction method is deprecated and will - be removed in the next release of the SmartRedis client. - :param cluster: True if connecting to a redis cluster, defaults to False :type cluster: bool :param address: Address of the database @@ -106,12 +117,6 @@ def __deprecated_construction(self, cluster, address=None, logger_name="Default" :type logger_name: str :raises RedisConnectionError: if connection initialization fails """ - w.warn( - 'This construction method is deprecated and will be removed in the next ' + - 'release of the SmartRedis client.', - DeprecationWarning, - stacklevel=3 - ) if address: self.__set_address(address) if "SSDB" not in os.environ: @@ -121,7 +126,11 @@ def __deprecated_construction(self, cluster, address=None, logger_name="Default" except (PybindRedisReplyError, RuntimeError) as e: raise RedisConnectionError(str(e)) from None - def __new_construction(self, config_options=None, logger_name="Default"): # pylint: disable=no-self-use + @staticmethod + def __standard_construction( + config_options: t.Optional[ConfigOptions] = None, + logger_name: str = "Default" + ) -> PyClient: """Initialize a RedisAI client The address of the Redis database is expected to be found in the diff --git a/tests/python/test_prefixing.py b/tests/python/test_prefixing.py index 503ab7b8e..023470633 100644 --- a/tests/python/test_prefixing.py +++ b/tests/python/test_prefixing.py @@ -25,6 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import numpy as np +import os from smartredis import Client, Dataset @@ -34,11 +35,11 @@ def test_prefixing(context, monkeypatch): monkeypatch.setenv("SSKEYIN", "prefix_test,prefix_ignore") # Set up client - c = Client(address=None, logger_name=context) + c = Client(logger_name=context) c.use_dataset_ensemble_prefix(True) c.use_tensor_ensemble_prefix(True) c.set_data_source("prefix_test") - + # Create Dataset d = Dataset("test_dataset") data = np.uint16([1, 2, 3, 4]) @@ -53,7 +54,6 @@ def test_prefixing(context, monkeypatch): assert c.tensor_exists("test_tensor") assert c.key_exists("prefix_test.test_tensor") assert not c.key_exists("test_tensor") - def test_model_prefixing(mock_model, context, monkeypatch): # configure prefix variables @@ -61,10 +61,10 @@ def test_model_prefixing(mock_model, context, monkeypatch): monkeypatch.setenv("SSKEYIN", "prefix_test,prefix_ignore") # Set up client - c = Client(address=None, logger_name=context) + c = Client(logger_name=context) c.use_model_ensemble_prefix(True) c.set_data_source("prefix_test") - + # Create model model = mock_model.create_torch_cnn() c.set_model("simple_cnn", model, "TORCH", "CPU") @@ -80,27 +80,27 @@ def test_list_prefixing(context, monkeypatch): monkeypatch.setenv("SSKEYIN", "prefix_test,prefix_ignore") # Set up client - c = Client(address=None, logger_name=context) + c = Client(logger_name=context) c.use_list_ensemble_prefix(True) c.set_data_source("prefix_test") # Build datasets num_datasets = 4 original_datasets = [create_dataset(f"dataset_{i}") for i in range(num_datasets)] - + # Make sure the list is cleared list_name = "dataset_test_list" c.delete_list(list_name) - + # Put datasets into the list for i in range(num_datasets): c.put_dataset(original_datasets[i]) c.append_to_list(list_name, original_datasets[i]) - + # Validate keys to see whether prefixing was applied properly assert c.key_exists("prefix_test.dataset_test_list") assert not c.key_exists("dataset_test_list") - + # ------------ helper functions --------------------------------- def create_dataset(name): From 0467d8aa1576b2aac2c2e87c41c5dfcfa50f29e3 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Wed, 25 Oct 2023 10:59:28 -0500 Subject: [PATCH 16/25] Consolidate requirements lists (#420) Move all requirements to setup.cfg [ committed by @billschereriii ] [ reviewed by @MattToast ] --- doc/changelog.rst | 3 +++ requirements-dev.txt | 12 ------------ requirements.txt | 4 +++- setup.cfg | 6 ++++-- 4 files changed, 10 insertions(+), 15 deletions(-) delete mode 100644 requirements-dev.txt diff --git a/doc/changelog.rst b/doc/changelog.rst index a280040d2..d64019e1b 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Centralized dependency tracking to setup.cfg - Improved robustness of Python client construction - Updated Client and Dataset documentation - Expanded list of allowed characters in the SSDB address @@ -27,6 +28,7 @@ Description Detailed Notes +- Merged dependency lists from requirements.txt and requirements-dev.txt into setup.cfg to have only one set of dependencies going forward (PR420_) - Improved robustness of Python client construction by adding detection of invalid kwargs (PR419_) - Updated the Client and Dataset API documentation to clarify which interacts with the backend db (PR416_) - The SSDB address can now include '-' and '_' as special characters in the name. This gives users more options for naming the UDS socket file (PR415_) @@ -43,6 +45,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR420: https://github.com/CrayLabs/SmartRedis/pull/420 .. _PR419: https://github.com/CrayLabs/SmartRedis/pull/419 .. _PR416: https://github.com/CrayLabs/SmartRedis/pull/416 .. _PR415: https://github.com/CrayLabs/SmartRedis/pull/415 diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 85379568d..000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,12 +0,0 @@ -sphinx==3.1.1 -sphinx-book-theme==0.2.0 -pytest>=6.0.0 -pytest-cov==2.10.1 -black==23.3.0 -isort==5.6.4 -pylint>=2.10.0 -breathe==4.26.0 -torch>=1.7.1 -sphinx-fortran==1.1.1 -jinja2==3.0.3 -mypy>=1.4.0 diff --git a/requirements.txt b/requirements.txt index ad21d06a8..bb27ece74 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ -numpy>=1.18.2 +# requirements.txt +# Dependencies are maintained in setup.cfg +-e . diff --git a/setup.cfg b/setup.cfg index 3ca4bd846..c27151f4c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,17 +41,19 @@ python_requires = >=3.7,<3.11 dev = pytest>=6.0.0 pytest-cov==2.10.1 - black==20.8b1 + black==23.3.0 isort==5.6.4 pylint>=2.10.0 torch>=1.7.1 mypy>=1.4.0 + jinja2==3.0.3 doc= sphinx==3.1.1 sphinx-fortran==1.1.1 sphinx_rtd_theme>=0.5.0 - breathe==4.25.1 + sphinx-book-theme==0.2.0 + breathe==4.26.0 xarray= xarray>=0.14.1 From 4b6c9cd5800f9a51d6c152a30b4cbc7eef41c3cc Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:49:08 -0500 Subject: [PATCH 17/25] Add cluster to KW check in client constructor (#421) Add cluster to KW check in client constructor [ committed by @billschereriii ] [ reviewed by @MattToast ] --- doc/changelog.rst | 3 ++- src/cpp/client.cpp | 9 --------- src/python/module/smartredis/client.py | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index d64019e1b..fdda0ac5a 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -29,7 +29,7 @@ Description Detailed Notes - Merged dependency lists from requirements.txt and requirements-dev.txt into setup.cfg to have only one set of dependencies going forward (PR420_) -- Improved robustness of Python client construction by adding detection of invalid kwargs (PR419_) +- Improved robustness of Python client construction by adding detection of invalid kwargs (PR419_), (PR421_) - Updated the Client and Dataset API documentation to clarify which interacts with the backend db (PR416_) - The SSDB address can now include '-' and '_' as special characters in the name. This gives users more options for naming the UDS socket file (PR415_) - Added tests to increase Python code coverage @@ -45,6 +45,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR421: https://github.com/CrayLabs/SmartRedis/pull/421 .. _PR420: https://github.com/CrayLabs/SmartRedis/pull/420 .. _PR419: https://github.com/CrayLabs/SmartRedis/pull/419 .. _PR416: https://github.com/CrayLabs/SmartRedis/pull/416 diff --git a/src/cpp/client.cpp b/src/cpp/client.cpp index 2e4581762..86b8dc583 100644 --- a/src/cpp/client.cpp +++ b/src/cpp/client.cpp @@ -107,18 +107,9 @@ void Client::_establish_server_connection() Client::Client(bool cluster, const std::string& logger_name) : SRObject(logger_name) { - std::cout << "In deprecated constructor" << std::endl; // Log that a new client has been instantiated log_data(LLDebug, "New client created"); - // Log deprecation warning for this method - log_data( - LLInfo, - "Deprecation Notice: Client::Client(bool, std::string) constructor " - "should not be used. Please migrate your code to use the " - "Client::Client(ConfigOptions*) constructor and set the server type " - "in the SR_DB_TYPE environment variable."); - // Create our ConfigOptions object (default = no suffixing) auto cfgopts = ConfigOptions::create_from_environment(""); _cfgopts = cfgopts.release(); diff --git a/src/python/module/smartredis/client.py b/src/python/module/smartredis/client.py index 985045ea5..1b1c830a2 100644 --- a/src/python/module/smartredis/client.py +++ b/src/python/module/smartredis/client.py @@ -84,7 +84,7 @@ def __init__(self, *a: t.Any, **kw: t.Any): raise TypeError(f"Invalid type for argument 0: {type(a[0])}") else: # Only kwargs in the call - if "address" in kw: + if "address" in kw or "cluster" in kw: pyclient = self.__address_construction(*a, **kw) else: pyclient = self.__standard_construction(*a, **kw) From 6bfcf1deb0d726df75610fe8a352b9ecb13c44f7 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Thu, 26 Oct 2023 17:09:08 -0500 Subject: [PATCH 18/25] Add tests for client parameter combinatorics (#422) Add tests for client parameter combinatorics [ committed by @billschereriii ] [ reviewed by @MattToast ] --- doc/changelog.rst | 3 + tests/python/test_address.py | 48 --------------- tests/python/test_client.py | 110 +++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 48 deletions(-) delete mode 100644 tests/python/test_address.py create mode 100644 tests/python/test_client.py diff --git a/doc/changelog.rst b/doc/changelog.rst index fdda0ac5a..1be07cd30 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Added test cases for all Client construction parameter combinations - Centralized dependency tracking to setup.cfg - Improved robustness of Python client construction - Updated Client and Dataset documentation @@ -28,6 +29,7 @@ Description Detailed Notes +- Added test cases for all Client construction parameter combinations (PR422_) - Merged dependency lists from requirements.txt and requirements-dev.txt into setup.cfg to have only one set of dependencies going forward (PR420_) - Improved robustness of Python client construction by adding detection of invalid kwargs (PR419_), (PR421_) - Updated the Client and Dataset API documentation to clarify which interacts with the backend db (PR416_) @@ -45,6 +47,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR422: https://github.com/CrayLabs/SmartRedis/pull/422 .. _PR421: https://github.com/CrayLabs/SmartRedis/pull/421 .. _PR420: https://github.com/CrayLabs/SmartRedis/pull/420 .. _PR419: https://github.com/CrayLabs/SmartRedis/pull/419 diff --git a/tests/python/test_address.py b/tests/python/test_address.py deleted file mode 100644 index 9b8ed4cf6..000000000 --- a/tests/python/test_address.py +++ /dev/null @@ -1,48 +0,0 @@ -# BSD 2-Clause License -# -# Copyright (c) 2021-2023, Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os - -from smartredis import Client - -def test_serialization(context): - c = Client(None, logger_name=context) - assert str(c) != repr(c) - - -def test_address(context): - # This test evaluates deprecated behavior and will be removed - # in the next release when the deprecated Client construction - # is removed. - # get env var to set through client init - ssdb = os.environ["SSDB"] - del os.environ["SSDB"] - - # client init should fail if SSDB not set - _ = Client(False, address=ssdb, logger_name=context) - - # check if SSDB was set anyway - assert os.environ["SSDB"] == ssdb diff --git a/tests/python/test_client.py b/tests/python/test_client.py new file mode 100644 index 000000000..5ccf7f53a --- /dev/null +++ b/tests/python/test_client.py @@ -0,0 +1,110 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import pytest + +from smartredis import Client, ConfigOptions + +def test_serialization(context): + c = Client(None, logger_name=context) + assert str(c) != repr(c) + + +def test_address(context): + # get env var to set through client init + ssdb = os.environ["SSDB"] + del os.environ["SSDB"] + + # client init should fail if SSDB not set + _ = Client(False, address=ssdb, logger_name=context) + + # check if SSDB was set anyway + assert os.environ["SSDB"] == ssdb + +# Globals for Client constructor testing +ac_original = Client._Client__address_construction +sc_original = Client._Client__standard_construction +cluster_mode = os.environ["SR_DB_TYPE"] == "Clustered" +target_address = os.environ["SSDB"] +co_envt = ConfigOptions.create_from_environment("") + +@pytest.mark.parametrize( + "args, kwargs, expected_constructor", [ + # address constructions + [(False,), {}, "address"], + [(False,), {"address": target_address}, "address"], + [(False,), {"address": target_address, "logger_name": "log_name"}, "address"], + [(False,), {"logger_name": "log_name"}, "address"], + [(False, target_address), {}, "address"], + [(False, target_address), {"logger_name": "log_name"}, "address"], + [(False, target_address, "log_name"), {}, "address"], + [(), {"cluster": cluster_mode}, "address"], + [(), {"cluster": cluster_mode, "address": target_address}, "address"], + [(), {"cluster": cluster_mode, "address": target_address, "logger_name": "log_name"}, "address"], + [(), {"cluster": cluster_mode, "logger_name": "log_name"}, "address"], + # standard constructions + [(None,), {}, "standard"], + [(None,), {"logger_name": "log_name"}, "standard"], + [(None, "log_name"), {}, "standard"], + [(co_envt,), {}, "standard"], + [(co_envt,), {"logger_name": "log_name"}, "standard"], + [(co_envt, "log_name"), {}, "standard"], + [(), {}, "standard"], + [(), {"config_options": None}, "standard"], + [(), {"config_options": None, "logger_name": "log_name"}, "standard"], + [(), {"config_options": co_envt}, "standard"], + [(), {"config_options": co_envt, "logger_name": "log_name"}, "standard"], + [(), {"logger_name": "log_name"}, "standard"], +]) +def test_client_constructor(args, kwargs, expected_constructor, monkeypatch): + ac_got_called = False + sc_got_called = False + + def mock_address_constructor(self, *a, **kw): + nonlocal ac_got_called + ac_got_called = True + return ac_original(self, *a, **kw) + + @staticmethod + def mock_standard_constructor(*a, **kw): + nonlocal sc_got_called + sc_got_called = True + return sc_original(*a, **kw) + + monkeypatch.setattr( + Client, "_Client__address_construction", mock_address_constructor) + monkeypatch.setattr( + Client, "_Client__standard_construction", mock_standard_constructor) + + Client(*args, **kwargs) + + if expected_constructor == "address": + assert ac_got_called + assert not sc_got_called + if expected_constructor == "standard": + assert not ac_got_called + assert sc_got_called From 2507bb6333581e9e6f743d5d9b0af20581b2308e Mon Sep 17 00:00:00 2001 From: Al Rigazzi Date: Thu, 9 Nov 2023 10:12:39 +0100 Subject: [PATCH 19/25] Update documentation (#423) Fix some broken links in documentation and update section on how to run tests. [ committed by @al-rigazzi ] [ reviewed by @billschereriii ] --- doc/advanced_topics.rst | 2 +- doc/changelog.rst | 3 + doc/clients/c-plus.rst | 35 ++++++----- doc/clients/c.rst | 35 ++++++----- doc/clients/fortran.rst | 34 +++++------ doc/clients/python.rst | 2 +- doc/conf.py | 2 +- doc/data_structures.rst | 81 +++++++++++++------------- doc/developer/testing.rst | 17 ++---- doc/examples/cpp_api_examples.rst | 4 +- doc/examples/python_api_examples.rst | 4 +- doc/overview.rst | 14 +++-- src/python/module/smartredis/client.py | 3 +- 13 files changed, 116 insertions(+), 120 deletions(-) diff --git a/doc/advanced_topics.rst b/doc/advanced_topics.rst index a84d090af..e2c2dcd14 100644 --- a/doc/advanced_topics.rst +++ b/doc/advanced_topics.rst @@ -125,7 +125,7 @@ lead to race conditions: // Delete an aggregation list void delete_list(const std::string& list_name); -.. _advanced-topics-dataset-aggregation: +.. _advanced-topics-multi-db: Multiple Database Support ========================= diff --git a/doc/changelog.rst b/doc/changelog.rst index 1be07cd30..bd4cbba40 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Updated documentation - Added test cases for all Client construction parameter combinations - Centralized dependency tracking to setup.cfg - Improved robustness of Python client construction @@ -29,6 +30,7 @@ Description Detailed Notes +- Some broken links in the documentation were fixed, and the instructions to run the tests were updated (PR423_) - Added test cases for all Client construction parameter combinations (PR422_) - Merged dependency lists from requirements.txt and requirements-dev.txt into setup.cfg to have only one set of dependencies going forward (PR420_) - Improved robustness of Python client construction by adding detection of invalid kwargs (PR419_), (PR421_) @@ -47,6 +49,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR423: https://github.com/CrayLabs/SmartRedis/pull/423 .. _PR422: https://github.com/CrayLabs/SmartRedis/pull/422 .. _PR421: https://github.com/CrayLabs/SmartRedis/pull/421 .. _PR420: https://github.com/CrayLabs/SmartRedis/pull/420 diff --git a/doc/clients/c-plus.rst b/doc/clients/c-plus.rst index 67844c8db..f8e39bd23 100644 --- a/doc/clients/c-plus.rst +++ b/doc/clients/c-plus.rst @@ -2,23 +2,23 @@ C++ APIs ******** -The following page provides a comprehensive overview of the SmartRedis C++ -Client and Dataset APIs. +The following page provides a comprehensive overview of the SmartRedis C++ +Client and Dataset APIs. Further explanation and details of each are presented below. Client API ========== -The Client API is purpose-built for interaction with the backend database, -which extends the capabilities of the Redis in-memory data store. -It's important to note that the SmartRedis Client API is the exclusive -means for altering, transmitting, and receiving data within the backend -database. More specifically, the Client API is responsible for both -creating and modifying data structures, which encompass :ref:`Models `, -:ref:`Scripts `, and :ref:`Tensors `. -It also handles the transmission and reception of -the aforementioned data structures in addition to :ref:`Dataset ` -data structure. Creating and modifying the ``DataSet`` object +The Client API is purpose-built for interaction with the backend database, +which extends the capabilities of the Redis in-memory data store. +It's important to note that the SmartRedis Client API is the exclusive +means for altering, transmitting, and receiving data within the backend +database. More specifically, the Client API is responsible for both +creating and modifying data structures, which encompass :ref:`Models `, +:ref:`Scripts `, and :ref:`Tensors `. +It also handles the transmission and reception of +the aforementioned data structures in addition to :ref:`Dataset ` +data structure. Creating and modifying the ``DataSet`` object is confined to local operation by the DataSet API. .. doxygenclass:: SmartRedis::Client @@ -30,15 +30,14 @@ is confined to local operation by the DataSet API. Dataset API =========== -The C++ DataSet API enables a user to manage a group of tensors -and associated metadata within a datastructure called a ``DataSet`` object. -The DataSet API operates independently of the database and solely -maintains the dataset object in-memory. The actual interaction with the Redis database, -where a snapshot of the DataSet object is sent, is handled by the Client API. For more +The C++ DataSet API enables a user to manage a group of tensors +and associated metadata within a datastructure called a ``DataSet`` object. +The DataSet API operates independently of the database and solely +maintains the dataset object in-memory. The actual interaction with the Redis database, +where a snapshot of the DataSet object is sent, is handled by the Client API. For more information on the ``DataSet`` object, click :ref:`here `. .. doxygenclass:: SmartRedis::DataSet :project: cpp_client :members: :undoc-members: - diff --git a/doc/clients/c.rst b/doc/clients/c.rst index 91b3ebb29..0312aa75b 100644 --- a/doc/clients/c.rst +++ b/doc/clients/c.rst @@ -2,23 +2,23 @@ C APIs ******* -The following page provides a comprehensive overview of the SmartRedis C -Client and Dataset APIs. +The following page provides a comprehensive overview of the SmartRedis C +Client and Dataset APIs. Further explanation and details of each are presented below. Client API ========== -The Client API is purpose-built for interaction with the backend database, -which extends the capabilities of the Redis in-memory data store. -It's important to note that the SmartRedis Client API is the exclusive -means for altering, transmitting, and receiving data within the backend -database. More specifically, the Client API is responsible for both -creating and modifying data structures, which encompass :ref:`Models `, -:ref:`Scripts `, and :ref:`Tensors `. -It also handles the transmission and reception of -the aforementioned data structures in addition to :ref:`Dataset ` -data structure. Creating and modifying the ``DataSet`` object +The Client API is purpose-built for interaction with the backend database, +which extends the capabilities of the Redis in-memory data store. +It's important to note that the SmartRedis Client API is the exclusive +means for altering, transmitting, and receiving data within the backend +database. More specifically, the Client API is responsible for both +creating and modifying data structures, which encompass :ref:`Models `, +:ref:`Scripts `, and :ref:`Tensors `. +It also handles the transmission and reception of +the aforementioned data structures in addition to :ref:`Dataset ` +data structure. Creating and modifying the ``DataSet`` object is confined to local operation by the DataSet API. .. doxygenfile:: c_client.h @@ -28,13 +28,12 @@ is confined to local operation by the DataSet API. Dataset API =========== -The C DataSet API enables a user to manage a group of tensors -and associated metadata within a datastructure called a ``DataSet`` object. -The DataSet API operates independently of the database and solely -maintains the dataset object in-memory. The actual interaction with the Redis database, -where a snapshot of the DataSet object is sent, is handled by the Client API. For more +The C DataSet API enables a user to manage a group of tensors +and associated metadata within a datastructure called a ``DataSet`` object. +The DataSet API operates independently of the database and solely +maintains the dataset object in-memory. The actual interaction with the Redis database, +where a snapshot of the DataSet object is sent, is handled by the Client API. For more information on the ``DataSet`` object, click :ref:`here `. .. doxygenfile:: c_dataset.h :project: c_client - diff --git a/doc/clients/fortran.rst b/doc/clients/fortran.rst index 0a31c2a3f..e701de974 100644 --- a/doc/clients/fortran.rst +++ b/doc/clients/fortran.rst @@ -2,23 +2,23 @@ Fortran APIs ************ -The following page provides a comprehensive overview of the SmartRedis Fortran -Client and Dataset APIs. +The following page provides a comprehensive overview of the SmartRedis Fortran +Client and Dataset APIs. Further explanation and details of each are presented below. Client API ========== -The Client API is purpose-built for interaction with the backend database, -which extends the capabilities of the Redis in-memory data store. -It's important to note that the SmartRedis Client API is the exclusive -means for altering, transmitting, and receiving data within the backend -database. More specifically, the Client API is responsible for both -creating and modifying data structures, which encompass :ref:`Models `, -:ref:`Scripts `, and :ref:`Tensors `. -It also handles the transmission and reception of -the aforementioned data structures in addition to :ref:`Dataset ` -data structure. Creating and modifying the ``DataSet`` object +The Client API is purpose-built for interaction with the backend database, +which extends the capabilities of the Redis in-memory data store. +It's important to note that the SmartRedis Client API is the exclusive +means for altering, transmitting, and receiving data within the backend +database. More specifically, the Client API is responsible for both +creating and modifying data structures, which encompass :ref:`Models `, +:ref:`Scripts `, and :ref:`Tensors `. +It also handles the transmission and reception of +the aforementioned data structures in addition to :ref:`Dataset ` +data structure. Creating and modifying the ``DataSet`` object is confined to local operation by the DataSet API. The following are overloaded interfaces which support @@ -32,11 +32,11 @@ The following are overloaded interfaces which support Dataset API =========== -The Fortran DataSet API enables a user to manage a group of tensors -and associated metadata within a datastructure called a ``DataSet`` object. -The DataSet API operates independently of the database and solely -maintains the dataset object in-memory. The actual interaction with the Redis database, -where a snapshot of the DataSet object is sent, is handled by the Client API. For more +The Fortran DataSet API enables a user to manage a group of tensors +and associated metadata within a datastructure called a ``DataSet`` object. +The DataSet API operates independently of the database and solely +maintains the dataset object in-memory. The actual interaction with the Redis database, +where a snapshot of the DataSet object is sent, is handled by the Client API. For more information on the ``DataSet`` object, click :ref:`here `. The following are overloaded interfaces which support diff --git a/doc/clients/python.rst b/doc/clients/python.rst index 7b446664f..3146b17a1 100644 --- a/doc/clients/python.rst +++ b/doc/clients/python.rst @@ -16,7 +16,7 @@ means for altering, transmitting, and receiving data within the backend database. More specifically, the Client API is responsible for both creating and modifying data structures, which encompass :ref:`Models `, :ref:`Scripts `, -and :ref:`Tensors `. +and :ref:`Tensors `. It also handles the transmission and reception of the aforementioned data structures in addition to :ref:`Dataset ` data structure. Creating and modifying the ``DataSet`` object is confined to local diff --git a/doc/conf.py b/doc/conf.py index e461faabe..612ecea60 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -13,7 +13,7 @@ # import os # import sys # sys.path.insert(0, os.path.abspath('.')) - +# pylint: skip-file # -- Project information ----------------------------------------------------- diff --git a/doc/data_structures.rst b/doc/data_structures.rst index bd4441e86..61ad0a9a2 100644 --- a/doc/data_structures.rst +++ b/doc/data_structures.rst @@ -2,22 +2,22 @@ Data Structures *************** -SmartSim defines primary three data structures designed for use within backend databases: +SmartSim defines primary three data structures designed for use within backend databases: * ``Tensor`` : represents an n-dimensional array of values. * ``Model`` : represents a computational ML model for one of the supported backend frameworks. * ``Script`` : represents a TorchScript program. In addition, SmartRedis defines a data -structure named ``DataSet`` that enables a user to manage a group of tensors -and associated metadata in-memory. In this section, we will provide an explanation -of the SmartRedis API used to interact with these four data structures, +structure named ``DataSet`` that enables a user to manage a group of tensors +and associated metadata in-memory. In this section, we will provide an explanation +of the SmartRedis API used to interact with these four data structures, along with relevant insights on performance and best practices. -We illustrate concepts and capabilities of the Python -and C++ SmartRedis APIs. The C and Fortran function signatures closely -mirror the C++ API, and for brevity, we won't delve -into them. For full discussion of the C and Fortran APIs, +We illustrate concepts and capabilities of the Python +and C++ SmartRedis APIs. The C and Fortran function signatures closely +mirror the C++ API, and for brevity, we won't delve +into them. For full discussion of the C and Fortran APIs, please refer to their respective documentation pages. @@ -178,14 +178,14 @@ SmartSim ensemble capabilities. Dataset ======= -When dealing with multi-modal data or complex data sets, -one may have different types of tensors (e.g., images, text embeddings, -numerical data) and metadata for each data point. Grouping them into a +When dealing with multi-modal data or complex data sets, +one may have different types of tensors (e.g., images, text embeddings, +numerical data) and metadata for each data point. Grouping them into a collection represents each data point as a cohesive unit. The ``DataSet`` data structure provides this functionality to stage tensors and metadata -in-memory via the ``DataSet API``. After the creation of a -``DataSet`` object, the grouped data can be efficiently stored in the backend database -by the ``Client API`` and subsequently retrieved using the assigned ``DataSet`` name. +in-memory via the ``DataSet API``. After the creation of a +``DataSet`` object, the grouped data can be efficiently stored in the backend database +by the ``Client API`` and subsequently retrieved using the assigned ``DataSet`` name. In the upcoming sections, we outline the process of building, sending, and retrieving a ``DataSet``. Listed below are the supported tensor and metadata types. @@ -237,31 +237,31 @@ Build and Send a DataSet When building a ``DataSet`` object in-memory, a user can group various combinations of tensors and metadata that -constrain to the supported data types in the table above. To illustrate, -tensors can be inserted into a ``dataset`` object via the ``Dataset.add_tensor()`` method. -The SmartRedis DataSet API functions -are available in C, C++, Python, and Fortran. The ``DataSet.add_tensor()`` function, -operates independently of the database and solely -maintains the dataset object. Storing the dataset in the backend +constrain to the supported data types in the table above. To illustrate, +tensors can be inserted into a ``dataset`` object via the ``Dataset.add_tensor()`` method. +The SmartRedis DataSet API functions +are available in C, C++, Python, and Fortran. The ``DataSet.add_tensor()`` function, +operates independently of the database and solely +maintains the dataset object. Storing the dataset in the backend database is done via the Client API ``put_dataset()`` method. .. note:: - The ``DataSet.add_tensor()`` function copies user-provided - tensor data; this prevents potential issues arising from the user's - data being cleared or deallocated. Any additional memory allocated + The ``DataSet.add_tensor()`` function copies user-provided + tensor data; this prevents potential issues arising from the user's + data being cleared or deallocated. Any additional memory allocated for this purpose will be released when the DataSet object is destroyed. Metadata can be added to an in-memory ``DataSet`` object with the ``DataSet.add_meta_scalar()`` and ``DataSet.add_meta_string()`` -functions. Methods exist for adding scalar metadata (e.g., double) and string metadata. +functions. Methods exist for adding scalar metadata (e.g., double) and string metadata. For both functions, the first input -parameter is the name of the metadata field. -The field name serves as an internal identifier within the ``DataSet`` -for grouped metadata values. It's used to retrieve metadata in the future. -Since it's an internal identifier, users don't need to be concerned -about conflicts with keys in the database. In other words, multiple -``DataSet`` objects can use the same metadata field names without causing -issues because these names are managed within the ``DataSet`` and won't +parameter is the name of the metadata field. +The field name serves as an internal identifier within the ``DataSet`` +for grouped metadata values. It's used to retrieve metadata in the future. +Since it's an internal identifier, users don't need to be concerned +about conflicts with keys in the database. In other words, multiple +``DataSet`` objects can use the same metadata field names without causing +issues because these names are managed within the ``DataSet`` and won't interfere with external database keys. The C++ interface for adding metadata is shown below: @@ -280,13 +280,13 @@ metadata is shown below: When adding a scalar or string metadata value, the value is copied by the ``DataSet``, and as a result, the user does not need to ensure that the metadata values provided -are still in-memory. In other words, -the ``DataSet`` handles the memory management of these metadata values, -and you don't need to retain or manage the original copies separately +are still in-memory. In other words, +the ``DataSet`` handles the memory management of these metadata values, +and you don't need to retain or manage the original copies separately once they have been included in the ``DataSet`` object. Additionally, multiple metadata values can be added to a single field name, and the default behavior is to append the value to the -field name (creating the field if not already present). This behavior allows the ``DataSet`` metadata +field name (creating the field if not already present). This behavior allows the ``DataSet`` metadata to function like one-dimensional arrays. Also, note that in the above C++ example, @@ -296,7 +296,7 @@ requirements exist for C and Fortran ``DataSet`` implementations. Finally, the ``DataSet`` object is sent to the database using the ``Client.put_dataset()`` function, which is uniform across all clients. -To emphasize once more, all interactions with the backend database are handle by +To emphasize once more, all interactions with the backend database are handle by the Client API, not the DataSet API. @@ -320,11 +320,11 @@ the previous section for details on tensor retrieve function calls. There are four functions for retrieving metadata information from a ``DataSet`` object in-memory: -``get_meta_scalars()``, ``get_meta_strings()``, ``get_metadata_field_names()`` +``get_meta_scalars()``, ``get_meta_strings()``, ``get_metadata_field_names()`` and ``get_metadata_field_type()``. As the names suggest, the ``get_meta_scalars()`` function is used for retrieving numerical metadata values, while the ``get_meta_strings()`` function -is for retrieving metadata string values. The ``get_metadata_field_names()`` function -retrieves a list of all metadata field names in the ``DataSet`` object. Lastly, +is for retrieving metadata string values. The ``get_metadata_field_names()`` function +retrieves a list of all metadata field names in the ``DataSet`` object. Lastly, the ``get_metadata_field_type()`` function returns the type (scalar or string) of the metadata attached to the specified field name. The metadata retrieval function prototypes vary across the clients based on programming language constraints, @@ -339,8 +339,7 @@ Aggregating ----------- SmartRedis also supports an advanced API for working with aggregate -lists of DataSets; details may be found -:ref:`here `. +lists of DataSets; details may be found :ref:`here `. .. _data-structures-model: diff --git a/doc/developer/testing.rst b/doc/developer/testing.rst index d0b899a0b..c4487db28 100644 --- a/doc/developer/testing.rst +++ b/doc/developer/testing.rst @@ -6,15 +6,8 @@ Testing Description Quick instructions ################## -To run the tests, assuming that all requirements have been installed -1. Activate your environment with SmartSim and SmartRedis installed -2. Modify `SR_DB_TYPE` (`Clustered` or `Standalone`) and - `SMARTREDIS_TEST_DEVICE` (`gpu` or `cpu`) as necessary in - `setup_test_env.sh`. -3. `source setup_test_env.sh` -4. `pushd utils/create_cluster; python local_cluster.py; popd` -5. `export SSDB="127.0.0.1:6379,127.0.0.1:6380,127.0.0.1:6381"` -5. Run the desired tests (see `make help` for more details) +To run the tests, simply execute the following command. Omit ``SR_PYTHON=On`` or ``SR_FORTRAN=On`` if you do not wish to test those languages: + ``make test SR_PYTHON=On SR_FORTRAN=On`` ################### Unit Test Framework @@ -32,6 +25,6 @@ file exists, then it is preferred that a new file (prefixed with *test_*) is cre In Summary =========== - - New unit tests should be placed in ``tests/cpp/unit-tests/`` - - Testing files should be prefixed with *test_* - - It is preferred that new unit tests are in a new ``SCENARIO`` +- New unit tests should be placed in ``tests/cpp/unit-tests/`` +- Testing files should be prefixed with *test_* +- It is preferred that new unit tests are in a new ``SCENARIO`` diff --git a/doc/examples/cpp_api_examples.rst b/doc/examples/cpp_api_examples.rst index a24efa003..550553fe7 100644 --- a/doc/examples/cpp_api_examples.rst +++ b/doc/examples/cpp_api_examples.rst @@ -35,10 +35,10 @@ DataSets The C++ ``Client`` API stores and retrieve datasets from the backend database. The C++ ``DataSet`` API can store and retrieve tensors and metadata from an in-memory ``DataSet`` object. -To reiterate, the actual interaction with the backend database, +To reiterate, the actual interaction with the backend database, where a snapshot of the ``DataSet`` object is sent, is handled by the Client API. For further information about datasets, please refer to the :ref:`Dataset -section of the Data Structures documentation page `. +section of the Data Structures documentation page `. The code below shows how to store and retrieve tensors and metadata which belong to a ``DataSet``. diff --git a/doc/examples/python_api_examples.rst b/doc/examples/python_api_examples.rst index 1adf575ab..dcdbfd1ef 100644 --- a/doc/examples/python_api_examples.rst +++ b/doc/examples/python_api_examples.rst @@ -39,10 +39,10 @@ Datasets The Python ``Client`` API stores and retrieve datasets from the backend database. The Python ``DataSet`` API can store and retrieve tensors and metadata from an in-memory ``DataSet`` object. -To reiterate, the actual interaction with the backend database, +To reiterate, the actual interaction with the backend database, where a snapshot of the ``DataSet`` object is sent, is handled by the Client API. For further information about datasets, please refer to the :ref:`Dataset -section of the Data Structures documentation page `. +section of the Data Structures documentation page `. The code below shows how to store and retrieve tensors which belong to a ``DataSet``. diff --git a/doc/overview.rst b/doc/overview.rst index a17cbdd31..b0c3e9b8e 100644 --- a/doc/overview.rst +++ b/doc/overview.rst @@ -3,22 +3,24 @@ Overview ******** -SmartRedis is a collection of Redis clients that support -RedisAI capabilities and include additional +SmartRedis is a collection of `Redis `__ clients that support +`RedisAI `__ capabilities and include additional features for high performance computing (HPC) applications. Key features of RedisAI that are supported by SmartRedis include: - A tensor data type in Redis - TensorFlow, TensorFlow Lite, Torch, - and ONNXRuntime backends for model evaluations -- TorchScript storage and evaluation + and `ONNXRuntime `__ backends for model evaluations +- `TorchScript `__ storage and evaluation In addition to the RedisAI capabilities above, SmartRedis includes the following features developed for large, distributed HPC architectures: -- Redis cluster support for RedisAI data types (tensors, - models, and scripts). +- Redis cluster support for RedisAI data types + (`tensors `__, + `models `__, + and `scripts `__). - Distributed model and script placement for parallel evaluation that maximizes hardware utilization and throughput - A ``DataSet`` storage format to aggregate multiple tensors diff --git a/src/python/module/smartredis/client.py b/src/python/module/smartredis/client.py index 1b1c830a2..943e162db 100644 --- a/src/python/module/smartredis/client.py +++ b/src/python/module/smartredis/client.py @@ -1560,7 +1560,8 @@ def set_model_chunk_size(self, chunk_size: int) -> None: NOTE: The default size of 511MB should be fine for most applications, so it is expected to be very rare that a client calls this method. It is not necessary to call - this method a model to be chunked. + this method for a model to be chunked. + :param chunk_size: The new chunk size in bytes :type addresses: int :raises RedisReplyError: if there is an error From 1f7d478ed87c0a2b15f6a2216e63f4ff46307fe2 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Thu, 16 Nov 2023 14:53:52 -0600 Subject: [PATCH 20/25] Make Dataset::add_tensor() const correct (#427) Make Dataset::add_tensor() const correct [ committed by @billschereriii ] [ reviewed by @MattToast @al-rigazzi ] --- .github/workflows/run_static_and_examples.yml | 5 +++-- .github/workflows/run_tests.yml | 5 +++-- doc/changelog.rst | 3 +++ include/dataset.h | 4 ++-- include/tensor.h | 14 +++++++------- include/tensor.tcc | 16 ++++++++-------- include/tensorbase.h | 4 ++-- include/tensorpack.h | 2 +- src/cpp/dataset.cpp | 4 ++-- src/cpp/tensorbase.cpp | 2 +- src/cpp/tensorpack.cpp | 2 +- 11 files changed, 33 insertions(+), 28 deletions(-) diff --git a/.github/workflows/run_static_and_examples.yml b/.github/workflows/run_static_and_examples.yml index 5687f7733..e1d73b55e 100644 --- a/.github/workflows/run_static_and_examples.yml +++ b/.github/workflows/run_static_and_examples.yml @@ -96,12 +96,13 @@ jobs: rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB && echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list && sudo apt-get update -y && - sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic + sudo apt-get install -y intel-oneapi-compiler-fortran=2023.2.2-47 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic source /opt/intel/oneapi/setvars.sh && + sudo apt-get install -y intel-oneapi-common-vars && printenv >> $GITHUB_ENV && echo "CC=icx" >> $GITHUB_ENV && echo "CXX=icpx" >> $GITHUB_ENV && - echo "FC=ifort" >> $GITHUB_ENV + echo "FC=ifx" >> $GITHUB_ENV # Install additional dependencies - name: Install Cmake Linux diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index b527cde00..676e3f188 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -108,12 +108,13 @@ jobs: rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB && echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list && sudo apt-get update -y && - sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic + sudo apt-get install -y intel-oneapi-compiler-fortran=2023.2.2-47 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic source /opt/intel/oneapi/setvars.sh && + sudo apt-get install -y intel-oneapi-common-vars && printenv >> $GITHUB_ENV && echo "CC=icx" >> $GITHUB_ENV && echo "CXX=icpx" >> $GITHUB_ENV && - echo "FC=ifort" >> $GITHUB_ENV + echo "FC=ifx" >> $GITHUB_ENV # Set up perl environment for LCOV - uses: actions/checkout@v3 diff --git a/doc/changelog.rst b/doc/changelog.rst index bd4cbba40..0fc750f87 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Improved const correctness of Dataset - Updated documentation - Added test cases for all Client construction parameter combinations - Centralized dependency tracking to setup.cfg @@ -30,6 +31,7 @@ Description Detailed Notes +- The Dataset add_tensor method is now const correct, as are all internal the methods it calls (PR427_) - Some broken links in the documentation were fixed, and the instructions to run the tests were updated (PR423_) - Added test cases for all Client construction parameter combinations (PR422_) - Merged dependency lists from requirements.txt and requirements-dev.txt into setup.cfg to have only one set of dependencies going forward (PR420_) @@ -49,6 +51,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR427: https://github.com/CrayLabs/SmartRedis/pull/427 .. _PR423: https://github.com/CrayLabs/SmartRedis/pull/423 .. _PR422: https://github.com/CrayLabs/SmartRedis/pull/422 .. _PR421: https://github.com/CrayLabs/SmartRedis/pull/421 diff --git a/include/dataset.h b/include/dataset.h index 0f72471ff..5fb5be85d 100644 --- a/include/dataset.h +++ b/include/dataset.h @@ -105,7 +105,7 @@ class DataSet : public SRObject * \throw SmartRedis::Exception if add_tensor operation fails */ void add_tensor(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, const SRMemoryLayout mem_layout); @@ -423,7 +423,7 @@ class DataSet : public SRObject * \param mem_layout The memory layout for the provided tensor data */ void _add_to_tensorpack(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, const SRMemoryLayout mem_layout); diff --git a/include/tensor.h b/include/tensor.h index 42d6c9623..e2e67dbc0 100644 --- a/include/tensor.h +++ b/include/tensor.h @@ -59,7 +59,7 @@ class Tensor : public TensorBase * \param mem_layout The memory layout of the source data */ Tensor(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, const SRMemoryLayout mem_layout); @@ -157,7 +157,7 @@ class Tensor : public TensorBase * only. The initial caller SHOULD NOT use * this pointer. */ - void* _copy_nested_to_contiguous(void* src_data, + void* _copy_nested_to_contiguous(const void* src_data, const size_t* dims, const size_t n_dims, void* dest_data); @@ -213,7 +213,7 @@ class Tensor : public TensorBase * \param mem_layout The layout of the source data * memory structure */ - virtual void _set_tensor_data(void* src_data, + virtual void _set_tensor_data(const void* src_data, const std::vector& dims, const SRMemoryLayout mem_layout); @@ -226,7 +226,7 @@ class Tensor : public TensorBase * \param dims The dimensions of the tensor */ void _f_to_c_memcpy(T* c_data, - T* f_data, + const T* f_data, const std::vector& dims); /*! @@ -238,7 +238,7 @@ class Tensor : public TensorBase * \param dims The dimensions of the tensor */ void _c_to_f_memcpy(T* f_data, - T* c_data, + const T* c_data, const std::vector& dims); /*! @@ -253,7 +253,7 @@ class Tensor : public TensorBase * \param current_dim The index of the current dimension */ void _f_to_c(T* c_data, - T* f_data, + const T* f_data, const std::vector& dims, std::vector dim_positions, size_t current_dim); @@ -270,7 +270,7 @@ class Tensor : public TensorBase * \param current_dim The index of the current dimension */ void _c_to_f(T* f_data, - T* c_data, + const T* c_data, const std::vector& dims, std::vector dim_positions, size_t current_dim); diff --git a/include/tensor.tcc b/include/tensor.tcc index d23bbd4ea..2248677f8 100644 --- a/include/tensor.tcc +++ b/include/tensor.tcc @@ -34,7 +34,7 @@ // Tensor constructor template Tensor::Tensor(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, const SRMemoryLayout mem_layout) : @@ -218,7 +218,7 @@ void Tensor::fill_mem_space(void* data, // copy values from nested memory structure to contiguous memory structure template -void* Tensor::_copy_nested_to_contiguous(void* src_data, +void* Tensor::_copy_nested_to_contiguous(const void* src_data, const size_t* dims, const size_t n_dims, void* dest_data) @@ -293,7 +293,7 @@ T* Tensor::_build_nested_memory(void** data, // Set the tensor data from a src memory location. template -void Tensor::_set_tensor_data(void* src_data, +void Tensor::_set_tensor_data(const void* src_data, const std::vector& dims, const SRMemoryLayout mem_layout) { @@ -311,7 +311,7 @@ void Tensor::_set_tensor_data(void* src_data, std::memcpy(_data, src_data, n_bytes); break; case SRMemLayoutFortranContiguous: - _f_to_c_memcpy((T*)_data, (T*) src_data, dims); + _f_to_c_memcpy((T*)_data, (const T*)src_data, dims); break; case SRMemLayoutNested: _copy_nested_to_contiguous( @@ -333,7 +333,7 @@ size_t Tensor::_n_data_bytes() // c-style array memory space (row major) template void Tensor::_f_to_c_memcpy(T* c_data, - T* f_data, + const T* f_data, const std::vector& dims) { if (c_data == NULL || f_data == NULL) { @@ -347,7 +347,7 @@ void Tensor::_f_to_c_memcpy(T* c_data, // fortran memory space layout (col major) template void Tensor::_c_to_f_memcpy(T* f_data, - T* c_data, + const T* c_data, const std::vector& dims) { if (c_data == NULL || f_data == NULL) { @@ -360,7 +360,7 @@ void Tensor::_c_to_f_memcpy(T* f_data, // Copy fortran column major memory to c-style row major memory recursively template void Tensor::_f_to_c(T* c_data, - T* f_data, + const T* f_data, const std::vector& dims, std::vector dim_positions, size_t current_dim) @@ -388,7 +388,7 @@ void Tensor::_f_to_c(T* c_data, // Copy c-style row major memory to fortran column major memory recursively template void Tensor::_c_to_f(T* f_data, - T* c_data, + const T* c_data, const std::vector& dims, std::vector dim_positions, size_t current_dim) diff --git a/include/tensorbase.h b/include/tensorbase.h index 7bea067b8..19e794cfd 100644 --- a/include/tensorbase.h +++ b/include/tensorbase.h @@ -95,7 +95,7 @@ class TensorBase{ * \param mem_layout The memory layout of the source data */ TensorBase(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, const SRMemoryLayout mem_layout); @@ -259,7 +259,7 @@ class TensorBase{ * \param dims The dimensions of the data * \param mem_layout The memory layout of the source data */ - virtual void _set_tensor_data(void* src_data, + virtual void _set_tensor_data(const void* src_data, const std::vector& dims, const SRMemoryLayout mem_layout) = 0; diff --git a/include/tensorpack.h b/include/tensorpack.h index c4f72942f..032813a36 100644 --- a/include/tensorpack.h +++ b/include/tensorpack.h @@ -97,7 +97,7 @@ class TensorPack * \param mem_layout The memory layout of the data */ void add_tensor(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, const SRMemoryLayout mem_layout); diff --git a/src/cpp/dataset.cpp b/src/cpp/dataset.cpp index 1a71b9da3..93ab3ab74 100644 --- a/src/cpp/dataset.cpp +++ b/src/cpp/dataset.cpp @@ -51,7 +51,7 @@ DataSet::~DataSet() // Add a tensor to the DataSet. void DataSet::add_tensor(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, SRMemoryLayout mem_layout) @@ -321,7 +321,7 @@ SRMetaDataType DataSet::get_metadata_field_type( // Add a Tensor (not yet allocated) to the TensorPack void DataSet::_add_to_tensorpack(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, const SRMemoryLayout mem_layout) diff --git a/src/cpp/tensorbase.cpp b/src/cpp/tensorbase.cpp index 2ffd4391b..656174739 100644 --- a/src/cpp/tensorbase.cpp +++ b/src/cpp/tensorbase.cpp @@ -34,7 +34,7 @@ using namespace SmartRedis; // TensorBase constructor TensorBase::TensorBase(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, const SRMemoryLayout mem_layout) diff --git a/src/cpp/tensorpack.cpp b/src/cpp/tensorpack.cpp index 85c670b81..6216edae9 100644 --- a/src/cpp/tensorpack.cpp +++ b/src/cpp/tensorpack.cpp @@ -62,7 +62,7 @@ TensorPack::~TensorPack() // Add a tensor to the dataset void TensorPack::add_tensor(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, const SRMemoryLayout mem_layout) From 9fb15076e5ea48af58ffb3b2c3ef012201a8af81 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:30:22 -0600 Subject: [PATCH 21/25] More const correctness fixes (#430) Const correctness fixes for the C++ Client class [ committed by @billschereriii ] [ reviewed by @MattToast ] --- doc/changelog.rst | 5 +++- include/client.h | 71 ++++++++++++++++++++++++---------------------- src/cpp/client.cpp | 49 ++++++++++++++++++-------------- 3 files changed, 69 insertions(+), 56 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 0fc750f87..1dd0ac64a 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,7 +8,8 @@ To be released at some future point in time Description -- Improved const correctness of Dataset +- Improved const correctness of C++ Client +- Improved const correctness of C++ Dataset - Updated documentation - Added test cases for all Client construction parameter combinations - Centralized dependency tracking to setup.cfg @@ -31,6 +32,7 @@ Description Detailed Notes +- Fields in several C++ API methods are now properly marked as const (PR430_) - The Dataset add_tensor method is now const correct, as are all internal the methods it calls (PR427_) - Some broken links in the documentation were fixed, and the instructions to run the tests were updated (PR423_) - Added test cases for all Client construction parameter combinations (PR422_) @@ -51,6 +53,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR430: https://github.com/CrayLabs/SmartRedis/pull/430 .. _PR427: https://github.com/CrayLabs/SmartRedis/pull/427 .. _PR423: https://github.com/CrayLabs/SmartRedis/pull/423 .. _PR422: https://github.com/CrayLabs/SmartRedis/pull/422 diff --git a/include/client.h b/include/client.h index 78c0b40e0..123efcb81 100644 --- a/include/client.h +++ b/include/client.h @@ -218,7 +218,7 @@ class Client : public SRObject * \throw SmartRedis::Exception if put tensor command fails */ void put_tensor(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, const SRMemoryLayout mem_layout); @@ -647,8 +647,8 @@ class Client : public SRObject * \throw SmartRedis::Exception if run model command fails */ void run_model(const std::string& name, - std::vector inputs, - std::vector outputs); + const std::vector inputs, + const std::vector outputs); /*! * \brief Run a model in the database using the @@ -674,8 +674,8 @@ class Client : public SRObject * \throw SmartRedis::Exception if run model command fails */ void run_model_multigpu(const std::string& name, - std::vector inputs, - std::vector outputs, + const std::vector inputs, + const std::vector outputs, int offset, int first_gpu, int num_gpus); @@ -699,8 +699,8 @@ class Client : public SRObject */ void run_script(const std::string& name, const std::string& function, - std::vector inputs, - std::vector outputs); + const std::vector inputs, + const std::vector outputs); /*! * \brief Run a script function in the database using the @@ -724,8 +724,8 @@ class Client : public SRObject */ void run_script_multigpu(const std::string& name, const std::string& function, - std::vector inputs, - std::vector outputs, + const std::vector inputs, + const std::vector outputs, int offset, int first_gpu, int num_gpus); @@ -755,7 +755,8 @@ class Client : public SRObject * \param num_gpus the number of gpus for which the model was stored * \throw SmartRedis::Exception if model deletion fails */ - void delete_model_multigpu(const std::string& name, int first_gpu, int num_gpus); + void delete_model_multigpu( + const std::string& name, int first_gpu, int num_gpus); /*! * \brief Remove a script from the database @@ -782,7 +783,8 @@ class Client : public SRObject * \param num_gpus the number of gpus for which the script was stored * \throw SmartRedis::Exception if script deletion fails */ - void delete_script_multigpu(const std::string& name, int first_gpu, int num_gpus); + void delete_script_multigpu( + const std::string& name, int first_gpu, int num_gpus); /*! * \brief Check if a key exists in the database @@ -1024,7 +1026,7 @@ class Client : public SRObject * CLUSTER SLOTS commands will lead to SmartRedis::Exception * being thrown. */ - parsed_reply_nested_map get_db_node_info(std::string address); + parsed_reply_nested_map get_db_node_info(const std::string address); /*! * \brief Returns the response from a CLUSTER INFO command @@ -1041,7 +1043,7 @@ class Client : public SRObject * CLUSTER SLOTS commands will lead to SmartRedis::Exception * being thrown. */ - parsed_reply_map get_db_cluster_info(std::string address); + parsed_reply_map get_db_cluster_info(const std::string address); /*! * \brief Returns the response from an AI.INFO command sent to @@ -1063,7 +1065,7 @@ class Client : public SRObject */ parsed_reply_map get_ai_info(const std::string& address, const std::string& key, - const bool reset_stat); + bool reset_stat); /*! * \brief Flush the database shard at the provided address @@ -1078,7 +1080,7 @@ class Client : public SRObject * CLUSTER SLOTS commands will lead to SmartRedis::Exception * being thrown. */ - void flush_db(std::string address); + void flush_db(const std::string address); /*! * \brief Read the configuration parameters of a running server. @@ -1101,8 +1103,8 @@ class Client : public SRObject * CLUSTER SLOTS commands will lead to SmartRedis::Exception * being thrown. */ - std::unordered_map config_get(std::string expression, - std::string address); + std::unordered_map config_get( + const std::string expression, const std::string address); /*! * \brief Reconfigure the server. It can change both trivial @@ -1123,7 +1125,10 @@ class Client : public SRObject * CLUSTER SLOTS commands will lead to SmartRedis::Exception * being thrown. */ - void config_set(std::string config_param, std::string value, std::string address); + void config_set( + const std::string config_param, + const std::string value, + const std::string address); /*! * \brief Performs a synchronous save of the database shard, @@ -1140,7 +1145,7 @@ class Client : public SRObject * CLUSTER SLOTS commands will lead to SmartRedis::Exception * being thrown. */ - void save(std::string address); + void save(const std::string address); /*! * \brief Appends a dataset to the aggregation list @@ -1305,8 +1310,8 @@ class Client : public SRObject * input parameters are invalid */ std::vector get_dataset_list_range(const std::string& list_name, - const int start_index, - const int end_index); + int start_index, + int end_index); /*! * \brief Reconfigure the chunking size that Redis uses for model @@ -1466,11 +1471,9 @@ class Client : public SRObject */ inline std::vector _get_dataset_list_range(const std::string& list_name, - const int start_index, - const int end_index); + int start_index, + int end_index); - - // Add a retrieved tensor to a dataset /*! * \brief Add a tensor retrieved via get_tensor() to a dataset * \param dataset The dataset which will receive the tensor @@ -1579,7 +1582,7 @@ class Client : public SRObject * which is already in the database. */ inline std::string _build_tensor_key(const std::string& name, - const bool on_db); + bool on_db); /*! * \brief Build full formatted key of a model or a script, @@ -1589,7 +1592,7 @@ class Client : public SRObject * which is already in the database. */ inline std::string _build_model_key(const std::string& name, - const bool on_db); + bool on_db); /*! * \brief Build full formatted key of a dataset, based @@ -1600,7 +1603,7 @@ class Client : public SRObject * \returns Formatted key. */ inline std::string _build_dataset_key(const std::string& dataset_name, - const bool on_db); + bool on_db); /*! * \brief Create the key for putting or getting a DataSet tensor @@ -1613,7 +1616,7 @@ class Client : public SRObject */ inline std::string _build_dataset_tensor_key(const std::string& dataset_name, const std::string& tensor_name, - const bool on_db); + bool on_db); /*! * \brief Create keys for putting or getting a DataSet tensors @@ -1627,7 +1630,7 @@ class Client : public SRObject inline std::vector _build_dataset_tensor_keys(const std::string& dataset_name, const std::vector& tensor_names, - const bool on_db); + bool on_db); /*! * \brief Create the key for putting or getting DataSet metadata @@ -1638,7 +1641,7 @@ class Client : public SRObject * \returns A string of the key for the metadata */ inline std::string _build_dataset_meta_key(const std::string& dataset_name, - const bool on_db); + bool on_db); /*! * \brief Create the key to place an indicator in the database @@ -1649,7 +1652,7 @@ class Client : public SRObject * \returns A string of the key for the ack key */ inline std::string _build_dataset_ack_key(const std::string& dataset_name, - const bool on_db); + bool on_db); /*! * \brief Create the key for putting or getting aggregation list @@ -1660,7 +1663,7 @@ class Client : public SRObject * \returns A string of the key for aggregation list */ inline std::string _build_list_key(const std::string& list_name, - const bool on_db); + bool on_db); /*! * \brief Append the Command associated with @@ -1720,7 +1723,7 @@ class Client : public SRObject * ensemble_member.{dataset_name} * \returns DataSet name */ - std::string _get_dataset_name_from_list_entry(std::string& dataset_key); + std::string _get_dataset_name_from_list_entry(const std::string& dataset_key); /*! * \brief Poll aggregation list length using a custom comparison function diff --git a/src/cpp/client.cpp b/src/cpp/client.cpp index 86b8dc583..f105b9fc5 100644 --- a/src/cpp/client.cpp +++ b/src/cpp/client.cpp @@ -293,7 +293,7 @@ void Client::delete_dataset(const std::string& name) // Put a tensor into the database void Client::put_tensor(const std::string& name, - void* data, + const void* data, const std::vector& dims, const SRTensorType type, const SRMemoryLayout mem_layout) @@ -1306,7 +1306,7 @@ void Client::use_dataset_ensemble_prefix(bool use_prefix) } // Returns information about the given database node -parsed_reply_nested_map Client::get_db_node_info(std::string address) +parsed_reply_nested_map Client::get_db_node_info(const std::string address) { // Track calls to this API function LOG_API_FUNCTION(); @@ -1326,7 +1326,7 @@ parsed_reply_nested_map Client::get_db_node_info(std::string address) } // Returns the CLUSTER INFO command reply addressed to a single cluster node. -parsed_reply_map Client::get_db_cluster_info(std::string address) +parsed_reply_map Client::get_db_cluster_info(const std::string address) { // Track calls to this API function LOG_API_FUNCTION(); @@ -1351,7 +1351,7 @@ parsed_reply_map Client::get_db_cluster_info(std::string address) // Returns the AI.INFO command reply parsed_reply_map Client::get_ai_info(const std::string& address, const std::string& key, - const bool reset_stat) + bool reset_stat) { // Track calls to this API function LOG_API_FUNCTION(); @@ -1395,7 +1395,8 @@ parsed_reply_map Client::get_ai_info(const std::string& address, } // Delete all the keys of the given database -void Client::flush_db(std::string address) +void Client::flush_db( + const std::string address) { // Track calls to this API function LOG_API_FUNCTION(); @@ -1412,7 +1413,9 @@ void Client::flush_db(std::string address) // Read the configuration parameters of a running server std::unordered_map -Client::config_get(std::string expression, std::string address) +Client::config_get( + const std::string expression, + const std::string address) { // Track calls to this API function LOG_API_FUNCTION(); @@ -1437,7 +1440,10 @@ Client::config_get(std::string expression, std::string address) } // Reconfigure the server -void Client::config_set(std::string config_param, std::string value, std::string address) +void Client::config_set( + const std::string config_param, + const std::string value, + const std::string address) { // Track calls to this API function LOG_API_FUNCTION(); @@ -1452,7 +1458,7 @@ void Client::config_set(std::string config_param, std::string value, std::string throw SRRuntimeException("CONFIG SET command failed"); } -void Client::save(std::string address) +void Client::save(const std::string address) { // Track calls to this API function LOG_API_FUNCTION(); @@ -1728,8 +1734,8 @@ std::vector Client::get_datasets_from_list(const std::string& list_name // Retrieve a subset of datsets in the aggregation list std::vector Client::get_dataset_list_range(const std::string& list_name, - const int start_index, - const int end_index) + int start_index, + int end_index) { // Track calls to this API function LOG_API_FUNCTION(); @@ -1841,8 +1847,8 @@ inline void Client::_add_dataset_tensor( inline std::vector Client::_get_dataset_list_range(const std::string& list_name, - const int start_index, - const int end_index) + int start_index, + int end_index) { // Build the list key std::string list_key = _build_list_key(list_name, true); @@ -1984,7 +1990,7 @@ Client::_get_dataset_list_range(const std::string& list_name, // Build full formatted key of a tensor, based on current prefix settings. inline std::string Client::_build_tensor_key(const std::string& key, - const bool on_db) + bool on_db) { std::string prefix(""); if (_use_tensor_prefix) @@ -1996,7 +2002,7 @@ inline std::string Client::_build_tensor_key(const std::string& key, // Build full formatted key of a model or a script, // based on current prefix settings. inline std::string Client::_build_model_key(const std::string& key, - const bool on_db) + bool on_db) { std::string prefix(""); if (_use_model_prefix) @@ -2007,7 +2013,7 @@ inline std::string Client::_build_model_key(const std::string& key, // Build full formatted key of a dataset, based on current prefix settings. inline std::string Client::_build_dataset_key(const std::string& dataset_name, - const bool on_db) + bool on_db) { std::string prefix(""); if (_use_dataset_prefix) @@ -2020,7 +2026,7 @@ inline std::string Client::_build_dataset_key(const std::string& dataset_name, inline std::string Client::_build_dataset_tensor_key(const std::string& dataset_name, const std::string& tensor_name, - const bool on_db) + bool on_db) { return _build_dataset_key(dataset_name, on_db) + "." + tensor_name; } @@ -2029,7 +2035,7 @@ Client::_build_dataset_tensor_key(const std::string& dataset_name, inline std::vector Client::_build_dataset_tensor_keys(const std::string& dataset_name, const std::vector& tensor_names, - const bool on_db) + bool on_db) { std::vector dataset_tensor_keys; for (size_t i = 0; i < tensor_names.size(); i++) { @@ -2043,7 +2049,7 @@ Client::_build_dataset_tensor_keys(const std::string& dataset_name, // Create the key for putting or getting DataSet metadata in the database inline std::string Client::_build_dataset_meta_key(const std::string& dataset_name, - const bool on_db) + bool on_db) { return _build_dataset_key(dataset_name, on_db) + ".meta"; } @@ -2051,7 +2057,7 @@ Client::_build_dataset_meta_key(const std::string& dataset_name, // Create the key for putting or getting aggregation list in the dataset inline std::string Client::_build_list_key(const std::string& list_name, - const bool on_db) + bool on_db) { std::string prefix; if (_use_list_prefix) @@ -2065,7 +2071,7 @@ Client::_build_list_key(const std::string& list_name, // dataset has been successfully stored. inline std::string Client::_build_dataset_ack_key(const std::string& dataset_name, - const bool on_db) + bool on_db) { return _build_dataset_meta_key(dataset_name, on_db); } @@ -2223,7 +2229,8 @@ TensorBase* Client::_get_tensorbase_obj(const std::string& name) } // Determine datset name from aggregation list entry -std::string Client::_get_dataset_name_from_list_entry(std::string& dataset_key) +std::string Client::_get_dataset_name_from_list_entry( + const std::string& dataset_key) { size_t open_brace_pos = dataset_key.find_first_of('{'); From 8ba2c40905ab3689afd056d2976d31710ebc503d Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Wed, 6 Dec 2023 13:58:43 -0600 Subject: [PATCH 22/25] Make namespace declarations consistent (#434) Make py/SmartRedis namespace declarations consistent throughout the pybind layer [ committed by @billschereriii ] [ reviewed by @mellis13 ] --- doc/changelog.rst | 3 +++ include/pyclient.h | 4 ++-- include/pyconfigoptions.h | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 1dd0ac64a..f66f60535 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Improved consistency of namespace declarations for C++ pybind interface - Improved const correctness of C++ Client - Improved const correctness of C++ Dataset - Updated documentation @@ -32,6 +33,7 @@ Description Detailed Notes +- Made the declaration of the py namespace in py*.h consistently outside the SmartRedis namespace declaration (PR434_) - Fields in several C++ API methods are now properly marked as const (PR430_) - The Dataset add_tensor method is now const correct, as are all internal the methods it calls (PR427_) - Some broken links in the documentation were fixed, and the instructions to run the tests were updated (PR423_) @@ -53,6 +55,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR434: https://github.com/CrayLabs/SmartRedis/pull/434 .. _PR430: https://github.com/CrayLabs/SmartRedis/pull/430 .. _PR427: https://github.com/CrayLabs/SmartRedis/pull/427 .. _PR423: https://github.com/CrayLabs/SmartRedis/pull/423 diff --git a/include/pyclient.h b/include/pyclient.h index deb01f57a..96d30ef3a 100644 --- a/include/pyclient.h +++ b/include/pyclient.h @@ -43,10 +43,10 @@ ///@file -namespace SmartRedis { - namespace py = pybind11; +namespace SmartRedis { + /*! * \brief The PyClient class is a wrapper around the C++ client that is needed for the Python diff --git a/include/pyconfigoptions.h b/include/pyconfigoptions.h index 0eb887a3f..e8fbea84f 100644 --- a/include/pyconfigoptions.h +++ b/include/pyconfigoptions.h @@ -41,7 +41,6 @@ namespace py = pybind11; - namespace SmartRedis { From 821914ff0787cd44a0903df4d5352f9f148a86cf Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Thu, 7 Dec 2023 15:03:32 -0600 Subject: [PATCH 23/25] Fix missing space (#435) Add a space missing from an error message [ committed by @billschereriii ] [ reviewed by @MattToast ] --- doc/changelog.rst | 3 +++ src/cpp/client.cpp | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index f66f60535..1c9a1c5d7 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Added a missing space in an error message - Improved consistency of namespace declarations for C++ pybind interface - Improved const correctness of C++ Client - Improved const correctness of C++ Dataset @@ -33,6 +34,7 @@ Description Detailed Notes +- Added a missing space in an error message (PR435_) - Made the declaration of the py namespace in py*.h consistently outside the SmartRedis namespace declaration (PR434_) - Fields in several C++ API methods are now properly marked as const (PR430_) - The Dataset add_tensor method is now const correct, as are all internal the methods it calls (PR427_) @@ -55,6 +57,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR435: https://github.com/CrayLabs/SmartRedis/pull/435 .. _PR434: https://github.com/CrayLabs/SmartRedis/pull/434 .. _PR430: https://github.com/CrayLabs/SmartRedis/pull/430 .. _PR427: https://github.com/CrayLabs/SmartRedis/pull/427 diff --git a/src/cpp/client.cpp b/src/cpp/client.cpp index f105b9fc5..c6a181328 100644 --- a/src/cpp/client.cpp +++ b/src/cpp/client.cpp @@ -1238,7 +1238,7 @@ void Client::set_data_source(std::string source_id) if (!valid_prefix) { throw SRRuntimeException("Client error: data source " + std::string(source_id) + - "could not be found during client "+ + " could not be found during client "+ "initialization."); } From 0151e009fab13b960224457adc62ff1bdebdc792 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Fri, 15 Dec 2023 17:02:15 -0600 Subject: [PATCH 24/25] Unpin Intel compiler (#436) Apply a proper fix to enable use of the Intel OneAPI compiler. The underlying issue here is that the Intel Fortran compiler, ifx, is on release 2024.0, but the C/C++ compilers are on release 2023.3. Environment configuration for the compilers is done via /opt/intel/oneapi/setvars.sh. The problem is, the version of that script that sets up the right paths for the Fortran compiler is overwritten by the version that sets up the right paths for the C/C++ compilers when we install them. So we have to invoke setvars.sh TWICE: once after installing the Fortran compiler, and then once again after installing the C/C++ compilers. This gets all the right paths into play. Invoking it a second time generates an error unless you add the --force command-line paramater. [ committed by @billschereriii ] [ reviewed by @al-rigazzi @ashao ] --- .github/workflows/run_static_and_examples.yml | 4 +++- .github/workflows/run_tests.yml | 4 +++- doc/changelog.rst | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_static_and_examples.yml b/.github/workflows/run_static_and_examples.yml index e1d73b55e..64991ff9c 100644 --- a/.github/workflows/run_static_and_examples.yml +++ b/.github/workflows/run_static_and_examples.yml @@ -96,9 +96,11 @@ jobs: rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB && echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list && sudo apt-get update -y && - sudo apt-get install -y intel-oneapi-compiler-fortran=2023.2.2-47 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic + sudo apt-get install -y intel-oneapi-compiler-fortran && source /opt/intel/oneapi/setvars.sh && + sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic sudo apt-get install -y intel-oneapi-common-vars && + source /opt/intel/oneapi/setvars.sh --force && printenv >> $GITHUB_ENV && echo "CC=icx" >> $GITHUB_ENV && echo "CXX=icpx" >> $GITHUB_ENV && diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 676e3f188..7cbce6ae4 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -108,9 +108,11 @@ jobs: rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB && echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list && sudo apt-get update -y && - sudo apt-get install -y intel-oneapi-compiler-fortran=2023.2.2-47 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic + sudo apt-get install -y intel-oneapi-compiler-fortran && source /opt/intel/oneapi/setvars.sh && + sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic sudo apt-get install -y intel-oneapi-common-vars && + source /opt/intel/oneapi/setvars.sh --force && printenv >> $GITHUB_ENV && echo "CC=icx" >> $GITHUB_ENV && echo "CXX=icpx" >> $GITHUB_ENV && diff --git a/doc/changelog.rst b/doc/changelog.rst index 1c9a1c5d7..d4a758cad 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -88,6 +88,7 @@ Released on September 13, 2023 Description +- Unpin the Intel Fortran compiler in CI/CD - Reduced number of suppressed lint errors - Expanded documentation of aggregation lists - Updated third-party software dependencies to current versions @@ -102,6 +103,7 @@ Description Detailed Notes +- Unpin the Intel Fortran compiler in CI/CD. This requires running the compiler setup script twice, once for Fortran and once for other languages, since they're on different releases (PR436_) - Refactor factory for ConfigOptions to avoid using protected member outside an instance (PR393_) - Added a new advanced topics documentation page with a section on aggregation lists (PR390_) - Updated pybind (2.10.3 => 2.11.1), hiredis (1.1.0 => 1.2.0), and redis++ (1.3.5 => 1.3.10) dependencies to current versions (PR389_) @@ -115,6 +117,7 @@ Detailed Notes - Deleted obsolete build and testing files that are no longer needed with the new build and test system (PR366_) - Reuse existing redis connection when mapping the Redis cluster (PR364_) +.. _PR436: https://github.com/CrayLabs/SmartRedis/pull/436 .. _PR393: https://github.com/CrayLabs/SmartRedis/pull/393 .. _PR390: https://github.com/CrayLabs/SmartRedis/pull/390 .. _PR389: https://github.com/CrayLabs/SmartRedis/pull/389 From 6c3c5d322144f8106e8de1813c891234b35ddb12 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:20:47 -0600 Subject: [PATCH 25/25] SmartRedis 0.5.0 release (#437) Update for SmartRedis 0.5.0 release [ committed by @billschereriii ] [ reviewed by @al-rigazzi ] --- CMakeLists.txt | 2 +- doc/changelog.rst | 13 +++++++++---- setup.cfg | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ff6d1aa0..4d24abd77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,7 +31,7 @@ endif (POLICY CMP0048) # Project definition for the SmartRedis project cmake_minimum_required(VERSION 3.13) -project(SmartRedis VERSION "0.4.2") +project(SmartRedis VERSION "0.5.0") # Configure options for the SmartRedis project option(SR_PYTHON "Build the python module" OFF) diff --git a/doc/changelog.rst b/doc/changelog.rst index d4a758cad..11453946c 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -6,8 +6,14 @@ Development branch To be released at some future point in time +0.5.0 +----- + +Released on December 18, 2023 + Description +- Unpin the Intel Fortran compiler in CI/CD - Added a missing space in an error message - Improved consistency of namespace declarations for C++ pybind interface - Improved const correctness of C++ Client @@ -34,6 +40,7 @@ Description Detailed Notes +- Unpin the Intel Fortran compiler in CI/CD. This requires running the compiler setup script twice, once for Fortran and once for other languages, since they're on different releases (PR436_) - Added a missing space in an error message (PR435_) - Made the declaration of the py namespace in py*.h consistently outside the SmartRedis namespace declaration (PR434_) - Fields in several C++ API methods are now properly marked as const (PR430_) @@ -57,6 +64,7 @@ Detailed Notes - Create CONTRIBUTIONS.md file that points to the contribution guideline for both SmartSim and SmartRedis (PR395_) - Migrated to ConfigOptions-based Client construction, adding multiple database support (PR353_) +.. _PR436: https://github.com/CrayLabs/SmartRedis/pull/436 .. _PR435: https://github.com/CrayLabs/SmartRedis/pull/435 .. _PR434: https://github.com/CrayLabs/SmartRedis/pull/434 .. _PR430: https://github.com/CrayLabs/SmartRedis/pull/430 @@ -88,7 +96,6 @@ Released on September 13, 2023 Description -- Unpin the Intel Fortran compiler in CI/CD - Reduced number of suppressed lint errors - Expanded documentation of aggregation lists - Updated third-party software dependencies to current versions @@ -103,7 +110,6 @@ Description Detailed Notes -- Unpin the Intel Fortran compiler in CI/CD. This requires running the compiler setup script twice, once for Fortran and once for other languages, since they're on different releases (PR436_) - Refactor factory for ConfigOptions to avoid using protected member outside an instance (PR393_) - Added a new advanced topics documentation page with a section on aggregation lists (PR390_) - Updated pybind (2.10.3 => 2.11.1), hiredis (1.1.0 => 1.2.0), and redis++ (1.3.5 => 1.3.10) dependencies to current versions (PR389_) @@ -117,7 +123,6 @@ Detailed Notes - Deleted obsolete build and testing files that are no longer needed with the new build and test system (PR366_) - Reuse existing redis connection when mapping the Redis cluster (PR364_) -.. _PR436: https://github.com/CrayLabs/SmartRedis/pull/436 .. _PR393: https://github.com/CrayLabs/SmartRedis/pull/393 .. _PR390: https://github.com/CrayLabs/SmartRedis/pull/390 .. _PR389: https://github.com/CrayLabs/SmartRedis/pull/389 @@ -543,4 +548,4 @@ Released on April 1, 2021 Description -- Initial 0.1.0 release of SmartRedis \ No newline at end of file +- Initial 0.1.0 release of SmartRedis diff --git a/setup.cfg b/setup.cfg index c27151f4c..1e62e4a5b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = smartredis -version = 0.4.2 +version = 0.5.0 description = RedisAI clients for SmartSim long_description = file: README.md long_description_content_type=text/markdown