From 914c54932adc48fb31ca92f10695abab927452b7 Mon Sep 17 00:00:00 2001 From: Ray Douglass <3107146+raydouglass@users.noreply.github.com> Date: Mon, 4 Dec 2023 14:11:25 -0500 Subject: [PATCH 1/4] Pin actions/labeler to v4 [skip ci] (#2037) RAPIDS repos are using the `main` branch of https://github.com/actions/labeler which recently introduced [breaking changes](https://github.com/actions/labeler/releases/tag/v5.0.0). This PR pins to the latest v4 release of the labeler action until we can evaluate the changes required for v5. Authors: - Ray Douglass (https://github.com/raydouglass) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) --- .github/workflows/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 55117f774a..1ddd5b5cc3 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -6,6 +6,6 @@ jobs: triage: runs-on: ubuntu-latest steps: - - uses: actions/labeler@main + - uses: actions/labeler@v4 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" From 42e9f15044a3017c28f5b9321cf1b8c2954fe047 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 5 Dec 2023 17:43:20 -0500 Subject: [PATCH 2/4] Make device_resources accessed from device_resources_manager thread-safe (#2030) Update device_resources_manager to reuse only the memory manager, stream, and stream pools across threads. Create a unique resources object per device for each thread, since the resources object is not thread-safe. Authors: - William Hicks (https://github.com/wphicks) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2030 --- .../raft/core/device_resources_manager.hpp | 90 ++++++------------- 1 file changed, 28 insertions(+), 62 deletions(-) diff --git a/cpp/include/raft/core/device_resources_manager.hpp b/cpp/include/raft/core/device_resources_manager.hpp index ee4b151362..c3482b0c04 100644 --- a/cpp/include/raft/core/device_resources_manager.hpp +++ b/cpp/include/raft/core/device_resources_manager.hpp @@ -254,12 +254,6 @@ struct device_resources_manager { // Container for underlying device resources to be re-used across host // threads for each device std::vector per_device_components_; - // Container for device_resources objects shared among threads. The index - // of the outer vector is the thread id of the thread requesting resources - // modulo the total number of resources managed by this object. The inner - // vector contains all resources associated with that id across devices - // in any order. - std::vector> resources_{}; // Return a lock for accessing shared data [[nodiscard]] auto get_lock() const { return std::unique_lock{manager_mutex_}; } @@ -271,72 +265,44 @@ struct device_resources_manager { // all host threads. auto const& get_device_resources_(int device_id) { - // Each thread maintains an independent list of devices it has - // accessed. If it has not marked a device as initialized, it - // acquires a lock to initialize it exactly once. This means that each - // thread will lock once for a particular device and not proceed until - // some thread has actually generated the corresponding device - // components - thread_local auto initialized_devices = std::vector{}; - auto res_iter = decltype(std::end(resources_[0])){}; - if (std::find(std::begin(initialized_devices), std::end(initialized_devices), device_id) == - std::end(initialized_devices)) { + thread_local auto thread_resources = std::vector>([]() { + auto result = 0; + RAFT_CUDA_TRY(cudaGetDeviceCount(&result)); + RAFT_EXPECTS(result != 0, "No CUDA devices found"); + return result; + }()); + if (!thread_resources[device_id]) { // Only lock if we have not previously accessed this device on this // thread auto lock = get_lock(); - initialized_devices.push_back(device_id); // If we are building components, do not allow any further changes to // resource parameters. params_finalized_ = true; - if (resources_.empty()) { - // We will potentially need as many device_resources objects as there are combinations of - // streams and pools on a given device. - resources_.resize(std::max(params_.stream_count.value_or(1), std::size_t{1}) * - std::max(params_.pool_count, std::size_t{1})); - } - - auto res_idx = get_thread_id() % resources_.size(); - // Check to see if we have constructed device_resources for the - // requested device at the index assigned to this thread - res_iter = std::find_if(std::begin(resources_[res_idx]), - std::end(resources_[res_idx]), - [device_id](auto&& res) { return res.get_device() == device_id; }); + // Even if we have not yet built device_resources for the current + // device, we may have already built the underlying components, since + // multiple device_resources may point to the same components. + auto component_iter = std::find_if( + std::begin(per_device_components_), + std::end(per_device_components_), + [device_id](auto&& components) { return components.get_device_id() == device_id; }); - if (res_iter == std::end(resources_[res_idx])) { - // Even if we have not yet built device_resources for the current - // device, we may have already built the underlying components, since - // multiple device_resources may point to the same components. - auto component_iter = std::find_if( - std::begin(per_device_components_), - std::end(per_device_components_), - [device_id](auto&& components) { return components.get_device_id() == device_id; }); - if (component_iter == std::end(per_device_components_)) { - // Build components for this device if we have not yet done so on - // another thread - per_device_components_.emplace_back(device_id, params_); - component_iter = std::prev(std::end(per_device_components_)); - } - auto scoped_device = device_setter(device_id); - // Build the device_resources object for this thread out of shared - // components - resources_[res_idx].emplace_back(component_iter->get_stream(), - component_iter->get_pool(), - component_iter->get_workspace_memory_resource(), - component_iter->get_workspace_allocation_limit()); - res_iter = std::prev(std::end(resources_[res_idx])); + if (component_iter == std::end(per_device_components_)) { + // Build components for this device if we have not yet done so on + // another thread + per_device_components_.emplace_back(device_id, params_); + component_iter = std::prev(std::end(per_device_components_)); } - } else { - auto res_idx = get_thread_id() % resources_.size(); - // If we have previously accessed this device on this thread, we do not - // need to lock. We know that this thread already initialized the - // resources it requires for this device if no other thread had already done so, so we simply - // retrieve the previously-generated resources. - res_iter = std::find_if(std::begin(resources_[res_idx]), - std::end(resources_[res_idx]), - [device_id](auto&& res) { return res.get_device() == device_id; }); + auto scoped_device = device_setter(device_id); + // Build the device_resources object for this thread out of shared + // components + thread_resources[device_id].emplace(component_iter->get_stream(), + component_iter->get_pool(), + component_iter->get_workspace_memory_resource(), + component_iter->get_workspace_allocation_limit()); } - return *res_iter; + + return thread_resources[device_id].value(); } // Thread-safe setter for the number of streams From c3a9c781cada69945d283765b0ffef148b1e5d67 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 6 Dec 2023 02:44:58 +0100 Subject: [PATCH 3/4] Fixing small bug in raft-ann-bench (#2041) Authors: - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Divye Gala (https://github.com/divyegala) URL: https://github.com/rapidsai/raft/pull/2041 --- python/raft-ann-bench/src/raft-ann-bench/run/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/raft-ann-bench/src/raft-ann-bench/run/__main__.py b/python/raft-ann-bench/src/raft-ann-bench/run/__main__.py index 4611f39264..9841b47b98 100644 --- a/python/raft-ann-bench/src/raft-ann-bench/run/__main__.py +++ b/python/raft-ann-bench/src/raft-ann-bench/run/__main__.py @@ -574,8 +574,8 @@ def add_algo_group(group_list): index["search_params"].append(search_dict) executables_to_run[executable]["index"].append(index) - if len(index["search_params"]) == 0: - print("No search parameters were added to configuration") + if len(index["search_params"]) == 0: + print("No search parameters were added to configuration") run_build_and_search( conf_file, From ecd292b18194bd3af156107ee052fcfa0db0d3a8 Mon Sep 17 00:00:00 2001 From: Rui Lan Date: Tue, 5 Dec 2023 17:57:44 -0800 Subject: [PATCH 4/4] [REVIEW] Fix typos in parameter tuning guide (#2034) This PR fixes the typos in the ANN benchmark parameter tuning guide regarding `refine_ratio` and dataset location. Specifically, 1. Default refine ratio should be 1 instead of 0 - RAFT IVF-PQ. [link](https://github.com/rapidsai/raft/blob/branch-24.02/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h#L99) - FAISS GPU. [link](https://github.com/rapidsai/raft/blob/branch-24.02/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h#L89) 2. Default dataset location - RAFT IVF-Flat. Should be mmap instead of device. [link](https://github.com/rapidsai/raft/blob/branch-24.02/cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h#L81) - RAFT IVF-PQ. Should be host instead of device. [link](https://github.com/rapidsai/raft/blob/branch-24.02/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h#L84) - RAFT CAGRA. Should be mmap instead of device. [link](https://github.com/rapidsai/raft/blob/branch-24.02/cpp/bench/ann/src/raft/raft_cagra_wrapper.h#L113) And I think we can unify the dataset location to either mmap or host. Furthermore, to enable better copy performance and enable kernel/copy overlap, RAFT should also support `pinned_host` as one of the memory types. I can open a separate issue for it if people think it's reasonable. Authors: - Rui Lan (https://github.com/abc99lr) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2034 --- docs/source/ann_benchmarks_param_tuning.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/ann_benchmarks_param_tuning.md b/docs/source/ann_benchmarks_param_tuning.md index dd74f030ad..235346084c 100644 --- a/docs/source/ann_benchmarks_param_tuning.md +++ b/docs/source/ann_benchmarks_param_tuning.md @@ -20,7 +20,7 @@ IVF-flat is a simple algorithm which won't save any space, but it provides compe | `nlist` | `build_param` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | | `niter` | `build_param` | N | Positive Integer >0 | 20 | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | | `ratio` | `build_param` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `dataset_memory_type` | `build_param` | N | ["device", "host", "mmap"] | "device" | What memory type should the dataset reside? | +| `dataset_memory_type` | `build_param` | N | ["device", "host", "mmap"] | "mmap" | What memory type should the dataset reside? | | `query_memory_type` | `search_params` | N | ["device", "host", "mmap"] | "device | What memory type should the queries reside? | | `nprobe` | `search_params` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | @@ -37,12 +37,12 @@ IVF-pq is an inverted-file index, which partitions the vectors into a series of | `pq_dim` | `build_param` | N | Positive Integer. Multiple of 8. | 0 | Dimensionality of the vector after product quantization. When 0, a heuristic is used to select this value. `pq_dim` * `pq_bits` must be a multiple of 8. | | `pq_bits` | `build_param` | N | Positive Integer. [4-8] | 8 | Bit length of the vector element after quantization. | | `codebook_kind` | `build_param` | N | ["cluster", "subspace"] | "subspace" | Type of codebook. See the [API docs](https://docs.rapids.ai/api/raft/nightly/cpp_api/neighbors_ivf_pq/#_CPPv412codebook_gen) for more detail | -| `dataset_memory_type` | `build_param` | N | ["device", "host", "mmap"] | "device" | What memory type should the dataset reside? | +| `dataset_memory_type` | `build_param` | N | ["device", "host", "mmap"] | "host" | What memory type should the dataset reside? | | `query_memory_type` | `search_params` | N | ["device", "host", "mmap"] | "device | What memory type should the queries reside? | | `nprobe` | `search_params` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | | `internalDistanceDtype` | `search_params` | N | [`float`, `half`] | `half` | The precision to use for the distance computations. Lower precision can increase performance at the cost of accuracy. | | `smemLutDtype` | `search_params` | N | [`float`, `half`, `fp8`] | `half` | The precision to use for the lookup table in shared memory. Lower precision can increase performance at the cost of accuracy. | -| `refine_ratio` | `search_params` | N| Positive Number >=0 | 0 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | +| `refine_ratio` | `search_params` | N| Positive Number >=1 | 1 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | ### `raft_cagra` @@ -53,7 +53,7 @@ IVF-pq is an inverted-file index, which partitions the vectors into a series of | `graph_degree` | `build_param` | N | Positive Integer >0 | 64 | Degree of the final kNN graph index. | | `intermediate_graph_degree` | `build_param` | N | Positive Integer >0 | 128 | Degree of the intermediate kNN graph. | | `graph_build_algo` | `build_param` | N | ["IVF_PQ", "NN_DESCENT"] | "IVF_PQ" | Algorithm to use for search | -| `dataset_memory_type` | `build_param` | N | ["device", "host", "mmap"] | "device" | What memory type should the dataset reside while constructing the index? | +| `dataset_memory_type` | `build_param` | N | ["device", "host", "mmap"] | "mmap" | What memory type should the dataset reside while constructing the index? | | `query_memory_type` | `search_params` | N | ["device", "host", "mmap"] | "device | What memory type should the queries reside? | | `itopk` | `search_wdith` | N | Positive Integer >0 | 64 | Number of intermediate search results retained during the search. Higher values improve search accuracy at the cost of speed. | | `search_width` | `search_param` | N | Positive Integer >0 | 1 | Number of graph nodes to select as the starting point for the search in each iteration. | @@ -77,7 +77,7 @@ To fine tune CAGRA index building we can customize IVF-PQ index builder options | `ivf_pq_search_nprobe` | `build_params` | N | Positive Integer >0 | min(2*dim, nlist) | The closest number of clusters to search for each query vector. | | `ivf_pq_search_internalDistanceDtype` | `build_params` | N | [`float`, `half`] | `fp8` | The precision to use for the distance computations. Lower precision can increase performance at the cost of accuracy. | | `ivf_pq_search_smemLutDtype` | `build_params` | N | [`float`, `half`, `fp8`] | `half` | The precision to use for the lookup table in shared memory. Lower precision can increase performance at the cost of accuracy. | -| `ivf_pq_search_refine_ratio` | `build_params` | N| Positive Number >=0 | 2 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | +| `ivf_pq_search_refine_ratio` | `build_params` | N| Positive Number >=1 | 2 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | Alternatively, if `graph_build_algo == "NN_DESCENT"`, then we can customize the following parameters @@ -125,7 +125,7 @@ IVF-pq is an inverted-file index, which partitions the vectors into a series of | `usePrecomputed` | `build_param` | N | Boolean. Default=`false` | `false` | Use pre-computed lookup tables to speed up search at the cost of increased memory usage. | | `useFloat16` | `build_param` | N | Boolean. Default=`false` | `false` | Use half-precision floats for clustering step. | | `nprobe` | `search_params` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | -| `refine_ratio` | `search_params` | N| Positive Number >=0 | 0 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | +| `refine_ratio` | `search_params` | N| Positive Number >=1 | 1 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | ### `faiss_cpu_flat` @@ -159,7 +159,7 @@ Use FAISS IVF-PQ index on CPU | `usePrecomputed` | `build_param` | N | Boolean. Default=`false` | `false` | Use pre-computed lookup tables to speed up search at the cost of increased memory usage. | | `bitsPerCode` | `build_param` | N | Positive Integer [4-8] | 8 | Number of bits to use for each code. | | `nprobe` | `search_params` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | -| `refine_ratio` | `search_params` | N| Positive Number >=0 | 0 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | +| `refine_ratio` | `search_params` | N| Positive Number >=1 | 1 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | | `numThreads` | `search_params` | N | Positive Integer >0 | 1 | Number of threads to use for queries. |