Adding file mode option to serialize() methods

cpp/cmake/modules/ConfigureCUDA.cmake

@@ -45,6 +45,7 @@ if(CUDA_LOG_COMPILE_TIME)
 endif()
 
 list(APPEND CUVS_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr)
+list(APPEND CUVS_CUDA_FLAGS -allow-unsupported-compiler)
 list(APPEND CUVS_CXX_FLAGS "-DCUDA_API_PER_THREAD_DEFAULT_STREAM")
 list(APPEND CUVS_CUDA_FLAGS "-DCUDA_API_PER_THREAD_DEFAULT_STREAM")
 # make sure we produce smallest binary size

cpp/include/cuvs/neighbors/brute_force.h

@@ -184,6 +184,34 @@ cuvsError_t cuvsBruteForceSearch(cuvsResources_t res,
  * cuvsError_t res_create_status = cuvsResourcesCreate(&res);
  *
  * // create an index with `cuvsBruteforceBuild`
+ * cuvsBruteForceSerializeWithMode(res, "/path/to/index", index, 'w');
+ * @endcode
+ *
+ * @param[in] res cuvsResources_t opaque C handle
+ * @param[in] filename the file name for saving the index
+ * @param[in] index BRUTEFORCE index
+ * @param[in] file_mode File mode: 'w' for write (ios::out), 'a' for append (ios::app)
+ *
+ */
+cuvsError_t cuvsBruteForceSerializeWithMode(cuvsResources_t res,
+                                             const char* filename,
+                                             cuvsBruteForceIndex_t index,
+                                             char file_mode);
+
+/**
+ * Save the index to file (backward compatibility version - writes to file).
+ * The serialization format can be subject to changes, therefore loading
+ * an index saved with a previous version of cuvs is not guaranteed
+ * to work.
+ *
+ * @code{.c}
+ * #include <cuvs/neighbors/brute_force.h>
+ *
+ * // Create cuvsResources_t
+ * cuvsResources_t res;
+ * cuvsError_t res_create_status = cuvsResourcesCreate(&res);
+ *
+ * // create an index with `cuvsBruteforceBuild`
  * cuvsBruteForceSerialize(res, "/path/to/index", index);
  * @endcode
  *

cpp/include/cuvs/neighbors/brute_force.hpp

@@ -731,11 +731,13 @@ void search(raft::resources const& handle,
  * @param[in] index brute force index
  * @param[in] include_dataset whether to include the dataset in the serialized
  * output
+ * @param[in] file_mode File mode: 'w' for write (ios::out), 'a' for append (ios::app)
  */
 void serialize(raft::resources const& handle,
                const std::string& filename,
                const cuvs::neighbors::brute_force::index<half, float>& index,
-               bool include_dataset = true);
+               bool include_dataset = true,
+               char file_mode = 'w');
 /**
  * Save the index to file.
  * The serialization format can be subject to changes, therefore loading
@@ -761,12 +763,14 @@ void serialize(raft::resources const& handle,
  * @param[in] index brute force index
  * @param[in] include_dataset whether to include the dataset in the serialized
  * output
+ * @param[in] file_mode File mode: 'w' for write (ios::out), 'a' for append (ios::app)
  *
  */
 void serialize(raft::resources const& handle,
                const std::string& filename,
                const cuvs::neighbors::brute_force::index<float, float>& index,
-               bool include_dataset = true);
+               bool include_dataset = true,
+               char file_mode = 'w');
 
 /**
  * Write the index to an output stream

cpp/include/cuvs/neighbors/cagra.h

@@ -560,6 +560,35 @@ cuvsError_t cuvsCagraSearch(cuvsResources_t res,
  * cuvsError_t res_create_status = cuvsResourcesCreate(&res);
  *
  * // create an index with `cuvsCagraBuild`
+ * cuvsCagraSerializeWithMode(res, "/path/to/index", index, true, 'w');
+ * @endcode
+ *
+ * @param[in] res cuvsResources_t opaque C handle
+ * @param[in] filename the file name for saving the index
+ * @param[in] index CAGRA index
+ * @param[in] include_dataset Whether or not to write out the dataset to the file.
+ * @param[in] file_mode File mode: 'w' for write (ios::out), 'a' for append (ios::app)
+ *
+ */
+cuvsError_t cuvsCagraSerializeWithMode(cuvsResources_t res,
+                                        const char* filename,
+                                        cuvsCagraIndex_t index,
+                                        bool include_dataset,
+                                        char file_mode);
+
+/**
+ * Save the index to file (backward compatibility version - writes to file).
+ *
+ * Experimental, both the API and the serialization format are subject to change.
+ *
+ * @code{.c}
+ * #include <cuvs/neighbors/cagra.h>
+ *
+ * // Create cuvsResources_t
+ * cuvsResources_t res;
+ * cuvsError_t res_create_status = cuvsResourcesCreate(&res);
+ *
+ * // create an index with `cuvsCagraBuild`
  * cuvsCagraSerialize(res, "/path/to/index", index, true);
  * @endcode
  *
@@ -590,7 +619,7 @@ cuvsError_t cuvsCagraSerialize(cuvsResources_t res,
  * cuvsError_t res_create_status = cuvsResourcesCreate(&res);
  *
  * // create an index with `cuvsCagraBuild`
- * cuvsCagraSerializeHnswlib(res, "/path/to/index", index);
+ * cuvsCagraSerializeToHnswlib(res, "/path/to/index", index);
  * @endcode
  *
  * @param[in] res cuvsResources_t opaque C handle
@@ -602,6 +631,36 @@ cuvsError_t cuvsCagraSerializeToHnswlib(cuvsResources_t res,
                                         const char* filename,
                                         cuvsCagraIndex_t index);
 
+/**
+ * Save the CAGRA index to file in hnswlib format with file mode control.
+ * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS,
+ *       as the serialization format is not compatible with the original hnswlib.
+ *
+ * Experimental, both the API and the serialization format are subject to change.
+ *
+ * @code{.c}
+ * #include <cuvs/core/c_api.h>
+ * #include <cuvs/neighbors/cagra.h>
+ *
+ * // Create cuvsResources_t
+ * cuvsResources_t res;
+ * cuvsError_t res_create_status = cuvsResourcesCreate(&res);
+ *
+ * // create an index with `cuvsCagraBuild`
+ * cuvsCagraSerializeToHnswlibWithMode(res, "/path/to/index", index, 'w');
+ * @endcode
+ *
+ * @param[in] res cuvsResources_t opaque C handle
+ * @param[in] filename the file name for saving the index
+ * @param[in] index CAGRA index
+ * @param[in] file_mode File mode: 'w' for write (ios::out), 'a' for append (ios::app)
+ *
+ */
+cuvsError_t cuvsCagraSerializeToHnswlibWithMode(cuvsResources_t res,
+                                                const char* filename,
+                                                cuvsCagraIndex_t index,
+                                                char file_mode);
+
 /**
  * Load index from file.
  *

cpp/include/cuvs/neighbors/cagra.hpp

@@ -1294,12 +1294,14 @@ void search(raft::resources const& res,
  * @param[in] filename the file name for saving the index
  * @param[in] index CAGRA index
  * @param[in] include_dataset Whether or not to write out the dataset to the file.
+ * @param[in] file_mode File mode: 'w' for write (ios::out), 'a' for append (ios::app)
  *
  */
 void serialize(raft::resources const& handle,
                const std::string& filename,
                const cuvs::neighbors::cagra::index<float, uint32_t>& index,
-               bool include_dataset = true);
+               bool include_dataset = true,
+               char file_mode = 'w');
 
 /**
  * Load index from file.
@@ -1399,12 +1401,14 @@ void deserialize(raft::resources const& handle,
  * @param[in] filename the file name for saving the index
  * @param[in] index CAGRA index
  * @param[in] include_dataset Whether or not to write out the dataset to the file.
+ * @param[in] file_mode File mode: 'w' for write (ios::out), 'a' for append (ios::app)
  *
  */
 void serialize(raft::resources const& handle,
                const std::string& filename,
                const cuvs::neighbors::cagra::index<half, uint32_t>& index,
-               bool include_dataset = true);
+               bool include_dataset = true,
+               char file_mode = 'w');
 
 /**
  * Load index from file.
@@ -1505,11 +1509,13 @@ void deserialize(raft::resources const& handle,
  * @param[in] filename the file name for saving the index
  * @param[in] index CAGRA index
  * @param[in] include_dataset Whether or not to write out the dataset to the file.
+ * @param[in] file_mode File mode: 'w' for write (ios::out), 'a' for append (ios::app)
  */
 void serialize(raft::resources const& handle,
                const std::string& filename,
                const cuvs::neighbors::cagra::index<int8_t, uint32_t>& index,
-               bool include_dataset = true);
+               bool include_dataset = true,
+               char file_mode = 'w');
 
 /**
  * Load index from file.
@@ -1610,11 +1616,13 @@ void deserialize(raft::resources const& handle,
  * @param[in] filename the file name for saving the index
  * @param[in] index CAGRA index
  * @param[in] include_dataset Whether or not to write out the dataset to the file.
+ * @param[in] file_mode File mode: 'w' for write (ios::out), 'a' for append (ios::app)
  */
 void serialize(raft::resources const& handle,
                const std::string& filename,
                const cuvs::neighbors::cagra::index<uint8_t, uint32_t>& index,
-               bool include_dataset = true);
+               bool include_dataset = true,
+               char file_mode = 'w');
 
 /**
  * Load index from file.

cpp/src/neighbors/brute_force_c.cpp

@@ -107,11 +107,11 @@ void _search(cuvsResources_t res,
 }
 
 template <typename T, typename DistT = float>
-void _serialize(cuvsResources_t res, const char* filename, cuvsBruteForceIndex index)
+void _serialize(cuvsResources_t res, const char* filename, cuvsBruteForceIndex index, char file_mode)
 {
   auto res_ptr   = reinterpret_cast<raft::resources*>(res);
   auto index_ptr = reinterpret_cast<cuvs::neighbors::brute_force::index<T, DistT>*>(index.addr);
-  cuvs::neighbors::brute_force::serialize(*res_ptr, std::string(filename), *index_ptr);
+  cuvs::neighbors::brute_force::serialize(*res_ptr, std::string(filename), *index_ptr, true, file_mode);
 }
 
 template <typename T, typename DistT = float>
@@ -263,17 +263,25 @@ extern "C" cuvsError_t cuvsBruteForceDeserialize(cuvsResources_t res,
   });
 }
 
-extern "C" cuvsError_t cuvsBruteForceSerialize(cuvsResources_t res,
-                                               const char* filename,
-                                               cuvsBruteForceIndex_t index)
+extern "C" cuvsError_t cuvsBruteForceSerializeWithMode(cuvsResources_t res,
+                                                        const char* filename,
+                                                        cuvsBruteForceIndex_t index,
+                                                        char file_mode)
 {
   return cuvs::core::translate_exceptions([=] {
     if (index->dtype.code == kDLFloat && index->dtype.bits == 32) {
-      _serialize<float>(res, filename, *index);
+      _serialize<float>(res, filename, *index, file_mode);
     } else if (index->dtype.code == kDLFloat && index->dtype.bits == 16) {
-      _serialize<half>(res, filename, *index);
+      _serialize<half>(res, filename, *index, file_mode);
     } else {
       RAFT_FAIL("Unsupported index dtype: %d and bits: %d", index->dtype.code, index->dtype.bits);
     }
   });
 }
+
+extern "C" cuvsError_t cuvsBruteForceSerialize(cuvsResources_t res,
+                                               const char* filename,
+                                               cuvsBruteForceIndex_t index)
+{
+  return cuvsBruteForceSerializeWithMode(res, filename, index, 'w');
+}

cpp/src/neighbors/brute_force_serialize.cu

@@ -55,19 +55,33 @@ void serialize(raft::resources const& handle,
 void serialize(raft::resources const& handle,
                const std::string& filename,
                const index<half, float>& index,
-               bool include_dataset)
+               bool include_dataset,
+               char file_mode)
 {
-  auto os = std::ofstream{filename, std::ios::out | std::ios::binary};
+  std::ios::openmode mode = std::ios::binary;
+  if (file_mode == 'a') {
+    mode |= std::ios::app;
+  } else {
+    mode |= std::ios::out;
+  }
+  auto os = std::ofstream{filename, mode};
   RAFT_EXPECTS(os, "Cannot open file %s", filename.c_str());
   serialize<half, float>(handle, os, index, include_dataset);
 }
 
 void serialize(raft::resources const& handle,
                const std::string& filename,
                const index<float, float>& index,
-               bool include_dataset)
+               bool include_dataset,
+               char file_mode)
 {
-  auto os = std::ofstream{filename, std::ios::out | std::ios::binary};
+  std::ios::openmode mode = std::ios::binary;
+  if (file_mode == 'a') {
+    mode |= std::ios::app;
+  } else {
+    mode |= std::ios::out;
+  }
+  auto os = std::ofstream{filename, mode};
   RAFT_EXPECTS(os, "Cannot open file %s", filename.c_str());
   serialize<float, float>(handle, os, index, include_dataset);
 }
@@ -88,6 +102,23 @@ void serialize(raft::resources const& handle,
   serialize<float, float>(handle, os, index, include_dataset);
 }
 
+// Backward compatibility functions - use default 'w' mode
+void serialize(raft::resources const& handle,
+               const std::string& filename,
+               const index<half, float>& index,
+               bool include_dataset)
+{
+  serialize(handle, filename, index, include_dataset, 'w');
+}
+
+void serialize(raft::resources const& handle,
+               const std::string& filename,
+               const index<float, float>& index,
+               bool include_dataset)
+{
+  serialize(handle, filename, index, include_dataset, 'w');
+}
+
 template <typename T, typename DistT>
 auto deserialize(raft::resources const& handle, std::istream& is)
 {