From 667724a6f00925597dbbbc0bd5c5e4aaee7207c9 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 16 Sep 2025 13:50:29 -0500 Subject: [PATCH 01/43] Optimize VDS operations with r-tree --- doxygen/examples/tables/propertyLists.dox | 4 + fortran/src/H5Pff.F90 | 70 ++ fortran/src/hdf5_fortrandll.def.in | 2 + java/src/hdf/hdf5lib/H5.java | 35 + java/src/jni/h5pDAPLImp.c | 45 ++ java/src/jni/h5pDAPLImp.h | 14 + release_docs/release_archive.txt | 20 + src/H5Dprivate.h | 1 + src/H5Dvirtual.c | 766 ++++++++++++++++------ src/H5Olayout.c | 3 +- src/H5Oprivate.h | 4 + src/H5Pdapl.c | 86 +++ src/H5Pdcpl.c | 16 +- src/H5Ppublic.h | 47 ++ test/dsets.c | 2 + test/rtree.c | 278 ++++++++ 16 files changed, 1177 insertions(+), 216 deletions(-) diff --git a/doxygen/examples/tables/propertyLists.dox b/doxygen/examples/tables/propertyLists.dox index 4480cabc64f..dde08728d6e 100644 --- a/doxygen/examples/tables/propertyLists.dox +++ b/doxygen/examples/tables/propertyLists.dox @@ -616,6 +616,10 @@ encoding for object names. #H5Pset_virtual_view/#H5Pget_virtual_view Sets/gets the view of the virtual dataset (VDS) to include or exclude missing mapped elements. + +#H5Pset_dset_use_spatial_tree/#H5Pget_dset_use_spatial_tree +Sets/gets the flag to use spatial trees when searching VDS mappings + //! [dapl_table] * diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index 41529a17180..6b1e5840d97 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -4365,6 +4365,76 @@ END FUNCTION h5pget_chunk_cache_c END SUBROUTINE h5pget_chunk_cache_f +!> +!! \ingroup FH5P +!! +!! \brief Gets the value of the "use spatial tree" flag which enables spatial tree +!! construction and usage for VDS mapping searches. +!! +!! \param dapl_id Target dataset access property list identifier. +!! \param use_tree Value of the setting. +!! \param hdferr \fortran_error +!! +!! See C API: @ref H5Pget_dset_use_spatial_tree() +!! + SUBROUTINE h5pget_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) + IMPLICIT NONE + INTEGER(HID_T) , INTENT(IN) :: dapl_id + LOGICAL , INTENT(OUT) :: use_tree + INTEGER , INTENT(OUT) :: hdferr + LOGICAL(C_BOOL) :: c_use_tree + + INTERFACE + INTEGER(C_INT) FUNCTION H5Pget_dset_use_spatial_tree_c(dapl_id, use_tree) BIND(C, NAME='H5Pget_dset_use_spatial_tree') + IMPORT :: C_INT, HID_T, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), INTENT(IN), VALUE :: dapl_id + LOGICAL(C_BOOL), INTENT(OUT) :: use_tree + END FUNCTION H5Pget_dset_use_spatial_tree_c + END INTERFACE + + hdferr = INT(H5Pget_dset_use_spatial_tree_c(dapl_id, c_use_tree)) + + ! Transfer value of C C_BOOL type to Fortran LOGICAL + use_tree = c_use_tree + + END SUBROUTINE h5pget_dset_use_spatial_tree_f + +!> +!! \ingroup FH5P +!! +!! \brief Sets the value of the "use spatial tree" flag which enables spatial tree +!! construction and usage for VDS mapping searches. +!! +!! \param dapl_id Target dataset access property list identifier. +!! \param use_tree Value of the setting. +!! \param hdferr \fortran_error +!! +!! See C API: @ref H5Pset_dset_use_spatial_tree() +!! + SUBROUTINE h5pset_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) + IMPLICIT NONE + INTEGER(HID_T) , INTENT(IN) :: dapl_id + LOGICAL , INTENT(IN) :: use_tree + INTEGER , INTENT(OUT) :: hdferr + LOGICAL(C_BOOL) :: c_use_tree + + INTERFACE + INTEGER FUNCTION h5pset_dset_use_spatial_tree_c(dapl_id, use_tree) BIND(C, NAME='H5Pset_dset_use_spatial_tree') + IMPORT :: HID_T, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), INTENT(IN), VALUE :: dapl_id + LOGICAL(C_BOOL), INTENT(IN), VALUE :: use_tree + END FUNCTION h5pset_dset_use_spatial_tree_c + END INTERFACE + + ! Transfer value of Fortran LOGICAL to C C_BOOL type + c_use_tree = use_tree + + hdferr = INT(h5pset_dset_use_spatial_tree_c(dapl_id, c_use_tree)) + + END SUBROUTINE h5pset_dset_use_spatial_tree_f + #ifdef H5_DOXYGEN !> !! \ingroup FH5P diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index 1769264001c..4ecf34ea8d5 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -420,6 +420,8 @@ H5P_mp_H5PGET_VIRTUAL_VSPACE_F H5P_mp_H5PGET_VIRTUAL_SRCSPACE_F H5P_mp_H5PGET_VIRTUAL_FILENAME_F H5P_mp_H5PGET_VIRTUAL_DSETNAME_F +H5P_mp_H5PGET_DSET_USE_SPATIAL_TREE_F +H5P_mp_H5PSET_DSET_USE_SPATIAL_TREE_F H5P_mp_H5PGET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_VOL_F diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 2fd8dcf8544..b399c6080a7 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -10675,6 +10675,41 @@ public synchronized static native void H5Pset_virtual_prefix(long dapl_id, Strin public synchronized static native void H5Pset_efile_prefix(long dapl_id, String prefix) throws HDF5LibraryException, NullPointerException; + /** + * @ingroup JH5P + * + * H5Pget_dset_use_spatial_tree accesses the flag for whether or not datasets accessed with the given dapl + * will use a spatial tree for mappings + * + * @param dapl_id + * IN: Dataset access property list + * + * @return true if the given dapl is set to use a spatial tree, false if not. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + **/ + public synchronized static native boolean H5Pget_dset_use_spatial_tree(long dapl_id) + throws HDF5LibraryException; + + /** + * @ingroup JH5P + * + * H5Pset_dset_use_spatial_tree sets the dapl to not use (or explicitly use) a spatial tree + * during mapping operations + * + * @param dapl_id + * IN: Dataset access property list + * + * @param use_tree + * IN: the use_tree flag setting + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + **/ + public synchronized static native void H5Pset_dset_use_spatial_tree(long dapl_id, boolean use_tree) + throws HDF5LibraryException; + // public synchronized static native void H5Pset_append_flush(long plist_id, int ndims, long[] boundary, // H5D_append_cb func, H5D_append_t udata) throws HDF5LibraryException; diff --git a/java/src/jni/h5pDAPLImp.c b/java/src/jni/h5pDAPLImp.c index b6ae1836bb9..995bc1eb743 100644 --- a/java/src/jni/h5pDAPLImp.c +++ b/java/src/jni/h5pDAPLImp.c @@ -312,6 +312,51 @@ H5D_append_cb(hid_t dataset_id, hsize_t *cur_dims, void *cb_data) return (herr_t)status; } /* end H5D_append_cb */ +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pset_dset_use_spatial_tree + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id, jboolean use_tree) +{ + bool use_tree_val; + herr_t retVal = FAIL; + + UNUSED(clss); + + use_tree_val = (JNI_TRUE == use_tree) ? true : false; + + if ((retVal = H5Pset_dset_use_spatial_tree((hid_t)dapl_id, (bool)use_tree_val)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + +done: + return; +} /* end Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_dset_use_spatial_tree + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id) +{ + bool use_tree = false; + jboolean bval = JNI_FALSE; + + UNUSED(clss); + + if (H5Pget_dset_use_spatial_tree((hid_t)dapl_id, (bool *)&use_tree) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + if (use_tree == true) + bval = JNI_TRUE; + +done: + return bval; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1dset_1use_1spatial_1tree */ + #ifdef __cplusplus } /* end extern "C" */ #endif /* __cplusplus */ diff --git a/java/src/jni/h5pDAPLImp.h b/java/src/jni/h5pDAPLImp.h index ac4828b8403..ec4b1049263 100644 --- a/java/src/jni/h5pDAPLImp.h +++ b/java/src/jni/h5pDAPLImp.h @@ -89,6 +89,20 @@ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1virtual_1printf_1gap(JNIEnv * */ JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1printf_1gap(JNIEnv *, jclass, jlong); +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pset_dset_use_spatial_tree + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_dset_use_spatial_tree + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1dset_1use_1spatial_1tree(JNIEnv *, jclass, jlong); + #ifdef __cplusplus } /* end extern "C" */ #endif /* __cplusplus */ diff --git a/release_docs/release_archive.txt b/release_docs/release_archive.txt index d7082a87139..20d450af590 100644 --- a/release_docs/release_archive.txt +++ b/release_docs/release_archive.txt @@ -700,6 +700,26 @@ New Features library behavior, and the connector ID and information could not be read back from that plist later. + - Virtual datasets use a spatial tree to optimize searches + + Virtual dataset operations with many (>1,000) mappings were much slower than + corresponding operations on normal datasets. This was due to the need + to iterate through every source dataset's dataspace and check for an intersection + with the user-selected region for a read/write in the virtual dataset. + + Virtual datasets now use an r-tree (defined in H5RT.c) to perform a spatial search. + This allows the dataspaces that intersect the user-selection to be computed with, + in most cases, vastly fewer costly intersection checks, improving the speed of VDS + read/write operations. + + Virtual datasets will use the r-tree by default, since the majority of use cases, + should see improvements from use of the tree. However, because some workflows may + find that the overhead of the tree outweighs the time saved on searches, there is + a new Dataset Access Property List (DAPL) property to control use of the spatial tree. + + This property can be set or queried with the new API functions + H5Pset_dset_use_spatial_tree()/H5Pget_dset_use_spatial_tree(). + Parallel Library: ----------------- - diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 7088443f6a2..3bcfb7850ef 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -54,6 +54,7 @@ #define H5D_ACS_VDS_PREFIX_NAME "vds_prefix" /* VDS file prefix */ #define H5D_ACS_APPEND_FLUSH_NAME "append_flush" /* Append flush actions */ #define H5D_ACS_EFILE_PREFIX_NAME "external file prefix" /* External file prefix */ +#define H5D_ACS_USE_TREE_NAME "tree" /* Whether to use spatial tree */ /* ======== Data transfer properties ======== */ #define H5D_XFER_MAX_TEMP_BUF_NAME "max_temp_buf" /* Maximum temp buffer size */ diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 62e1d39cbfb..6d49894a924 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -60,6 +60,7 @@ #include "H5MMprivate.h" /* Memory management */ #include "H5Oprivate.h" /* Object headers */ #include "H5Pprivate.h" /* Property Lists */ +#include "H5RTprivate.h" /* R-trees */ #include "H5Sprivate.h" /* Dataspaces */ #include "H5VLprivate.h" /* Virtual Object Layer */ @@ -108,6 +109,12 @@ static herr_t H5D__virtual_read_one(H5D_dset_io_info_t *dset_info, static herr_t H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset); + +/* R-tree helper functions */ +static herr_t H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank); +static herr_t H5D__rtree_should_insert(void *mapping_entry, bool *should_insert); +static herr_t H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, + H5RT_leaf_t **leaves_out, bool **is_in_tree_out, size_t *leaf_count); /*********************/ /* Package Variables */ /*********************/ @@ -921,6 +928,8 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) H5P_genplist_t *plist; size_t i; herr_t ret_value = SUCCEED; + bool *new_in_tree = NULL; + H5RT_t *new_tree = NULL; FUNC_ENTER_PACKAGE @@ -1053,6 +1062,23 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) virt->list_nalloc = 0; } /* end else */ + /* Copy spatial tree if it exists */ + if (virt->tree) { + if ((new_tree = H5RT_copy(virt->tree)) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy spatial tree"); + virt->tree = new_tree; + + if ((new_in_tree = H5MM_calloc(virt->list_nalloc * sizeof(bool))) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate in-tree array"); + + /* Copy is_in_tree array */ + memcpy(new_in_tree, virt->is_in_tree, virt->list_nused * sizeof(bool)); + virt->is_in_tree = new_in_tree; + } else { + virt->tree = NULL; + virt->is_in_tree = NULL; + } + /* Copy property lists */ if (orig_source_fapl >= 0) { if (NULL == (plist = (H5P_genplist_t *)H5I_object_verify(orig_source_fapl, H5I_GENPROP_LST))) @@ -1145,6 +1171,19 @@ H5D__virtual_free_layout_mappings(H5O_storage_virtual_t *virt) virt->list_nused = (size_t)0; (void)memset(virt->min_dims, 0, sizeof(virt->min_dims)); + /* Destroy the spatial tree, if it exists */ + if (virt->tree) { + if (H5RT_free(virt->tree) < 0) { + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to destroy spatial tree"); + } + virt->tree = NULL; + } + + /* Destroy the list tracking which indices are in spatial tree */ + if (virt->is_in_tree) { + virt->is_in_tree = H5MM_xfree(virt->is_in_tree); + } + /* Note the lack of a done: label. This is because there are no HGOTO_ERROR * calls. If one is added, a done: label must also be added */ FUNC_LEAVE_NOAPI(ret_value) @@ -2776,267 +2815,345 @@ H5D__virtual_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t H5_ATTR_UNUSED * FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__virtual_io_init() */ -/*------------------------------------------------------------------------- - * Function: H5D__virtual_pre_io - * - * Purpose: Project all virtual mappings onto mem_space, with the - * results stored in projected_mem_space for each mapping. - * Opens all source datasets if possible. The total number - * of elements is stored in tot_nelmts. - * - * Return: Non-negative on success/Negative on failure - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, H5S_t *file_space, - H5S_t *mem_space, hsize_t *tot_nelmts) -{ +// TODO +static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, H5S_t *file_space, + H5S_t *mem_space, hsize_t *tot_nelmts, H5O_storage_virtual_ent_t *curr_mapping) { const H5D_t *dset = dset_info->dset; /* Local pointer to dataset info */ hssize_t select_nelmts; /* Number of elements in selection */ hsize_t bounds_start[H5S_MAX_RANK]; /* Selection bounds start */ hsize_t bounds_end[H5S_MAX_RANK]; /* Selection bounds end */ int rank = 0; bool bounds_init = false; /* Whether bounds_start, bounds_end, and rank are valid */ - size_t i, j, k; /* Local index variables */ + size_t j, k; /* Local index variables */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE - /* Sanity check */ - assert(storage); - assert(mem_space); - assert(file_space); - assert(tot_nelmts); + /* Sanity check that the virtual space has been patched by now */ + assert(curr_mapping->virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); - /* Initialize layout if necessary */ - if (!storage->init) - if (H5D__virtual_init_all(dset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize virtual layout"); + /* Check for "printf" source dataset resolution */ + if (curr_mapping->psfn_nsubs || curr_mapping->psdn_nsubs) { + bool partial_block; - /* Initialize tot_nelmts */ - *tot_nelmts = 0; + assert(curr_mapping->unlim_dim_virtual >= 0); - /* Iterate over mappings */ - for (i = 0; i < storage->list_nused; i++) { - /* Sanity check that the virtual space has been patched by now */ - assert(storage->list[i].virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); - - /* Check for "printf" source dataset resolution */ - if (storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { - bool partial_block; + /* Get selection bounds if necessary */ + if (!bounds_init) { + /* Get rank of VDS */ + if ((rank = H5S_GET_EXTENT_NDIMS(dset->shared->space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions"); - assert(storage->list[i].unlim_dim_virtual >= 0); + /* Get selection bounds */ + if (H5S_SELECT_BOUNDS(file_space, bounds_start, bounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); - /* Get selection bounds if necessary */ - if (!bounds_init) { - /* Get rank of VDS */ - if ((rank = H5S_GET_EXTENT_NDIMS(dset->shared->space)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions"); + /* Adjust bounds_end to represent the extent just enclosing them + * (add 1) */ + for (j = 0; j < (size_t)rank; j++) + bounds_end[j]++; - /* Get selection bounds */ - if (H5S_SELECT_BOUNDS(file_space, bounds_start, bounds_end) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); - - /* Adjust bounds_end to represent the extent just enclosing them - * (add 1) */ - for (j = 0; j < (size_t)rank; j++) - bounds_end[j]++; - - /* Bounds are now initialized */ - bounds_init = true; - } /* end if */ + /* Bounds are now initialized */ + bounds_init = true; + } /* end if */ - /* Get index of first block in virtual selection */ - storage->list[i].sub_dset_io_start = - (size_t)H5S_hyper_get_first_inc_block(storage->list[i].source_dset.virtual_select, - bounds_start[storage->list[i].unlim_dim_virtual], NULL); + /* Get index of first block in virtual selection */ + curr_mapping->sub_dset_io_start = + (size_t)H5S_hyper_get_first_inc_block(curr_mapping->source_dset.virtual_select, + bounds_start[curr_mapping->unlim_dim_virtual], NULL); + + /* Get index of first block outside of virtual selection */ + curr_mapping->sub_dset_io_end = (size_t)H5S_hyper_get_first_inc_block( + curr_mapping->source_dset.virtual_select, bounds_end[curr_mapping->unlim_dim_virtual], + &partial_block); + if (partial_block) + curr_mapping->sub_dset_io_end++; + if (curr_mapping->sub_dset_io_end > curr_mapping->sub_dset_nused) + curr_mapping->sub_dset_io_end = curr_mapping->sub_dset_nused; + + /* Iterate over sub-source dsets */ + for (j = curr_mapping->sub_dset_io_start; j < curr_mapping->sub_dset_io_end; j++) { + /* Check for clipped virtual selection */ + if (!curr_mapping->sub_dset[j].clipped_virtual_select) { + hsize_t start[H5S_MAX_RANK]; + /* This should only be NULL if this is a partial block */ + assert((j == (curr_mapping->sub_dset_io_end - 1)) && partial_block); + + /* If the source space status is not correct, we must try to + * open the source dataset to patch it */ + if (curr_mapping->source_space_status != H5O_VIRTUAL_STATUS_CORRECT) { + assert(!curr_mapping->sub_dset[j].dset); + if (H5D__virtual_open_source_dset(dset, curr_mapping, + &curr_mapping->sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset"); + } /* end if */ - /* Get index of first block outside of virtual selection */ - storage->list[i].sub_dset_io_end = (size_t)H5S_hyper_get_first_inc_block( - storage->list[i].source_dset.virtual_select, bounds_end[storage->list[i].unlim_dim_virtual], - &partial_block); - if (partial_block) - storage->list[i].sub_dset_io_end++; - if (storage->list[i].sub_dset_io_end > storage->list[i].sub_dset_nused) - storage->list[i].sub_dset_io_end = storage->list[i].sub_dset_nused; + /* If we obtained a valid source space, we must create + * clipped source and virtual selections, otherwise we + * cannot do this and we will leave them NULL. This doesn't + * hurt anything because we can't do I/O because the dataset + * must not have been found. */ + if (curr_mapping->source_space_status == H5O_VIRTUAL_STATUS_CORRECT) { + hsize_t tmp_dims[H5S_MAX_RANK]; + hsize_t vbounds_end[H5S_MAX_RANK]; - /* Iterate over sub-source dsets */ - for (j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++) { - /* Check for clipped virtual selection */ - if (!storage->list[i].sub_dset[j].clipped_virtual_select) { - hsize_t start[H5S_MAX_RANK]; - /* This should only be NULL if this is a partial block */ - assert((j == (storage->list[i].sub_dset_io_end - 1)) && partial_block); - - /* If the source space status is not correct, we must try to - * open the source dataset to patch it */ - if (storage->list[i].source_space_status != H5O_VIRTUAL_STATUS_CORRECT) { - assert(!storage->list[i].sub_dset[j].dset); - if (H5D__virtual_open_source_dset(dset, &storage->list[i], - &storage->list[i].sub_dset[j]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset"); - } /* end if */ + /* Get bounds of virtual selection */ + if (H5S_SELECT_BOUNDS(curr_mapping->sub_dset[j].virtual_select, tmp_dims, + vbounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); - /* If we obtained a valid source space, we must create - * clipped source and virtual selections, otherwise we - * cannot do this and we will leave them NULL. This doesn't - * hurt anything because we can't do I/O because the dataset - * must not have been found. */ - if (storage->list[i].source_space_status == H5O_VIRTUAL_STATUS_CORRECT) { - hsize_t tmp_dims[H5S_MAX_RANK]; - hsize_t vbounds_end[H5S_MAX_RANK]; + assert(bounds_init); - /* Get bounds of virtual selection */ - if (H5S_SELECT_BOUNDS(storage->list[i].sub_dset[j].virtual_select, tmp_dims, - vbounds_end) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); + /* Convert bounds to extent (add 1) */ + for (k = 0; k < (size_t)rank; k++) + vbounds_end[k]++; - assert(bounds_init); + /* Temporarily set extent of virtual selection to bounds */ + if (H5S_set_extent(curr_mapping->sub_dset[j].virtual_select, vbounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to modify size of dataspace"); - /* Convert bounds to extent (add 1) */ - for (k = 0; k < (size_t)rank; k++) - vbounds_end[k]++; + /* Get current VDS dimensions */ + if (H5S_get_simple_extent_dims(dset->shared->space, tmp_dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get VDS dimensions"); - /* Temporarily set extent of virtual selection to bounds */ - if (H5S_set_extent(storage->list[i].sub_dset[j].virtual_select, vbounds_end) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to modify size of dataspace"); + /* Copy virtual selection */ + if (NULL == (curr_mapping->sub_dset[j].clipped_virtual_select = + H5S_copy(curr_mapping->sub_dset[j].virtual_select, false, true))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection"); - /* Get current VDS dimensions */ - if (H5S_get_simple_extent_dims(dset->shared->space, tmp_dims, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get VDS dimensions"); - - /* Copy virtual selection */ - if (NULL == (storage->list[i].sub_dset[j].clipped_virtual_select = - H5S_copy(storage->list[i].sub_dset[j].virtual_select, false, true))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection"); - - /* Clip virtual selection to real virtual extent */ - (void)memset(start, 0, sizeof(start)); - if (H5S_select_hyperslab(storage->list[i].sub_dset[j].clipped_virtual_select, - H5S_SELECT_AND, start, NULL, tmp_dims, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTSELECT, FAIL, "unable to clip hyperslab"); - - /* Project intersection of virtual space and clipped - * virtual space onto source space (create - * clipped_source_select) */ - if (H5S_select_project_intersection( - storage->list[i].sub_dset[j].virtual_select, storage->list[i].source_select, - storage->list[i].sub_dset[j].clipped_virtual_select, - &storage->list[i].sub_dset[j].clipped_source_select, true) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, - "can't project virtual intersection onto memory space"); + /* Clip virtual selection to real virtual extent */ + (void)memset(start, 0, sizeof(start)); + if (H5S_select_hyperslab(curr_mapping->sub_dset[j].clipped_virtual_select, + H5S_SELECT_AND, start, NULL, tmp_dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSELECT, FAIL, "unable to clip hyperslab"); - /* Set extents of virtual_select and - * clipped_virtual_select to virtual extent */ - if (H5S_set_extent(storage->list[i].sub_dset[j].virtual_select, tmp_dims) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to modify size of dataspace"); - if (H5S_set_extent(storage->list[i].sub_dset[j].clipped_virtual_select, tmp_dims) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to modify size of dataspace"); - } /* end if */ - } /* end if */ - - /* Only continue if we managed to obtain a - * clipped_virtual_select */ - if (storage->list[i].sub_dset[j].clipped_virtual_select) { - /* Project intersection of file space and mapping virtual space - * onto memory space */ + /* Project intersection of virtual space and clipped + * virtual space onto source space (create + * clipped_source_select) */ if (H5S_select_project_intersection( - file_space, mem_space, storage->list[i].sub_dset[j].clipped_virtual_select, - &storage->list[i].sub_dset[j].projected_mem_space, true) < 0) + curr_mapping->sub_dset[j].virtual_select, curr_mapping->source_select, + curr_mapping->sub_dset[j].clipped_virtual_select, + &curr_mapping->sub_dset[j].clipped_source_select, true) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space"); - /* Check number of elements selected */ - if ((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS( - storage->list[i].sub_dset[j].projected_mem_space)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, - "unable to get number of elements in selection"); - - /* Check if anything is selected */ - if (select_nelmts > (hssize_t)0) { - /* Open source dataset */ - if (!storage->list[i].sub_dset[j].dset) - /* Try to open dataset */ - if (H5D__virtual_open_source_dset(dset, &storage->list[i], - &storage->list[i].sub_dset[j]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, - "unable to open source dataset"); - - /* If the source dataset is not open, mark the selected - * elements as zero so projected_mem_space is freed */ - if (!storage->list[i].sub_dset[j].dset) - select_nelmts = (hssize_t)0; - } /* end if */ - - /* If there are not elements selected in this mapping, free - * projected_mem_space, otherwise update tot_nelmts */ - if (select_nelmts == (hssize_t)0) { - if (H5S_close(storage->list[i].sub_dset[j].projected_mem_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, - "can't close projected memory space"); - storage->list[i].sub_dset[j].projected_mem_space = NULL; - } /* end if */ - else - *tot_nelmts += (hsize_t)select_nelmts; + /* Set extents of virtual_select and + * clipped_virtual_select to virtual extent */ + if (H5S_set_extent(curr_mapping->sub_dset[j].virtual_select, tmp_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to modify size of dataspace"); + if (H5S_set_extent(curr_mapping->sub_dset[j].clipped_virtual_select, tmp_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to modify size of dataspace"); } /* end if */ - } /* end for */ - } /* end if */ - else { - if (storage->list[i].source_dset.clipped_virtual_select) { - /* Project intersection of file space and mapping virtual space onto - * memory space */ + } /* end if */ + + /* Only continue if we managed to obtain a + * clipped_virtual_select */ + if (curr_mapping->sub_dset[j].clipped_virtual_select) { + /* Project intersection of file space and mapping virtual space + * onto memory space */ if (H5S_select_project_intersection( - file_space, mem_space, storage->list[i].source_dset.clipped_virtual_select, - &storage->list[i].source_dset.projected_mem_space, true) < 0) + file_space, mem_space, curr_mapping->sub_dset[j].clipped_virtual_select, + &curr_mapping->sub_dset[j].projected_mem_space, true) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space"); - /* Check number of elements selected, add to tot_nelmts */ + /* Check number of elements selected */ if ((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS( - storage->list[i].source_dset.projected_mem_space)) < 0) + curr_mapping->sub_dset[j].projected_mem_space)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to get number of elements in selection"); /* Check if anything is selected */ if (select_nelmts > (hssize_t)0) { /* Open source dataset */ - if (!storage->list[i].source_dset.dset) + if (!curr_mapping->sub_dset[j].dset) /* Try to open dataset */ - if (H5D__virtual_open_source_dset(dset, &storage->list[i], - &storage->list[i].source_dset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset"); - - /* If the source dataset is not open, mark the selected elements - * as zero so projected_mem_space is freed */ - if (!storage->list[i].source_dset.dset) + if (H5D__virtual_open_source_dset(dset, curr_mapping, + &curr_mapping->sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, + "unable to open source dataset"); + + /* If the source dataset is not open, mark the selected + * elements as zero so projected_mem_space is freed */ + if (!curr_mapping->sub_dset[j].dset) select_nelmts = (hssize_t)0; } /* end if */ /* If there are not elements selected in this mapping, free - * projected_mem_space, otherwise update tot_nelmts */ + * projected_mem_space, otherwise update tot_nelmts */ if (select_nelmts == (hssize_t)0) { - if (H5S_close(storage->list[i].source_dset.projected_mem_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space"); - storage->list[i].source_dset.projected_mem_space = NULL; + if (H5S_close(curr_mapping->sub_dset[j].projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, + "can't close projected memory space"); + curr_mapping->sub_dset[j].projected_mem_space = NULL; } /* end if */ else *tot_nelmts += (hsize_t)select_nelmts; } /* end if */ - else { - /* If there is no clipped_dim_virtual, this must be an unlimited - * selection whose dataset was not found in the last call to - * H5Dget_space(). Do not attempt to open it as this might - * affect the extent and we are not going to recalculate it - * here. */ - assert(storage->list[i].unlim_dim_virtual >= 0); - assert(!storage->list[i].source_dset.dset); - } /* end else */ - } /* end else */ - } /* end for */ + } /* end for */ + } /* end if */ + else { + if (curr_mapping->source_dset.clipped_virtual_select) { + /* Project intersection of file space and mapping virtual space onto + * memory space */ + if (H5S_select_project_intersection( + file_space, mem_space, curr_mapping->source_dset.clipped_virtual_select, + &curr_mapping->source_dset.projected_mem_space, true) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, + "can't project virtual intersection onto memory space"); + + /* Check number of elements selected, add to tot_nelmts */ + if ((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS( + curr_mapping->source_dset.projected_mem_space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, + "unable to get number of elements in selection"); + + /* Check if anything is selected */ + if (select_nelmts > (hssize_t)0) { + /* Open source dataset */ + if (!curr_mapping->source_dset.dset) + /* Try to open dataset */ + if (H5D__virtual_open_source_dset(dset, curr_mapping, + &curr_mapping->source_dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset"); + + /* If the source dataset is not open, mark the selected elements + * as zero so projected_mem_space is freed */ + if (!curr_mapping->source_dset.dset) + select_nelmts = (hssize_t)0; + } /* end if */ + + /* If there are not elements selected in this mapping, free + * projected_mem_space, otherwise update tot_nelmts */ + if (select_nelmts == (hssize_t)0) { + if (H5S_close(curr_mapping->source_dset.projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space"); + curr_mapping->source_dset.projected_mem_space = NULL; + } /* end if */ + else + *tot_nelmts += (hsize_t)select_nelmts; + } /* end if */ + else { + /* If there is no clipped_dim_virtual, this must be an unlimited + * selection whose dataset was not found in the last call to + * H5Dget_space(). Do not attempt to open it as this might + * affect the extent and we are not going to recalculate it + * here. */ + assert(curr_mapping->unlim_dim_virtual >= 0); + assert(!curr_mapping->source_dset.dset); + } /* end else */ + } /* end else */ +done: + FUNC_LEAVE_NOAPI(ret_value) +} + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_pre_io + * + * Purpose: Project all virtual mappings onto mem_space, with the + * results stored in projected_mem_space for each mapping. + * Opens all source datasets if possible. The total number + * of elements is stored in tot_nelmts. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, H5S_t *file_space, + H5S_t *mem_space, hsize_t *tot_nelmts) +{ + const H5D_t *dset = dset_info->dset; /* Local pointer to dataset info */ + herr_t ret_value = SUCCEED; /* Return value */ + bool tree_enabled = false; + H5P_genplist_t *dapl_plist = NULL; /* Dataset access property list */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + assert(storage); + assert(mem_space); + assert(file_space); + assert(tot_nelmts); + + /* Initialize layout if necessary */ + if (!storage->init) + if (H5D__virtual_init_all(dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize virtual layout"); + + /* Initialize tot_nelmts */ + *tot_nelmts = 0; + + /* If r-tree use is enabled and no tree currently exists, build the tree */ + if (NULL == (dapl_plist = (H5P_genplist_t *)H5I_object(dset->shared->dapl_id))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for dapl ID"); + + if (H5P_get(dapl_plist, H5D_ACS_USE_TREE_NAME, &tree_enabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual use tree flag"); + + if (tree_enabled && !storage->tree) { + int rank = 0; + assert(!storage->is_in_tree); + /* Get the rank of the dataset */ + if ((rank = H5S_GET_EXTENT_NDIMS(dset_info->dset->shared->space)) < 0 || rank >= H5S_MAX_RANK) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset rank"); + + if (rank == 0) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "virtual dataset has no rank"); + + if (H5D__virtual_build_tree(storage, rank) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build virtual mapping tree"); + } + + /* Iterate over the mappings */ + if (storage->tree) { + /* Perform a spatial tree search to get a list of mappings + * whose virtual selection intersects the IO operation */ + H5RT_result_t *search_results = NULL; /* Search results from R-tree */ + H5RT_result_t *curr_result = NULL; /* Current result in search results */ + hsize_t min[H5S_MAX_RANK]; + hsize_t max[H5S_MAX_RANK]; + + memset(min, 0, sizeof(min)); + memset(max, 0, sizeof(max)); + + if (H5S_SELECT_BOUNDS(file_space, min, max) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); + + if (H5RT_search(storage->tree, min, max, &search_results) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "R-tree search failed"); + + /* First, iterate over the mappings with an intersection found via the tree */ + curr_result = search_results; + while (curr_result) { + H5RT_leaf_t *curr_leaf = curr_result->leaf; + + assert(curr_leaf); + + size_t mapping_index = (size_t)curr_leaf->record; + + if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, + &storage->list[mapping_index]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + + curr_result = curr_result->next; + } + } + + /* Iterate over the mappings that are not stored in the tree */ + /* Index of each boolean in the 'is in tree' list = index of the mapping it describes in the mapping list */ + for (size_t i = 0; i < storage->list_nused; i++) { + /* Skip any mappings that would have been searched by the tree */ + if (storage->is_in_tree && storage->is_in_tree[i]) + continue; + + if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, + &storage->list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -3672,3 +3789,234 @@ H5D__virtual_release_source_dset_files(H5D_virtual_held_file_t *head) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_release_source_dset_files() */ + +/*------------------------------------------------------------------------- + * Function: H5D__rtree_should_insert + * + * Purpose: Determine whether the given mapping is valid for + * insertion into a spatial tree + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__rtree_should_insert(void *mapping_entry, bool *should_insert) +{ + herr_t ret_value = SUCCEED; + H5S_t *vspace = NULL; + H5S_t *src_space = NULL; + hsize_t virt_nelems = 0; + hsize_t src_nelems = 0; + H5O_storage_virtual_ent_t *entry = NULL; + + FUNC_ENTER_PACKAGE_NOERR + + if (!mapping_entry || !should_insert) { + ret_value = FAIL; + goto done; + } + + entry = (H5O_storage_virtual_ent_t *)mapping_entry; + + /* Do not insert mappings with an unlimited dimension */ + if ((vspace = entry->source_dset.virtual_select) != NULL) + virt_nelems = (hsize_t)H5S_GET_SELECT_NPOINTS(vspace); + + if ((src_space = entry->source_dset.clipped_source_select) != NULL) + src_nelems = (hsize_t)H5S_GET_SELECT_NPOINTS(src_space); + + if (virt_nelems == H5S_UNLIMITED || src_nelems == H5S_UNLIMITED) { + *should_insert = false; + goto done; + } + + /* Do not insert printf-style mappings */ + if (entry->psfn_nsubs > 0 || entry->psdn_nsubs > 0) { + *should_insert = false; + goto done; + } + + /* Do not insert zero-dim mappings */ + if ((H5S_GET_EXTENT_NDIMS(vspace)) < 1 || + (H5S_GET_EXTENT_NDIMS(src_space)) < 1) { + *should_insert = false; + goto done; + } + + /* Otherwise, we can insert it */ + *should_insert = true; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} + +/*------------------------------------------------------------------------- + * Function: H5D__mappings_to_leaves + * + * Purpose: Allocate leaf array and boolean array for construction of a + * spatial tree from a list of mappings + * + * Parameters: mappings : Pointer to array of mappings to be inserted + * num_mappings: Number of mappings in the array + * leaves_out: Pointer to array of leaves, one per mapping that should be inserted + * Allocated on success and must be freed by caller. + * is_in_tree_out: Pointer to boolean array, one per mapping, indicating + * whether the mapping is in the list of leaves. + * Allocated on success and must be freed by caller. + * leaf_count: Pointer to number of leaves allocated in leaves_out. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, H5RT_leaf_t **leaves_out, bool **is_in_tree_out, size_t *leaf_count) { + herr_t ret_value = SUCCEED; + + H5RT_leaf_t *leaves_temp = NULL; + bool *is_in_tree = NULL; + + bool should_insert_space = false; + + H5O_storage_virtual_ent_t *curr_mapping = NULL; + H5RT_leaf_t *curr_leaf = NULL; + size_t curr_leaf_count = 0; + H5S_t *curr_space = NULL; + + int rank = 0; + + FUNC_ENTER_PACKAGE + + assert(mappings); + assert(num_mappings > 0); + assert(leaf_count); + assert(leaves_out); + assert(is_in_tree_out); + + /* TODO - For now, assume that bulk allocation of too much memory is faster than + * piecemeal allocation of the exactly correct amount */ + if ((leaves_temp = (H5RT_leaf_t *)calloc(num_mappings, sizeof(H5RT_leaf_t))) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate leaves array"); + + if ((is_in_tree = (bool *)H5MM_calloc(num_mappings * sizeof(bool))) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate is_in_tree array"); + + for (size_t i = 0; i < num_mappings; i++) { + curr_mapping = &mappings[i]; + + if (H5D__rtree_should_insert((void*)curr_mapping, &should_insert_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "error checking if mapping should be inserted"); + + if (!should_insert_space) + continue; + + is_in_tree[i] = true; + + /* Leaf is already allocated, populate its fields */ + curr_leaf = &leaves_temp[curr_leaf_count]; + + /* Store the index into the mapping list */ + curr_leaf->record = (uintptr_t) i; + + if ((curr_space = mappings[i].source_dset.virtual_select) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "mapping has no virtual space"); + + if ((rank = H5S_GET_EXTENT_NDIMS(curr_space)) < 0 || rank > H5S_MAX_RANK) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get rank of dataspace"); + + if (rank == 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "mapping has zero-dimensional space"); + + /* Get selection bounds */ + if (H5S_SELECT_BOUNDS(curr_space, curr_leaf->min, curr_leaf->max) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get selection bounds"); + + for (int d = 0; d < rank; d++) { + /* Validate bounds and compute midpoint safely */ + if (curr_leaf->min[d] > curr_leaf->max[d]) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "invalid selection bounds: min > max"); + curr_leaf->mid[d] = curr_leaf->min[d] + (curr_leaf->max[d] - curr_leaf->min[d]) / 2; + } + + curr_leaf_count++; + } + + *leaves_out = leaves_temp; + *leaf_count = curr_leaf_count; + *is_in_tree_out = is_in_tree; +done: + if (ret_value < 0) { + if (leaves_temp) + free(leaves_temp); + if (is_in_tree) + H5MM_free(is_in_tree); + } + + FUNC_LEAVE_NOAPI(ret_value); +} + +// attempt to build a tree and a boolean list and store them on the storage object +/*------------------------------------------------------------------------- + * Function: H5D__virtual_build_tree + * + * Purpose: Build a spatial tree of mapping indices, and a list of + * which mapping indices are in the tree, and store them on + * the provided virtual layout + * + * Parameters: virt: The virtual layout with the mapping to build the + * tree from. The tree will be stored at virt->tree, + * and the list of stored indices will be stored at + * virt->is_in_tree. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) { + H5O_storage_virtual_ent_t *mappings = virt->list; + size_t num_mappings = virt->list_nused; + + H5RT_leaf_t *leaves = NULL; + size_t num_leaves = 0; + bool *is_in_tree = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + assert(virt); + + if (H5D__mappings_to_leaves(mappings, num_mappings, &leaves, &is_in_tree, &num_leaves) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get leaves from mappings"); + + if (num_leaves == 0) { + /* Don't build a tree, release allocated memory */ + free(leaves); + leaves = NULL; + free(is_in_tree); + is_in_tree = NULL; + + virt->tree = NULL; + virt->is_in_tree = NULL; + + } else { + virt->is_in_tree = is_in_tree; + + /* Build the tree */ + if ((virt->tree = H5RT_create(rank, leaves, num_leaves)) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to create mapping tree"); + /* Tree takes ownership of leaves */ + leaves = NULL; + } + +done: + if (ret_value < 0) { + if (leaves) + free(leaves); + if (is_in_tree) + free(is_in_tree); + } + + FUNC_LEAVE_NOAPI(ret_value) +} \ No newline at end of file diff --git a/src/H5Olayout.c b/src/H5Olayout.c index 4bf9d69ebad..9828bc39e51 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -565,13 +565,14 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU ret_value = mesg; done: - if (ret_value == NULL) + if (ret_value == NULL) { if (mesg) { if (mesg->type == H5D_VIRTUAL) if (H5D__virtual_reset_layout(mesg) < 0) HDONE_ERROR(H5E_OHDR, H5E_CANTFREE, NULL, "unable to reset virtual layout"); H5FL_FREE(H5O_layout_t, mesg); } + } FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__layout_decode() */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 74f14e01e1a..5b216b20a99 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -42,6 +42,7 @@ typedef struct H5O_fill_t H5O_fill_t; #include "H5Tprivate.h" /* Datatype functions */ #include "H5VLprivate.h" /* Virtual Object Layer */ #include "H5Zprivate.h" /* I/O pipeline filters */ +#include "H5RTprivate.h" /* R-tree for virtual dataspaces */ /* Forward references of package typedefs */ typedef struct H5O_msg_class_t H5O_msg_class_t; @@ -603,6 +604,9 @@ typedef struct H5O_storage_virtual_t { H5O_storage_virtual_ent_t *source_dset_hash_table; /* Hash table of virtual entries sorted by source dataset name. Only the first occurrence of each source dataset name is stored. */ + H5RT_t* tree; + bool *is_in_tree; /* List of the indices in 'list' that are stored in tree for quick access + * Some mappings cannot be stored in the tree and must be searched manually */ } H5O_storage_virtual_t; typedef struct H5O_storage_t { diff --git a/src/H5Pdapl.c b/src/H5Pdapl.c index e9c46746965..83003276f93 100644 --- a/src/H5Pdapl.c +++ b/src/H5Pdapl.c @@ -97,6 +97,12 @@ #define H5D_ACS_EFILE_PREFIX_CMP H5P__dapl_efile_pref_cmp #define H5D_ACS_EFILE_PREFIX_CLOSE H5P__dapl_efile_pref_close +/* Definitions for use of VDS mapping spatial tree */ +#define H5D_ACS_USE_TREE_SIZE sizeof(bool) +#define H5D_ACS_USE_TREE_DEF true +#define H5D_ACS_USE_TREE_ENC H5P__encode_bool +#define H5D_ACS_USE_TREE_DEC H5P__decode_bool + /******************/ /* Local Typedefs */ /******************/ @@ -175,6 +181,7 @@ static const H5D_append_flush_t H5D_def_append_flush_g = static const char *H5D_def_efile_prefix_g = H5D_ACS_EFILE_PREFIX_DEF; /* Default external file prefix string */ static const char *H5D_def_vds_prefix_g = H5D_ACS_VDS_PREFIX_DEF; /* Default vds prefix string */ +static const bool H5D_def_tree_g = H5D_ACS_USE_TREE_DEF; /* Default use of spatial tree for VDS mappings */ /*------------------------------------------------------------------------- * Function: H5P__dacc_reg_prop @@ -247,6 +254,11 @@ H5P__dacc_reg_prop(H5P_genclass_t *pclass) H5D_ACS_EFILE_PREFIX_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class"); + /* Register the spatial tree use property */ + if (H5P__register_real(pclass, H5D_ACS_USE_TREE_NAME, H5D_ACS_USE_TREE_SIZE, &H5D_def_tree_g, NULL, NULL, + NULL, H5D_ACS_USE_TREE_ENC, H5D_ACS_USE_TREE_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class"); + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__dacc_reg_prop() */ @@ -1543,3 +1555,77 @@ H5Pget_virtual_prefix(hid_t plist_id, char *prefix /*out*/, size_t size) done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_virtual_prefix() */ + +/*----------------------------------------------------------------------------- + * Function: H5Pget_dset_use_spatial_tree + * + * Purpose: + * + * Access the flag for whether or not datasets created by the given dcpl + * construct a spatial tree and use it when searching over VDS mappings + * + * Return: + * + * Failure: Negative value (FAIL) + * Success: Non-negative value (SUCCEED) + * + *----------------------------------------------------------------------------- + */ +herr_t H5Pget_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) { + bool setting = false; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + + if (NULL == use_tree) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "receiving pointer cannot be NULL"); + + plist = H5P_object_verify(dcpl_id, H5P_DATASET_ACCESS, true); + if (NULL == plist) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + + if (H5P_peek(plist, H5D_ACS_USE_TREE_NAME, &setting) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get dset use spatial tree flag value"); + + *use_tree = setting; + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_dset_use_spatial_tree() */ + +/*----------------------------------------------------------------------------- + * Function: H5Pset_dset_use_spatial_tree + * + * Purpose: + * + * Set the DAPL to construct a spatial tree and use it when searching over + * VDS mappings + * + * Return: + * + * Failure: Negative value (FAIL) + * Success: Non-negative value (SUCCEED) + * + *----------------------------------------------------------------------------- + */ +herr_t H5Pset_dset_use_spatial_tree(hid_t dapl_id, bool use_tree) { + H5P_genplist_t *plist = NULL; + bool prev_set = false; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + + plist = H5P_object_verify(dapl_id, H5P_DATASET_ACCESS, false); + if (NULL == plist) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + + if (H5P_peek(plist, H5D_ACS_USE_TREE_NAME, &prev_set) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get extant dset use spatial tree flag value"); + + if (H5P_poke(plist, H5D_ACS_USE_TREE_NAME, &use_tree) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set dset use spatial tree flag value"); + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_dset_use_spatial_tree() */ diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index df51df30237..50cee816de4 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -87,9 +87,9 @@ } #define H5D_DEF_STORAGE_VIRTUAL_INIT \ { \ - {HADDR_UNDEF, 0}, 0, NULL, 0, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ - H5D_VDS_ERROR, HSIZE_UNDEF, -1, -1, false, NULL, NULL \ + {HADDR_UNDEF, 0}, 0, NULL, 0, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ + H5D_VDS_ERROR, HSIZE_UNDEF, -1, -1, false, NULL, NULL, NULL, NULL \ } #define H5D_DEF_STORAGE_COMPACT \ { \ @@ -2119,6 +2119,7 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const /* Expand list if necessary */ if (virtual_layout.storage.u.virt.list_nused == virtual_layout.storage.u.virt.list_nalloc) { H5O_storage_virtual_ent_t *x; /* Pointer to the new list */ + bool *y; /* Pointer to the new is_in_tree list */ size_t new_alloc = MAX(H5D_VIRTUAL_DEF_LIST_SIZE, virtual_layout.storage.u.virt.list_nalloc * 2); ptrdiff_t buf_diff; @@ -2130,6 +2131,11 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const virtual_layout.storage.u.virt.list = x; virtual_layout.storage.u.virt.list_nalloc = new_alloc; + /* Expand size of is_in_tree list */ + if (NULL == (y = (bool *)H5MM_realloc(virtual_layout.storage.u.virt.is_in_tree, new_alloc * sizeof(bool)))) + HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't reallocate virtual dataset mapping is_in_tree list"); + virtual_layout.storage.u.virt.is_in_tree = y; + /* Adjust pointers in the hash tables in case realloc moved the buffers, and hence all the elements * and hash handles in the hash tables */ HASH_ADJUST_PTRS(hh_source_file, virtual_layout.storage.u.virt.source_file_hash_table, buf_diff); @@ -2213,9 +2219,8 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const if (H5D_virtual_update_min_dims(&virtual_layout, virtual_layout.storage.u.virt.list_nused) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "unable to update virtual dataset minimum dimensions"); - /* Finish adding entry */ + /* Finish adding entry to list */ virtual_layout.storage.u.virt.list_nused++; - done: /* Set VDS layout information in property list */ /* (Even on failure, so there's not a mangled layout struct in the list) */ @@ -2257,7 +2262,6 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const virtual_layout.storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_xfree(virtual_layout.storage.u.virt.list); } /* end if */ - FUNC_LEAVE_API(ret_value) } /* end H5Pset_virtual() */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index e454669f6fe..40ab7d5f154 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -6010,6 +6010,31 @@ H5_DLL herr_t H5Pget_chunk_opts(hid_t plist_id, unsigned *opts); * */ H5_DLL herr_t H5Pget_dset_no_attrs_hint(hid_t dcpl_id, bool *minimize); + /** + * \ingroup DCPL + * + * \brief Retrieves the setting for whether or not to use a spatial tree + * for VDS mappings + * + * \dcpl_id + * \param[out] use_tree Flag indicating whether the dataset will or + * will not use a spatial tree + * + * \return \herr_t + * + * \details H5Pget_dset_use_spatial_tree() retrieves the + * use spatial tree flag setting for the dataset + * creation property list \p dcpl_id. This setting determines + * whether a dataset created with the dataset creation + * property list \p dcpl_id will construct a spatial tree + * and use it in an attempt to optimize certain operations. + * The setting value is returned in the boolean pointer + * \p use_tree. + * + * \since 2.0.0 + * + */ +H5_DLL herr_t H5Pget_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree); /** * \ingroup DCPL * @@ -6493,6 +6518,28 @@ H5_DLL herr_t H5Pset_chunk_opts(hid_t plist_id, unsigned opts); * */ H5_DLL herr_t H5Pset_dset_no_attrs_hint(hid_t dcpl_id, bool minimize); +/** + * \ingroup DCPL + * + * \brief Sets the flag to use a spatial tree for mappings + * + * \dcpl_id + * \param[in] use_tree Flag for indicating whether or not a dataset + * should use a spatial tree for mappings + * + * \return \herr_t + * + * \details H5Pset_dset_use_spatial_tree() sets the use-tree flag + * for the dataset creation property list \p dcpl_id. + * Datasets created with the dataset creation property + * list \p dcpl_id will construct a spatial tree and use + * it in an attempt to optimize intersection-check + * operations on mappings when \p use_tree is set to true. + * + * \since 2.0.0 + * + */ +H5_DLL herr_t H5Pset_dset_use_spatial_tree(hid_t dcpl_id, bool use_tree); /** * \ingroup DCPL * diff --git a/test/dsets.c b/test/dsets.c index b9e1d342a7d..9047973c95c 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -83,6 +83,8 @@ static const char *FILENAME[] = {"dataset", /* 0 */ #define OHMIN_FILENAME_A "ohdr_min_a" +#define SPATIAL_TREE_FILENAME "spatial_tree" + #define FILENAME_BUF_SIZE 1024 #define KB 1024 diff --git a/test/rtree.c b/test/rtree.c index 4674dbeadbf..a7c5f782e13 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -30,12 +30,24 @@ #include "H5CXprivate.h" /* API Contexts */ #include "H5VLprivate.h" /* Virtual Object Layer */ +#define H5D_FRIEND /*suppress error about including H5Dpkg */ +#define H5D_TESTING +#include "H5Dpkg.h" /* Datasets */ + #define RTREE_TEST_BASE_COORD 10000 #define RTREE_TEST_BASE_SIZE 1000 #define RTREE_TEST_CREATE_RANK 8 #define RTREE_TEST_CREATE_NUM_COUNTS 4 +#define RTREE_DAPL_FILENAME "vds_rtree_test.h5" +#define RTREE_DAPL_SRC_FILENAME "vds_src_rtree_test.h5" +#define RTREE_DAPL_VDS_NAME "vdset" +#define RTREE_DAPL_SRC_DATASET_NAME "src_dset" + +#define RTREE_DAPL_DATASET_DIM1 10 +#define RTREE_DAPL_DATASET_DIM2 10 + static const size_t test_counts[RTREE_TEST_CREATE_NUM_COUNTS] = {1, 100, 500, 10000}; /* Helper function to generate leaf data */ @@ -52,6 +64,9 @@ static H5RT_leaf_t **manual_search(H5RT_leaf_t *leaves, size_t leaf_count, int r static herr_t verify_rtree_search(H5RT_result_set_t *result_set, H5RT_leaf_t *leaves, size_t leaf_count, hsize_t min[], hsize_t max[], int rank); +/* Helper to create and initialize virtual dset in a file */ +static hid_t create_virtual_dataset(hid_t file_id, hid_t dapl_id); + static herr_t verify_rtree_search(H5RT_result_set_t *result_set, H5RT_leaf_t *leaves, size_t leaf_count, hsize_t min[], hsize_t max[], int rank) @@ -415,6 +430,265 @@ test_rtree_copy(void) return FAIL; } +/*------------------------------------------------------------------------- + * Function: create_virtual_dataset + * + * Purpose: Helper function to create a virtual dataset with mappings + * + * Return: Success: dataset ID + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +static hid_t +create_virtual_dataset(hid_t file_id, hid_t dapl_id) +{ + hid_t vspace_id = H5I_INVALID_HID; + hid_t srcspace_id = H5I_INVALID_HID; + hid_t srcfile_id = H5I_INVALID_HID; + hid_t srcdset_id = H5I_INVALID_HID; + hid_t vdset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hsize_t dims[2] = {RTREE_DAPL_DATASET_DIM1, RTREE_DAPL_DATASET_DIM2}; + int wbuf[RTREE_DAPL_DATASET_DIM1][RTREE_DAPL_DATASET_DIM2]; + int i, j; + + if ((vspace_id = H5Screate_simple(2, dims, NULL)) < 0) + goto error; + + if ((srcspace_id = H5Screate_simple(2, dims, NULL)) < 0) + goto error; + + /* Create source file */ + if ((srcfile_id = H5Fcreate(RTREE_DAPL_SRC_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* Create source dataset */ + if ((srcdset_id = H5Dcreate2(srcfile_id, RTREE_DAPL_SRC_DATASET_NAME, H5T_NATIVE_INT, srcspace_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + if (H5Pset_virtual(dcpl_id, vspace_id, "vds_src_rtree_test.h5", RTREE_DAPL_SRC_DATASET_NAME, srcspace_id) < 0) + goto error; + + /* Create virtual dataset */ + if ((vdset_id = H5Dcreate2(file_id, RTREE_DAPL_VDS_NAME, H5T_NATIVE_INT, vspace_id, + H5P_DEFAULT, dcpl_id, dapl_id)) < 0) + goto error; + + /* Initialize and write data to source dataset */ + for (i = 0; i < RTREE_DAPL_DATASET_DIM1; i++) + for (j = 0; j < RTREE_DAPL_DATASET_DIM2; j++) + wbuf[i][j] = i * RTREE_DAPL_DATASET_DIM2 + j; + + if (H5Dwrite(srcdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) + goto error; + + /* Cleanup */ + if (H5Sclose(vspace_id) < 0) + goto error; + if (H5Sclose(srcspace_id) < 0) + goto error; + if (H5Dclose(srcdset_id) < 0) + goto error; + if (H5Fclose(srcfile_id) < 0) + goto error; + if (H5Pclose(dcpl_id) < 0) + goto error; + + return vdset_id; +error: + /* Cleanup */ + H5E_BEGIN_TRY { + H5Sclose(vspace_id); + H5Sclose(srcspace_id); + H5Dclose(srcdset_id); + H5Fclose(srcfile_id); + H5Dclose(vdset_id); + H5Pclose(dcpl_id); + } H5E_END_TRY; + + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: test_rtree_dapl + * + * Purpose: Test R-tree options on the DCPL + * + * Return: Success: SUCCEED + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_rtree_dapl(bool use_tree) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t dapl_id = H5I_INVALID_HID; + hid_t vdset_id = H5I_INVALID_HID; + + int rbuf[RTREE_DAPL_DATASET_DIM1][RTREE_DAPL_DATASET_DIM2]; + const char *test_str = NULL; + + /* Internal values for introspection */ + H5D_t *dset = NULL; + H5O_storage_virtual_t *storage = NULL; + + /* Inverse of use_tree for re-open part of test */ + bool use_tree_inverse = !use_tree; + + if (use_tree) { + test_str = "spatial tree option enabled"; + } else { + test_str = "spatial tree option disabled"; + } + + TESTING(test_str); + + /* Create file */ + if ((file_id = H5Fcreate(RTREE_DAPL_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Create DAPL */ + if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + + /* Set the spatial tree property */ + if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree) < 0) + FAIL_STACK_ERROR; + + /* Create virtual dataset with some mappings */ + if ((vdset_id = create_virtual_dataset(file_id, dapl_id)) < 0) + FAIL_STACK_ERROR; + + /* Read the entire virtual dataset to force tree initialization */ + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify read data matches expected pattern from create_virtual_dataset */ + for (int i = 0; i < RTREE_DAPL_DATASET_DIM1; i++) { + for (int j = 0; j < RTREE_DAPL_DATASET_DIM2; j++) { + int expected = i * RTREE_DAPL_DATASET_DIM2 + j; + if (rbuf[i][j] != expected) { + printf("Data mismatch at [%d][%d]: expected %d, got %d\n", i, j, expected, rbuf[i][j]); + FAIL_STACK_ERROR; + } + } + } + + /* Get the dataset object - this is using internal API for testing */ + if (NULL == (dset = (H5D_t *)H5VL_object(vdset_id))) + FAIL_STACK_ERROR; + + if (dset->shared->layout.type != H5D_VIRTUAL) + FAIL_STACK_ERROR; + + /* Get the virtual storage structure */ + storage = &(dset->shared->layout.storage.u.virt); + + /* Verify tree existence matches expectation */ + if (use_tree) { + if (storage->tree == NULL) { + puts("Expected spatial tree to exist but it was NULL"); + FAIL_STACK_ERROR; + } + if (storage->is_in_tree == NULL) { + puts("Expected is_in_tree array to exist but it was NULL"); + FAIL_STACK_ERROR; + } + } else { + if (storage->tree != NULL) { + puts("Expected spatial tree to be NULL but it exists"); + FAIL_STACK_ERROR; + } + if (storage->is_in_tree != NULL) { + puts("Expected is_in_tree array to be NULL but it exists"); + FAIL_STACK_ERROR; + } + } + + /* Close the dataset and re-open it with the opposite value set in DAPL */ + if (H5Dclose(vdset_id) < 0) + FAIL_STACK_ERROR; + vdset_id = H5I_INVALID_HID; + + if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree_inverse) < 0) + FAIL_STACK_ERROR; + + if ((vdset_id = H5Dopen2(file_id, RTREE_DAPL_VDS_NAME, dapl_id)) < 0) + FAIL_STACK_ERROR; + + /* Read the entire virtual dataset to force tree initialization */ + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify read data matches expected pattern from create_virtual_dataset */ + for (int i = 0; i < RTREE_DAPL_DATASET_DIM1; i++) { + for (int j = 0; j < RTREE_DAPL_DATASET_DIM2; j++) { + int expected = i * RTREE_DAPL_DATASET_DIM2 + j; + if (rbuf[i][j] != expected) { + printf("Data mismatch after re-open at [%d][%d]: expected %d, got %d\n", i, j, expected, rbuf[i][j]); + FAIL_STACK_ERROR; + } + } + } + + /* Get the dataset object - this is using internal API for testing */ + if (NULL == (dset = (H5D_t *)H5VL_object(vdset_id))) + FAIL_STACK_ERROR; + + if (dset->shared->layout.type != H5D_VIRTUAL) + FAIL_STACK_ERROR; + + + storage = &(dset->shared->layout.storage.u.virt); + + /* Verify tree existence matches expectation */ + if (use_tree_inverse) { + if (storage->tree == NULL) { + puts("Expected spatial tree to exist but it was NULL"); + FAIL_STACK_ERROR; + } + if (storage->is_in_tree == NULL) { + puts("Expected is_in_tree array to exist but it was NULL"); + FAIL_STACK_ERROR; + } + } else { + if (storage->tree != NULL) { + puts("Expected spatial tree to be NULL but it exists"); + FAIL_STACK_ERROR; + } + if (storage->is_in_tree != NULL) { + puts("Expected is_in_tree array to be NULL but it exists"); + FAIL_STACK_ERROR; + } + } + + /* Cleanup */ + if (H5Dclose(vdset_id) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dapl_id) < 0) + FAIL_STACK_ERROR; + if (H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(vdset_id); + H5Pclose(dapl_id); + H5Fclose(file_id); + } H5E_END_TRY; + + return FAIL; +} + /*------------------------------------------------------------------------- * Function: main * @@ -439,6 +713,10 @@ main(void) nerrors += test_rtree_search() < 0 ? 1 : 0; nerrors += test_rtree_copy() < 0 ? 1 : 0; + /* Test spatial tree with DCPL property enabled */ + nerrors += test_rtree_dapl(true) < 0 ? 1 : 0; + nerrors += test_rtree_dapl(false) < 0 ? 1 : 0; + if (nerrors) goto error; From 8d10f8cda7172e822d61b9a5ab6fd30f593b6a81 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Wed, 17 Sep 2025 11:10:53 -0500 Subject: [PATCH 02/43] Changes for reworked tree interface --- src/H5Dvirtual.c | 61 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 6d49894a924..bcaf3a07f5f 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -3130,7 +3130,6 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag curr_result = search_results; while (curr_result) { H5RT_leaf_t *curr_leaf = curr_result->leaf; - assert(curr_leaf); size_t mapping_index = (size_t)curr_leaf->record; @@ -3141,6 +3140,10 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag curr_result = curr_result->next; } + + /* Free search results */ + if (H5RT_free_results(search_results) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't free R-tree search results"); } /* Iterate over the mappings that are not stored in the tree */ @@ -3894,8 +3897,17 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings assert(leaves_out); assert(is_in_tree_out); - /* TODO - For now, assume that bulk allocation of too much memory is faster than - * piecemeal allocation of the exactly correct amount */ + /* Get rank from the first mapping's virtual selection */ + if ((curr_space = mappings[0].source_dset.virtual_select) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "first mapping has no virtual space"); + + if ((rank = H5S_GET_EXTENT_NDIMS(curr_space)) < 0 || rank > H5S_MAX_RANK) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get rank of dataspace"); + + if (rank == 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "mapping has zero-dimensional space"); + + /* Allocate array of leaf structures */ if ((leaves_temp = (H5RT_leaf_t *)calloc(num_mappings, sizeof(H5RT_leaf_t))) == NULL) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate leaves array"); @@ -3913,20 +3925,15 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings is_in_tree[i] = true; - /* Leaf is already allocated, populate its fields */ + /* Initialize leaf with dynamic coordinate allocation */ curr_leaf = &leaves_temp[curr_leaf_count]; + if (H5RT_leaf_init(curr_leaf, rank, (uintptr_t)i) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't initialize R-tree leaf"); - /* Store the index into the mapping list */ - curr_leaf->record = (uintptr_t) i; + /* Record is already set by H5RT_leaf_init */ if ((curr_space = mappings[i].source_dset.virtual_select) == NULL) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "mapping has no virtual space"); - - if ((rank = H5S_GET_EXTENT_NDIMS(curr_space)) < 0 || rank > H5S_MAX_RANK) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get rank of dataspace"); - - if (rank == 0) - HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "mapping has zero-dimensional space"); /* Get selection bounds */ if (H5S_SELECT_BOUNDS(curr_space, curr_leaf->min, curr_leaf->max) < 0) @@ -3947,8 +3954,13 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings *is_in_tree_out = is_in_tree; done: if (ret_value < 0) { - if (leaves_temp) + if (leaves_temp) { + /* Clean up coordinate arrays for initialized leaves */ + for (size_t j = 0; j < curr_leaf_count; j++) { + H5RT_leaf_cleanup(&leaves_temp[j]); + } free(leaves_temp); + } if (is_in_tree) H5MM_free(is_in_tree); } @@ -3992,10 +4004,14 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) { if (num_leaves == 0) { /* Don't build a tree, release allocated memory */ - free(leaves); - leaves = NULL; - free(is_in_tree); - is_in_tree = NULL; + if (leaves) { + free(leaves); + leaves = NULL; + } + if (is_in_tree) { + H5MM_free(is_in_tree); + is_in_tree = NULL; + } virt->tree = NULL; virt->is_in_tree = NULL; @@ -4006,16 +4022,21 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) { /* Build the tree */ if ((virt->tree = H5RT_create(rank, leaves, num_leaves)) == NULL) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to create mapping tree"); - /* Tree takes ownership of leaves */ + /* Tree takes ownership of leaves array and coordinate arrays */ leaves = NULL; } done: if (ret_value < 0) { - if (leaves) + if (leaves) { + /* Clean up coordinate arrays and the leaf array */ + for (size_t i = 0; i < num_leaves; i++) { + H5RT_leaf_cleanup(&leaves[i]); + } free(leaves); + } if (is_in_tree) - free(is_in_tree); + H5MM_free(is_in_tree); } FUNC_LEAVE_NOAPI(ret_value) From af9e0ffe4d2172ad66d941a68ebfe79439b94f70 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Thu, 18 Sep 2025 14:46:28 -0500 Subject: [PATCH 03/43] Clean up search results on failure --- src/H5Dvirtual.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index bcaf3a07f5f..a28d0dc103e 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -3070,6 +3070,7 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag herr_t ret_value = SUCCEED; /* Return value */ bool tree_enabled = false; H5P_genplist_t *dapl_plist = NULL; /* Dataset access property list */ + H5RT_result_t *search_results = NULL; /* Search results from R-tree */ FUNC_ENTER_PACKAGE @@ -3112,7 +3113,6 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag if (storage->tree) { /* Perform a spatial tree search to get a list of mappings * whose virtual selection intersects the IO operation */ - H5RT_result_t *search_results = NULL; /* Search results from R-tree */ H5RT_result_t *curr_result = NULL; /* Current result in search results */ hsize_t min[H5S_MAX_RANK]; hsize_t max[H5S_MAX_RANK]; @@ -3144,6 +3144,7 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag /* Free search results */ if (H5RT_free_results(search_results) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't free R-tree search results"); + search_results = NULL; } /* Iterate over the mappings that are not stored in the tree */ @@ -3159,6 +3160,11 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag } done: + if (ret_value < 0) + if (search_results) + if (H5RT_free_results(search_results) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't free R-tree search results"); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_pre_io() */ From 9d14f0993d70ba239aeebdfaad130bb6c7be3f69 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 18 Sep 2025 19:47:56 +0000 Subject: [PATCH 04/43] Committing clang-format changes --- java/src/jni/h5pDAPLImp.c | 9 +- java/src/jni/h5pDAPLImp.h | 3 +- src/H5Dprivate.h | 2 +- src/H5Dvirtual.c | 206 +++++++++++++++++++------------------- src/H5Oprivate.h | 6 +- src/H5Pdapl.c | 22 ++-- src/H5Pdcpl.c | 14 +-- src/H5Ppublic.h | 2 +- test/rtree.c | 68 +++++++------ 9 files changed, 172 insertions(+), 160 deletions(-) diff --git a/java/src/jni/h5pDAPLImp.c b/java/src/jni/h5pDAPLImp.c index 995bc1eb743..9a0c29b83ca 100644 --- a/java/src/jni/h5pDAPLImp.c +++ b/java/src/jni/h5pDAPLImp.c @@ -318,9 +318,10 @@ H5D_append_cb(hid_t dataset_id, hsize_t *cur_dims, void *cb_data) * Signature: (JZ)V */ JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id, jboolean use_tree) +Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id, + jboolean use_tree) { - bool use_tree_val; + bool use_tree_val; herr_t retVal = FAIL; UNUSED(clss); @@ -342,8 +343,8 @@ Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, j JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id) { - bool use_tree = false; - jboolean bval = JNI_FALSE; + bool use_tree = false; + jboolean bval = JNI_FALSE; UNUSED(clss); diff --git a/java/src/jni/h5pDAPLImp.h b/java/src/jni/h5pDAPLImp.h index ec4b1049263..b6ce51d9f8b 100644 --- a/java/src/jni/h5pDAPLImp.h +++ b/java/src/jni/h5pDAPLImp.h @@ -94,7 +94,8 @@ JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1printf_1gap(JNIEnv * Method: H5Pset_dset_use_spatial_tree * Signature: (JZ)V */ -JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIEnv *, jclass, jlong, jboolean); +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIEnv *, jclass, jlong, + jboolean); /* * Class: hdf_hdf5lib_H5 diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 3bcfb7850ef..f8c78d9ab0a 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -54,7 +54,7 @@ #define H5D_ACS_VDS_PREFIX_NAME "vds_prefix" /* VDS file prefix */ #define H5D_ACS_APPEND_FLUSH_NAME "append_flush" /* Append flush actions */ #define H5D_ACS_EFILE_PREFIX_NAME "external file prefix" /* External file prefix */ -#define H5D_ACS_USE_TREE_NAME "tree" /* Whether to use spatial tree */ +#define H5D_ACS_USE_TREE_NAME "tree" /* Whether to use spatial tree */ /* ======== Data transfer properties ======== */ #define H5D_XFER_MAX_TEMP_BUF_NAME "max_temp_buf" /* Maximum temp buffer size */ diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index a28d0dc103e..c9ed2bd971b 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -109,12 +109,11 @@ static herr_t H5D__virtual_read_one(H5D_dset_io_info_t *dset_info, static herr_t H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset); - /* R-tree helper functions */ static herr_t H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank); static herr_t H5D__rtree_should_insert(void *mapping_entry, bool *should_insert); static herr_t H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, - H5RT_leaf_t **leaves_out, bool **is_in_tree_out, size_t *leaf_count); + H5RT_leaf_t **leaves_out, bool **is_in_tree_out, size_t *leaf_count); /*********************/ /* Package Variables */ /*********************/ @@ -927,9 +926,9 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) hid_t orig_source_dapl; H5P_genplist_t *plist; size_t i; - herr_t ret_value = SUCCEED; - bool *new_in_tree = NULL; - H5RT_t *new_tree = NULL; + herr_t ret_value = SUCCEED; + bool *new_in_tree = NULL; + H5RT_t *new_tree = NULL; FUNC_ENTER_PACKAGE @@ -1074,8 +1073,9 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) /* Copy is_in_tree array */ memcpy(new_in_tree, virt->is_in_tree, virt->list_nused * sizeof(bool)); virt->is_in_tree = new_in_tree; - } else { - virt->tree = NULL; + } + else { + virt->tree = NULL; virt->is_in_tree = NULL; } @@ -2816,15 +2816,17 @@ H5D__virtual_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t H5_ATTR_UNUSED * } /* end H5D__virtual_io_init() */ // TODO -static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, H5S_t *file_space, - H5S_t *mem_space, hsize_t *tot_nelmts, H5O_storage_virtual_ent_t *curr_mapping) { +static herr_t +H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, H5S_t *file_space, H5S_t *mem_space, + hsize_t *tot_nelmts, H5O_storage_virtual_ent_t *curr_mapping) +{ const H5D_t *dset = dset_info->dset; /* Local pointer to dataset info */ hssize_t select_nelmts; /* Number of elements in selection */ hsize_t bounds_start[H5S_MAX_RANK]; /* Selection bounds start */ hsize_t bounds_end[H5S_MAX_RANK]; /* Selection bounds end */ int rank = 0; bool bounds_init = false; /* Whether bounds_start, bounds_end, and rank are valid */ - size_t j, k; /* Local index variables */ + size_t j, k; /* Local index variables */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -2849,7 +2851,7 @@ static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); /* Adjust bounds_end to represent the extent just enclosing them - * (add 1) */ + * (add 1) */ for (j = 0; j < (size_t)rank; j++) bounds_end[j]++; @@ -2858,9 +2860,8 @@ static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, } /* end if */ /* Get index of first block in virtual selection */ - curr_mapping->sub_dset_io_start = - (size_t)H5S_hyper_get_first_inc_block(curr_mapping->source_dset.virtual_select, - bounds_start[curr_mapping->unlim_dim_virtual], NULL); + curr_mapping->sub_dset_io_start = (size_t)H5S_hyper_get_first_inc_block( + curr_mapping->source_dset.virtual_select, bounds_start[curr_mapping->unlim_dim_virtual], NULL); /* Get index of first block outside of virtual selection */ curr_mapping->sub_dset_io_end = (size_t)H5S_hyper_get_first_inc_block( @@ -2880,26 +2881,25 @@ static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, assert((j == (curr_mapping->sub_dset_io_end - 1)) && partial_block); /* If the source space status is not correct, we must try to - * open the source dataset to patch it */ + * open the source dataset to patch it */ if (curr_mapping->source_space_status != H5O_VIRTUAL_STATUS_CORRECT) { assert(!curr_mapping->sub_dset[j].dset); - if (H5D__virtual_open_source_dset(dset, curr_mapping, - &curr_mapping->sub_dset[j]) < 0) + if (H5D__virtual_open_source_dset(dset, curr_mapping, &curr_mapping->sub_dset[j]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset"); } /* end if */ /* If we obtained a valid source space, we must create - * clipped source and virtual selections, otherwise we - * cannot do this and we will leave them NULL. This doesn't - * hurt anything because we can't do I/O because the dataset - * must not have been found. */ + * clipped source and virtual selections, otherwise we + * cannot do this and we will leave them NULL. This doesn't + * hurt anything because we can't do I/O because the dataset + * must not have been found. */ if (curr_mapping->source_space_status == H5O_VIRTUAL_STATUS_CORRECT) { hsize_t tmp_dims[H5S_MAX_RANK]; hsize_t vbounds_end[H5S_MAX_RANK]; /* Get bounds of virtual selection */ - if (H5S_SELECT_BOUNDS(curr_mapping->sub_dset[j].virtual_select, tmp_dims, - vbounds_end) < 0) + if (H5S_SELECT_BOUNDS(curr_mapping->sub_dset[j].virtual_select, tmp_dims, vbounds_end) < + 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); assert(bounds_init); @@ -2910,8 +2910,7 @@ static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, /* Temporarily set extent of virtual selection to bounds */ if (H5S_set_extent(curr_mapping->sub_dset[j].virtual_select, vbounds_end) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to modify size of dataspace"); + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of dataspace"); /* Get current VDS dimensions */ if (H5S_get_simple_extent_dims(dset->shared->space, tmp_dims, NULL) < 0) @@ -2919,18 +2918,18 @@ static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, /* Copy virtual selection */ if (NULL == (curr_mapping->sub_dset[j].clipped_virtual_select = - H5S_copy(curr_mapping->sub_dset[j].virtual_select, false, true))) + H5S_copy(curr_mapping->sub_dset[j].virtual_select, false, true))) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection"); /* Clip virtual selection to real virtual extent */ (void)memset(start, 0, sizeof(start)); - if (H5S_select_hyperslab(curr_mapping->sub_dset[j].clipped_virtual_select, - H5S_SELECT_AND, start, NULL, tmp_dims, NULL) < 0) + if (H5S_select_hyperslab(curr_mapping->sub_dset[j].clipped_virtual_select, H5S_SELECT_AND, + start, NULL, tmp_dims, NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSELECT, FAIL, "unable to clip hyperslab"); /* Project intersection of virtual space and clipped - * virtual space onto source space (create - * clipped_source_select) */ + * virtual space onto source space (create + * clipped_source_select) */ if (H5S_select_project_intersection( curr_mapping->sub_dset[j].virtual_select, curr_mapping->source_select, curr_mapping->sub_dset[j].clipped_virtual_select, @@ -2939,30 +2938,28 @@ static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, "can't project virtual intersection onto memory space"); /* Set extents of virtual_select and - * clipped_virtual_select to virtual extent */ + * clipped_virtual_select to virtual extent */ if (H5S_set_extent(curr_mapping->sub_dset[j].virtual_select, tmp_dims) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to modify size of dataspace"); + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of dataspace"); if (H5S_set_extent(curr_mapping->sub_dset[j].clipped_virtual_select, tmp_dims) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to modify size of dataspace"); + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of dataspace"); } /* end if */ } /* end if */ /* Only continue if we managed to obtain a - * clipped_virtual_select */ + * clipped_virtual_select */ if (curr_mapping->sub_dset[j].clipped_virtual_select) { /* Project intersection of file space and mapping virtual space - * onto memory space */ - if (H5S_select_project_intersection( - file_space, mem_space, curr_mapping->sub_dset[j].clipped_virtual_select, - &curr_mapping->sub_dset[j].projected_mem_space, true) < 0) + * onto memory space */ + if (H5S_select_project_intersection(file_space, mem_space, + curr_mapping->sub_dset[j].clipped_virtual_select, + &curr_mapping->sub_dset[j].projected_mem_space, true) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space"); /* Check number of elements selected */ - if ((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS( - curr_mapping->sub_dset[j].projected_mem_space)) < 0) + if ((select_nelmts = + (hssize_t)H5S_GET_SELECT_NPOINTS(curr_mapping->sub_dset[j].projected_mem_space)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to get number of elements in selection"); @@ -2971,23 +2968,20 @@ static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, /* Open source dataset */ if (!curr_mapping->sub_dset[j].dset) /* Try to open dataset */ - if (H5D__virtual_open_source_dset(dset, curr_mapping, - &curr_mapping->sub_dset[j]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, - "unable to open source dataset"); + if (H5D__virtual_open_source_dset(dset, curr_mapping, &curr_mapping->sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset"); /* If the source dataset is not open, mark the selected - * elements as zero so projected_mem_space is freed */ + * elements as zero so projected_mem_space is freed */ if (!curr_mapping->sub_dset[j].dset) select_nelmts = (hssize_t)0; } /* end if */ /* If there are not elements selected in this mapping, free - * projected_mem_space, otherwise update tot_nelmts */ + * projected_mem_space, otherwise update tot_nelmts */ if (select_nelmts == (hssize_t)0) { if (H5S_close(curr_mapping->sub_dset[j].projected_mem_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, - "can't close projected memory space"); + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space"); curr_mapping->sub_dset[j].projected_mem_space = NULL; } /* end if */ else @@ -2998,16 +2992,16 @@ static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, else { if (curr_mapping->source_dset.clipped_virtual_select) { /* Project intersection of file space and mapping virtual space onto - * memory space */ - if (H5S_select_project_intersection( - file_space, mem_space, curr_mapping->source_dset.clipped_virtual_select, - &curr_mapping->source_dset.projected_mem_space, true) < 0) + * memory space */ + if (H5S_select_project_intersection(file_space, mem_space, + curr_mapping->source_dset.clipped_virtual_select, + &curr_mapping->source_dset.projected_mem_space, true) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space"); /* Check number of elements selected, add to tot_nelmts */ - if ((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS( - curr_mapping->source_dset.projected_mem_space)) < 0) + if ((select_nelmts = + (hssize_t)H5S_GET_SELECT_NPOINTS(curr_mapping->source_dset.projected_mem_space)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to get number of elements in selection"); @@ -3016,18 +3010,17 @@ static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, /* Open source dataset */ if (!curr_mapping->source_dset.dset) /* Try to open dataset */ - if (H5D__virtual_open_source_dset(dset, curr_mapping, - &curr_mapping->source_dset) < 0) + if (H5D__virtual_open_source_dset(dset, curr_mapping, &curr_mapping->source_dset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset"); /* If the source dataset is not open, mark the selected elements - * as zero so projected_mem_space is freed */ + * as zero so projected_mem_space is freed */ if (!curr_mapping->source_dset.dset) select_nelmts = (hssize_t)0; } /* end if */ /* If there are not elements selected in this mapping, free - * projected_mem_space, otherwise update tot_nelmts */ + * projected_mem_space, otherwise update tot_nelmts */ if (select_nelmts == (hssize_t)0) { if (H5S_close(curr_mapping->source_dset.projected_mem_space) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space"); @@ -3038,10 +3031,10 @@ static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, } /* end if */ else { /* If there is no clipped_dim_virtual, this must be an unlimited - * selection whose dataset was not found in the last call to - * H5Dget_space(). Do not attempt to open it as this might - * affect the extent and we are not going to recalculate it - * here. */ + * selection whose dataset was not found in the last call to + * H5Dget_space(). Do not attempt to open it as this might + * affect the extent and we are not going to recalculate it + * here. */ assert(curr_mapping->unlim_dim_virtual >= 0); assert(!curr_mapping->source_dset.dset); } /* end else */ @@ -3066,11 +3059,11 @@ static herr_t H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, H5S_t *file_space, H5S_t *mem_space, hsize_t *tot_nelmts) { - const H5D_t *dset = dset_info->dset; /* Local pointer to dataset info */ - herr_t ret_value = SUCCEED; /* Return value */ - bool tree_enabled = false; - H5P_genplist_t *dapl_plist = NULL; /* Dataset access property list */ - H5RT_result_t *search_results = NULL; /* Search results from R-tree */ + const H5D_t *dset = dset_info->dset; /* Local pointer to dataset info */ + herr_t ret_value = SUCCEED; /* Return value */ + bool tree_enabled = false; + H5P_genplist_t *dapl_plist = NULL; /* Dataset access property list */ + H5RT_result_t *search_results = NULL; /* Search results from R-tree */ FUNC_ENTER_PACKAGE @@ -3113,9 +3106,9 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag if (storage->tree) { /* Perform a spatial tree search to get a list of mappings * whose virtual selection intersects the IO operation */ - H5RT_result_t *curr_result = NULL; /* Current result in search results */ - hsize_t min[H5S_MAX_RANK]; - hsize_t max[H5S_MAX_RANK]; + H5RT_result_t *curr_result = NULL; /* Current result in search results */ + hsize_t min[H5S_MAX_RANK]; + hsize_t max[H5S_MAX_RANK]; memset(min, 0, sizeof(min)); memset(max, 0, sizeof(max)); @@ -3135,7 +3128,7 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag size_t mapping_index = (size_t)curr_leaf->record; if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, - &storage->list[mapping_index]) < 0) + &storage->list[mapping_index]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); curr_result = curr_result->next; @@ -3148,14 +3141,15 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag } /* Iterate over the mappings that are not stored in the tree */ - /* Index of each boolean in the 'is in tree' list = index of the mapping it describes in the mapping list */ + /* Index of each boolean in the 'is in tree' list = index of the mapping it describes in the mapping list + */ for (size_t i = 0; i < storage->list_nused; i++) { /* Skip any mappings that would have been searched by the tree */ if (storage->is_in_tree && storage->is_in_tree[i]) continue; if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, - &storage->list[i]) < 0) + &storage->list[i]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); } @@ -3812,12 +3806,12 @@ H5D__virtual_release_source_dset_files(H5D_virtual_held_file_t *head) static herr_t H5D__rtree_should_insert(void *mapping_entry, bool *should_insert) { - herr_t ret_value = SUCCEED; - H5S_t *vspace = NULL; - H5S_t *src_space = NULL; - hsize_t virt_nelems = 0; - hsize_t src_nelems = 0; - H5O_storage_virtual_ent_t *entry = NULL; + herr_t ret_value = SUCCEED; + H5S_t *vspace = NULL; + H5S_t *src_space = NULL; + hsize_t virt_nelems = 0; + hsize_t src_nelems = 0; + H5O_storage_virtual_ent_t *entry = NULL; FUNC_ENTER_PACKAGE_NOERR @@ -3847,8 +3841,7 @@ H5D__rtree_should_insert(void *mapping_entry, bool *should_insert) } /* Do not insert zero-dim mappings */ - if ((H5S_GET_EXTENT_NDIMS(vspace)) < 1 || - (H5S_GET_EXTENT_NDIMS(src_space)) < 1) { + if ((H5S_GET_EXTENT_NDIMS(vspace)) < 1 || (H5S_GET_EXTENT_NDIMS(src_space)) < 1) { *should_insert = false; goto done; } @@ -3880,18 +3873,20 @@ H5D__rtree_should_insert(void *mapping_entry, bool *should_insert) *------------------------------------------------------------------------- */ static herr_t -H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, H5RT_leaf_t **leaves_out, bool **is_in_tree_out, size_t *leaf_count) { +H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, H5RT_leaf_t **leaves_out, + bool **is_in_tree_out, size_t *leaf_count) +{ herr_t ret_value = SUCCEED; H5RT_leaf_t *leaves_temp = NULL; - bool *is_in_tree = NULL; + bool *is_in_tree = NULL; bool should_insert_space = false; - H5O_storage_virtual_ent_t *curr_mapping = NULL; - H5RT_leaf_t *curr_leaf = NULL; - size_t curr_leaf_count = 0; - H5S_t *curr_space = NULL; + H5O_storage_virtual_ent_t *curr_mapping = NULL; + H5RT_leaf_t *curr_leaf = NULL; + size_t curr_leaf_count = 0; + H5S_t *curr_space = NULL; int rank = 0; @@ -3923,19 +3918,19 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings for (size_t i = 0; i < num_mappings; i++) { curr_mapping = &mappings[i]; - if (H5D__rtree_should_insert((void*)curr_mapping, &should_insert_space) < 0) + if (H5D__rtree_should_insert((void *)curr_mapping, &should_insert_space) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "error checking if mapping should be inserted"); - + if (!should_insert_space) continue; - + is_in_tree[i] = true; /* Initialize leaf with dynamic coordinate allocation */ curr_leaf = &leaves_temp[curr_leaf_count]; if (H5RT_leaf_init(curr_leaf, rank, (uintptr_t)i) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't initialize R-tree leaf"); - + /* Record is already set by H5RT_leaf_init */ if ((curr_space = mappings[i].source_dset.virtual_select) == NULL) @@ -3955,8 +3950,8 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings curr_leaf_count++; } - *leaves_out = leaves_temp; - *leaf_count = curr_leaf_count; + *leaves_out = leaves_temp; + *leaf_count = curr_leaf_count; *is_in_tree_out = is_in_tree; done: if (ret_value < 0) { @@ -3971,7 +3966,7 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings H5MM_free(is_in_tree); } - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value); } // attempt to build a tree and a boolean list and store them on the storage object @@ -3992,14 +3987,15 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) { - H5O_storage_virtual_ent_t *mappings = virt->list; - size_t num_mappings = virt->list_nused; +H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) +{ + H5O_storage_virtual_ent_t *mappings = virt->list; + size_t num_mappings = virt->list_nused; - H5RT_leaf_t *leaves = NULL; + H5RT_leaf_t *leaves = NULL; size_t num_leaves = 0; - bool *is_in_tree = NULL; - herr_t ret_value = SUCCEED; + bool *is_in_tree = NULL; + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE @@ -4019,10 +4015,10 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) { is_in_tree = NULL; } - virt->tree = NULL; + virt->tree = NULL; virt->is_in_tree = NULL; - - } else { + } + else { virt->is_in_tree = is_in_tree; /* Build the tree */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 5b216b20a99..3245bab64f8 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -604,9 +604,9 @@ typedef struct H5O_storage_virtual_t { H5O_storage_virtual_ent_t *source_dset_hash_table; /* Hash table of virtual entries sorted by source dataset name. Only the first occurrence of each source dataset name is stored. */ - H5RT_t* tree; - bool *is_in_tree; /* List of the indices in 'list' that are stored in tree for quick access - * Some mappings cannot be stored in the tree and must be searched manually */ + H5RT_t *tree; + bool *is_in_tree; /* List of the indices in 'list' that are stored in tree for quick access + * Some mappings cannot be stored in the tree and must be searched manually */ } H5O_storage_virtual_t; typedef struct H5O_storage_t { diff --git a/src/H5Pdapl.c b/src/H5Pdapl.c index 83003276f93..268de2f8223 100644 --- a/src/H5Pdapl.c +++ b/src/H5Pdapl.c @@ -181,7 +181,7 @@ static const H5D_append_flush_t H5D_def_append_flush_g = static const char *H5D_def_efile_prefix_g = H5D_ACS_EFILE_PREFIX_DEF; /* Default external file prefix string */ static const char *H5D_def_vds_prefix_g = H5D_ACS_VDS_PREFIX_DEF; /* Default vds prefix string */ -static const bool H5D_def_tree_g = H5D_ACS_USE_TREE_DEF; /* Default use of spatial tree for VDS mappings */ +static const bool H5D_def_tree_g = H5D_ACS_USE_TREE_DEF; /* Default use of spatial tree for VDS mappings */ /*------------------------------------------------------------------------- * Function: H5P__dacc_reg_prop @@ -1571,10 +1571,12 @@ H5Pget_virtual_prefix(hid_t plist_id, char *prefix /*out*/, size_t size) * *----------------------------------------------------------------------------- */ -herr_t H5Pget_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) { - bool setting = false; - H5P_genplist_t *plist = NULL; - herr_t ret_value = SUCCEED; +herr_t +H5Pget_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) +{ + bool setting = false; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; FUNC_ENTER_API(FAIL) @@ -1609,10 +1611,12 @@ herr_t H5Pget_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) { * *----------------------------------------------------------------------------- */ -herr_t H5Pset_dset_use_spatial_tree(hid_t dapl_id, bool use_tree) { - H5P_genplist_t *plist = NULL; - bool prev_set = false; - herr_t ret_value = SUCCEED; +herr_t +H5Pset_dset_use_spatial_tree(hid_t dapl_id, bool use_tree) +{ + H5P_genplist_t *plist = NULL; + bool prev_set = false; + herr_t ret_value = SUCCEED; FUNC_ENTER_API(FAIL) diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 50cee816de4..443944644cc 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -87,8 +87,8 @@ } #define H5D_DEF_STORAGE_VIRTUAL_INIT \ { \ - {HADDR_UNDEF, 0}, 0, NULL, 0, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ + {HADDR_UNDEF, 0}, 0, NULL, 0, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ H5D_VDS_ERROR, HSIZE_UNDEF, -1, -1, false, NULL, NULL, NULL, NULL \ } #define H5D_DEF_STORAGE_COMPACT \ @@ -2119,7 +2119,7 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const /* Expand list if necessary */ if (virtual_layout.storage.u.virt.list_nused == virtual_layout.storage.u.virt.list_nalloc) { H5O_storage_virtual_ent_t *x; /* Pointer to the new list */ - bool *y; /* Pointer to the new is_in_tree list */ + bool *y; /* Pointer to the new is_in_tree list */ size_t new_alloc = MAX(H5D_VIRTUAL_DEF_LIST_SIZE, virtual_layout.storage.u.virt.list_nalloc * 2); ptrdiff_t buf_diff; @@ -2132,10 +2132,12 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const virtual_layout.storage.u.virt.list_nalloc = new_alloc; /* Expand size of is_in_tree list */ - if (NULL == (y = (bool *)H5MM_realloc(virtual_layout.storage.u.virt.is_in_tree, new_alloc * sizeof(bool)))) - HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't reallocate virtual dataset mapping is_in_tree list"); + if (NULL == + (y = (bool *)H5MM_realloc(virtual_layout.storage.u.virt.is_in_tree, new_alloc * sizeof(bool)))) + HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, + "can't reallocate virtual dataset mapping is_in_tree list"); virtual_layout.storage.u.virt.is_in_tree = y; - + /* Adjust pointers in the hash tables in case realloc moved the buffers, and hence all the elements * and hash handles in the hash tables */ HASH_ADJUST_PTRS(hh_source_file, virtual_layout.storage.u.virt.source_file_hash_table, buf_diff); diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 40ab7d5f154..d6bd3ea419a 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -6010,7 +6010,7 @@ H5_DLL herr_t H5Pget_chunk_opts(hid_t plist_id, unsigned *opts); * */ H5_DLL herr_t H5Pget_dset_no_attrs_hint(hid_t dcpl_id, bool *minimize); - /** +/** * \ingroup DCPL * * \brief Retrieves the setting for whether or not to use a spatial tree diff --git a/test/rtree.c b/test/rtree.c index a7c5f782e13..077b290034d 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -32,7 +32,7 @@ #define H5D_FRIEND /*suppress error about including H5Dpkg */ #define H5D_TESTING -#include "H5Dpkg.h" /* Datasets */ +#include "H5Dpkg.h" /* Datasets */ #define RTREE_TEST_BASE_COORD 10000 #define RTREE_TEST_BASE_SIZE 1000 @@ -40,9 +40,9 @@ #define RTREE_TEST_CREATE_RANK 8 #define RTREE_TEST_CREATE_NUM_COUNTS 4 -#define RTREE_DAPL_FILENAME "vds_rtree_test.h5" -#define RTREE_DAPL_SRC_FILENAME "vds_src_rtree_test.h5" -#define RTREE_DAPL_VDS_NAME "vdset" +#define RTREE_DAPL_FILENAME "vds_rtree_test.h5" +#define RTREE_DAPL_SRC_FILENAME "vds_src_rtree_test.h5" +#define RTREE_DAPL_VDS_NAME "vdset" #define RTREE_DAPL_SRC_DATASET_NAME "src_dset" #define RTREE_DAPL_DATASET_DIM1 10 @@ -443,13 +443,13 @@ test_rtree_copy(void) static hid_t create_virtual_dataset(hid_t file_id, hid_t dapl_id) { - hid_t vspace_id = H5I_INVALID_HID; + hid_t vspace_id = H5I_INVALID_HID; hid_t srcspace_id = H5I_INVALID_HID; - hid_t srcfile_id = H5I_INVALID_HID; - hid_t srcdset_id = H5I_INVALID_HID; - hid_t vdset_id = H5I_INVALID_HID; - hid_t dcpl_id = H5I_INVALID_HID; - hsize_t dims[2] = {RTREE_DAPL_DATASET_DIM1, RTREE_DAPL_DATASET_DIM2}; + hid_t srcfile_id = H5I_INVALID_HID; + hid_t srcdset_id = H5I_INVALID_HID; + hid_t vdset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hsize_t dims[2] = {RTREE_DAPL_DATASET_DIM1, RTREE_DAPL_DATASET_DIM2}; int wbuf[RTREE_DAPL_DATASET_DIM1][RTREE_DAPL_DATASET_DIM2]; int i, j; @@ -465,18 +465,19 @@ create_virtual_dataset(hid_t file_id, hid_t dapl_id) /* Create source dataset */ if ((srcdset_id = H5Dcreate2(srcfile_id, RTREE_DAPL_SRC_DATASET_NAME, H5T_NATIVE_INT, srcspace_id, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error; - if (H5Pset_virtual(dcpl_id, vspace_id, "vds_src_rtree_test.h5", RTREE_DAPL_SRC_DATASET_NAME, srcspace_id) < 0) + if (H5Pset_virtual(dcpl_id, vspace_id, "vds_src_rtree_test.h5", RTREE_DAPL_SRC_DATASET_NAME, + srcspace_id) < 0) goto error; /* Create virtual dataset */ - if ((vdset_id = H5Dcreate2(file_id, RTREE_DAPL_VDS_NAME, H5T_NATIVE_INT, vspace_id, - H5P_DEFAULT, dcpl_id, dapl_id)) < 0) + if ((vdset_id = H5Dcreate2(file_id, RTREE_DAPL_VDS_NAME, H5T_NATIVE_INT, vspace_id, H5P_DEFAULT, dcpl_id, + dapl_id)) < 0) goto error; /* Initialize and write data to source dataset */ @@ -502,14 +503,16 @@ create_virtual_dataset(hid_t file_id, hid_t dapl_id) return vdset_id; error: /* Cleanup */ - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Sclose(vspace_id); H5Sclose(srcspace_id); H5Dclose(srcdset_id); H5Fclose(srcfile_id); H5Dclose(vdset_id); H5Pclose(dcpl_id); - } H5E_END_TRY; + } + H5E_END_TRY; return FAIL; } @@ -527,15 +530,15 @@ create_virtual_dataset(hid_t file_id, hid_t dapl_id) static herr_t test_rtree_dapl(bool use_tree) { - hid_t file_id = H5I_INVALID_HID; - hid_t dapl_id = H5I_INVALID_HID; - hid_t vdset_id = H5I_INVALID_HID; + hid_t file_id = H5I_INVALID_HID; + hid_t dapl_id = H5I_INVALID_HID; + hid_t vdset_id = H5I_INVALID_HID; - int rbuf[RTREE_DAPL_DATASET_DIM1][RTREE_DAPL_DATASET_DIM2]; + int rbuf[RTREE_DAPL_DATASET_DIM1][RTREE_DAPL_DATASET_DIM2]; const char *test_str = NULL; /* Internal values for introspection */ - H5D_t *dset = NULL; + H5D_t *dset = NULL; H5O_storage_virtual_t *storage = NULL; /* Inverse of use_tree for re-open part of test */ @@ -543,7 +546,8 @@ test_rtree_dapl(bool use_tree) if (use_tree) { test_str = "spatial tree option enabled"; - } else { + } + else { test_str = "spatial tree option disabled"; } @@ -600,7 +604,8 @@ test_rtree_dapl(bool use_tree) puts("Expected is_in_tree array to exist but it was NULL"); FAIL_STACK_ERROR; } - } else { + } + else { if (storage->tree != NULL) { puts("Expected spatial tree to be NULL but it exists"); FAIL_STACK_ERROR; @@ -631,7 +636,8 @@ test_rtree_dapl(bool use_tree) for (int j = 0; j < RTREE_DAPL_DATASET_DIM2; j++) { int expected = i * RTREE_DAPL_DATASET_DIM2 + j; if (rbuf[i][j] != expected) { - printf("Data mismatch after re-open at [%d][%d]: expected %d, got %d\n", i, j, expected, rbuf[i][j]); + printf("Data mismatch after re-open at [%d][%d]: expected %d, got %d\n", i, j, expected, + rbuf[i][j]); FAIL_STACK_ERROR; } } @@ -644,7 +650,6 @@ test_rtree_dapl(bool use_tree) if (dset->shared->layout.type != H5D_VIRTUAL) FAIL_STACK_ERROR; - storage = &(dset->shared->layout.storage.u.virt); /* Verify tree existence matches expectation */ @@ -657,7 +662,8 @@ test_rtree_dapl(bool use_tree) puts("Expected is_in_tree array to exist but it was NULL"); FAIL_STACK_ERROR; } - } else { + } + else { if (storage->tree != NULL) { puts("Expected spatial tree to be NULL but it exists"); FAIL_STACK_ERROR; @@ -672,19 +678,21 @@ test_rtree_dapl(bool use_tree) if (H5Dclose(vdset_id) < 0) FAIL_STACK_ERROR; if (H5Pclose(dapl_id) < 0) - FAIL_STACK_ERROR; + FAIL_STACK_ERROR; if (H5Fclose(file_id) < 0) - FAIL_STACK_ERROR; + FAIL_STACK_ERROR; PASSED(); return SUCCEED; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Dclose(vdset_id); H5Pclose(dapl_id); H5Fclose(file_id); - } H5E_END_TRY; + } + H5E_END_TRY; return FAIL; } From cda3af8144a003554c3d88c6b862e8d4a3b0cdd7 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Thu, 18 Sep 2025 14:51:12 -0500 Subject: [PATCH 05/43] Move release note --- release_docs/CHANGELOG.md | 20 ++++++++++++++++++++ release_docs/release_archive.txt | 20 -------------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/release_docs/CHANGELOG.md b/release_docs/CHANGELOG.md index ce89a350b8f..c512c00c6af 100644 --- a/release_docs/CHANGELOG.md +++ b/release_docs/CHANGELOG.md @@ -461,6 +461,26 @@ Simple example programs showing how to use complex number datatypes have been ad This layout copy is now delayed until either a user requests the DCPL, or until the start of an operation that needs to read the layout from the DCPL. +### Virtual datasets now use a spatial tree to optimize searches + + Virtual dataset operations with many (>1,000) mappings were much slower than + corresponding operations on normal datasets. This was due to the need + to iterate through every source dataset's dataspace and check for an intersection + with the user-selected region for a read/write in the virtual dataset. + + Virtual datasets now use an r-tree (defined in H5RT.c) to perform a spatial search. + This allows the dataspaces that intersect the user-selection to be computed with, + in most cases, vastly fewer costly intersection checks, improving the speed of VDS + read/write operations. + + Virtual datasets will use the r-tree by default, since the majority of use cases, + should see improvements from use of the tree. However, because some workflows may + find that the overhead of the tree outweighs the time saved on searches, there is + a new Dataset Access Property List (DAPL) property to control use of the spatial tree. + + This property can be set or queried with the new API functions + H5Pset_dset_use_spatial_tree()/H5Pget_dset_use_spatial_tree(). + ## Parallel Library ### Added H5FDsubfiling_get_file_mapping() API function for subfiling VFD diff --git a/release_docs/release_archive.txt b/release_docs/release_archive.txt index 20d450af590..d7082a87139 100644 --- a/release_docs/release_archive.txt +++ b/release_docs/release_archive.txt @@ -700,26 +700,6 @@ New Features library behavior, and the connector ID and information could not be read back from that plist later. - - Virtual datasets use a spatial tree to optimize searches - - Virtual dataset operations with many (>1,000) mappings were much slower than - corresponding operations on normal datasets. This was due to the need - to iterate through every source dataset's dataspace and check for an intersection - with the user-selected region for a read/write in the virtual dataset. - - Virtual datasets now use an r-tree (defined in H5RT.c) to perform a spatial search. - This allows the dataspaces that intersect the user-selection to be computed with, - in most cases, vastly fewer costly intersection checks, improving the speed of VDS - read/write operations. - - Virtual datasets will use the r-tree by default, since the majority of use cases, - should see improvements from use of the tree. However, because some workflows may - find that the overhead of the tree outweighs the time saved on searches, there is - a new Dataset Access Property List (DAPL) property to control use of the spatial tree. - - This property can be set or queried with the new API functions - H5Pset_dset_use_spatial_tree()/H5Pget_dset_use_spatial_tree(). - Parallel Library: ----------------- - From 7a8e36312edabf39e6520ae037e2fbb94bf80a6e Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Mon, 22 Sep 2025 10:58:34 -0500 Subject: [PATCH 06/43] Add threshold for r-tree usage --- doxygen/examples/tables/propertyLists.dox | 2 +- fortran/src/H5Pff.F90 | 2 +- release_docs/CHANGELOG.md | 8 +- src/H5Dprivate.h | 3 + src/H5Dvirtual.c | 2 +- test/rtree.c | 259 +++++++++++++++++----- 6 files changed, 216 insertions(+), 60 deletions(-) diff --git a/doxygen/examples/tables/propertyLists.dox b/doxygen/examples/tables/propertyLists.dox index dde08728d6e..1983b5adce4 100644 --- a/doxygen/examples/tables/propertyLists.dox +++ b/doxygen/examples/tables/propertyLists.dox @@ -618,7 +618,7 @@ encoding for object names. #H5Pset_dset_use_spatial_tree/#H5Pget_dset_use_spatial_tree -Sets/gets the flag to use spatial trees when searching VDS mappings +Sets/gets the flag to use spatial trees when searching many VDS mappings //! [dapl_table] diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index 6b1e5840d97..6c775856f6e 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -4404,7 +4404,7 @@ END SUBROUTINE h5pget_dset_use_spatial_tree_f !! \ingroup FH5P !! !! \brief Sets the value of the "use spatial tree" flag which enables spatial tree -!! construction and usage for VDS mapping searches. +!! construction and usage for large VDS mapping searches. !! !! \param dapl_id Target dataset access property list identifier. !! \param use_tree Value of the setting. diff --git a/release_docs/CHANGELOG.md b/release_docs/CHANGELOG.md index c512c00c6af..887141afc39 100644 --- a/release_docs/CHANGELOG.md +++ b/release_docs/CHANGELOG.md @@ -468,10 +468,10 @@ Simple example programs showing how to use complex number datatypes have been ad to iterate through every source dataset's dataspace and check for an intersection with the user-selected region for a read/write in the virtual dataset. - Virtual datasets now use an r-tree (defined in H5RT.c) to perform a spatial search. - This allows the dataspaces that intersect the user-selection to be computed with, - in most cases, vastly fewer costly intersection checks, improving the speed of VDS - read/write operations. + Virtual datasets with many mappings now use an r-tree (defined in H5RT.c) to + perform a spatial search. This allows the dataspaces that intersect the + user-selection to be computed with, in most cases, much fewer intersection checks, + improving the speed of VDS read/write operations. Virtual datasets will use the r-tree by default, since the majority of use cases, should see improvements from use of the tree. However, because some workflows may diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index f8c78d9ab0a..a6175492016 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -125,6 +125,9 @@ /* Default virtual dataset list size */ #define H5D_VIRTUAL_DEF_LIST_SIZE 8 +/* Threshold for use of a tree for VDS mappings */ +#define H5D_VIRTUAL_TREE_THRESHOLD 50 + #ifdef H5D_MODULE #define H5D_OBJ_ID(D) (((H5D_obj_create_t *)(D))->dcpl_id) #else /* H5D_MODULE */ diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index c9ed2bd971b..fb305c14573 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -3088,7 +3088,7 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag if (H5P_get(dapl_plist, H5D_ACS_USE_TREE_NAME, &tree_enabled) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual use tree flag"); - if (tree_enabled && !storage->tree) { + if (tree_enabled && !storage->tree && storage->list_nused >= H5D_VIRTUAL_TREE_THRESHOLD) { int rank = 0; assert(!storage->is_in_tree); /* Get the rank of the dataset */ diff --git a/test/rtree.c b/test/rtree.c index 077b290034d..51afc32df7a 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -40,6 +40,8 @@ #define RTREE_TEST_CREATE_RANK 8 #define RTREE_TEST_CREATE_NUM_COUNTS 4 +#define RTREE_SRC_FILENAME "vds_src_file.h5" + #define RTREE_DAPL_FILENAME "vds_rtree_test.h5" #define RTREE_DAPL_SRC_FILENAME "vds_src_rtree_test.h5" #define RTREE_DAPL_VDS_NAME "vdset" @@ -48,7 +50,11 @@ #define RTREE_DAPL_DATASET_DIM1 10 #define RTREE_DAPL_DATASET_DIM2 10 -static const size_t test_counts[RTREE_TEST_CREATE_NUM_COUNTS] = {1, 100, 500, 10000}; +#define RTREE_THRESHOLD_FILENAME "vds_rtree_threshold_test.h5" +#define RTREE_MAX_TEST_MAPPINGS (H5D_VIRTUAL_TREE_THRESHOLD + 10) + +static const size_t test_counts[RTREE_TEST_CREATE_NUM_COUNTS] = {H5D_VIRTUAL_TREE_THRESHOLD, 100, 1000, + 10000}; /* Helper function to generate leaf data */ static H5RT_leaf_t *generate_leaves(int rank, size_t leaf_count); @@ -65,7 +71,7 @@ static herr_t verify_rtree_search(H5RT_result_set_t *result_set, H5RT_leaf_t *le hsize_t min[], hsize_t max[], int rank); /* Helper to create and initialize virtual dset in a file */ -static hid_t create_virtual_dataset(hid_t file_id, hid_t dapl_id); +static hid_t create_virtual_dataset(hid_t file_id, hid_t dapl_id, int num_mappings); static herr_t verify_rtree_search(H5RT_result_set_t *result_set, H5RT_leaf_t *leaves, size_t leaf_count, hsize_t min[], @@ -433,7 +439,7 @@ test_rtree_copy(void) /*------------------------------------------------------------------------- * Function: create_virtual_dataset * - * Purpose: Helper function to create a virtual dataset with mappings + * Purpose: Helper function to create a 1D virtual dataset with mappings * * Return: Success: dataset ID * Failure: H5I_INVALID_HID @@ -441,60 +447,85 @@ test_rtree_copy(void) *------------------------------------------------------------------------- */ static hid_t -create_virtual_dataset(hid_t file_id, hid_t dapl_id) +create_virtual_dataset(hid_t file_id, hid_t dapl_id, int num_mappings) { hid_t vspace_id = H5I_INVALID_HID; hid_t srcspace_id = H5I_INVALID_HID; + hid_t vsel_id = H5I_INVALID_HID; hid_t srcfile_id = H5I_INVALID_HID; hid_t srcdset_id = H5I_INVALID_HID; hid_t vdset_id = H5I_INVALID_HID; hid_t dcpl_id = H5I_INVALID_HID; - hsize_t dims[2] = {RTREE_DAPL_DATASET_DIM1, RTREE_DAPL_DATASET_DIM2}; - int wbuf[RTREE_DAPL_DATASET_DIM1][RTREE_DAPL_DATASET_DIM2]; - int i, j; - - if ((vspace_id = H5Screate_simple(2, dims, NULL)) < 0) - goto error; - - if ((srcspace_id = H5Screate_simple(2, dims, NULL)) < 0) - goto error; - - /* Create source file */ - if ((srcfile_id = H5Fcreate(RTREE_DAPL_SRC_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + hsize_t vdims[1] = {(hsize_t)num_mappings}; + hsize_t srcdims[1] = {1}; + hsize_t start[1], count[1]; + char srcdset_name[256]; + int wdata; + int i; + + /* Create 1D virtual dataset space */ + if ((vspace_id = H5Screate_simple(1, vdims, NULL)) < 0) goto error; - /* Create source dataset */ - if ((srcdset_id = H5Dcreate2(srcfile_id, RTREE_DAPL_SRC_DATASET_NAME, H5T_NATIVE_INT, srcspace_id, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + /* Create 1D source dataset space (single element) */ + if ((srcspace_id = H5Screate_simple(1, srcdims, NULL)) < 0) goto error; + /* Create dataset creation property list */ if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error; - if (H5Pset_virtual(dcpl_id, vspace_id, "vds_src_rtree_test.h5", RTREE_DAPL_SRC_DATASET_NAME, - srcspace_id) < 0) + /* Create source file */ + if ((srcfile_id = H5Fcreate(RTREE_SRC_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; + /* Create multiple source dsets and add virtual mappings */ + for (i = 0; i < num_mappings; i++) { + sprintf(srcdset_name, "src_dset_%d", i); + + /* Create source dataset */ + if ((srcdset_id = H5Dcreate2(srcfile_id, srcdset_name, H5T_NATIVE_INT, srcspace_id, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* Write data to source dataset (value equals index) */ + wdata = i; + if (H5Dwrite(srcdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &wdata) < 0) + goto error; + + /* Create hyperslab selection for virtual dataset (one element at position i) */ + if ((vsel_id = H5Scopy(vspace_id)) < 0) + goto error; + + start[0] = (hsize_t)i; + count[0] = 1; + if (H5Sselect_hyperslab(vsel_id, H5S_SELECT_SET, start, NULL, count, NULL) < 0) + goto error; + + /* Add virtual mapping */ + if (H5Pset_virtual(dcpl_id, vsel_id, RTREE_SRC_FILENAME, srcdset_name, srcspace_id) < 0) + goto error; + + /* Close source dataset and selection */ + if (H5Dclose(srcdset_id) < 0) + goto error; + if (H5Sclose(vsel_id) < 0) + goto error; + + srcdset_id = H5I_INVALID_HID; + vsel_id = H5I_INVALID_HID; + } + /* Create virtual dataset */ if ((vdset_id = H5Dcreate2(file_id, RTREE_DAPL_VDS_NAME, H5T_NATIVE_INT, vspace_id, H5P_DEFAULT, dcpl_id, dapl_id)) < 0) goto error; - /* Initialize and write data to source dataset */ - for (i = 0; i < RTREE_DAPL_DATASET_DIM1; i++) - for (j = 0; j < RTREE_DAPL_DATASET_DIM2; j++) - wbuf[i][j] = i * RTREE_DAPL_DATASET_DIM2 + j; - - if (H5Dwrite(srcdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) - goto error; - /* Cleanup */ if (H5Sclose(vspace_id) < 0) goto error; if (H5Sclose(srcspace_id) < 0) goto error; - if (H5Dclose(srcdset_id) < 0) - goto error; if (H5Fclose(srcfile_id) < 0) goto error; if (H5Pclose(dcpl_id) < 0) @@ -507,6 +538,7 @@ create_virtual_dataset(hid_t file_id, hid_t dapl_id) { H5Sclose(vspace_id); H5Sclose(srcspace_id); + H5Sclose(vsel_id); H5Dclose(srcdset_id); H5Fclose(srcfile_id); H5Dclose(vdset_id); @@ -520,7 +552,7 @@ create_virtual_dataset(hid_t file_id, hid_t dapl_id) /*------------------------------------------------------------------------- * Function: test_rtree_dapl * - * Purpose: Test R-tree options on the DCPL + * Purpose: Test R-tree options on the DAPL * * Return: Success: SUCCEED * Failure: FAIL @@ -534,7 +566,7 @@ test_rtree_dapl(bool use_tree) hid_t dapl_id = H5I_INVALID_HID; hid_t vdset_id = H5I_INVALID_HID; - int rbuf[RTREE_DAPL_DATASET_DIM1][RTREE_DAPL_DATASET_DIM2]; + int rbuf[H5D_VIRTUAL_TREE_THRESHOLD + 10]; const char *test_str = NULL; /* Internal values for introspection */ @@ -565,22 +597,19 @@ test_rtree_dapl(bool use_tree) if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree) < 0) FAIL_STACK_ERROR; - /* Create virtual dataset with some mappings */ - if ((vdset_id = create_virtual_dataset(file_id, dapl_id)) < 0) + /* Create virtual dataset with enough mappings to use tree */ + if ((vdset_id = create_virtual_dataset(file_id, dapl_id, RTREE_MAX_TEST_MAPPINGS)) < 0) FAIL_STACK_ERROR; /* Read the entire virtual dataset to force tree initialization */ if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) FAIL_STACK_ERROR; - /* Verify read data matches expected pattern from create_virtual_dataset */ - for (int i = 0; i < RTREE_DAPL_DATASET_DIM1; i++) { - for (int j = 0; j < RTREE_DAPL_DATASET_DIM2; j++) { - int expected = i * RTREE_DAPL_DATASET_DIM2 + j; - if (rbuf[i][j] != expected) { - printf("Data mismatch at [%d][%d]: expected %d, got %d\n", i, j, expected, rbuf[i][j]); - FAIL_STACK_ERROR; - } + /* Verify read data matches expected pattern */ + for (int i = 0; i < (H5D_VIRTUAL_TREE_THRESHOLD + 10); i++) { + if (rbuf[i] != i) { + printf("Data mismatch at [%d]: expected %d, got %d\n", i, i, rbuf[i]); + FAIL_STACK_ERROR; } } @@ -631,15 +660,11 @@ test_rtree_dapl(bool use_tree) if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) FAIL_STACK_ERROR; - /* Verify read data matches expected pattern from create_virtual_dataset */ - for (int i = 0; i < RTREE_DAPL_DATASET_DIM1; i++) { - for (int j = 0; j < RTREE_DAPL_DATASET_DIM2; j++) { - int expected = i * RTREE_DAPL_DATASET_DIM2 + j; - if (rbuf[i][j] != expected) { - printf("Data mismatch after re-open at [%d][%d]: expected %d, got %d\n", i, j, expected, - rbuf[i][j]); - FAIL_STACK_ERROR; - } + /* Verify read data matches expected pattern */ + for (int i = 0; i < (H5D_VIRTUAL_TREE_THRESHOLD + 10); i++) { + if (rbuf[i] != i) { + printf("Data mismatch after re-open at [%d]: expected %d, got %d\n", i, i, rbuf[i]); + FAIL_STACK_ERROR; } } @@ -697,6 +722,130 @@ test_rtree_dapl(bool use_tree) return FAIL; } +/*------------------------------------------------------------------------- + * Function: test_rtree_threshold + * + * Purpose: Test that threshold controls r-tree usage properly + * + * Return: Success: SUCCEED + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_rtree_threshold(bool use_tree) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t dapl_id = H5I_INVALID_HID; + hid_t vdset_id = H5I_INVALID_HID; + int rbuf[RTREE_MAX_TEST_MAPPINGS]; + + /* Internal values for introspection */ + H5D_t *dset = NULL; + H5O_storage_virtual_t *storage = NULL; + + const char *test_str = + use_tree ? "threshold behavior with tree enabled" : "threshold behavior with tree disabled"; + + TESTING(test_str); + + /* Test cases: below threshold, at threshold, above threshold */ + int test_cases[3] = {H5D_VIRTUAL_TREE_THRESHOLD - 1, H5D_VIRTUAL_TREE_THRESHOLD, RTREE_MAX_TEST_MAPPINGS}; + + for (int test_idx = 0; test_idx < 3; test_idx++) { + int num_mappings = test_cases[test_idx]; + bool expect_tree; + + /* Determine expected tree behavior based on threshold and use_tree setting */ + /* Tree is created only when: tree_enabled AND num_mappings >= threshold */ + expect_tree = (use_tree && (num_mappings >= H5D_VIRTUAL_TREE_THRESHOLD)); + + if ((file_id = H5Fcreate(RTREE_THRESHOLD_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + + /* Set the spatial tree property */ + if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree) < 0) + FAIL_STACK_ERROR; + + /* Create virtual dataset with specified number of mappings */ + if ((vdset_id = create_virtual_dataset(file_id, dapl_id, num_mappings)) < 0) + FAIL_STACK_ERROR; + + /* Read the virtual dataset to force initialization */ + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data pattern (each element should equal its index) */ + for (int i = 0; i < num_mappings; i++) { + if (rbuf[i] != i) { + printf("%d mappings: Data mismatch at [%d]: expected %d, got %d\n", num_mappings, i, i, + rbuf[i]); + FAIL_STACK_ERROR; + } + } + + /* Get the dataset object for introspection */ + if (NULL == (dset = (H5D_t *)H5VL_object(vdset_id))) + FAIL_STACK_ERROR; + + if (dset->shared->layout.type != H5D_VIRTUAL) + FAIL_STACK_ERROR; + + /* Get the virtual storage structure */ + storage = &(dset->shared->layout.storage.u.virt); + + /* Verify tree existence matches expectation */ + if (expect_tree) { + if (storage->tree == NULL) { + printf("%d mappings: Expected spatial tree to exist but it was NULL\n", num_mappings); + FAIL_STACK_ERROR; + } + if (storage->is_in_tree == NULL) { + printf("%d mappings: Expected is_in_tree array to exist but it was NULL\n", num_mappings); + FAIL_STACK_ERROR; + } + } + else { + if (storage->tree != NULL) { + printf("%d mappings: Expected spatial tree to be NULL but it exists\n", num_mappings); + FAIL_STACK_ERROR; + } + if (storage->is_in_tree != NULL) { + printf("%d mappings: Expected is_in_tree array to be NULL but it exists\n", num_mappings); + FAIL_STACK_ERROR; + } + } + + /* Cleanup */ + if (H5Dclose(vdset_id) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dapl_id) < 0) + FAIL_STACK_ERROR; + if (H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + vdset_id = H5I_INVALID_HID; + dapl_id = H5I_INVALID_HID; + file_id = H5I_INVALID_HID; + } + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Dclose(vdset_id); + H5Pclose(dapl_id); + H5Fclose(file_id); + } + H5E_END_TRY; + + return FAIL; +} /*------------------------------------------------------------------------- * Function: main * @@ -721,10 +870,14 @@ main(void) nerrors += test_rtree_search() < 0 ? 1 : 0; nerrors += test_rtree_copy() < 0 ? 1 : 0; - /* Test spatial tree with DCPL property enabled */ + /* Test spatial tree with DAPL property enabled */ nerrors += test_rtree_dapl(true) < 0 ? 1 : 0; nerrors += test_rtree_dapl(false) < 0 ? 1 : 0; + /* Test the mapping count threshold */ + nerrors += test_rtree_threshold(true) < 0 ? 1 : 0; + nerrors += test_rtree_threshold(false) < 0 ? 1 : 0; + if (nerrors) goto error; From 217be3455f3346472315b1d76222069102e4024e Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 23 Sep 2025 09:42:19 -0500 Subject: [PATCH 07/43] Add Fortran API test --- fortran/test/tH5P.F90 | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/fortran/test/tH5P.F90 b/fortran/test/tH5P.F90 index 5a317d5593c..5e066b28c9d 100644 --- a/fortran/test/tH5P.F90 +++ b/fortran/test/tH5P.F90 @@ -777,8 +777,10 @@ SUBROUTINE test_misc_properties(total_error) INTEGER, INTENT(INOUT) :: total_error INTEGER(hid_t) :: fapl_id = -1 ! Local fapl + INTEGER(hid_t) :: dapl_id = -1 ! Local dapl LOGICAL :: use_file_locking ! (H5Pset/get_file_locking_f) LOGICAL :: ignore_disabled_locks ! (H5Pset/get_file_locking_f) + LOGICAL :: use_spatial_tree ! (H5Pset/get_dset_use_spatial_tree_f) INTEGER :: error ! Create a default fapl @@ -826,6 +828,39 @@ SUBROUTINE test_misc_properties(total_error) CALL H5Pclose_f(fapl_id, error) CALL check("H5Pclose_f", error, total_error) + ! Create a dataset access property list + CALL H5Pcreate_f(H5P_DATASET_ACCESS_F, dapl_id, error) + CALL check("H5Pcreate_f", error, total_error) + + ! Test H5Pset/get_dset_use_spatial_tree_f + ! true value + use_spatial_tree = .TRUE. + CALL h5pset_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pset_dset_use_spatial_tree_f", error, total_error) + use_spatial_tree = .FALSE. + CALL h5pget_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pget_dset_use_spatial_tree_f", error, total_error) + if(use_spatial_tree .neqv. .TRUE.) then + total_error = total_error + 1 + write(*,*) "Got wrong use_spatial_tree flag from h5pget_dset_use_spatial_tree_f" + endif + + ! false value + use_spatial_tree = .FALSE. + CALL h5pset_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pset_dset_use_spatial_tree_f", error, total_error) + use_spatial_tree = .TRUE. + CALL h5pget_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pget_dset_use_spatial_tree_f", error, total_error) + if(use_spatial_tree .neqv. .FALSE.) then + total_error = total_error + 1 + write(*,*) "Got wrong use_spatial_tree flag from h5pget_dset_use_spatial_tree_f" + endif + + ! Close the dapl + CALL H5Pclose_f(dapl_id, error) + CALL check("H5Pclose_f", error, total_error) + END SUBROUTINE test_misc_properties !------------------------------------------------------------------------- From 9a84834922fd3000b871065a3dc074623429a807 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 23 Sep 2025 09:46:19 -0500 Subject: [PATCH 08/43] Delay DAPL check --- src/H5Dvirtual.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index fb305c14573..48c0cdbe11b 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -3082,24 +3082,28 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag *tot_nelmts = 0; /* If r-tree use is enabled and no tree currently exists, build the tree */ - if (NULL == (dapl_plist = (H5P_genplist_t *)H5I_object(dset->shared->dapl_id))) - HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for dapl ID"); - - if (H5P_get(dapl_plist, H5D_ACS_USE_TREE_NAME, &tree_enabled) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual use tree flag"); - - if (tree_enabled && !storage->tree && storage->list_nused >= H5D_VIRTUAL_TREE_THRESHOLD) { + if (!storage->tree && storage->list_nused >= H5D_VIRTUAL_TREE_THRESHOLD) { int rank = 0; assert(!storage->is_in_tree); - /* Get the rank of the dataset */ - if ((rank = H5S_GET_EXTENT_NDIMS(dset_info->dset->shared->space)) < 0 || rank >= H5S_MAX_RANK) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset rank"); - if (rank == 0) - HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "virtual dataset has no rank"); + /* Check that building the tree is enabled */ + if (NULL == (dapl_plist = (H5P_genplist_t *)H5I_object(dset->shared->dapl_id))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for dapl ID"); + + if (H5P_get(dapl_plist, H5D_ACS_USE_TREE_NAME, &tree_enabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual use tree flag"); - if (H5D__virtual_build_tree(storage, rank) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build virtual mapping tree"); + if (tree_enabled) { + /* Get the rank of the dataset */ + if ((rank = H5S_GET_EXTENT_NDIMS(dset_info->dset->shared->space)) < 0 || rank >= H5S_MAX_RANK) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset rank"); + + if (rank == 0) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "virtual dataset has no rank"); + + if (H5D__virtual_build_tree(storage, rank) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build virtual mapping tree"); + } } /* Iterate over the mappings */ From 391b001d122b5713b1a0f98d20ebdd167187586f Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 23 Sep 2025 09:47:26 -0500 Subject: [PATCH 09/43] Convert error check to assert --- src/H5Dvirtual.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 48c0cdbe11b..0d76636f4e2 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -3819,10 +3819,8 @@ H5D__rtree_should_insert(void *mapping_entry, bool *should_insert) FUNC_ENTER_PACKAGE_NOERR - if (!mapping_entry || !should_insert) { - ret_value = FAIL; - goto done; - } + assert(mapping_entry); + assert(should_insert); entry = (H5O_storage_virtual_ent_t *)mapping_entry; @@ -3936,9 +3934,8 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't initialize R-tree leaf"); /* Record is already set by H5RT_leaf_init */ - - if ((curr_space = mappings[i].source_dset.virtual_select) == NULL) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "mapping has no virtual space"); + assert(mappings[i].source_dset.virtual_select); + curr_space = mappings[i].source_dset.virtual_select; /* Get selection bounds */ if (H5S_SELECT_BOUNDS(curr_space, curr_leaf->min, curr_leaf->max) < 0) From c37bad580d209daf7b0888c8d802a46d875f6a27 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 23 Sep 2025 09:57:40 -0500 Subject: [PATCH 10/43] Convert should_insert helper to macro --- src/H5Dvirtual.c | 107 ++++++++++++++++++++--------------------------- 1 file changed, 46 insertions(+), 61 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 0d76636f4e2..7a37dcd423a 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -71,6 +71,51 @@ /* Default size for sub_dset array */ #define H5D_VIRTUAL_DEF_SUB_DSET_SIZE 128 +/* + * Determines whether a virtual dataset mapping entry should be inserted + * into the R-tree spatial index. + * + * Parameters: + * entry - H5O_storage_virtual_ent_t* mapping entry to check + * should_insert - bool output parameter set to true/false + * + */ +#define H5D_RTREE_SHOULD_INSERT(entry, should_insert) \ + do { \ + H5S_t *_vspace = NULL; \ + H5S_t *_src_space = NULL; \ + hsize_t _virt_nelems = 0; \ + hsize_t _src_nelems = 0; \ + \ + assert(entry); \ + \ + /* Get virtual space and element count */ \ + if ((_vspace = (entry)->source_dset.virtual_select) != NULL) \ + _virt_nelems = (hsize_t)H5S_GET_SELECT_NPOINTS(_vspace); \ + \ + /* Get source space and element count */ \ + if ((_src_space = (entry)->source_dset.clipped_source_select) != NULL) \ + _src_nelems = (hsize_t)H5S_GET_SELECT_NPOINTS(_src_space); \ + \ + /* Do not insert mappings with an unlimited dimension */ \ + if (_virt_nelems == H5S_UNLIMITED || _src_nelems == H5S_UNLIMITED) { \ + should_insert = false; \ + } \ + /* Do not insert printf-style mappings */ \ + else if ((entry)->psfn_nsubs > 0 || (entry)->psdn_nsubs > 0) { \ + should_insert = false; \ + } \ + /* Do not insert zero-dim mappings */ \ + else if ((_vspace && H5S_GET_EXTENT_NDIMS(_vspace) < 1) || \ + (_src_space && H5S_GET_EXTENT_NDIMS(_src_space) < 1)) { \ + should_insert = false; \ + } \ + /* Otherwise, we can insert it */ \ + else { \ + should_insert = true; \ + } \ + } while (0) + /******************/ /* Local Typedefs */ /******************/ @@ -111,7 +156,6 @@ static herr_t H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, /* R-tree helper functions */ static herr_t H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank); -static herr_t H5D__rtree_should_insert(void *mapping_entry, bool *should_insert); static herr_t H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, H5RT_leaf_t **leaves_out, bool **is_in_tree_out, size_t *leaf_count); /*********************/ @@ -3797,64 +3841,6 @@ H5D__virtual_release_source_dset_files(H5D_virtual_held_file_t *head) FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_release_source_dset_files() */ -/*------------------------------------------------------------------------- - * Function: H5D__rtree_should_insert - * - * Purpose: Determine whether the given mapping is valid for - * insertion into a spatial tree - * - * Return: Non-negative on success/Negative on failure - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D__rtree_should_insert(void *mapping_entry, bool *should_insert) -{ - herr_t ret_value = SUCCEED; - H5S_t *vspace = NULL; - H5S_t *src_space = NULL; - hsize_t virt_nelems = 0; - hsize_t src_nelems = 0; - H5O_storage_virtual_ent_t *entry = NULL; - - FUNC_ENTER_PACKAGE_NOERR - - assert(mapping_entry); - assert(should_insert); - - entry = (H5O_storage_virtual_ent_t *)mapping_entry; - - /* Do not insert mappings with an unlimited dimension */ - if ((vspace = entry->source_dset.virtual_select) != NULL) - virt_nelems = (hsize_t)H5S_GET_SELECT_NPOINTS(vspace); - - if ((src_space = entry->source_dset.clipped_source_select) != NULL) - src_nelems = (hsize_t)H5S_GET_SELECT_NPOINTS(src_space); - - if (virt_nelems == H5S_UNLIMITED || src_nelems == H5S_UNLIMITED) { - *should_insert = false; - goto done; - } - - /* Do not insert printf-style mappings */ - if (entry->psfn_nsubs > 0 || entry->psdn_nsubs > 0) { - *should_insert = false; - goto done; - } - - /* Do not insert zero-dim mappings */ - if ((H5S_GET_EXTENT_NDIMS(vspace)) < 1 || (H5S_GET_EXTENT_NDIMS(src_space)) < 1) { - *should_insert = false; - goto done; - } - - /* Otherwise, we can insert it */ - *should_insert = true; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} - /*------------------------------------------------------------------------- * Function: H5D__mappings_to_leaves * @@ -3920,8 +3906,7 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings for (size_t i = 0; i < num_mappings; i++) { curr_mapping = &mappings[i]; - if (H5D__rtree_should_insert((void *)curr_mapping, &should_insert_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "error checking if mapping should be inserted"); + H5D_RTREE_SHOULD_INSERT(curr_mapping, should_insert_space); if (!should_insert_space) continue; From 604143960692f32d8c43877bf15051d27de5d5c1 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 23 Sep 2025 10:08:18 -0500 Subject: [PATCH 11/43] Convert record to void pointer --- src/H5Dvirtual.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 7a37dcd423a..274ef76ada1 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -3173,10 +3173,8 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag H5RT_leaf_t *curr_leaf = curr_result->leaf; assert(curr_leaf); - size_t mapping_index = (size_t)curr_leaf->record; - if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, - &storage->list[mapping_index]) < 0) + (H5O_storage_virtual_ent_t *)curr_leaf->record) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); curr_result = curr_result->next; @@ -3915,7 +3913,7 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings /* Initialize leaf with dynamic coordinate allocation */ curr_leaf = &leaves_temp[curr_leaf_count]; - if (H5RT_leaf_init(curr_leaf, rank, (uintptr_t)i) < 0) + if (H5RT_leaf_init(curr_leaf, rank, (void *)curr_mapping) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't initialize R-tree leaf"); /* Record is already set by H5RT_leaf_init */ From 24877768fcf99f5d0197310354a06a31d0b8d36b Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 23 Sep 2025 13:27:17 -0500 Subject: [PATCH 12/43] Update results handling --- src/H5Dvirtual.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 274ef76ada1..f11652d8f34 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -3103,11 +3103,11 @@ static herr_t H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, H5S_t *file_space, H5S_t *mem_space, hsize_t *tot_nelmts) { - const H5D_t *dset = dset_info->dset; /* Local pointer to dataset info */ - herr_t ret_value = SUCCEED; /* Return value */ - bool tree_enabled = false; - H5P_genplist_t *dapl_plist = NULL; /* Dataset access property list */ - H5RT_result_t *search_results = NULL; /* Search results from R-tree */ + const H5D_t *dset = dset_info->dset; /* Local pointer to dataset info */ + herr_t ret_value = SUCCEED; /* Return value */ + bool tree_enabled = false; + H5P_genplist_t *dapl_plist = NULL; /* Dataset access property list */ + H5RT_result_set_t *search_results = NULL; /* Search results from R-tree */ FUNC_ENTER_PACKAGE @@ -3154,9 +3154,8 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag if (storage->tree) { /* Perform a spatial tree search to get a list of mappings * whose virtual selection intersects the IO operation */ - H5RT_result_t *curr_result = NULL; /* Current result in search results */ - hsize_t min[H5S_MAX_RANK]; - hsize_t max[H5S_MAX_RANK]; + hsize_t min[H5S_MAX_RANK]; + hsize_t max[H5S_MAX_RANK]; memset(min, 0, sizeof(min)); memset(max, 0, sizeof(max)); @@ -3168,16 +3167,13 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "R-tree search failed"); /* First, iterate over the mappings with an intersection found via the tree */ - curr_result = search_results; - while (curr_result) { - H5RT_leaf_t *curr_leaf = curr_result->leaf; + for (size_t i = 0; i < search_results->count; i++) { + H5RT_leaf_t *curr_leaf = search_results->results[i].leaf; assert(curr_leaf); if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, (H5O_storage_virtual_ent_t *)curr_leaf->record) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); - - curr_result = curr_result->next; } /* Free search results */ From e59e47e2873f13bbc5f56b49c5663357733c86f1 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 23 Sep 2025 15:08:12 -0500 Subject: [PATCH 13/43] Rework is_in_tree to not_in_tree_list --- src/H5Dvirtual.c | 306 +++++++++++++++++++++++++++++++---------------- src/H5Oprivate.h | 7 +- src/H5Pdcpl.c | 10 +- test/rtree.c | 31 +++-- 4 files changed, 225 insertions(+), 129 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index f11652d8f34..e540ab9663e 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -157,7 +157,9 @@ static herr_t H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, /* R-tree helper functions */ static herr_t H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank); static herr_t H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, - H5RT_leaf_t **leaves_out, bool **is_in_tree_out, size_t *leaf_count); + H5RT_leaf_t **leaves_out, H5O_storage_virtual_ent_t ***not_in_tree_out, + size_t *leaf_count, size_t *not_in_tree_count); +static herr_t H5D__should_build_tree(H5O_storage_virtual_t *storage, hid_t dapl_id, bool *should_build_tree); /*********************/ /* Package Variables */ /*********************/ @@ -964,15 +966,15 @@ H5D__virtual_load_layout(H5F_t *f, H5O_layout_t *layout) herr_t H5D__virtual_copy_layout(H5O_layout_t *layout) { - H5O_storage_virtual_ent_t *orig_list = NULL; - H5O_storage_virtual_t *virt = &layout->storage.u.virt; - hid_t orig_source_fapl; - hid_t orig_source_dapl; - H5P_genplist_t *plist; - size_t i; - herr_t ret_value = SUCCEED; - bool *new_in_tree = NULL; - H5RT_t *new_tree = NULL; + H5O_storage_virtual_ent_t *orig_list = NULL; + H5O_storage_virtual_ent_t **orig_not_in_tree_list = NULL; + H5O_storage_virtual_t *virt = &layout->storage.u.virt; + hid_t orig_source_fapl; + hid_t orig_source_dapl; + H5P_genplist_t *plist; + size_t i; + herr_t ret_value = SUCCEED; + H5RT_t *new_tree = NULL; FUNC_ENTER_PACKAGE @@ -987,12 +989,14 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) /* Save original entry list and top-level property lists and reset in layout * so the originals aren't closed on error */ - orig_source_fapl = virt->source_fapl; - virt->source_fapl = -1; - orig_source_dapl = virt->source_dapl; - virt->source_dapl = -1; - orig_list = virt->list; - virt->list = NULL; + orig_source_fapl = virt->source_fapl; + virt->source_fapl = -1; + orig_source_dapl = virt->source_dapl; + virt->source_dapl = -1; + orig_list = virt->list; + virt->list = NULL; + orig_not_in_tree_list = virt->not_in_tree_list; + virt->not_in_tree_list = NULL; /* Copy entry list */ if (virt->list_nused > 0) { @@ -1111,16 +1115,29 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy spatial tree"); virt->tree = new_tree; - if ((new_in_tree = H5MM_calloc(virt->list_nalloc * sizeof(bool))) == NULL) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate in-tree array"); - - /* Copy is_in_tree array */ - memcpy(new_in_tree, virt->is_in_tree, virt->list_nused * sizeof(bool)); - virt->is_in_tree = new_in_tree; + /* Copy not_in_tree_list (pointer array) */ + if (virt->not_in_tree_nused > 0) { + assert(orig_not_in_tree_list); + + /* Allocate new pointer array */ + if ((virt->not_in_tree_list = (H5O_storage_virtual_ent_t **)H5MM_calloc( + virt->not_in_tree_nalloc * sizeof(H5O_storage_virtual_ent_t *))) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "unable to allocate not_in_tree_list pointer array"); + + /* Point to corresponding entries in the new list */ + for (i = 0; i < virt->not_in_tree_nused; i++) { + ptrdiff_t offset = orig_not_in_tree_list[i] - orig_list; /* Calculate original offset */ + assert(offset >= 0 && (size_t)offset < virt->list_nused); /* Validate offset */ + virt->not_in_tree_list[i] = &virt->list[offset]; /* Point to new list entry */ + } + } } else { - virt->tree = NULL; - virt->is_in_tree = NULL; + virt->tree = NULL; + virt->not_in_tree_list = NULL; + virt->not_in_tree_nused = 0; + virt->not_in_tree_nalloc = 0; } /* Copy property lists */ @@ -1223,9 +1240,12 @@ H5D__virtual_free_layout_mappings(H5O_storage_virtual_t *virt) virt->tree = NULL; } - /* Destroy the list tracking which indices are in spatial tree */ - if (virt->is_in_tree) { - virt->is_in_tree = H5MM_xfree(virt->is_in_tree); + /* Destroy the pointer array tracking which mappings are not in spatial tree */ + if (virt->not_in_tree_list) { + /* Only free the pointer array itself - the entries are owned by the main list */ + virt->not_in_tree_list = H5MM_xfree(virt->not_in_tree_list); + virt->not_in_tree_nused = 0; + virt->not_in_tree_nalloc = 0; } /* Note the lack of a done: label. This is because there are no HGOTO_ERROR @@ -3103,11 +3123,10 @@ static herr_t H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, H5S_t *file_space, H5S_t *mem_space, hsize_t *tot_nelmts) { - const H5D_t *dset = dset_info->dset; /* Local pointer to dataset info */ - herr_t ret_value = SUCCEED; /* Return value */ - bool tree_enabled = false; - H5P_genplist_t *dapl_plist = NULL; /* Dataset access property list */ - H5RT_result_set_t *search_results = NULL; /* Search results from R-tree */ + const H5D_t *dset = dset_info->dset; /* Local pointer to dataset info */ + herr_t ret_value = SUCCEED; /* Return value */ + bool should_build_tree = true; + H5RT_result_set_t *search_results = NULL; /* Search results from R-tree */ FUNC_ENTER_PACKAGE @@ -3125,29 +3144,21 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag /* Initialize tot_nelmts */ *tot_nelmts = 0; - /* If r-tree use is enabled and no tree currently exists, build the tree */ - if (!storage->tree && storage->list_nused >= H5D_VIRTUAL_TREE_THRESHOLD) { - int rank = 0; - assert(!storage->is_in_tree); + if (H5D__should_build_tree(storage, dset->shared->dapl_id, &should_build_tree) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't determine if should build VDS tree"); - /* Check that building the tree is enabled */ - if (NULL == (dapl_plist = (H5P_genplist_t *)H5I_object(dset->shared->dapl_id))) - HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for dapl ID"); - - if (H5P_get(dapl_plist, H5D_ACS_USE_TREE_NAME, &tree_enabled) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual use tree flag"); + if (should_build_tree) { + int rank = 0; - if (tree_enabled) { - /* Get the rank of the dataset */ - if ((rank = H5S_GET_EXTENT_NDIMS(dset_info->dset->shared->space)) < 0 || rank >= H5S_MAX_RANK) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset rank"); + /* Get the rank of the dataset */ + if ((rank = H5S_GET_EXTENT_NDIMS(dset_info->dset->shared->space)) < 0 || rank >= H5S_MAX_RANK) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset rank"); - if (rank == 0) - HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "virtual dataset has no rank"); + if (rank == 0) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "virtual dataset has no rank"); - if (H5D__virtual_build_tree(storage, rank) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build virtual mapping tree"); - } + if (H5D__virtual_build_tree(storage, rank) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build virtual mapping tree"); } /* Iterate over the mappings */ @@ -3180,19 +3191,21 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag if (H5RT_free_results(search_results) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't free R-tree search results"); search_results = NULL; - } - - /* Iterate over the mappings that are not stored in the tree */ - /* Index of each boolean in the 'is in tree' list = index of the mapping it describes in the mapping list - */ - for (size_t i = 0; i < storage->list_nused; i++) { - /* Skip any mappings that would have been searched by the tree */ - if (storage->is_in_tree && storage->is_in_tree[i]) - continue; - if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, - &storage->list[i]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + /* Iterate over the mappings that are not stored in the tree */ + for (size_t i = 0; i < storage->not_in_tree_nused; i++) { + if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, + storage->not_in_tree_list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + } + } + else { + /* No tree - iterate over all mappings directly */ + for (size_t i = 0; i < storage->list_nused; i++) { + if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, + &storage->list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + } } done: @@ -3845,10 +3858,10 @@ H5D__virtual_release_source_dset_files(H5D_virtual_held_file_t *head) * num_mappings: Number of mappings in the array * leaves_out: Pointer to array of leaves, one per mapping that should be inserted * Allocated on success and must be freed by caller. - * is_in_tree_out: Pointer to boolean array, one per mapping, indicating - * whether the mapping is in the list of leaves. - * Allocated on success and must be freed by caller. + * not_in_tree_out: Pointer to array of pointers to mappings NOT in tree. + * Allocated on success and must be freed by caller. * leaf_count: Pointer to number of leaves allocated in leaves_out. + * not_in_tree_count: Pointer to number of entries in not_in_tree_out. * * Return: Non-negative on success/Negative on failure * @@ -3856,19 +3869,21 @@ H5D__virtual_release_source_dset_files(H5D_virtual_held_file_t *head) */ static herr_t H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, H5RT_leaf_t **leaves_out, - bool **is_in_tree_out, size_t *leaf_count) + H5O_storage_virtual_ent_t ***not_in_tree_out, size_t *leaf_count, + size_t *not_in_tree_count) { herr_t ret_value = SUCCEED; - H5RT_leaf_t *leaves_temp = NULL; - bool *is_in_tree = NULL; + H5RT_leaf_t *leaves_temp = NULL; + H5O_storage_virtual_ent_t **not_in_tree = NULL; bool should_insert_space = false; - H5O_storage_virtual_ent_t *curr_mapping = NULL; - H5RT_leaf_t *curr_leaf = NULL; - size_t curr_leaf_count = 0; - H5S_t *curr_space = NULL; + H5O_storage_virtual_ent_t *curr_mapping = NULL; + H5RT_leaf_t *curr_leaf = NULL; + size_t curr_leaf_count = 0; + size_t curr_not_tree_count = 0; + H5S_t *curr_space = NULL; int rank = 0; @@ -3878,7 +3893,8 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings assert(num_mappings > 0); assert(leaf_count); assert(leaves_out); - assert(is_in_tree_out); + assert(not_in_tree_out); + assert(not_in_tree_count); /* Get rank from the first mapping's virtual selection */ if ((curr_space = mappings[0].source_dset.virtual_select) == NULL) @@ -3890,22 +3906,25 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings if (rank == 0) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "mapping has zero-dimensional space"); - /* Allocate array of leaf structures */ + /* Allocate array of leaf structures and not-in-tree mappings */ if ((leaves_temp = (H5RT_leaf_t *)calloc(num_mappings, sizeof(H5RT_leaf_t))) == NULL) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate leaves array"); - if ((is_in_tree = (bool *)H5MM_calloc(num_mappings * sizeof(bool))) == NULL) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate is_in_tree array"); + if ((not_in_tree = (H5O_storage_virtual_ent_t **)H5MM_calloc( + num_mappings * sizeof(H5O_storage_virtual_ent_t *))) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate not_in_tree array"); for (size_t i = 0; i < num_mappings; i++) { curr_mapping = &mappings[i]; H5D_RTREE_SHOULD_INSERT(curr_mapping, should_insert_space); - if (!should_insert_space) + if (!should_insert_space) { + /* Store pointer to mapping in not_in_tree array */ + not_in_tree[curr_not_tree_count] = curr_mapping; + curr_not_tree_count++; continue; - - is_in_tree[i] = true; + } /* Initialize leaf with dynamic coordinate allocation */ curr_leaf = &leaves_temp[curr_leaf_count]; @@ -3930,9 +3949,10 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings curr_leaf_count++; } - *leaves_out = leaves_temp; - *leaf_count = curr_leaf_count; - *is_in_tree_out = is_in_tree; + *leaves_out = leaves_temp; + *leaf_count = curr_leaf_count; + *not_in_tree_out = not_in_tree; + *not_in_tree_count = curr_not_tree_count; done: if (ret_value < 0) { if (leaves_temp) { @@ -3942,25 +3962,24 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings } free(leaves_temp); } - if (is_in_tree) - H5MM_free(is_in_tree); + if (not_in_tree) + H5MM_free(not_in_tree); } FUNC_LEAVE_NOAPI(ret_value); } -// attempt to build a tree and a boolean list and store them on the storage object /*------------------------------------------------------------------------- * Function: H5D__virtual_build_tree * * Purpose: Build a spatial tree of mapping indices, and a list of - * which mapping indices are in the tree, and store them on + * mappings not in the tree, and store them on * the provided virtual layout * * Parameters: virt: The virtual layout with the mapping to build the * tree from. The tree will be stored at virt->tree, - * and the list of stored indices will be stored at - * virt->is_in_tree. + * and the list of non-tree mappings will be stored at + * virt->not_in_tree_list. * * Return: Non-negative on success/Negative on failure * @@ -3972,35 +3991,29 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) H5O_storage_virtual_ent_t *mappings = virt->list; size_t num_mappings = virt->list_nused; - H5RT_leaf_t *leaves = NULL; - size_t num_leaves = 0; - bool *is_in_tree = NULL; - herr_t ret_value = SUCCEED; + H5RT_leaf_t *leaves = NULL; + size_t num_leaves = 0; + H5O_storage_virtual_ent_t **not_in_tree_mappings = NULL; + size_t not_in_tree_count = 0; + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE assert(virt); - if (H5D__mappings_to_leaves(mappings, num_mappings, &leaves, &is_in_tree, &num_leaves) < 0) + if (H5D__mappings_to_leaves(mappings, num_mappings, &leaves, ¬_in_tree_mappings, &num_leaves, + ¬_in_tree_count) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get leaves from mappings"); if (num_leaves == 0) { - /* Don't build a tree, release allocated memory */ + /* No tree to build */ + virt->tree = NULL; if (leaves) { free(leaves); leaves = NULL; } - if (is_in_tree) { - H5MM_free(is_in_tree); - is_in_tree = NULL; - } - - virt->tree = NULL; - virt->is_in_tree = NULL; } else { - virt->is_in_tree = is_in_tree; - /* Build the tree */ if ((virt->tree = H5RT_create(rank, leaves, num_leaves)) == NULL) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to create mapping tree"); @@ -4008,6 +4021,27 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) leaves = NULL; } + /* Store not-in-tree mappings (regardless of whether tree was built) */ + if (not_in_tree_count > 0) { + virt->not_in_tree_list = not_in_tree_mappings; + virt->not_in_tree_nused = not_in_tree_count; + virt->not_in_tree_nalloc = not_in_tree_count; /* Currently allocated exactly what we need */ + not_in_tree_mappings = NULL; /* Transfer ownership to virt */ + } + else { + /* Clean up any existing allocation */ + if (virt->not_in_tree_list) { + H5MM_free(virt->not_in_tree_list); + } + virt->not_in_tree_list = NULL; + virt->not_in_tree_nused = 0; + virt->not_in_tree_nalloc = 0; + if (not_in_tree_mappings) { + H5MM_free(not_in_tree_mappings); + not_in_tree_mappings = NULL; + } + } + done: if (ret_value < 0) { if (leaves) { @@ -4017,9 +4051,69 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) } free(leaves); } - if (is_in_tree) - H5MM_free(is_in_tree); + if (not_in_tree_mappings) + H5MM_free(not_in_tree_mappings); } FUNC_LEAVE_NOAPI(ret_value) +} + +/*------------------------------------------------------------------------- + * Function: H5D__should_build_tree + * + * Purpose: Determine whether to build a spatial tree of mapping indice + * for the provided dataset layout + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__should_build_tree(H5O_storage_virtual_t *storage, hid_t dapl_id, bool *should_build_tree) +{ + herr_t ret_value = SUCCEED; + H5P_genplist_t *dapl_plist = NULL; + bool tree_enabled = false; + + FUNC_ENTER_PACKAGE + + assert(storage); + assert(should_build_tree); + assert(dapl_id != H5I_INVALID_HID); + + /* Don't build if already exists */ + if (storage->tree) { + *should_build_tree = false; + HGOTO_DONE(SUCCEED); + } + + /* Don't build if too few mappings */ + if (storage->list_nused < H5D_VIRTUAL_TREE_THRESHOLD) { + *should_build_tree = false; + HGOTO_DONE(SUCCEED); + } + + /* Don't build if DAPL property has disabled the tree */ + if (*should_build_tree) { + if (NULL == (dapl_plist = (H5P_genplist_t *)H5I_object(dapl_id))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for dapl ID"); + + if (H5P_get(dapl_plist, H5D_ACS_USE_TREE_NAME, &tree_enabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual use tree flag"); + + if (!tree_enabled) { + *should_build_tree = false; + HGOTO_DONE(SUCCEED); + } + } + + /* Don't build tree if we previously assembled ONLY a not_in_tree_list + * - this indicates that all mappings are not in the tree */ + if (*should_build_tree && storage->not_in_tree_list && !storage->tree) { + *should_build_tree = false; + HGOTO_DONE(SUCCEED); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); } \ No newline at end of file diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 3245bab64f8..bf2580758a2 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -605,8 +605,11 @@ typedef struct H5O_storage_virtual_t { *source_dset_hash_table; /* Hash table of virtual entries sorted by source dataset name. Only the first occurrence of each source dataset name is stored. */ H5RT_t *tree; - bool *is_in_tree; /* List of the indices in 'list' that are stored in tree for quick access - * Some mappings cannot be stored in the tree and must be searched manually */ + size_t not_in_tree_nused; /* Number of entries in not_in_tree_list */ + size_t not_in_tree_nalloc; /* Allocated size of not_in_tree_list */ + H5O_storage_virtual_ent_t * + *not_in_tree_list; /* Array of POINTERS to mappings NOT in tree for quick access + * Some mappings cannot be stored in the tree and must be searched manually */ } H5O_storage_virtual_t; typedef struct H5O_storage_t { diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 443944644cc..0e0689e2df4 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -89,7 +89,7 @@ { \ {HADDR_UNDEF, 0}, 0, NULL, 0, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ - H5D_VDS_ERROR, HSIZE_UNDEF, -1, -1, false, NULL, NULL, NULL, NULL \ + H5D_VDS_ERROR, HSIZE_UNDEF, -1, -1, false, NULL, NULL, NULL, 0, 0, NULL, \ } #define H5D_DEF_STORAGE_COMPACT \ { \ @@ -2119,7 +2119,6 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const /* Expand list if necessary */ if (virtual_layout.storage.u.virt.list_nused == virtual_layout.storage.u.virt.list_nalloc) { H5O_storage_virtual_ent_t *x; /* Pointer to the new list */ - bool *y; /* Pointer to the new is_in_tree list */ size_t new_alloc = MAX(H5D_VIRTUAL_DEF_LIST_SIZE, virtual_layout.storage.u.virt.list_nalloc * 2); ptrdiff_t buf_diff; @@ -2131,13 +2130,6 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const virtual_layout.storage.u.virt.list = x; virtual_layout.storage.u.virt.list_nalloc = new_alloc; - /* Expand size of is_in_tree list */ - if (NULL == - (y = (bool *)H5MM_realloc(virtual_layout.storage.u.virt.is_in_tree, new_alloc * sizeof(bool)))) - HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, - "can't reallocate virtual dataset mapping is_in_tree list"); - virtual_layout.storage.u.virt.is_in_tree = y; - /* Adjust pointers in the hash tables in case realloc moved the buffers, and hence all the elements * and hash handles in the hash tables */ HASH_ADJUST_PTRS(hh_source_file, virtual_layout.storage.u.virt.source_file_hash_table, buf_diff); diff --git a/test/rtree.c b/test/rtree.c index 51afc32df7a..8afb6fea494 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -629,8 +629,8 @@ test_rtree_dapl(bool use_tree) puts("Expected spatial tree to exist but it was NULL"); FAIL_STACK_ERROR; } - if (storage->is_in_tree == NULL) { - puts("Expected is_in_tree array to exist but it was NULL"); + if (storage->not_in_tree_nused > 0 && storage->not_in_tree_list == NULL) { + puts("Expected not_in_tree_list array to exist but it was NULL"); FAIL_STACK_ERROR; } } @@ -639,8 +639,8 @@ test_rtree_dapl(bool use_tree) puts("Expected spatial tree to be NULL but it exists"); FAIL_STACK_ERROR; } - if (storage->is_in_tree != NULL) { - puts("Expected is_in_tree array to be NULL but it exists"); + if (storage->not_in_tree_list != NULL || storage->not_in_tree_nused > 0) { + puts("Expected not_in_tree_list to be empty but it exists"); FAIL_STACK_ERROR; } } @@ -683,18 +683,21 @@ test_rtree_dapl(bool use_tree) puts("Expected spatial tree to exist but it was NULL"); FAIL_STACK_ERROR; } - if (storage->is_in_tree == NULL) { - puts("Expected is_in_tree array to exist but it was NULL"); + /* not_in_tree_list can be NULL if all mappings fit in tree - this is OK */ + /* Just verify consistency: if nused > 0, then list should exist */ + if (storage->not_in_tree_nused > 0 && storage->not_in_tree_list == NULL) { + puts("Expected not_in_tree_list array to exist but it was NULL"); FAIL_STACK_ERROR; } + /* When tree is enabled, we just verify tree exists - not_in_tree_list may or may not exist */ } else { if (storage->tree != NULL) { puts("Expected spatial tree to be NULL but it exists"); FAIL_STACK_ERROR; } - if (storage->is_in_tree != NULL) { - puts("Expected is_in_tree array to be NULL but it exists"); + if (storage->not_in_tree_list != NULL || storage->not_in_tree_nused > 0) { + puts("Expected not_in_tree_list to be empty but it exists"); FAIL_STACK_ERROR; } } @@ -803,8 +806,11 @@ test_rtree_threshold(bool use_tree) printf("%d mappings: Expected spatial tree to exist but it was NULL\n", num_mappings); FAIL_STACK_ERROR; } - if (storage->is_in_tree == NULL) { - printf("%d mappings: Expected is_in_tree array to exist but it was NULL\n", num_mappings); + /* not_in_tree_list can be NULL if all mappings fit in tree - this is OK */ + /* Just verify consistency: if nused > 0, then list should exist */ + if (storage->not_in_tree_nused > 0 && storage->not_in_tree_list == NULL) { + printf("%d mappings: Expected not_in_tree_list array to exist but it was NULL\n", + num_mappings); FAIL_STACK_ERROR; } } @@ -813,8 +819,9 @@ test_rtree_threshold(bool use_tree) printf("%d mappings: Expected spatial tree to be NULL but it exists\n", num_mappings); FAIL_STACK_ERROR; } - if (storage->is_in_tree != NULL) { - printf("%d mappings: Expected is_in_tree array to be NULL but it exists\n", num_mappings); + if (storage->not_in_tree_list != NULL) { + printf("%d mappings: Expected not_in_tree_list array to be NULL but it exists\n", + num_mappings); FAIL_STACK_ERROR; } } From 1de7b6e4c731bf2625ff6673bdefed921dec2bc4 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 23 Sep 2025 15:30:14 -0500 Subject: [PATCH 14/43] Update results handling --- src/H5Dvirtual.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index e540ab9663e..f91ede379ff 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -3179,7 +3179,7 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag /* First, iterate over the mappings with an intersection found via the tree */ for (size_t i = 0; i < search_results->count; i++) { - H5RT_leaf_t *curr_leaf = search_results->results[i].leaf; + H5RT_leaf_t *curr_leaf = search_results->results[i]; assert(curr_leaf); if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, From f6eef6790c2134c8233e8045827096800d7e9d7e Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Thu, 25 Sep 2025 15:23:32 -0500 Subject: [PATCH 15/43] Add bullet point to CHANGELOG executive summary --- release_docs/CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/release_docs/CHANGELOG.md b/release_docs/CHANGELOG.md index 887141afc39..49630e41a01 100644 --- a/release_docs/CHANGELOG.md +++ b/release_docs/CHANGELOG.md @@ -25,8 +25,14 @@ For releases prior to version 2.0.0, please see the release.txt file and for mor ## Performance Enhancements: +<<<<<<< HEAD - [30% faster opening](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#layoutcopydelay) and [25% faster closing](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#fileformat) of virtual datasets. - [Reduced memory overhead](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#fileformat) via shared name strings and optimized spatial search algorithms for virtual datasets. +======= +- Up to [2500% faster](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#rtree) Virtual Dataset read/write operations +- 30% faster opening and 25% faster closing of virtual datasets. +- Reduced memory overhead via shared name strings and optimized spatial search algorithms for virtual datasets. +>>>>>>> 7ae02e1c82 (Add bullet point to CHANGELOG executive summary) ## Significant Advancements: @@ -461,7 +467,7 @@ Simple example programs showing how to use complex number datatypes have been ad This layout copy is now delayed until either a user requests the DCPL, or until the start of an operation that needs to read the layout from the DCPL. -### Virtual datasets now use a spatial tree to optimize searches +### Virtual datasets now use a spatial tree to optimize searches Virtual dataset operations with many (>1,000) mappings were much slower than corresponding operations on normal datasets. This was due to the need From eab9a6f6f7a3165e46b4e24a3a0baf4b59dced44 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Fri, 26 Sep 2025 16:16:25 -0500 Subject: [PATCH 16/43] Move tree creation higher up --- src/H5Dvirtual.c | 171 +++++++++++++++++++++++++++-------------------- test/rtree.c | 40 ++++++++--- 2 files changed, 130 insertions(+), 81 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index f91ede379ff..5c9c00a872b 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -147,7 +147,8 @@ static herr_t H5D__virtual_build_source_name(char char **built_name); static herr_t H5D__virtual_init_all(const H5D_t *dset); static herr_t H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, - H5S_t *file_space, H5S_t *mem_space, hsize_t *tot_nelmts); + H5S_t *file_space, H5S_t *mem_space, hsize_t *tot_nelmts, + H5RT_result_set_t *mappings); static herr_t H5D__virtual_post_io(H5O_storage_virtual_t *storage); static herr_t H5D__virtual_read_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset); @@ -3121,12 +3122,9 @@ H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, H5S_t *file_s */ static herr_t H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, H5S_t *file_space, - H5S_t *mem_space, hsize_t *tot_nelmts) + H5S_t *mem_space, hsize_t *tot_nelmts, H5RT_result_set_t *mappings) { - const H5D_t *dset = dset_info->dset; /* Local pointer to dataset info */ herr_t ret_value = SUCCEED; /* Return value */ - bool should_build_tree = true; - H5RT_result_set_t *search_results = NULL; /* Search results from R-tree */ FUNC_ENTER_PACKAGE @@ -3136,50 +3134,14 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag assert(file_space); assert(tot_nelmts); - /* Initialize layout if necessary */ - if (!storage->init) - if (H5D__virtual_init_all(dset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize virtual layout"); - /* Initialize tot_nelmts */ *tot_nelmts = 0; - if (H5D__should_build_tree(storage, dset->shared->dapl_id, &should_build_tree) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't determine if should build VDS tree"); - - if (should_build_tree) { - int rank = 0; - - /* Get the rank of the dataset */ - if ((rank = H5S_GET_EXTENT_NDIMS(dset_info->dset->shared->space)) < 0 || rank >= H5S_MAX_RANK) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset rank"); - - if (rank == 0) - HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "virtual dataset has no rank"); - - if (H5D__virtual_build_tree(storage, rank) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build virtual mapping tree"); - } - /* Iterate over the mappings */ - if (storage->tree) { - /* Perform a spatial tree search to get a list of mappings - * whose virtual selection intersects the IO operation */ - hsize_t min[H5S_MAX_RANK]; - hsize_t max[H5S_MAX_RANK]; - - memset(min, 0, sizeof(min)); - memset(max, 0, sizeof(max)); - - if (H5S_SELECT_BOUNDS(file_space, min, max) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); - - if (H5RT_search(storage->tree, min, max, &search_results) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "R-tree search failed"); - + if (mappings) { /* First, iterate over the mappings with an intersection found via the tree */ - for (size_t i = 0; i < search_results->count; i++) { - H5RT_leaf_t *curr_leaf = search_results->results[i]; + for (size_t i = 0; i < mappings->count; i++) { + H5RT_leaf_t *curr_leaf = mappings->results[i]; assert(curr_leaf); if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, @@ -3187,11 +3149,6 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); } - /* Free search results */ - if (H5RT_free_results(search_results) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't free R-tree search results"); - search_results = NULL; - /* Iterate over the mappings that are not stored in the tree */ for (size_t i = 0; i < storage->not_in_tree_nused; i++) { if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, @@ -3209,11 +3166,6 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag } done: - if (ret_value < 0) - if (search_results) - if (H5RT_free_results(search_results) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't free R-tree search results"); - FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_pre_io() */ @@ -3237,7 +3189,7 @@ H5D__virtual_post_io(H5O_storage_virtual_t *storage) /* Sanity check */ assert(storage); - /* Iterate over mappings */ + /* Iterate over all mappings */ for (i = 0; i < storage->list_nused; i++) /* Check for "printf" source dataset resolution */ if (storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { @@ -3346,6 +3298,10 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info size_t nelmts; /* Number of elements to process */ size_t i, j; /* Local index variables */ herr_t ret_value = SUCCEED; /* Return value */ + bool should_build_tree = false; /* Whether to build a spatial tree */ + H5RT_result_set_t *mappings = NULL; /* Search results from R-tree */ + hsize_t min[H5S_MAX_RANK]; + hsize_t max[H5S_MAX_RANK]; FUNC_ENTER_PACKAGE @@ -3362,14 +3318,48 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info /* Initialize nelmts */ nelmts = H5S_GET_SELECT_NPOINTS(dset_info->file_space); + memset(min, 0, sizeof(min)); + memset(max, 0, sizeof(max)); #ifdef H5_HAVE_PARALLEL /* Parallel reads are not supported (yet) */ if (H5F_HAS_FEATURE(dset_info->dset->oloc.file, H5FD_FEAT_HAS_MPI)) HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "parallel reads not supported on virtual datasets"); #endif /* H5_HAVE_PARALLEL */ + /* Initialize layout if necessary */ + if (!storage->init) + if (H5D__virtual_init_all(dset_info->dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize virtual layout"); + + if (H5D__should_build_tree(storage, dset_info->dset->shared->dapl_id, &should_build_tree) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't determine if should build VDS tree"); + + if (should_build_tree) { + int rank = 0; + + /* Get the rank of the dataset */ + if ((rank = H5S_GET_EXTENT_NDIMS(dset_info->dset->shared->space)) < 0 || rank >= H5S_MAX_RANK) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset rank"); + + if (rank == 0) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "virtual dataset has no rank"); + + if (H5D__virtual_build_tree(storage, rank) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build virtual mapping tree"); + } + + if (storage->tree) { + /* Perform a spatial tree search to get a list of mappings + * whose virtual selection intersects the IO operation */ + if (H5S_SELECT_BOUNDS(dset_info->file_space, min, max) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); + + if (H5RT_search(storage->tree, min, max, &mappings) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "R-tree search failed"); + } + /* Prepare for I/O operation */ - if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts) < 0) + if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts, mappings) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to prepare for I/O operation"); /* Iterate over mappings */ @@ -3446,6 +3436,10 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info } /* end if */ done: + if (mappings) + if (H5RT_free_results(mappings) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't free R-tree search results"); + /* Cleanup I/O operation */ if (H5D__virtual_post_io(storage) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't cleanup I/O operation"); @@ -3542,6 +3536,10 @@ H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_inf size_t nelmts; /* Number of elements to process */ size_t i, j; /* Local index variables */ herr_t ret_value = SUCCEED; /* Return value */ + bool should_build_tree = false; /* Whether to build a spatial tree */ + H5RT_result_set_t *mappings = NULL; /* Search results from R-tree */ + hsize_t min[H5S_MAX_RANK]; + hsize_t max[H5S_MAX_RANK]; FUNC_ENTER_PACKAGE @@ -3558,14 +3556,48 @@ H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_inf /* Initialize nelmts */ nelmts = H5S_GET_SELECT_NPOINTS(dset_info->file_space); + memset(min, 0, sizeof(min)); + memset(max, 0, sizeof(max)); #ifdef H5_HAVE_PARALLEL /* Parallel writes are not supported (yet) */ if (H5F_HAS_FEATURE(dset_info->dset->oloc.file, H5FD_FEAT_HAS_MPI)) HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "parallel writes not supported on virtual datasets"); #endif /* H5_HAVE_PARALLEL */ + /* Initialize layout if necessary */ + if (!storage->init) + if (H5D__virtual_init_all(dset_info->dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize virtual layout"); + + if (H5D__should_build_tree(storage, dset_info->dset->shared->dapl_id, &should_build_tree) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't determine if should build VDS tree"); + + if (should_build_tree) { + int rank = 0; + + /* Get the rank of the dataset */ + if ((rank = H5S_GET_EXTENT_NDIMS(dset_info->dset->shared->space)) < 0 || rank >= H5S_MAX_RANK) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset rank"); + + if (rank == 0) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "virtual dataset has no rank"); + + if (H5D__virtual_build_tree(storage, rank) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build virtual mapping tree"); + } + + if (storage->tree) { + /* Perform a spatial tree search to get a list of mappings + * whose virtual selection intersects the IO operation */ + if (H5S_SELECT_BOUNDS(dset_info->file_space, min, max) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); + + if (H5RT_search(storage->tree, min, max, &mappings) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "R-tree search failed"); + } + /* Prepare for I/O operation */ - if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts) < 0) + if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts, mappings) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to prepare for I/O operation"); /* Fail if there are unmapped parts of the selection as they would not be @@ -3593,6 +3625,9 @@ H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_inf } /* end for */ done: + if (mappings) + H5RT_free_results(mappings); + /* Cleanup I/O operation */ if (H5D__virtual_post_io(storage) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't cleanup I/O operation"); @@ -4073,7 +4108,7 @@ H5D__should_build_tree(H5O_storage_virtual_t *storage, hid_t dapl_id, bool *shou { herr_t ret_value = SUCCEED; H5P_genplist_t *dapl_plist = NULL; - bool tree_enabled = false; + bool tree_enabled_dapl = false; FUNC_ENTER_PACKAGE @@ -4094,26 +4129,18 @@ H5D__should_build_tree(H5O_storage_virtual_t *storage, hid_t dapl_id, bool *shou } /* Don't build if DAPL property has disabled the tree */ - if (*should_build_tree) { - if (NULL == (dapl_plist = (H5P_genplist_t *)H5I_object(dapl_id))) - HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for dapl ID"); - - if (H5P_get(dapl_plist, H5D_ACS_USE_TREE_NAME, &tree_enabled) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual use tree flag"); + if (NULL == (dapl_plist = (H5P_genplist_t *)H5I_object(dapl_id))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for dapl ID"); - if (!tree_enabled) { - *should_build_tree = false; - HGOTO_DONE(SUCCEED); - } - } + if (H5P_get(dapl_plist, H5D_ACS_USE_TREE_NAME, &tree_enabled_dapl) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual use tree flag"); - /* Don't build tree if we previously assembled ONLY a not_in_tree_list - * - this indicates that all mappings are not in the tree */ - if (*should_build_tree && storage->not_in_tree_list && !storage->tree) { + if (!tree_enabled_dapl) { *should_build_tree = false; HGOTO_DONE(SUCCEED); } + *should_build_tree = true; done: FUNC_LEAVE_NOAPI(ret_value); } \ No newline at end of file diff --git a/test/rtree.c b/test/rtree.c index 8afb6fea494..1cc2c7478dd 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -51,7 +51,7 @@ #define RTREE_DAPL_DATASET_DIM2 10 #define RTREE_THRESHOLD_FILENAME "vds_rtree_threshold_test.h5" -#define RTREE_MAX_TEST_MAPPINGS (H5D_VIRTUAL_TREE_THRESHOLD + 10) +#define RTREE_MAX_TEST_MAPPINGS (H5D_VIRTUAL_TREE_THRESHOLD + 100) static const size_t test_counts[RTREE_TEST_CREATE_NUM_COUNTS] = {H5D_VIRTUAL_TREE_THRESHOLD, 100, 1000, 10000}; @@ -566,7 +566,7 @@ test_rtree_dapl(bool use_tree) hid_t dapl_id = H5I_INVALID_HID; hid_t vdset_id = H5I_INVALID_HID; - int rbuf[H5D_VIRTUAL_TREE_THRESHOLD + 10]; + int rbuf[RTREE_MAX_TEST_MAPPINGS]; const char *test_str = NULL; /* Internal values for introspection */ @@ -606,7 +606,7 @@ test_rtree_dapl(bool use_tree) FAIL_STACK_ERROR; /* Verify read data matches expected pattern */ - for (int i = 0; i < (H5D_VIRTUAL_TREE_THRESHOLD + 10); i++) { + for (int i = 0; i < (RTREE_MAX_TEST_MAPPINGS); i++) { if (rbuf[i] != i) { printf("Data mismatch at [%d]: expected %d, got %d\n", i, i, rbuf[i]); FAIL_STACK_ERROR; @@ -648,11 +648,33 @@ test_rtree_dapl(bool use_tree) /* Close the dataset and re-open it with the opposite value set in DAPL */ if (H5Dclose(vdset_id) < 0) FAIL_STACK_ERROR; + vdset_id = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + file_id = H5I_INVALID_HID; + + if (H5Pclose(dapl_id) < 0) + FAIL_STACK_ERROR; + + dapl_id = H5I_INVALID_HID; + + memset(rbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); + + H5close(); + H5open(); + + if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree_inverse) < 0) FAIL_STACK_ERROR; + if ((file_id = H5Fopen(RTREE_DAPL_FILENAME, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + if ((vdset_id = H5Dopen2(file_id, RTREE_DAPL_VDS_NAME, dapl_id)) < 0) FAIL_STACK_ERROR; @@ -661,7 +683,7 @@ test_rtree_dapl(bool use_tree) FAIL_STACK_ERROR; /* Verify read data matches expected pattern */ - for (int i = 0; i < (H5D_VIRTUAL_TREE_THRESHOLD + 10); i++) { + for (int i = 0; i < (RTREE_MAX_TEST_MAPPINGS); i++) { if (rbuf[i] != i) { printf("Data mismatch after re-open at [%d]: expected %d, got %d\n", i, i, rbuf[i]); FAIL_STACK_ERROR; @@ -677,27 +699,27 @@ test_rtree_dapl(bool use_tree) storage = &(dset->shared->layout.storage.u.virt); - /* Verify tree existence matches expectation */ + /* Verify tree existence matches expectation after re-open */ if (use_tree_inverse) { if (storage->tree == NULL) { - puts("Expected spatial tree to exist but it was NULL"); + puts("Expected spatial tree to exist but it was NULL after re-open"); FAIL_STACK_ERROR; } /* not_in_tree_list can be NULL if all mappings fit in tree - this is OK */ /* Just verify consistency: if nused > 0, then list should exist */ if (storage->not_in_tree_nused > 0 && storage->not_in_tree_list == NULL) { - puts("Expected not_in_tree_list array to exist but it was NULL"); + puts("Expected not_in_tree_list array to exist but it was NULL after re-open"); FAIL_STACK_ERROR; } /* When tree is enabled, we just verify tree exists - not_in_tree_list may or may not exist */ } else { if (storage->tree != NULL) { - puts("Expected spatial tree to be NULL but it exists"); + puts("Expected spatial tree to be NULL but it exists after re-open"); FAIL_STACK_ERROR; } if (storage->not_in_tree_list != NULL || storage->not_in_tree_nused > 0) { - puts("Expected not_in_tree_list to be empty but it exists"); + puts("Expected not_in_tree_list to be empty but it exists after re-open"); FAIL_STACK_ERROR; } } From f59f054cf363f5cba2af2c219b56b0941ff410a0 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Fri, 26 Sep 2025 17:21:06 -0500 Subject: [PATCH 17/43] Limit number of H5D__virtual_read_one calls --- src/H5Dvirtual.c | 81 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 18 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 5c9c00a872b..8077616e49a 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -150,11 +150,14 @@ static herr_t H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_vir H5S_t *file_space, H5S_t *mem_space, hsize_t *tot_nelmts, H5RT_result_set_t *mappings); static herr_t H5D__virtual_post_io(H5O_storage_virtual_t *storage); -static herr_t H5D__virtual_read_one(H5D_dset_io_info_t *dset_info, +static herr_t H5D__virtual_read_one_src(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset); static herr_t H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset); +static herr_t +H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping); + /* R-tree helper functions */ static herr_t H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank); static herr_t H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, @@ -3216,7 +3219,7 @@ H5D__virtual_post_io(H5O_storage_virtual_t *storage) } /* end H5D__virtual_post_io() */ /*------------------------------------------------------------------------- - * Function: H5D__virtual_read_one + * Function: H5D__virtual_read_one_src * * Purpose: Read from a single source dataset in a virtual dataset. * @@ -3225,7 +3228,7 @@ H5D__virtual_post_io(H5O_storage_virtual_t *storage) *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_read_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset) +H5D__virtual_read_one_src(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset) { H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ H5D_dset_io_info_t source_dinfo; /* Dataset info for source dataset read */ @@ -3278,7 +3281,7 @@ H5D__virtual_read_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__virtual_read_one() */ +} /* end H5D__virtual_read_one_src() */ /*------------------------------------------------------------------------- * Function: H5D__virtual_read @@ -3363,23 +3366,30 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to prepare for I/O operation"); /* Iterate over mappings */ - for (i = 0; i < storage->list_nused; i++) { - /* Sanity check that the virtual space has been patched by now */ - assert(storage->list[i].virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); + if (mappings) { + /* Iterate over intersections in tree */ + for (i = 0; i < mappings->count; i++) { + H5RT_leaf_t *curr_leaf = mappings->results[i]; + assert(curr_leaf); - /* Check for "printf" source dataset resolution */ - if (storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { - /* Iterate over sub-source dsets */ - for (j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++) - if (H5D__virtual_read_one(dset_info, &storage->list[i].sub_dset[j]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); - } /* end if */ - else - /* Read from source dataset */ - if (H5D__virtual_read_one(dset_info, &storage->list[i].source_dset) < 0) + if (H5D__virtual_read_one_mapping(dset_info, curr_leaf->record) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); - } /* end for */ + } + /* Iterate over not-in-tree mappings */ + for (i = 0; i < storage->not_in_tree_nused; i++) { + if (H5D__virtual_read_one_mapping(dset_info, storage->not_in_tree_list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + } + } else { + /* Iterate over all mappings */ + for (i = 0; i < storage->list_nused; i++) { + if (H5D__virtual_read_one_mapping(dset_info, &storage->list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + } /* end for */ + + } + /* Fill unmapped part of buffer with fill value */ if (tot_nelmts < nelmts) { H5D_fill_value_t fill_status; /* Fill value status */ @@ -4143,4 +4153,39 @@ H5D__should_build_tree(H5O_storage_virtual_t *storage, hid_t dapl_id, bool *shou *should_build_tree = true; done: FUNC_LEAVE_NOAPI(ret_value); +} + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_read_one_mapping + * + * Purpose: Read from a single mapping entry in a virtual dataset + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping) { + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Sanity check that the virtual space has been patched by now */ + assert(mapping->virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); + + /* Check for "printf" source dataset resolution */ + if (mapping->psfn_nsubs || mapping->psdn_nsubs) { + /* Iterate over sub-source dsets */ + for (size_t j = mapping->sub_dset_io_start; + j < mapping->sub_dset_io_end; j++) + if (H5D__virtual_read_one_src(dset_info, &mapping->sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + } /* end if */ + else + /* Read from source dataset */ + if (H5D__virtual_read_one_src(dset_info, &mapping->source_dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + +done: + FUNC_LEAVE_NOAPI(ret_value) } \ No newline at end of file From e2dfe89a5c6a90ffcc55051d5c0c359d4d6297f4 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Fri, 26 Sep 2025 17:33:11 -0500 Subject: [PATCH 18/43] Limit number of operations in post_io --- src/H5Dvirtual.c | 110 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 75 insertions(+), 35 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 8077616e49a..0e6f8995d7b 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -149,14 +149,15 @@ static herr_t H5D__virtual_init_all(const H5D_t *dset); static herr_t H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, H5S_t *file_space, H5S_t *mem_space, hsize_t *tot_nelmts, H5RT_result_set_t *mappings); -static herr_t H5D__virtual_post_io(H5O_storage_virtual_t *storage); +static herr_t H5D__virtual_post_io(H5O_storage_virtual_t *storage, H5RT_result_set_t *mappings); +static herr_t H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping); +static herr_t H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping); static herr_t H5D__virtual_read_one_src(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset); static herr_t H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset); -static herr_t -H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping); + /* R-tree helper functions */ static herr_t H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank); @@ -3182,9 +3183,9 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_post_io(H5O_storage_virtual_t *storage) +H5D__virtual_post_io(H5O_storage_virtual_t *storage, H5RT_result_set_t *mappings) { - size_t i, j; /* Local index variables */ + size_t i; /* Local index variables */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -3192,29 +3193,30 @@ H5D__virtual_post_io(H5O_storage_virtual_t *storage) /* Sanity check */ assert(storage); - /* Iterate over all mappings */ - for (i = 0; i < storage->list_nused; i++) - /* Check for "printf" source dataset resolution */ - if (storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { - /* Iterate over sub-source dsets */ - for (j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++) - /* Close projected memory space */ - if (storage->list[i].sub_dset[j].projected_mem_space) { - if (H5S_close(storage->list[i].sub_dset[j].projected_mem_space) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); - storage->list[i].sub_dset[j].projected_mem_space = NULL; - } /* end if */ - } /* end if */ - else - /* Close projected memory space */ - if (storage->list[i].source_dset.projected_mem_space) { - if (H5S_close(storage->list[i].source_dset.projected_mem_space) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); - storage->list[i].source_dset.projected_mem_space = NULL; - } /* end if */ + if (mappings) { + /* Iterate over mappings in tree */ + for (i = 0; i < mappings->count; i++) { + H5RT_leaf_t *curr_leaf = mappings->results[i]; + assert(curr_leaf); - /* Note the lack of a done: label. This is because there are no HGOTO_ERROR - * calls. If one is added, a done: label must also be added */ + if (H5D__virtual_close_mapping(curr_leaf->record) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + } + + /* Iterate over the mappings that are not stored in the tree */ + for (i = 0; i < storage->not_in_tree_nused; i++) { + if (H5D__virtual_close_mapping(storage->not_in_tree_list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + } + } else { + /* Iterate over all mappings */ + for (i = 0; i < storage->list_nused; i++) + if (H5D__virtual_close_mapping(&storage->list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "failed to close mapping"); + } + + +done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_post_io() */ @@ -3446,14 +3448,14 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info } /* end if */ done: + /* Cleanup I/O operation */ + if (H5D__virtual_post_io(storage, mappings) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't cleanup I/O operation"); + if (mappings) if (H5RT_free_results(mappings) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't free R-tree search results"); - /* Cleanup I/O operation */ - if (H5D__virtual_post_io(storage) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't cleanup I/O operation"); - /* Close fill space */ if (fill_space) if (H5S_close(fill_space) < 0) @@ -3635,13 +3637,13 @@ H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_inf } /* end for */ done: - if (mappings) - H5RT_free_results(mappings); - /* Cleanup I/O operation */ - if (H5D__virtual_post_io(storage) < 0) + if (H5D__virtual_post_io(storage, mappings) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't cleanup I/O operation"); + if (mappings) + H5RT_free_results(mappings); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_write() */ @@ -4188,4 +4190,42 @@ H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual done: FUNC_LEAVE_NOAPI(ret_value) +} + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_close_mapping + * + * Purpose: Frees memory structures allocated by H5D__virtual_pre_io + * for a particular mapping + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping) { + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check for "printf" source dataset resolution */ + if (mapping->psfn_nsubs || mapping->psdn_nsubs) { + /* Iterate over sub-source dsets */ + for (size_t j = mapping->sub_dset_io_start; j < mapping->sub_dset_io_end; j++) + /* Close projected memory space */ + if (mapping->sub_dset[j].projected_mem_space) { + if (H5S_close(mapping->sub_dset[j].projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); + mapping->sub_dset[j].projected_mem_space = NULL; + } /* end if */ + } /* end if */ + else + /* Close projected memory space */ + if (mapping->source_dset.projected_mem_space) { + if (H5S_close(mapping->source_dset.projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); + mapping->source_dset.projected_mem_space = NULL; + } /* end if */ +done: + FUNC_LEAVE_NOAPI(ret_value); } \ No newline at end of file From 186c7b5c3edc6152de166ddbeb2771941f956c7e Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Fri, 26 Sep 2025 17:41:18 -0500 Subject: [PATCH 19/43] Limit number of write calls --- src/H5Dvirtual.c | 80 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 21 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 0e6f8995d7b..989e5551472 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -154,7 +154,8 @@ static herr_t H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping); static herr_t H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping); static herr_t H5D__virtual_read_one_src(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset); -static herr_t H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, +static herr_t H5D__virtual_write_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping); +static herr_t H5D__virtual_write_one_src(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset); @@ -3465,7 +3466,7 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info } /* end H5D__virtual_read() */ /*------------------------------------------------------------------------- - * Function: H5D__virtual_write_one + * Function: H5D__virtual_write_one_src * * Purpose: Write to a single source dataset in a virtual dataset. * @@ -3474,7 +3475,7 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset) +H5D__virtual_write_one_src(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset) { H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ H5D_dset_io_info_t source_dinfo; /* Dataset info for source dataset write */ @@ -3529,7 +3530,7 @@ H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdse } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__virtual_write_one() */ +} /* end H5D__virtual_write_one_src() */ /*------------------------------------------------------------------------- * Function: H5D__virtual_write @@ -3546,7 +3547,7 @@ H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_inf H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ size_t nelmts; /* Number of elements to process */ - size_t i, j; /* Local index variables */ + size_t i; /* Local index variables */ herr_t ret_value = SUCCEED; /* Return value */ bool should_build_tree = false; /* Whether to build a spatial tree */ H5RT_result_set_t *mappings = NULL; /* Search results from R-tree */ @@ -3618,23 +3619,28 @@ H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_inf HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "write requested to unmapped portion of virtual dataset"); - /* Iterate over mappings */ - for (i = 0; i < storage->list_nused; i++) { - /* Sanity check that virtual space has been patched by now */ - assert(storage->list[i].virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); + if (mappings) { + /* Iterate over intersections in tree */ + for (i = 0; i < mappings->count; i++) { + H5RT_leaf_t *curr_leaf = mappings->results[i]; + assert(curr_leaf); - /* Check for "printf" source dataset resolution */ - if (storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { - /* Iterate over sub-source dsets */ - for (j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++) - if (H5D__virtual_write_one(dset_info, &storage->list[i].sub_dset[j]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset"); - } /* end if */ - else - /* Write to source dataset */ - if (H5D__virtual_write_one(dset_info, &storage->list[i].source_dset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset"); - } /* end for */ + if (H5D__virtual_write_one_mapping(dset_info, curr_leaf->record) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + } + + /* Iterate over not-in-tree mappings */ + for (i = 0; i < storage->not_in_tree_nused; i++) { + if (H5D__virtual_write_one_mapping(dset_info, storage->not_in_tree_list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + } + } else { + /* Iterate over all mappings */ + for (i = 0; i < storage->list_nused; i++) { + if (H5D__virtual_write_one_mapping(dset_info, &storage->list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write to virtual mapping"); + } + } done: /* Cleanup I/O operation */ @@ -4192,6 +4198,38 @@ H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual FUNC_LEAVE_NOAPI(ret_value) } +/*------------------------------------------------------------------------- + * Function: H5D__virtual_read_one_mapping + * + * Purpose: Write to a single mapping entry in a virtual dataset + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t H5D__virtual_write_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping) { + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Sanity check that virtual space has been patched by now */ + assert(mapping->virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); + + /* Check for "printf" source dataset resolution */ + if (mapping->psfn_nsubs || mapping->psdn_nsubs) { + /* Iterate over sub-source dsets */ + for (size_t j = mapping->sub_dset_io_start; j < mapping->sub_dset_io_end; j++) + if (H5D__virtual_write_one_src(dset_info, &mapping->sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset"); + } else + /* Write to source dataset */ + if (H5D__virtual_write_one_src(dset_info, &mapping->source_dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} + /*------------------------------------------------------------------------- * Function: H5D__virtual_close_mapping * From 56883681a6eb49be907461f2db0c3f585942d2e8 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Fri, 26 Sep 2025 18:04:38 -0500 Subject: [PATCH 20/43] Add TODO notes --- test/rtree.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/rtree.c b/test/rtree.c index 1cc2c7478dd..0f70569b4fc 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -905,8 +905,10 @@ main(void) /* Test the mapping count threshold */ nerrors += test_rtree_threshold(true) < 0 ? 1 : 0; + // TODO - Fix failure nerrors += test_rtree_threshold(false) < 0 ? 1 : 0; + // TODO - Write test with tree on/off if (nerrors) goto error; From 9f809daad6a9b0cef74c3f9862fb1011d900a289 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Fri, 26 Sep 2025 18:05:02 -0500 Subject: [PATCH 21/43] Clang-format --- src/H5Dvirtual.c | 99 ++++++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 989e5551472..23c502845bd 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -151,14 +151,14 @@ static herr_t H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_vir H5RT_result_set_t *mappings); static herr_t H5D__virtual_post_io(H5O_storage_virtual_t *storage, H5RT_result_set_t *mappings); static herr_t H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping); -static herr_t H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping); +static herr_t H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, + H5O_storage_virtual_ent_t *mapping); static herr_t H5D__virtual_read_one_src(H5D_dset_io_info_t *dset_info, - H5O_storage_virtual_srcdset_t *source_dset); -static herr_t H5D__virtual_write_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping); + H5O_storage_virtual_srcdset_t *source_dset); +static herr_t H5D__virtual_write_one_mapping(H5D_dset_io_info_t *dset_info, + H5O_storage_virtual_ent_t *mapping); static herr_t H5D__virtual_write_one_src(H5D_dset_io_info_t *dset_info, - H5O_storage_virtual_srcdset_t *source_dset); - - + H5O_storage_virtual_srcdset_t *source_dset); /* R-tree helper functions */ static herr_t H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank); @@ -3129,7 +3129,7 @@ static herr_t H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, H5S_t *file_space, H5S_t *mem_space, hsize_t *tot_nelmts, H5RT_result_set_t *mappings) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -3186,7 +3186,7 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag static herr_t H5D__virtual_post_io(H5O_storage_virtual_t *storage, H5RT_result_set_t *mappings) { - size_t i; /* Local index variables */ + size_t i; /* Local index variables */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -3209,13 +3209,13 @@ H5D__virtual_post_io(H5O_storage_virtual_t *storage, H5RT_result_set_t *mappings if (H5D__virtual_close_mapping(storage->not_in_tree_list[i]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); } - } else { + } + else { /* Iterate over all mappings */ for (i = 0; i < storage->list_nused; i++) if (H5D__virtual_close_mapping(&storage->list[i]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "failed to close mapping"); } - done: FUNC_LEAVE_NOAPI(ret_value) @@ -3298,16 +3298,16 @@ H5D__virtual_read_one_src(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_src static herr_t H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info_t *dset_info) { - H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ - hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ - H5S_t *fill_space = NULL; /* Space to fill with fill value */ - size_t nelmts; /* Number of elements to process */ - size_t i, j; /* Local index variables */ - herr_t ret_value = SUCCEED; /* Return value */ - bool should_build_tree = false; /* Whether to build a spatial tree */ - H5RT_result_set_t *mappings = NULL; /* Search results from R-tree */ - hsize_t min[H5S_MAX_RANK]; - hsize_t max[H5S_MAX_RANK]; + H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ + hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ + H5S_t *fill_space = NULL; /* Space to fill with fill value */ + size_t nelmts; /* Number of elements to process */ + size_t i, j; /* Local index variables */ + herr_t ret_value = SUCCEED; /* Return value */ + bool should_build_tree = false; /* Whether to build a spatial tree */ + H5RT_result_set_t *mappings = NULL; /* Search results from R-tree */ + hsize_t min[H5S_MAX_RANK]; + hsize_t max[H5S_MAX_RANK]; FUNC_ENTER_PACKAGE @@ -3356,7 +3356,7 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info if (storage->tree) { /* Perform a spatial tree search to get a list of mappings - * whose virtual selection intersects the IO operation */ + * whose virtual selection intersects the IO operation */ if (H5S_SELECT_BOUNDS(dset_info->file_space, min, max) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); @@ -3365,7 +3365,8 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info } /* Prepare for I/O operation */ - if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts, mappings) < 0) + if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts, + mappings) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to prepare for I/O operation"); /* Iterate over mappings */ @@ -3384,15 +3385,15 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info if (H5D__virtual_read_one_mapping(dset_info, storage->not_in_tree_list[i]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); } - } else { + } + else { /* Iterate over all mappings */ for (i = 0; i < storage->list_nused; i++) { if (H5D__virtual_read_one_mapping(dset_info, &storage->list[i]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); } /* end for */ - } - + /* Fill unmapped part of buffer with fill value */ if (tot_nelmts < nelmts) { H5D_fill_value_t fill_status; /* Fill value status */ @@ -3544,15 +3545,15 @@ H5D__virtual_write_one_src(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_sr static herr_t H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info_t *dset_info) { - H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ - hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ - size_t nelmts; /* Number of elements to process */ - size_t i; /* Local index variables */ - herr_t ret_value = SUCCEED; /* Return value */ - bool should_build_tree = false; /* Whether to build a spatial tree */ - H5RT_result_set_t *mappings = NULL; /* Search results from R-tree */ - hsize_t min[H5S_MAX_RANK]; - hsize_t max[H5S_MAX_RANK]; + H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ + hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ + size_t nelmts; /* Number of elements to process */ + size_t i; /* Local index variables */ + herr_t ret_value = SUCCEED; /* Return value */ + bool should_build_tree = false; /* Whether to build a spatial tree */ + H5RT_result_set_t *mappings = NULL; /* Search results from R-tree */ + hsize_t min[H5S_MAX_RANK]; + hsize_t max[H5S_MAX_RANK]; FUNC_ENTER_PACKAGE @@ -3601,7 +3602,7 @@ H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_inf if (storage->tree) { /* Perform a spatial tree search to get a list of mappings - * whose virtual selection intersects the IO operation */ + * whose virtual selection intersects the IO operation */ if (H5S_SELECT_BOUNDS(dset_info->file_space, min, max) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); @@ -3610,7 +3611,8 @@ H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_inf } /* Prepare for I/O operation */ - if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts, mappings) < 0) + if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts, + mappings) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to prepare for I/O operation"); /* Fail if there are unmapped parts of the selection as they would not be @@ -3634,7 +3636,8 @@ H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_inf if (H5D__virtual_write_one_mapping(dset_info, storage->not_in_tree_list[i]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); } - } else { + } + else { /* Iterate over all mappings */ for (i = 0; i < storage->list_nused; i++) { if (H5D__virtual_write_one_mapping(dset_info, &storage->list[i]) < 0) @@ -4124,8 +4127,8 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) static herr_t H5D__should_build_tree(H5O_storage_virtual_t *storage, hid_t dapl_id, bool *should_build_tree) { - herr_t ret_value = SUCCEED; - H5P_genplist_t *dapl_plist = NULL; + herr_t ret_value = SUCCEED; + H5P_genplist_t *dapl_plist = NULL; bool tree_enabled_dapl = false; FUNC_ENTER_PACKAGE @@ -4173,19 +4176,19 @@ H5D__should_build_tree(H5O_storage_virtual_t *storage, hid_t dapl_id, bool *shou *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping) { +H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping) +{ herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Sanity check that the virtual space has been patched by now */ + /* Sanity check that the virtual space has been patched by now */ assert(mapping->virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); /* Check for "printf" source dataset resolution */ if (mapping->psfn_nsubs || mapping->psdn_nsubs) { /* Iterate over sub-source dsets */ - for (size_t j = mapping->sub_dset_io_start; - j < mapping->sub_dset_io_end; j++) + for (size_t j = mapping->sub_dset_io_start; j < mapping->sub_dset_io_end; j++) if (H5D__virtual_read_one_src(dset_info, &mapping->sub_dset[j]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); } /* end if */ @@ -4207,7 +4210,9 @@ H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual * *------------------------------------------------------------------------- */ -static herr_t H5D__virtual_write_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping) { +static herr_t +H5D__virtual_write_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping) +{ herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE @@ -4221,7 +4226,8 @@ static herr_t H5D__virtual_write_one_mapping(H5D_dset_io_info_t *dset_info, H5O_ for (size_t j = mapping->sub_dset_io_start; j < mapping->sub_dset_io_end; j++) if (H5D__virtual_write_one_src(dset_info, &mapping->sub_dset[j]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset"); - } else + } + else /* Write to source dataset */ if (H5D__virtual_write_one_src(dset_info, &mapping->source_dset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset"); @@ -4241,7 +4247,8 @@ static herr_t H5D__virtual_write_one_mapping(H5D_dset_io_info_t *dset_info, H5O_ *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping) { +H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping) +{ herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE From 2805ba85796d594cc507046237091ddc082c3deb Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Mon, 29 Sep 2025 10:10:06 -0500 Subject: [PATCH 22/43] Add rtree R/W test --- test/rtree.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 2 deletions(-) diff --git a/test/rtree.c b/test/rtree.c index 0f70569b4fc..67e30f516a0 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -53,6 +53,8 @@ #define RTREE_THRESHOLD_FILENAME "vds_rtree_threshold_test.h5" #define RTREE_MAX_TEST_MAPPINGS (H5D_VIRTUAL_TREE_THRESHOLD + 100) +#define RTREE_RW_FILENAME "vds_rtree_rw.h5" + static const size_t test_counts[RTREE_TEST_CREATE_NUM_COUNTS] = {H5D_VIRTUAL_TREE_THRESHOLD, 100, 1000, 10000}; @@ -875,6 +877,115 @@ test_rtree_threshold(bool use_tree) return FAIL; } + +/*------------------------------------------------------------------------- + * Function: test_rtree_rw + * + * Purpose: Test that dataset reads/writes produce correctly values + * with rtree on/off + * + * Return: Success: SUCCEED + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_rtree_rw(bool use_tree) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t dapl_id = H5I_INVALID_HID; + hid_t vdset_id = H5I_INVALID_HID; + hid_t space_id = H5I_INVALID_HID; + hsize_t wdims = RTREE_MAX_TEST_MAPPINGS / 2; + int rbuf[RTREE_MAX_TEST_MAPPINGS]; + int wbuf[RTREE_MAX_TEST_MAPPINGS]; + int num_mappings = RTREE_MAX_TEST_MAPPINGS; + + const char *test_str = use_tree ? "R/W behavior with tree enabled" : "R/W behavior with tree disabled"; + + TESTING(test_str); + + memset(rbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); + memset(wbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); + + if ((file_id = H5Fcreate(RTREE_RW_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + + /* Set the spatial tree property */ + if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree) < 0) + FAIL_STACK_ERROR; + + /* Create virtual dataset with specified number of mappings */ + if ((vdset_id = create_virtual_dataset(file_id, dapl_id, num_mappings)) < 0) + FAIL_STACK_ERROR; + + /* Verify initial read values (each element should equal its index) */ + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + for (int i = 0; i < num_mappings; i++) { + if (rbuf[i] != i) { + printf("%d mappings: Data mismatch at [%d]: expected %d, got %d\n", num_mappings, i, i, rbuf[i]); + FAIL_STACK_ERROR; + } + } + + /* Write to first half of dataset with 2*index */ + for (int i = 0; i < num_mappings / 2; i++) + wbuf[i] = 2 * i; + + if ((space_id = H5Screate_simple(1, (const hsize_t *)&wdims, NULL)) < 0) + FAIL_STACK_ERROR; + + if (H5Sselect_hyperslab(space_id, H5S_SELECT_SET, (const hsize_t *)&(hsize_t){0}, NULL, + (const hsize_t *)&wdims, NULL) < 0) + FAIL_STACK_ERROR; + + if (H5Dwrite(vdset_id, H5T_NATIVE_INT, space_id, space_id, H5P_DEFAULT, wbuf) < 0) + FAIL_STACK_ERROR; + + /* Read back entire dataset and verify values */ + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + for (int i = 0; i < num_mappings; i++) { + int expected = (i < num_mappings / 2) ? (2 * i) : i; + if (rbuf[i] != expected) { + printf("%d mappings: Post-write data mismatch at [%d]: expected %d, got %d\n", num_mappings, i, + expected, rbuf[i]); + // FAIL_STACK_ERROR; + } + } + + /* Cleanup */ + if (H5Dclose(vdset_id) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dapl_id) < 0) + FAIL_STACK_ERROR; + if (H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(space_id) < 0) + FAIL_STACK_ERROR; + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(space_id); + H5Dclose(vdset_id); + H5Pclose(dapl_id); + H5Fclose(file_id); + } + H5E_END_TRY; + + return FAIL; +} + /*------------------------------------------------------------------------- * Function: main * @@ -907,8 +1018,8 @@ main(void) nerrors += test_rtree_threshold(true) < 0 ? 1 : 0; // TODO - Fix failure nerrors += test_rtree_threshold(false) < 0 ? 1 : 0; - - // TODO - Write test with tree on/off + nerrors += test_rtree_rw(true) < 0 ? 1 : 0; + nerrors += test_rtree_rw(false) < 0 ? 1 : 0; if (nerrors) goto error; From 391b6e8795f743f4c1341cf768ac8fa5fe745a2c Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Mon, 29 Sep 2025 10:36:04 -0500 Subject: [PATCH 23/43] Factor out test check to helper --- test/rtree.c | 143 +++++++++++++++++++++++++++------------------------ 1 file changed, 77 insertions(+), 66 deletions(-) diff --git a/test/rtree.c b/test/rtree.c index 67e30f516a0..dfb42667cd5 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -551,6 +551,73 @@ create_virtual_dataset(hid_t file_id, hid_t dapl_id, int num_mappings) return FAIL; } +/*------------------------------------------------------------------------- + * Function: test_rtree_existence_helper + * + * Purpose: Test helper to verify that r-tree existence on a dataset + * matches what is expected + * + * Return: Success: SUCCEED + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_rtree_existence_helper(hid_t vdset_id, bool expect_tree, bool *correct_out) +{ + herr_t ret_value = SUCCEED; + H5D_t *dset = NULL; + H5O_storage_virtual_t *storage = NULL; + + assert(correct_out); + *correct_out = false; + + /* Get the dataset object - this is using internal API for testing */ + if (NULL == (dset = (H5D_t *)H5VL_object(vdset_id))) { + ret_value = FAIL; + goto done; + } + + if (dset->shared->layout.type != H5D_VIRTUAL) { + ret_value = FAIL; + goto done; + } + + /* Get the virtual storage structure */ + storage = &(dset->shared->layout.storage.u.virt); + + /* Verify tree existence matches expectation */ + if (expect_tree) { + if (storage->tree == NULL) { + puts("Expected spatial tree to exist but it was NULL"); + *correct_out = false; + goto done; + } + + if (storage->not_in_tree_nused > 0 && storage->not_in_tree_list == NULL) { + puts("Expected not_in_tree_list array to exist but it was NULL"); + *correct_out = false; + goto done; + } + } + else { + if (storage->tree != NULL) { + puts("Expected spatial tree to be NULL but it exists"); + *correct_out = false; + goto done; + } + if (storage->not_in_tree_list != NULL || storage->not_in_tree_nused > 0) { + puts("Expected not_in_tree_list to be empty but it exists"); + *correct_out = false; + goto done; + } + } + + *correct_out = true; +done: + return ret_value; +} + /*------------------------------------------------------------------------- * Function: test_rtree_dapl * @@ -569,11 +636,8 @@ test_rtree_dapl(bool use_tree) hid_t vdset_id = H5I_INVALID_HID; int rbuf[RTREE_MAX_TEST_MAPPINGS]; - const char *test_str = NULL; - - /* Internal values for introspection */ - H5D_t *dset = NULL; - H5O_storage_virtual_t *storage = NULL; + bool tree_correct = false; + const char *test_str = NULL; /* Inverse of use_tree for re-open part of test */ bool use_tree_inverse = !use_tree; @@ -615,38 +679,13 @@ test_rtree_dapl(bool use_tree) } } - /* Get the dataset object - this is using internal API for testing */ - if (NULL == (dset = (H5D_t *)H5VL_object(vdset_id))) + /* Verify tree existence matches expectation */ + if (test_rtree_existence_helper(vdset_id, use_tree, &tree_correct) < 0) FAIL_STACK_ERROR; - if (dset->shared->layout.type != H5D_VIRTUAL) + if (!tree_correct) FAIL_STACK_ERROR; - /* Get the virtual storage structure */ - storage = &(dset->shared->layout.storage.u.virt); - - /* Verify tree existence matches expectation */ - if (use_tree) { - if (storage->tree == NULL) { - puts("Expected spatial tree to exist but it was NULL"); - FAIL_STACK_ERROR; - } - if (storage->not_in_tree_nused > 0 && storage->not_in_tree_list == NULL) { - puts("Expected not_in_tree_list array to exist but it was NULL"); - FAIL_STACK_ERROR; - } - } - else { - if (storage->tree != NULL) { - puts("Expected spatial tree to be NULL but it exists"); - FAIL_STACK_ERROR; - } - if (storage->not_in_tree_list != NULL || storage->not_in_tree_nused > 0) { - puts("Expected not_in_tree_list to be empty but it exists"); - FAIL_STACK_ERROR; - } - } - /* Close the dataset and re-open it with the opposite value set in DAPL */ if (H5Dclose(vdset_id) < 0) FAIL_STACK_ERROR; @@ -661,12 +700,11 @@ test_rtree_dapl(bool use_tree) if (H5Pclose(dapl_id) < 0) FAIL_STACK_ERROR; - dapl_id = H5I_INVALID_HID; - + dapl_id = H5I_INVALID_HID; + tree_correct = false; memset(rbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); H5close(); - H5open(); if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) FAIL_STACK_ERROR; @@ -692,40 +730,13 @@ test_rtree_dapl(bool use_tree) } } - /* Get the dataset object - this is using internal API for testing */ - if (NULL == (dset = (H5D_t *)H5VL_object(vdset_id))) + /* Verify tree existence matches expectation after re-open */ + if (test_rtree_existence_helper(vdset_id, use_tree_inverse, &tree_correct) < 0) FAIL_STACK_ERROR; - if (dset->shared->layout.type != H5D_VIRTUAL) + if (!tree_correct) FAIL_STACK_ERROR; - storage = &(dset->shared->layout.storage.u.virt); - - /* Verify tree existence matches expectation after re-open */ - if (use_tree_inverse) { - if (storage->tree == NULL) { - puts("Expected spatial tree to exist but it was NULL after re-open"); - FAIL_STACK_ERROR; - } - /* not_in_tree_list can be NULL if all mappings fit in tree - this is OK */ - /* Just verify consistency: if nused > 0, then list should exist */ - if (storage->not_in_tree_nused > 0 && storage->not_in_tree_list == NULL) { - puts("Expected not_in_tree_list array to exist but it was NULL after re-open"); - FAIL_STACK_ERROR; - } - /* When tree is enabled, we just verify tree exists - not_in_tree_list may or may not exist */ - } - else { - if (storage->tree != NULL) { - puts("Expected spatial tree to be NULL but it exists after re-open"); - FAIL_STACK_ERROR; - } - if (storage->not_in_tree_list != NULL || storage->not_in_tree_nused > 0) { - puts("Expected not_in_tree_list to be empty but it exists after re-open"); - FAIL_STACK_ERROR; - } - } - /* Cleanup */ if (H5Dclose(vdset_id) < 0) FAIL_STACK_ERROR; From e6d689a34c11fc67d9f7cac65141d0e97442b63d Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Mon, 29 Sep 2025 10:55:26 -0500 Subject: [PATCH 24/43] Add write init test cases --- test/rtree.c | 77 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/test/rtree.c b/test/rtree.c index dfb42667cd5..e8d4b28b136 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -629,54 +629,66 @@ test_rtree_existence_helper(hid_t vdset_id, bool expect_tree, bool *correct_out) *------------------------------------------------------------------------- */ static herr_t -test_rtree_dapl(bool use_tree) +test_rtree_dapl(bool use_tree, bool read_init) { hid_t file_id = H5I_INVALID_HID; hid_t dapl_id = H5I_INVALID_HID; hid_t vdset_id = H5I_INVALID_HID; - int rbuf[RTREE_MAX_TEST_MAPPINGS]; - bool tree_correct = false; - const char *test_str = NULL; + int rbuf[RTREE_MAX_TEST_MAPPINGS]; + int wbuf[RTREE_MAX_TEST_MAPPINGS]; + bool tree_correct = false; + char test_str[256]; /* Inverse of use_tree for re-open part of test */ bool use_tree_inverse = !use_tree; - if (use_tree) { - test_str = "spatial tree option enabled"; + memset(test_str, 0, sizeof(test_str)); + + if (snprintf(test_str, sizeof(test_str), "spatial tree option %s", use_tree ? "enabled" : "disabled") < 0) + FAIL_STACK_ERROR; + + if (read_init) { + strncat(test_str, " with read initialization", sizeof(test_str) - strlen(test_str) - 1); } else { - test_str = "spatial tree option disabled"; + strncat(test_str, " with write initialization", sizeof(test_str) - strlen(test_str) - 1); } TESTING(test_str); - /* Create file */ + memset(rbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); + memset(wbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); + + /* One-time setup */ if ((file_id = H5Fcreate(RTREE_DAPL_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR; - /* Create DAPL */ if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) FAIL_STACK_ERROR; - /* Set the spatial tree property */ - if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree) < 0) - FAIL_STACK_ERROR; - /* Create virtual dataset with enough mappings to use tree */ if ((vdset_id = create_virtual_dataset(file_id, dapl_id, RTREE_MAX_TEST_MAPPINGS)) < 0) FAIL_STACK_ERROR; - /* Read the entire virtual dataset to force tree initialization */ - if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + if (H5Dclose(vdset_id) < 0) FAIL_STACK_ERROR; - /* Verify read data matches expected pattern */ - for (int i = 0; i < (RTREE_MAX_TEST_MAPPINGS); i++) { - if (rbuf[i] != i) { - printf("Data mismatch at [%d]: expected %d, got %d\n", i, i, rbuf[i]); + /* Set the spatial tree property */ + if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree) < 0) + FAIL_STACK_ERROR; + + if ((vdset_id = H5Dopen2(file_id, RTREE_DAPL_VDS_NAME, dapl_id)) < 0) + FAIL_STACK_ERROR; + + /* Read/write the entire virtual dataset to force tree initialization */ + if (read_init) { + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + } + else { + if (H5Dwrite(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) FAIL_STACK_ERROR; - } } /* Verify tree existence matches expectation */ @@ -704,8 +716,6 @@ test_rtree_dapl(bool use_tree) tree_correct = false; memset(rbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); - H5close(); - if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) FAIL_STACK_ERROR; @@ -718,16 +728,14 @@ test_rtree_dapl(bool use_tree) if ((vdset_id = H5Dopen2(file_id, RTREE_DAPL_VDS_NAME, dapl_id)) < 0) FAIL_STACK_ERROR; - /* Read the entire virtual dataset to force tree initialization */ - if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) - FAIL_STACK_ERROR; - - /* Verify read data matches expected pattern */ - for (int i = 0; i < (RTREE_MAX_TEST_MAPPINGS); i++) { - if (rbuf[i] != i) { - printf("Data mismatch after re-open at [%d]: expected %d, got %d\n", i, i, rbuf[i]); + /* Read/write the entire virtual dataset to force tree initialization */ + if (read_init) { + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + } + else { + if (H5Dwrite(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) FAIL_STACK_ERROR; - } } /* Verify tree existence matches expectation after re-open */ @@ -1022,8 +1030,10 @@ main(void) nerrors += test_rtree_copy() < 0 ? 1 : 0; /* Test spatial tree with DAPL property enabled */ - nerrors += test_rtree_dapl(true) < 0 ? 1 : 0; - nerrors += test_rtree_dapl(false) < 0 ? 1 : 0; + nerrors += test_rtree_dapl(true, true) < 0 ? 1 : 0; + nerrors += test_rtree_dapl(true, false) < 0 ? 1 : 0; + nerrors += test_rtree_dapl(false, true) < 0 ? 1 : 0; + nerrors += test_rtree_dapl(false, false) < 0 ? 1 : 0; /* Test the mapping count threshold */ nerrors += test_rtree_threshold(true) < 0 ? 1 : 0; @@ -1031,6 +1041,7 @@ main(void) nerrors += test_rtree_threshold(false) < 0 ? 1 : 0; nerrors += test_rtree_rw(true) < 0 ? 1 : 0; nerrors += test_rtree_rw(false) < 0 ? 1 : 0; + if (nerrors) goto error; From 1df23b511e5c959e5befe6ec9e297eac314fb914 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Mon, 29 Sep 2025 13:57:57 -0500 Subject: [PATCH 25/43] Remove extent check for src dataspaces --- src/H5Dvirtual.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 23c502845bd..eb9178ea633 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -106,8 +106,7 @@ should_insert = false; \ } \ /* Do not insert zero-dim mappings */ \ - else if ((_vspace && H5S_GET_EXTENT_NDIMS(_vspace) < 1) || \ - (_src_space && H5S_GET_EXTENT_NDIMS(_src_space) < 1)) { \ + else if ((_vspace && H5S_GET_EXTENT_NDIMS(_vspace) < 1)) { \ should_insert = false; \ } \ /* Otherwise, we can insert it */ \ From 5727f418be465190c4486906730da67734562c76 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 30 Sep 2025 09:03:44 -0500 Subject: [PATCH 26/43] Cleanup --- src/H5Dvirtual.c | 6 +++--- test/rtree.c | 11 ++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index eb9178ea633..e41aac3a704 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -106,7 +106,7 @@ should_insert = false; \ } \ /* Do not insert zero-dim mappings */ \ - else if ((_vspace && H5S_GET_EXTENT_NDIMS(_vspace) < 1)) { \ + else if ((_vspace && H5S_GET_EXTENT_NDIMS(_vspace) < 1)) { \ should_insert = false; \ } \ /* Otherwise, we can insert it */ \ @@ -4116,7 +4116,7 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) /*------------------------------------------------------------------------- * Function: H5D__should_build_tree * - * Purpose: Determine whether to build a spatial tree of mapping indice + * Purpose: Determine whether to build a spatial tree of mapping indices * for the provided dataset layout * * Return: Non-negative on success/Negative on failure @@ -4201,7 +4201,7 @@ H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual } /*------------------------------------------------------------------------- - * Function: H5D__virtual_read_one_mapping + * Function: H5D__virtual_write_one_mapping * * Purpose: Write to a single mapping entry in a virtual dataset * diff --git a/test/rtree.c b/test/rtree.c index e8d4b28b136..cb60ec35e14 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -975,7 +975,7 @@ test_rtree_rw(bool use_tree) if (rbuf[i] != expected) { printf("%d mappings: Post-write data mismatch at [%d]: expected %d, got %d\n", num_mappings, i, expected, rbuf[i]); - // FAIL_STACK_ERROR; + FAIL_STACK_ERROR; } } @@ -1045,6 +1045,15 @@ main(void) if (nerrors) goto error; + H5E_BEGIN_TRY + { + H5Fdelete(RTREE_SRC_FILENAME, H5P_DEFAULT); + H5Fdelete(RTREE_DAPL_FILENAME, H5P_DEFAULT); + H5Fdelete(RTREE_THRESHOLD_FILENAME, H5P_DEFAULT); + H5Fdelete(RTREE_RW_FILENAME, H5P_DEFAULT); + } + H5E_END_TRY; + printf("All R-tree tests passed.\n"); return EXIT_SUCCESS; From d5dd171710495a60c38e58a43bdf04e937ab7302 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 30 Sep 2025 15:19:14 -0500 Subject: [PATCH 27/43] Expand java documentation --- java/src/hdf/hdf5lib/H5.java | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index b399c6080a7..aafeb8ba1bd 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -10678,8 +10678,17 @@ public synchronized static native void H5Pset_efile_prefix(long dapl_id, String /** * @ingroup JH5P * - * H5Pget_dset_use_spatial_tree accesses the flag for whether or not datasets accessed with the given dapl - * will use a spatial tree for mappings + * H5Pget_dset_use_spatial_tree accesses the flag for whether to use/not use a spatial tree + * during mapping operations on a Virtual Dataset. The default value is true. + * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. * * @param dapl_id * IN: Dataset access property list @@ -10695,8 +10704,17 @@ public synchronized static native boolean H5Pget_dset_use_spatial_tree(long dapl /** * @ingroup JH5P * - * H5Pset_dset_use_spatial_tree sets the dapl to not use (or explicitly use) a spatial tree - * during mapping operations + * H5Pset_dset_use_spatial_tree sets the dapl to use/not use a spatial tree + * during mapping operations on a Virtual Dataset. The default value is true. + * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. * * @param dapl_id * IN: Dataset access property list From c8abd880074d1b80ce2acb6e8cda7ed68f70133c Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 30 Sep 2025 15:27:43 -0500 Subject: [PATCH 28/43] Streamline leaf validity check --- src/H5Dvirtual.c | 51 +++++++++--------------------------------------- 1 file changed, 9 insertions(+), 42 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index e41aac3a704..1c9f4228c7d 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -75,45 +75,16 @@ * Determines whether a virtual dataset mapping entry should be inserted * into the R-tree spatial index. * - * Parameters: - * entry - H5O_storage_virtual_ent_t* mapping entry to check - * should_insert - bool output parameter set to true/false + * The following mappings cannot be added to the tree: + * - Mappings with an unlimited dimension, since this would invalidate the spatial search process + * - Mappings with zero dimensions, since the tree relies on each entry having at least one dimension * */ -#define H5D_RTREE_SHOULD_INSERT(entry, should_insert) \ - do { \ - H5S_t *_vspace = NULL; \ - H5S_t *_src_space = NULL; \ - hsize_t _virt_nelems = 0; \ - hsize_t _src_nelems = 0; \ - \ - assert(entry); \ - \ - /* Get virtual space and element count */ \ - if ((_vspace = (entry)->source_dset.virtual_select) != NULL) \ - _virt_nelems = (hsize_t)H5S_GET_SELECT_NPOINTS(_vspace); \ - \ - /* Get source space and element count */ \ - if ((_src_space = (entry)->source_dset.clipped_source_select) != NULL) \ - _src_nelems = (hsize_t)H5S_GET_SELECT_NPOINTS(_src_space); \ - \ - /* Do not insert mappings with an unlimited dimension */ \ - if (_virt_nelems == H5S_UNLIMITED || _src_nelems == H5S_UNLIMITED) { \ - should_insert = false; \ - } \ - /* Do not insert printf-style mappings */ \ - else if ((entry)->psfn_nsubs > 0 || (entry)->psdn_nsubs > 0) { \ - should_insert = false; \ - } \ - /* Do not insert zero-dim mappings */ \ - else if ((_vspace && H5S_GET_EXTENT_NDIMS(_vspace) < 1)) { \ - should_insert = false; \ - } \ - /* Otherwise, we can insert it */ \ - else { \ - should_insert = true; \ - } \ - } while (0) + +/* Mappings with an unlimited dimension */ +#define H5D_RTREE_SHOULD_INSERT(entry) \ + !(((entry)->unlim_dim_virtual >= 0) || ((entry)->source_dset.virtual_select && \ + H5S_GET_EXTENT_NDIMS((entry)->source_dset.virtual_select) < 1)) /******************/ /* Local Typedefs */ @@ -3932,8 +3903,6 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings H5RT_leaf_t *leaves_temp = NULL; H5O_storage_virtual_ent_t **not_in_tree = NULL; - bool should_insert_space = false; - H5O_storage_virtual_ent_t *curr_mapping = NULL; H5RT_leaf_t *curr_leaf = NULL; size_t curr_leaf_count = 0; @@ -3972,9 +3941,7 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings for (size_t i = 0; i < num_mappings; i++) { curr_mapping = &mappings[i]; - H5D_RTREE_SHOULD_INSERT(curr_mapping, should_insert_space); - - if (!should_insert_space) { + if (!(H5D_RTREE_SHOULD_INSERT(curr_mapping))) { /* Store pointer to mapping in not_in_tree array */ not_in_tree[curr_not_tree_count] = curr_mapping; curr_not_tree_count++; From 8b3dcb02ea438e4cb0d35fa8477848dd143699ab Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 30 Sep 2025 15:30:36 -0500 Subject: [PATCH 29/43] Optimize allocation in layout copy --- src/H5Dvirtual.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 1c9f4228c7d..6838493363a 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -1097,10 +1097,12 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) /* Allocate new pointer array */ if ((virt->not_in_tree_list = (H5O_storage_virtual_ent_t **)H5MM_calloc( - virt->not_in_tree_nalloc * sizeof(H5O_storage_virtual_ent_t *))) == NULL) + virt->not_in_tree_nused * sizeof(H5O_storage_virtual_ent_t *))) == NULL) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate not_in_tree_list pointer array"); + virt->not_in_tree_nalloc = virt->not_in_tree_nused; + /* Point to corresponding entries in the new list */ for (i = 0; i < virt->not_in_tree_nused; i++) { ptrdiff_t offset = orig_not_in_tree_list[i] - orig_list; /* Calculate original offset */ From e0a7fd41c6c47f69a84e4379a9d0595a11bf9e99 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 30 Sep 2025 15:34:16 -0500 Subject: [PATCH 30/43] Add func description for H5D__virtual_pre_io_process_mapping --- src/H5Dvirtual.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 6838493363a..c5e24ebf12e 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -2857,7 +2857,18 @@ H5D__virtual_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t H5_ATTR_UNUSED * FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__virtual_io_init() */ -// TODO +/*------------------------------------------------------------------------- + * Function: H5D__virtual_pre_io_process_mapping + * + * Purpose: Process a single virtual mapping to prepare for I/O. + * This includes projecting the virtual mapping onto mem_space + * and opening source datasets. The number of elements included + * in this mapping selection is added to tot_nelmts. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ static herr_t H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, H5S_t *file_space, H5S_t *mem_space, hsize_t *tot_nelmts, H5O_storage_virtual_ent_t *curr_mapping) @@ -3083,7 +3094,7 @@ H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, H5S_t *file_s } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value) -} +} /* end H5D__virtual_pre_io_process_mapping() */ /*------------------------------------------------------------------------- * Function: H5D__virtual_pre_io From 93d64f2ec1618a7050bdf1e29eea2357233c852d Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 30 Sep 2025 15:36:57 -0500 Subject: [PATCH 31/43] Add function end comments --- src/H5Dvirtual.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index c5e24ebf12e..807de9aaf6f 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -4002,7 +4002,7 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings } FUNC_LEAVE_NOAPI(ret_value); -} +} /* end H5D__mappings_to_leaves() */ /*------------------------------------------------------------------------- * Function: H5D__virtual_build_tree @@ -4091,7 +4091,7 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) } FUNC_LEAVE_NOAPI(ret_value) -} +} /* end H5D__virtual_build_tree() */ /*------------------------------------------------------------------------- * Function: H5D__should_build_tree @@ -4143,7 +4143,7 @@ H5D__should_build_tree(H5O_storage_virtual_t *storage, hid_t dapl_id, bool *shou *should_build_tree = true; done: FUNC_LEAVE_NOAPI(ret_value); -} +} /* end H5D__should_build_tree() */ /*------------------------------------------------------------------------- * Function: H5D__virtual_read_one_mapping @@ -4178,7 +4178,7 @@ H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual done: FUNC_LEAVE_NOAPI(ret_value) -} +} /* end H5D__virtual_read_one_mapping() */ /*------------------------------------------------------------------------- * Function: H5D__virtual_write_one_mapping @@ -4213,7 +4213,7 @@ H5D__virtual_write_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtua done: FUNC_LEAVE_NOAPI(ret_value) -} +} /* end H5D__virtual_write_one_mapping() */ /*------------------------------------------------------------------------- * Function: H5D__virtual_close_mapping @@ -4252,4 +4252,4 @@ H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping) } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value); -} \ No newline at end of file +} /* end H5D__virtual_close_mapping() */ \ No newline at end of file From 358f39c29d448adc66d468159b5f59e65788c2e2 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 30 Sep 2025 16:01:15 -0500 Subject: [PATCH 32/43] Grow not_in_tree list allocation by powers of 2 --- src/H5Dvirtual.c | 133 +++++++++++++++++++++++++++++++++++++++-------- src/H5Oprivate.h | 2 +- 2 files changed, 113 insertions(+), 22 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 807de9aaf6f..f9cba28f052 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -71,6 +71,9 @@ /* Default size for sub_dset array */ #define H5D_VIRTUAL_DEF_SUB_DSET_SIZE 128 +/* Default size for not_in_tree list allocation */ +#define H5D_VIRTUAL_NOT_IN_TREE_INIT_SIZE 64 + /* * Determines whether a virtual dataset mapping entry should be inserted * into the R-tree spatial index. @@ -134,8 +137,12 @@ static herr_t H5D__virtual_write_one_src(H5D_dset_io_info_t *dset_inf static herr_t H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank); static herr_t H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, H5RT_leaf_t **leaves_out, H5O_storage_virtual_ent_t ***not_in_tree_out, - size_t *leaf_count, size_t *not_in_tree_count); + size_t *leaf_count, size_t *not_in_tree_count, + size_t *not_in_tree_nalloc); static herr_t H5D__should_build_tree(H5O_storage_virtual_t *storage, hid_t dapl_id, bool *should_build_tree); +static herr_t H5D__virtual_not_in_tree_grow(H5O_storage_virtual_ent_t ***list, size_t *nalloc); +static herr_t H5D__virtual_not_in_tree_add(H5O_storage_virtual_ent_t ***list, size_t *nused, size_t *nalloc, + H5O_storage_virtual_ent_t *mapping); /*********************/ /* Package Variables */ /*********************/ @@ -3901,6 +3908,7 @@ H5D__virtual_release_source_dset_files(H5D_virtual_held_file_t *head) * Allocated on success and must be freed by caller. * leaf_count: Pointer to number of leaves allocated in leaves_out. * not_in_tree_count: Pointer to number of entries in not_in_tree_out. + * not_in_tree_nalloc: Pointer to allocated capacity of not_in_tree_out. * * Return: Non-negative on success/Negative on failure * @@ -3909,18 +3917,19 @@ H5D__virtual_release_source_dset_files(H5D_virtual_held_file_t *head) static herr_t H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, H5RT_leaf_t **leaves_out, H5O_storage_virtual_ent_t ***not_in_tree_out, size_t *leaf_count, - size_t *not_in_tree_count) + size_t *not_in_tree_count, size_t *not_in_tree_nalloc) { herr_t ret_value = SUCCEED; H5RT_leaf_t *leaves_temp = NULL; H5O_storage_virtual_ent_t **not_in_tree = NULL; - H5O_storage_virtual_ent_t *curr_mapping = NULL; - H5RT_leaf_t *curr_leaf = NULL; - size_t curr_leaf_count = 0; - size_t curr_not_tree_count = 0; - H5S_t *curr_space = NULL; + H5O_storage_virtual_ent_t *curr_mapping = NULL; + H5RT_leaf_t *curr_leaf = NULL; + size_t curr_leaf_count = 0; + size_t curr_not_tree_count = 0; + size_t not_in_tree_capacity = 0; + H5S_t *curr_space = NULL; int rank = 0; @@ -3932,6 +3941,7 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings assert(leaves_out); assert(not_in_tree_out); assert(not_in_tree_count); + assert(not_in_tree_nalloc); /* Get rank from the first mapping's virtual selection */ if ((curr_space = mappings[0].source_dset.virtual_select) == NULL) @@ -3943,21 +3953,25 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings if (rank == 0) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "mapping has zero-dimensional space"); - /* Allocate array of leaf structures and not-in-tree mappings */ + /* Allocate array of leaf structures */ if ((leaves_temp = (H5RT_leaf_t *)calloc(num_mappings, sizeof(H5RT_leaf_t))) == NULL) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate leaves array"); - if ((not_in_tree = (H5O_storage_virtual_ent_t **)H5MM_calloc( - num_mappings * sizeof(H5O_storage_virtual_ent_t *))) == NULL) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate not_in_tree array"); + /* Initialize not_in_tree list with initial capacity */ + not_in_tree_capacity = H5D_VIRTUAL_NOT_IN_TREE_INIT_SIZE; + + if (NULL == (not_in_tree = (H5O_storage_virtual_ent_t **)H5MM_malloc( + not_in_tree_capacity * sizeof(H5O_storage_virtual_ent_t *)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failed to allocate not_in_tree_list"); for (size_t i = 0; i < num_mappings; i++) { curr_mapping = &mappings[i]; if (!(H5D_RTREE_SHOULD_INSERT(curr_mapping))) { - /* Store pointer to mapping in not_in_tree array */ - not_in_tree[curr_not_tree_count] = curr_mapping; - curr_not_tree_count++; + /* Add to not_in_tree list, growing if needed */ + if (H5D__virtual_not_in_tree_add(¬_in_tree, &curr_not_tree_count, ¬_in_tree_capacity, + curr_mapping) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failed to add to not_in_tree_list"); continue; } @@ -3984,10 +3998,11 @@ H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings curr_leaf_count++; } - *leaves_out = leaves_temp; - *leaf_count = curr_leaf_count; - *not_in_tree_out = not_in_tree; - *not_in_tree_count = curr_not_tree_count; + *leaves_out = leaves_temp; + *leaf_count = curr_leaf_count; + *not_in_tree_out = not_in_tree; + *not_in_tree_count = curr_not_tree_count; + *not_in_tree_nalloc = not_in_tree_capacity; done: if (ret_value < 0) { if (leaves_temp) { @@ -4030,6 +4045,7 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) size_t num_leaves = 0; H5O_storage_virtual_ent_t **not_in_tree_mappings = NULL; size_t not_in_tree_count = 0; + size_t not_in_tree_nalloc = 0; herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE @@ -4037,7 +4053,7 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) assert(virt); if (H5D__mappings_to_leaves(mappings, num_mappings, &leaves, ¬_in_tree_mappings, &num_leaves, - ¬_in_tree_count) < 0) + ¬_in_tree_count, ¬_in_tree_nalloc) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get leaves from mappings"); if (num_leaves == 0) { @@ -4060,8 +4076,8 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) if (not_in_tree_count > 0) { virt->not_in_tree_list = not_in_tree_mappings; virt->not_in_tree_nused = not_in_tree_count; - virt->not_in_tree_nalloc = not_in_tree_count; /* Currently allocated exactly what we need */ - not_in_tree_mappings = NULL; /* Transfer ownership to virt */ + virt->not_in_tree_nalloc = not_in_tree_nalloc; + not_in_tree_mappings = NULL; /* Transfer ownership to virt */ } else { /* Clean up any existing allocation */ @@ -4093,6 +4109,81 @@ H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_build_tree() */ +/*------------------------------------------------------------------------- + * Function: H5D__virtual_not_in_tree_grow + * + * Purpose: Double the capacity of the not_in_tree_list buffer + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_not_in_tree_grow(H5O_storage_virtual_ent_t ***list, size_t *nalloc) +{ + size_t new_capacity = 0; + H5O_storage_virtual_ent_t **new_list = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + assert(list); + assert(*list); + assert(nalloc); + + new_capacity = *nalloc * 2; + + /* Overflow check */ + if (new_capacity < *nalloc || new_capacity > (SIZE_MAX / sizeof(H5O_storage_virtual_ent_t *))) + HGOTO_ERROR(H5E_DATASET, H5E_OVERFLOW, FAIL, "not_in_tree_list capacity overflow"); + + if (NULL == (new_list = (H5O_storage_virtual_ent_t **)H5MM_realloc( + *list, new_capacity * sizeof(H5O_storage_virtual_ent_t *)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failed to grow not_in_tree_list"); + + *list = new_list; + *nalloc = new_capacity; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_not_in_tree_grow() */ + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_not_in_tree_add + * + * Purpose: Add a mapping to the not_in_tree_list, growing if necessary + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_not_in_tree_add(H5O_storage_virtual_ent_t ***list, size_t *nused, size_t *nalloc, + H5O_storage_virtual_ent_t *mapping) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + assert(list); + assert(*list); + assert(nused); + assert(nalloc); + assert(mapping); + + /* Grow buffer if full */ + if (*nused >= *nalloc) { + if (H5D__virtual_not_in_tree_grow(list, nalloc) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failed to grow not_in_tree_list"); + } + + (*list)[*nused] = mapping; + (*nused)++; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_not_in_tree_add() */ + /*------------------------------------------------------------------------- * Function: H5D__should_build_tree * diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index bf2580758a2..ba9901e28e8 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -606,7 +606,7 @@ typedef struct H5O_storage_virtual_t { first occurrence of each source dataset name is stored. */ H5RT_t *tree; size_t not_in_tree_nused; /* Number of entries in not_in_tree_list */ - size_t not_in_tree_nalloc; /* Allocated size of not_in_tree_list */ + size_t not_in_tree_nalloc; /* Allocated size of not_in_tree_list (grows by power of 2) */ H5O_storage_virtual_ent_t * *not_in_tree_list; /* Array of POINTERS to mappings NOT in tree for quick access * Some mappings cannot be stored in the tree and must be searched manually */ From debe75152f1e0edd3dd27e73d314f19b6382cfc3 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Wed, 1 Oct 2025 16:30:08 -0500 Subject: [PATCH 33/43] Add file end newline --- src/H5Dvirtual.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index f9cba28f052..3d063229316 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -4343,4 +4343,4 @@ H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping) } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value); -} /* end H5D__virtual_close_mapping() */ \ No newline at end of file +} /* end H5D__virtual_close_mapping() */ From 1f8070ebb9c4bab9e9a0356d5451eea9a0170aef Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Wed, 1 Oct 2025 16:33:00 -0500 Subject: [PATCH 34/43] Expand Fortran documentation --- fortran/src/H5Pff.F90 | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index 6c775856f6e..dc6a1557998 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -4368,8 +4368,17 @@ END SUBROUTINE h5pget_chunk_cache_f !> !! \ingroup FH5P !! -!! \brief Gets the value of the "use spatial tree" flag which enables spatial tree -!! construction and usage for VDS mapping searches. +!! \brief Retrieves the flag for whether to use/not use a spatial tree +!! during mapping operations on a Virtual Dataset. The default value is true. +!! +!! Use of a spatial tree will accelerate the process of searching through mappings +!! to determine which contain intersections with the user's selection region. +!! With the tree disabled, all mappings will simply be iterated through and +!! checked directly. +!! +!! Certain workflows may find that tree creation overhead outweighs the time saved +!! on reads. In this case, disabling this property will lead to a performance improvement, +!! though it is expected that almost all cases will benefit from the tree on net. !! !! \param dapl_id Target dataset access property list identifier. !! \param use_tree Value of the setting. @@ -4403,8 +4412,17 @@ END SUBROUTINE h5pget_dset_use_spatial_tree_f !> !! \ingroup FH5P !! -!! \brief Sets the value of the "use spatial tree" flag which enables spatial tree -!! construction and usage for large VDS mapping searches. +!! \brief Sets the dapl to use/not use a spatial tree +!! during mapping operations on a Virtual Dataset. The default value is true. +!! +!! Use of a spatial tree will accelerate the process of searching through mappings +!! to determine which contain intersections with the user's selection region. +!! With the tree disabled, all mappings will simply be iterated through and +!! checked directly. +!! +!! Certain workflows may find that tree creation overhead outweighs the time saved +!! on reads. In this case, disabling this property will lead to a performance improvement, +!! though it is expected that almost all cases will benefit from the tree on net. !! !! \param dapl_id Target dataset access property list identifier. !! \param use_tree Value of the setting. From d1450f837a13df1be477905aff4b59a17eae9940 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Wed, 1 Oct 2025 16:37:35 -0500 Subject: [PATCH 35/43] Update C documentation --- src/H5Pdapl.c | 18 ++++++++++++++++++ src/H5Ppublic.h | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/H5Pdapl.c b/src/H5Pdapl.c index 268de2f8223..4817263aa6e 100644 --- a/src/H5Pdapl.c +++ b/src/H5Pdapl.c @@ -1564,6 +1564,15 @@ H5Pget_virtual_prefix(hid_t plist_id, char *prefix /*out*/, size_t size) * Access the flag for whether or not datasets created by the given dcpl * construct a spatial tree and use it when searching over VDS mappings * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. + * * Return: * * Failure: Negative value (FAIL) @@ -1604,6 +1613,15 @@ H5Pget_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) * Set the DAPL to construct a spatial tree and use it when searching over * VDS mappings * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. + * * Return: * * Failure: Negative value (FAIL) diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index d6bd3ea419a..0ed4b20650b 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -6031,6 +6031,15 @@ H5_DLL herr_t H5Pget_dset_no_attrs_hint(hid_t dcpl_id, bool *minimize); * The setting value is returned in the boolean pointer * \p use_tree. * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. + * * \since 2.0.0 * */ @@ -6536,6 +6545,15 @@ H5_DLL herr_t H5Pset_dset_no_attrs_hint(hid_t dcpl_id, bool minimize); * it in an attempt to optimize intersection-check * operations on mappings when \p use_tree is set to true. * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. + * * \since 2.0.0 * */ From 039c90c323c9c618cfbf6afcbffcc6e147dfe2a2 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Thu, 2 Oct 2025 14:06:33 -0500 Subject: [PATCH 36/43] Rename DAPL functions --- doxygen/examples/tables/propertyLists.dox | 2 +- fortran/src/H5Pff.F90 | 24 +++++++++++------------ fortran/src/hdf5_fortrandll.def.in | 4 ++-- fortran/test/tH5P.F90 | 20 +++++++++---------- java/src/hdf/hdf5lib/H5.java | 8 ++++---- java/src/jni/h5pDAPLImp.c | 16 +++++++-------- java/src/jni/h5pDAPLImp.h | 4 ++-- release_docs/CHANGELOG.md | 2 +- src/H5Pdapl.c | 12 ++++++------ src/H5Ppublic.h | 8 ++++---- test/rtree.c | 8 ++++---- 11 files changed, 54 insertions(+), 54 deletions(-) diff --git a/doxygen/examples/tables/propertyLists.dox b/doxygen/examples/tables/propertyLists.dox index 1983b5adce4..588c1dd1b4b 100644 --- a/doxygen/examples/tables/propertyLists.dox +++ b/doxygen/examples/tables/propertyLists.dox @@ -617,7 +617,7 @@ encoding for object names. Sets/gets the view of the virtual dataset (VDS) to include or exclude missing mapped elements. -#H5Pset_dset_use_spatial_tree/#H5Pget_dset_use_spatial_tree +#H5Pset_virtual_dset_use_spatial_tree/#H5Pget_virtual_dset_use_spatial_tree Sets/gets the flag to use spatial trees when searching many VDS mappings diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index dc6a1557998..b0a91dc39d0 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -4384,9 +4384,9 @@ END SUBROUTINE h5pget_chunk_cache_f !! \param use_tree Value of the setting. !! \param hdferr \fortran_error !! -!! See C API: @ref H5Pget_dset_use_spatial_tree() +!! See C API: @ref H5Pget_virtual_dset_use_spatial_tree() !! - SUBROUTINE h5pget_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) + SUBROUTINE h5pget_virtual_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) IMPLICIT NONE INTEGER(HID_T) , INTENT(IN) :: dapl_id LOGICAL , INTENT(OUT) :: use_tree @@ -4394,20 +4394,20 @@ SUBROUTINE h5pget_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) LOGICAL(C_BOOL) :: c_use_tree INTERFACE - INTEGER(C_INT) FUNCTION H5Pget_dset_use_spatial_tree_c(dapl_id, use_tree) BIND(C, NAME='H5Pget_dset_use_spatial_tree') + INTEGER(C_INT) FUNCTION H5Pget_virtual_dset_use_spatial_tree_c(dapl_id, use_tree) BIND(C, NAME='H5Pget_virtual_dset_use_spatial_tree') IMPORT :: C_INT, HID_T, C_BOOL IMPLICIT NONE INTEGER(HID_T), INTENT(IN), VALUE :: dapl_id LOGICAL(C_BOOL), INTENT(OUT) :: use_tree - END FUNCTION H5Pget_dset_use_spatial_tree_c + END FUNCTION H5Pget_virtual_dset_use_spatial_tree_c END INTERFACE - hdferr = INT(H5Pget_dset_use_spatial_tree_c(dapl_id, c_use_tree)) + hdferr = INT(H5Pget_virtual_dset_use_spatial_tree_c(dapl_id, c_use_tree)) ! Transfer value of C C_BOOL type to Fortran LOGICAL use_tree = c_use_tree - END SUBROUTINE h5pget_dset_use_spatial_tree_f + END SUBROUTINE h5pget_virtual_dset_use_spatial_tree_f !> !! \ingroup FH5P @@ -4428,9 +4428,9 @@ END SUBROUTINE h5pget_dset_use_spatial_tree_f !! \param use_tree Value of the setting. !! \param hdferr \fortran_error !! -!! See C API: @ref H5Pset_dset_use_spatial_tree() +!! See C API: @ref H5Pset_virtual_dset_use_spatial_tree() !! - SUBROUTINE h5pset_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) + SUBROUTINE h5pset_virtual_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) IMPLICIT NONE INTEGER(HID_T) , INTENT(IN) :: dapl_id LOGICAL , INTENT(IN) :: use_tree @@ -4438,20 +4438,20 @@ SUBROUTINE h5pset_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) LOGICAL(C_BOOL) :: c_use_tree INTERFACE - INTEGER FUNCTION h5pset_dset_use_spatial_tree_c(dapl_id, use_tree) BIND(C, NAME='H5Pset_dset_use_spatial_tree') + INTEGER FUNCTION h5pset_virtual_dset_use_spatial_tree_c(dapl_id, use_tree) BIND(C, NAME='H5Pset_virtual_dset_use_spatial_tree') IMPORT :: HID_T, C_BOOL IMPLICIT NONE INTEGER(HID_T), INTENT(IN), VALUE :: dapl_id LOGICAL(C_BOOL), INTENT(IN), VALUE :: use_tree - END FUNCTION h5pset_dset_use_spatial_tree_c + END FUNCTION h5pset_virtual_dset_use_spatial_tree_c END INTERFACE ! Transfer value of Fortran LOGICAL to C C_BOOL type c_use_tree = use_tree - hdferr = INT(h5pset_dset_use_spatial_tree_c(dapl_id, c_use_tree)) + hdferr = INT(h5pset_virtual_dset_use_spatial_tree_c(dapl_id, c_use_tree)) - END SUBROUTINE h5pset_dset_use_spatial_tree_f + END SUBROUTINE h5pset_virtual_dset_use_spatial_tree_f #ifdef H5_DOXYGEN !> diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index 4ecf34ea8d5..01797266d06 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -420,8 +420,8 @@ H5P_mp_H5PGET_VIRTUAL_VSPACE_F H5P_mp_H5PGET_VIRTUAL_SRCSPACE_F H5P_mp_H5PGET_VIRTUAL_FILENAME_F H5P_mp_H5PGET_VIRTUAL_DSETNAME_F -H5P_mp_H5PGET_DSET_USE_SPATIAL_TREE_F -H5P_mp_H5PSET_DSET_USE_SPATIAL_TREE_F +H5P_mp_h5pget_virtual_dset_use_spatial_tree_f +H5P_mp_h5pset_virtual_dset_use_spatial_tree_f H5P_mp_H5PGET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_VOL_F diff --git a/fortran/test/tH5P.F90 b/fortran/test/tH5P.F90 index 5e066b28c9d..e2ee1589eba 100644 --- a/fortran/test/tH5P.F90 +++ b/fortran/test/tH5P.F90 @@ -835,26 +835,26 @@ SUBROUTINE test_misc_properties(total_error) ! Test H5Pset/get_dset_use_spatial_tree_f ! true value use_spatial_tree = .TRUE. - CALL h5pset_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) - CALL check("h5pset_dset_use_spatial_tree_f", error, total_error) + CALL h5pset_virtual_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pset_virtual_dset_use_spatial_tree_f", error, total_error) use_spatial_tree = .FALSE. - CALL h5pget_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) - CALL check("h5pget_dset_use_spatial_tree_f", error, total_error) + CALL h5pget_virtual_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pget_virtual_dset_use_spatial_tree_f", error, total_error) if(use_spatial_tree .neqv. .TRUE.) then total_error = total_error + 1 - write(*,*) "Got wrong use_spatial_tree flag from h5pget_dset_use_spatial_tree_f" + write(*,*) "Got wrong use_spatial_tree flag from h5pget_virtual_dset_use_spatial_tree_f" endif ! false value use_spatial_tree = .FALSE. - CALL h5pset_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) - CALL check("h5pset_dset_use_spatial_tree_f", error, total_error) + CALL h5pset_virtual_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pset_virtual_dset_use_spatial_tree_f", error, total_error) use_spatial_tree = .TRUE. - CALL h5pget_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) - CALL check("h5pget_dset_use_spatial_tree_f", error, total_error) + CALL h5pget_virtual_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pget_virtual_dset_use_spatial_tree_f", error, total_error) if(use_spatial_tree .neqv. .FALSE.) then total_error = total_error + 1 - write(*,*) "Got wrong use_spatial_tree flag from h5pget_dset_use_spatial_tree_f" + write(*,*) "Got wrong use_spatial_tree flag from h5pget_virtual_dset_use_spatial_tree_f" endif ! Close the dapl diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index aafeb8ba1bd..3e06f8a91c7 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -10678,7 +10678,7 @@ public synchronized static native void H5Pset_efile_prefix(long dapl_id, String /** * @ingroup JH5P * - * H5Pget_dset_use_spatial_tree accesses the flag for whether to use/not use a spatial tree + * H5Pget_virtual_dset_use_spatial_tree accesses the flag for whether to use/not use a spatial tree * during mapping operations on a Virtual Dataset. The default value is true. * * Use of a spatial tree will accelerate the process of searching through mappings @@ -10698,13 +10698,13 @@ public synchronized static native void H5Pset_efile_prefix(long dapl_id, String * @exception HDF5LibraryException * Error from the HDF5 Library. **/ - public synchronized static native boolean H5Pget_dset_use_spatial_tree(long dapl_id) + public synchronized static native boolean H5Pget_virtual_dset_use_spatial_tree(long dapl_id) throws HDF5LibraryException; /** * @ingroup JH5P * - * H5Pset_dset_use_spatial_tree sets the dapl to use/not use a spatial tree + * H5Pset_virtual_dset_use_spatial_tree sets the dapl to use/not use a spatial tree * during mapping operations on a Virtual Dataset. The default value is true. * * Use of a spatial tree will accelerate the process of searching through mappings @@ -10725,7 +10725,7 @@ public synchronized static native boolean H5Pget_dset_use_spatial_tree(long dapl * @exception HDF5LibraryException * Error from the HDF5 Library. **/ - public synchronized static native void H5Pset_dset_use_spatial_tree(long dapl_id, boolean use_tree) + public synchronized static native void H5Pset_virtual_dset_use_spatial_tree(long dapl_id, boolean use_tree) throws HDF5LibraryException; // public synchronized static native void H5Pset_append_flush(long plist_id, int ndims, long[] boundary, diff --git a/java/src/jni/h5pDAPLImp.c b/java/src/jni/h5pDAPLImp.c index 9a0c29b83ca..a2a0b3da339 100644 --- a/java/src/jni/h5pDAPLImp.c +++ b/java/src/jni/h5pDAPLImp.c @@ -314,11 +314,11 @@ H5D_append_cb(hid_t dataset_id, hsize_t *cur_dims, void *cb_data) /* * Class: hdf_hdf5lib_H5 - * Method: H5Pset_dset_use_spatial_tree + * Method: H5Pset_virtual_dset_use_spatial_tree * Signature: (JZ)V */ JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id, +Java_hdf_hdf5lib_H5_H5Pset_1virtual_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id, jboolean use_tree) { bool use_tree_val; @@ -328,27 +328,27 @@ Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, j use_tree_val = (JNI_TRUE == use_tree) ? true : false; - if ((retVal = H5Pset_dset_use_spatial_tree((hid_t)dapl_id, (bool)use_tree_val)) < 0) + if ((retVal = H5Pset_virtual_dset_use_spatial_tree((hid_t)dapl_id, (bool)use_tree_val)) < 0) H5_LIBRARY_ERROR(ENVONLY); done: return; -} /* end Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree */ +} /* end Java_hdf_hdf5lib_H5_H5Pset_1virtual_1dset_1use_1spatial_1tree */ /* * Class: hdf_hdf5lib_H5 - * Method: H5Pget_dset_use_spatial_tree + * Method: H5Pget_virtual_dset_use_spatial_tree * Signature: (J)Z */ JNIEXPORT jboolean JNICALL -Java_hdf_hdf5lib_H5_H5Pget_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id) +Java_hdf_hdf5lib_H5_H5Pget_1virtual_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id) { bool use_tree = false; jboolean bval = JNI_FALSE; UNUSED(clss); - if (H5Pget_dset_use_spatial_tree((hid_t)dapl_id, (bool *)&use_tree) < 0) + if (H5Pget_virtual_dset_use_spatial_tree((hid_t)dapl_id, (bool *)&use_tree) < 0) H5_LIBRARY_ERROR(ENVONLY); if (use_tree == true) @@ -356,7 +356,7 @@ Java_hdf_hdf5lib_H5_H5Pget_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, j done: return bval; -} /* end Java_hdf_hdf5lib_H5_H5Pget_1dset_1use_1spatial_1tree */ +} /* end Java_hdf_hdf5lib_H5_H5Pget_1virtual_1dset_1use_1spatial_1tree */ #ifdef __cplusplus } /* end extern "C" */ diff --git a/java/src/jni/h5pDAPLImp.h b/java/src/jni/h5pDAPLImp.h index b6ce51d9f8b..92a5cbf91d9 100644 --- a/java/src/jni/h5pDAPLImp.h +++ b/java/src/jni/h5pDAPLImp.h @@ -91,7 +91,7 @@ JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1printf_1gap(JNIEnv /* * Class: hdf_hdf5lib_H5 - * Method: H5Pset_dset_use_spatial_tree + * Method: H5Pset_virtual_dset_use_spatial_tree * Signature: (JZ)V */ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIEnv *, jclass, jlong, @@ -99,7 +99,7 @@ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIE /* * Class: hdf_hdf5lib_H5 - * Method: H5Pget_dset_use_spatial_tree + * Method: H5Pget_virtual_dset_use_spatial_tree * Signature: (J)Z */ JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1dset_1use_1spatial_1tree(JNIEnv *, jclass, jlong); diff --git a/release_docs/CHANGELOG.md b/release_docs/CHANGELOG.md index 49630e41a01..3a2469eba30 100644 --- a/release_docs/CHANGELOG.md +++ b/release_docs/CHANGELOG.md @@ -485,7 +485,7 @@ Simple example programs showing how to use complex number datatypes have been ad a new Dataset Access Property List (DAPL) property to control use of the spatial tree. This property can be set or queried with the new API functions - H5Pset_dset_use_spatial_tree()/H5Pget_dset_use_spatial_tree(). + H5Pset_virtual_dset_use_spatial_tree()/H5Pget_virtual_dset_use_spatial_tree(). ## Parallel Library diff --git a/src/H5Pdapl.c b/src/H5Pdapl.c index 4817263aa6e..a33f0e757e6 100644 --- a/src/H5Pdapl.c +++ b/src/H5Pdapl.c @@ -1557,7 +1557,7 @@ H5Pget_virtual_prefix(hid_t plist_id, char *prefix /*out*/, size_t size) } /* end H5Pget_virtual_prefix() */ /*----------------------------------------------------------------------------- - * Function: H5Pget_dset_use_spatial_tree + * Function: H5Pget_virtual_dset_use_spatial_tree * * Purpose: * @@ -1581,7 +1581,7 @@ H5Pget_virtual_prefix(hid_t plist_id, char *prefix /*out*/, size_t size) *----------------------------------------------------------------------------- */ herr_t -H5Pget_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) +H5Pget_virtual_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) { bool setting = false; H5P_genplist_t *plist = NULL; @@ -1603,10 +1603,10 @@ H5Pget_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) done: FUNC_LEAVE_API(ret_value) -} /* H5Pget_dset_use_spatial_tree() */ +} /* H5Pget_virtual_dset_use_spatial_tree() */ /*----------------------------------------------------------------------------- - * Function: H5Pset_dset_use_spatial_tree + * Function: H5Pset_virtual_dset_use_spatial_tree * * Purpose: * @@ -1630,7 +1630,7 @@ H5Pget_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) *----------------------------------------------------------------------------- */ herr_t -H5Pset_dset_use_spatial_tree(hid_t dapl_id, bool use_tree) +H5Pset_virtual_dset_use_spatial_tree(hid_t dapl_id, bool use_tree) { H5P_genplist_t *plist = NULL; bool prev_set = false; @@ -1650,4 +1650,4 @@ H5Pset_dset_use_spatial_tree(hid_t dapl_id, bool use_tree) done: FUNC_LEAVE_API(ret_value) -} /* H5Pset_dset_use_spatial_tree() */ +} /* H5Pset_virtual_dset_use_spatial_tree() */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 0ed4b20650b..902ce59b99e 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -6022,7 +6022,7 @@ H5_DLL herr_t H5Pget_dset_no_attrs_hint(hid_t dcpl_id, bool *minimize); * * \return \herr_t * - * \details H5Pget_dset_use_spatial_tree() retrieves the + * \details H5Pget_virtual_dset_use_spatial_tree() retrieves the * use spatial tree flag setting for the dataset * creation property list \p dcpl_id. This setting determines * whether a dataset created with the dataset creation @@ -6043,7 +6043,7 @@ H5_DLL herr_t H5Pget_dset_no_attrs_hint(hid_t dcpl_id, bool *minimize); * \since 2.0.0 * */ -H5_DLL herr_t H5Pget_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree); +H5_DLL herr_t H5Pget_virtual_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree); /** * \ingroup DCPL * @@ -6538,7 +6538,7 @@ H5_DLL herr_t H5Pset_dset_no_attrs_hint(hid_t dcpl_id, bool minimize); * * \return \herr_t * - * \details H5Pset_dset_use_spatial_tree() sets the use-tree flag + * \details H5Pset_virtual_dset_use_spatial_tree() sets the use-tree flag * for the dataset creation property list \p dcpl_id. * Datasets created with the dataset creation property * list \p dcpl_id will construct a spatial tree and use @@ -6557,7 +6557,7 @@ H5_DLL herr_t H5Pset_dset_no_attrs_hint(hid_t dcpl_id, bool minimize); * \since 2.0.0 * */ -H5_DLL herr_t H5Pset_dset_use_spatial_tree(hid_t dcpl_id, bool use_tree); +H5_DLL herr_t H5Pset_virtual_dset_use_spatial_tree(hid_t dcpl_id, bool use_tree); /** * \ingroup DCPL * diff --git a/test/rtree.c b/test/rtree.c index cb60ec35e14..1d2c57e3f7e 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -675,7 +675,7 @@ test_rtree_dapl(bool use_tree, bool read_init) FAIL_STACK_ERROR; /* Set the spatial tree property */ - if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree) < 0) + if (H5Pset_virtual_dset_use_spatial_tree(dapl_id, use_tree) < 0) FAIL_STACK_ERROR; if ((vdset_id = H5Dopen2(file_id, RTREE_DAPL_VDS_NAME, dapl_id)) < 0) @@ -719,7 +719,7 @@ test_rtree_dapl(bool use_tree, bool read_init) if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) FAIL_STACK_ERROR; - if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree_inverse) < 0) + if (H5Pset_virtual_dset_use_spatial_tree(dapl_id, use_tree_inverse) < 0) FAIL_STACK_ERROR; if ((file_id = H5Fopen(RTREE_DAPL_FILENAME, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) @@ -813,7 +813,7 @@ test_rtree_threshold(bool use_tree) FAIL_STACK_ERROR; /* Set the spatial tree property */ - if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree) < 0) + if (H5Pset_virtual_dset_use_spatial_tree(dapl_id, use_tree) < 0) FAIL_STACK_ERROR; /* Create virtual dataset with specified number of mappings */ @@ -934,7 +934,7 @@ test_rtree_rw(bool use_tree) FAIL_STACK_ERROR; /* Set the spatial tree property */ - if (H5Pset_dset_use_spatial_tree(dapl_id, use_tree) < 0) + if (H5Pset_virtual_dset_use_spatial_tree(dapl_id, use_tree) < 0) FAIL_STACK_ERROR; /* Create virtual dataset with specified number of mappings */ From 0f7216128c693e13f93f83aa773deb2bbb9bd3de Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Thu, 2 Oct 2025 14:15:07 -0500 Subject: [PATCH 37/43] Remove un-needed defines --- test/rtree.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/rtree.c b/test/rtree.c index 1d2c57e3f7e..4a6f48b0898 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -23,12 +23,7 @@ * This file needs to access private datatypes from the H5RT package. */ #define H5RT_FRIEND /*suppress error about including H5RTpkg */ -#define H5RT_TESTING -#include "H5RTpkg.h" - -/* Other private headers */ -#include "H5CXprivate.h" /* API Contexts */ -#include "H5VLprivate.h" /* Virtual Object Layer */ +#include "H5RTpkg.h" /* R-tree package */ #define H5D_FRIEND /*suppress error about including H5Dpkg */ #define H5D_TESTING From ca2c3fa35fb1edfbd156d1097b4ce91e62de8944 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Oct 2025 19:20:59 +0000 Subject: [PATCH 38/43] Committing clang-format changes --- java/src/hdf/hdf5lib/H5.java | 3 ++- java/src/jni/h5pDAPLImp.c | 2 +- test/rtree.c | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 3e06f8a91c7..2a83ede0216 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -10725,7 +10725,8 @@ public synchronized static native boolean H5Pget_virtual_dset_use_spatial_tree(l * @exception HDF5LibraryException * Error from the HDF5 Library. **/ - public synchronized static native void H5Pset_virtual_dset_use_spatial_tree(long dapl_id, boolean use_tree) + public synchronized static native void H5Pset_virtual_dset_use_spatial_tree(long dapl_id, + boolean use_tree) throws HDF5LibraryException; // public synchronized static native void H5Pset_append_flush(long plist_id, int ndims, long[] boundary, diff --git a/java/src/jni/h5pDAPLImp.c b/java/src/jni/h5pDAPLImp.c index a2a0b3da339..478c315372a 100644 --- a/java/src/jni/h5pDAPLImp.c +++ b/java/src/jni/h5pDAPLImp.c @@ -319,7 +319,7 @@ H5D_append_cb(hid_t dataset_id, hsize_t *cur_dims, void *cb_data) */ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1virtual_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id, - jboolean use_tree) + jboolean use_tree) { bool use_tree_val; herr_t retVal = FAIL; diff --git a/test/rtree.c b/test/rtree.c index 4a6f48b0898..02db3a37ddf 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -22,7 +22,7 @@ /* * This file needs to access private datatypes from the H5RT package. */ -#define H5RT_FRIEND /*suppress error about including H5RTpkg */ +#define H5RT_FRIEND /*suppress error about including H5RTpkg */ #include "H5RTpkg.h" /* R-tree package */ #define H5D_FRIEND /*suppress error about including H5Dpkg */ From 8a45e21746a578bbb33445d44e96cfc69a67588f Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Thu, 2 Oct 2025 14:53:29 -0500 Subject: [PATCH 39/43] Correct un-renamed functions in java/fortran wrappers --- fortran/src/H5Pff.F90 | 6 ++++-- fortran/src/hdf5_fortrandll.def.in | 4 ++-- java/src/jni/h5pDAPLImp.h | 7 ++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index b0a91dc39d0..a44592134c4 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -4394,7 +4394,8 @@ SUBROUTINE h5pget_virtual_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) LOGICAL(C_BOOL) :: c_use_tree INTERFACE - INTEGER(C_INT) FUNCTION H5Pget_virtual_dset_use_spatial_tree_c(dapl_id, use_tree) BIND(C, NAME='H5Pget_virtual_dset_use_spatial_tree') + INTEGER(C_INT) FUNCTION H5Pget_virtual_dset_use_spatial_tree_c(dapl_id, use_tree) & + BIND(C, NAME='H5Pget_virtual_dset_use_spatial_tree') IMPORT :: C_INT, HID_T, C_BOOL IMPLICIT NONE INTEGER(HID_T), INTENT(IN), VALUE :: dapl_id @@ -4438,7 +4439,8 @@ SUBROUTINE h5pset_virtual_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) LOGICAL(C_BOOL) :: c_use_tree INTERFACE - INTEGER FUNCTION h5pset_virtual_dset_use_spatial_tree_c(dapl_id, use_tree) BIND(C, NAME='H5Pset_virtual_dset_use_spatial_tree') + INTEGER FUNCTION h5pset_virtual_dset_use_spatial_tree_c(dapl_id, use_tree) & + BIND(C, NAME='H5Pset_virtual_dset_use_spatial_tree') IMPORT :: HID_T, C_BOOL IMPLICIT NONE INTEGER(HID_T), INTENT(IN), VALUE :: dapl_id diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index 01797266d06..4d7b339a2d7 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -420,8 +420,8 @@ H5P_mp_H5PGET_VIRTUAL_VSPACE_F H5P_mp_H5PGET_VIRTUAL_SRCSPACE_F H5P_mp_H5PGET_VIRTUAL_FILENAME_F H5P_mp_H5PGET_VIRTUAL_DSETNAME_F -H5P_mp_h5pget_virtual_dset_use_spatial_tree_f -H5P_mp_h5pset_virtual_dset_use_spatial_tree_f +H5P_mp_H5PGET_VIRTUAL_DSET_USE_SPATIAL_TREE_F +H5P_mp_H5PSET_VIRTUAL_DSET_USE_SPATIAL_TREE_F H5P_mp_H5PGET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_VOL_F diff --git a/java/src/jni/h5pDAPLImp.h b/java/src/jni/h5pDAPLImp.h index 92a5cbf91d9..bd6a149d15d 100644 --- a/java/src/jni/h5pDAPLImp.h +++ b/java/src/jni/h5pDAPLImp.h @@ -94,15 +94,16 @@ JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1printf_1gap(JNIEnv * Method: H5Pset_virtual_dset_use_spatial_tree * Signature: (JZ)V */ -JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1dset_1use_1spatial_1tree(JNIEnv *, jclass, jlong, - jboolean); +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1virtual_1dset_1use_1spatial_1tree(JNIEnv *, jclass, jlong, + jboolean); /* * Class: hdf_hdf5lib_H5 * Method: H5Pget_virtual_dset_use_spatial_tree * Signature: (J)Z */ -JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1dset_1use_1spatial_1tree(JNIEnv *, jclass, jlong); +JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1dset_1use_1spatial_1tree(JNIEnv *, jclass, + jlong); #ifdef __cplusplus } /* end extern "C" */ From 1a4ed86ab471ecd87dd9fe8e1716d73e689a0ef9 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Fri, 3 Oct 2025 11:24:55 -0500 Subject: [PATCH 40/43] Remove git conflict markers from CHANGELOG --- release_docs/CHANGELOG.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/release_docs/CHANGELOG.md b/release_docs/CHANGELOG.md index 3a2469eba30..e02a0b03506 100644 --- a/release_docs/CHANGELOG.md +++ b/release_docs/CHANGELOG.md @@ -25,14 +25,9 @@ For releases prior to version 2.0.0, please see the release.txt file and for mor ## Performance Enhancements: -<<<<<<< HEAD +- Up to [2500% faster](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#rtree) Virtual Dataset read/write operations - [30% faster opening](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#layoutcopydelay) and [25% faster closing](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#fileformat) of virtual datasets. - [Reduced memory overhead](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#fileformat) via shared name strings and optimized spatial search algorithms for virtual datasets. -======= -- Up to [2500% faster](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#rtree) Virtual Dataset read/write operations -- 30% faster opening and 25% faster closing of virtual datasets. -- Reduced memory overhead via shared name strings and optimized spatial search algorithms for virtual datasets. ->>>>>>> 7ae02e1c82 (Add bullet point to CHANGELOG executive summary) ## Significant Advancements: From b1b43be2af00571574c0256ecc374219dbc6d171 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Fri, 3 Oct 2025 14:46:40 -0500 Subject: [PATCH 41/43] H5Pset/get_virtual_dset_use_spatial_tree() -> H5Pset/get_virtual_spatial_tree() --- doxygen/examples/tables/propertyLists.dox | 2 +- fortran/src/H5Pff.F90 | 28 +++++++++++------------ fortran/src/hdf5_fortrandll.def.in | 4 ++-- fortran/test/tH5P.F90 | 22 +++++++++--------- java/src/hdf/hdf5lib/H5.java | 10 ++++---- java/src/jni/h5pDAPLImp.c | 18 +++++++-------- java/src/jni/h5pDAPLImp.h | 12 +++++----- release_docs/CHANGELOG.md | 2 +- src/H5Pdapl.c | 12 +++++----- src/H5Ppublic.h | 8 +++---- test/rtree.c | 8 +++---- 11 files changed, 63 insertions(+), 63 deletions(-) diff --git a/doxygen/examples/tables/propertyLists.dox b/doxygen/examples/tables/propertyLists.dox index 588c1dd1b4b..6323bd740a9 100644 --- a/doxygen/examples/tables/propertyLists.dox +++ b/doxygen/examples/tables/propertyLists.dox @@ -617,7 +617,7 @@ encoding for object names. Sets/gets the view of the virtual dataset (VDS) to include or exclude missing mapped elements. -#H5Pset_virtual_dset_use_spatial_tree/#H5Pget_virtual_dset_use_spatial_tree +#H5Pset_virtual_spatial_tree/#H5Pget_virtual_spatial_tree Sets/gets the flag to use spatial trees when searching many VDS mappings diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index a44592134c4..a8a0e91c193 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -4384,9 +4384,9 @@ END SUBROUTINE h5pget_chunk_cache_f !! \param use_tree Value of the setting. !! \param hdferr \fortran_error !! -!! See C API: @ref H5Pget_virtual_dset_use_spatial_tree() +!! See C API: @ref H5Pget_virtual_spatial_tree() !! - SUBROUTINE h5pget_virtual_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) + SUBROUTINE h5pget_virtual_spatial_tree_f(dapl_id, use_tree, hdferr) IMPLICIT NONE INTEGER(HID_T) , INTENT(IN) :: dapl_id LOGICAL , INTENT(OUT) :: use_tree @@ -4394,21 +4394,21 @@ SUBROUTINE h5pget_virtual_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) LOGICAL(C_BOOL) :: c_use_tree INTERFACE - INTEGER(C_INT) FUNCTION H5Pget_virtual_dset_use_spatial_tree_c(dapl_id, use_tree) & - BIND(C, NAME='H5Pget_virtual_dset_use_spatial_tree') + INTEGER(C_INT) FUNCTION H5Pget_virtual_spatial_tree_c(dapl_id, use_tree) & + BIND(C, NAME='H5Pget_virtual_spatial_tree') IMPORT :: C_INT, HID_T, C_BOOL IMPLICIT NONE INTEGER(HID_T), INTENT(IN), VALUE :: dapl_id LOGICAL(C_BOOL), INTENT(OUT) :: use_tree - END FUNCTION H5Pget_virtual_dset_use_spatial_tree_c + END FUNCTION H5Pget_virtual_spatial_tree_c END INTERFACE - hdferr = INT(H5Pget_virtual_dset_use_spatial_tree_c(dapl_id, c_use_tree)) + hdferr = INT(H5Pget_virtual_spatial_tree_c(dapl_id, c_use_tree)) ! Transfer value of C C_BOOL type to Fortran LOGICAL use_tree = c_use_tree - END SUBROUTINE h5pget_virtual_dset_use_spatial_tree_f + END SUBROUTINE h5pget_virtual_spatial_tree_f !> !! \ingroup FH5P @@ -4429,9 +4429,9 @@ END SUBROUTINE h5pget_virtual_dset_use_spatial_tree_f !! \param use_tree Value of the setting. !! \param hdferr \fortran_error !! -!! See C API: @ref H5Pset_virtual_dset_use_spatial_tree() +!! See C API: @ref H5Pset_virtual_spatial_tree() !! - SUBROUTINE h5pset_virtual_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) + SUBROUTINE h5pset_virtual_spatial_tree_f(dapl_id, use_tree, hdferr) IMPLICIT NONE INTEGER(HID_T) , INTENT(IN) :: dapl_id LOGICAL , INTENT(IN) :: use_tree @@ -4439,21 +4439,21 @@ SUBROUTINE h5pset_virtual_dset_use_spatial_tree_f(dapl_id, use_tree, hdferr) LOGICAL(C_BOOL) :: c_use_tree INTERFACE - INTEGER FUNCTION h5pset_virtual_dset_use_spatial_tree_c(dapl_id, use_tree) & - BIND(C, NAME='H5Pset_virtual_dset_use_spatial_tree') + INTEGER FUNCTION h5pset_virtual_spatial_tree_c(dapl_id, use_tree) & + BIND(C, NAME='H5Pset_virtual_spatial_tree') IMPORT :: HID_T, C_BOOL IMPLICIT NONE INTEGER(HID_T), INTENT(IN), VALUE :: dapl_id LOGICAL(C_BOOL), INTENT(IN), VALUE :: use_tree - END FUNCTION h5pset_virtual_dset_use_spatial_tree_c + END FUNCTION h5pset_virtual_spatial_tree_c END INTERFACE ! Transfer value of Fortran LOGICAL to C C_BOOL type c_use_tree = use_tree - hdferr = INT(h5pset_virtual_dset_use_spatial_tree_c(dapl_id, c_use_tree)) + hdferr = INT(h5pset_virtual_spatial_tree_c(dapl_id, c_use_tree)) - END SUBROUTINE h5pset_virtual_dset_use_spatial_tree_f + END SUBROUTINE h5pset_virtual_spatial_tree_f #ifdef H5_DOXYGEN !> diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index 4d7b339a2d7..8a6ff9cbe0e 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -420,8 +420,8 @@ H5P_mp_H5PGET_VIRTUAL_VSPACE_F H5P_mp_H5PGET_VIRTUAL_SRCSPACE_F H5P_mp_H5PGET_VIRTUAL_FILENAME_F H5P_mp_H5PGET_VIRTUAL_DSETNAME_F -H5P_mp_H5PGET_VIRTUAL_DSET_USE_SPATIAL_TREE_F -H5P_mp_H5PSET_VIRTUAL_DSET_USE_SPATIAL_TREE_F +H5P_mp_H5PGET_VIRTUAL_SPATIAL_TREE_F +H5P_mp_H5PSET_VIRTUAL_SPATIAL_TREE_F H5P_mp_H5PGET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_VOL_F diff --git a/fortran/test/tH5P.F90 b/fortran/test/tH5P.F90 index e2ee1589eba..477b43060e5 100644 --- a/fortran/test/tH5P.F90 +++ b/fortran/test/tH5P.F90 @@ -832,29 +832,29 @@ SUBROUTINE test_misc_properties(total_error) CALL H5Pcreate_f(H5P_DATASET_ACCESS_F, dapl_id, error) CALL check("H5Pcreate_f", error, total_error) - ! Test H5Pset/get_dset_use_spatial_tree_f + ! Test H5Pset/get_virtual_spatial_tree_f ! true value use_spatial_tree = .TRUE. - CALL h5pset_virtual_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) - CALL check("h5pset_virtual_dset_use_spatial_tree_f", error, total_error) + CALL h5pset_virtual_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pset_virtual_spatial_tree_f", error, total_error) use_spatial_tree = .FALSE. - CALL h5pget_virtual_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) - CALL check("h5pget_virtual_dset_use_spatial_tree_f", error, total_error) + CALL h5pget_virtual_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pget_virtual_spatial_tree_f", error, total_error) if(use_spatial_tree .neqv. .TRUE.) then total_error = total_error + 1 - write(*,*) "Got wrong use_spatial_tree flag from h5pget_virtual_dset_use_spatial_tree_f" + write(*,*) "Got wrong use_spatial_tree flag from h5pget_virtual_spatial_tree_f" endif ! false value use_spatial_tree = .FALSE. - CALL h5pset_virtual_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) - CALL check("h5pset_virtual_dset_use_spatial_tree_f", error, total_error) + CALL h5pset_virtual_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pset_virtual_spatial_tree_f", error, total_error) use_spatial_tree = .TRUE. - CALL h5pget_virtual_dset_use_spatial_tree_f(dapl_id, use_spatial_tree, error) - CALL check("h5pget_virtual_dset_use_spatial_tree_f", error, total_error) + CALL h5pget_virtual_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pget_virtual_spatial_tree_f", error, total_error) if(use_spatial_tree .neqv. .FALSE.) then total_error = total_error + 1 - write(*,*) "Got wrong use_spatial_tree flag from h5pget_virtual_dset_use_spatial_tree_f" + write(*,*) "Got wrong use_spatial_tree flag from h5pget_virtual_spatial_tree_f" endif ! Close the dapl diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 2a83ede0216..74a84d3b537 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -10678,7 +10678,7 @@ public synchronized static native void H5Pset_efile_prefix(long dapl_id, String /** * @ingroup JH5P * - * H5Pget_virtual_dset_use_spatial_tree accesses the flag for whether to use/not use a spatial tree + * H5Pget_virtual_spatial_tree accesses the flag for whether to use/not use a spatial tree * during mapping operations on a Virtual Dataset. The default value is true. * * Use of a spatial tree will accelerate the process of searching through mappings @@ -10698,13 +10698,13 @@ public synchronized static native void H5Pset_efile_prefix(long dapl_id, String * @exception HDF5LibraryException * Error from the HDF5 Library. **/ - public synchronized static native boolean H5Pget_virtual_dset_use_spatial_tree(long dapl_id) + public synchronized static native boolean H5Pget_virtual_spatial_tree(long dapl_id) throws HDF5LibraryException; /** * @ingroup JH5P * - * H5Pset_virtual_dset_use_spatial_tree sets the dapl to use/not use a spatial tree + * H5Pset_virtual_spatial_tree sets the dapl to use/not use a spatial tree * during mapping operations on a Virtual Dataset. The default value is true. * * Use of a spatial tree will accelerate the process of searching through mappings @@ -10725,8 +10725,8 @@ public synchronized static native boolean H5Pget_virtual_dset_use_spatial_tree(l * @exception HDF5LibraryException * Error from the HDF5 Library. **/ - public synchronized static native void H5Pset_virtual_dset_use_spatial_tree(long dapl_id, - boolean use_tree) + public synchronized static native void H5Pset_virtual_spatial_tree(long dapl_id, + boolean use_tree) throws HDF5LibraryException; // public synchronized static native void H5Pset_append_flush(long plist_id, int ndims, long[] boundary, diff --git a/java/src/jni/h5pDAPLImp.c b/java/src/jni/h5pDAPLImp.c index 478c315372a..cab7c14f02c 100644 --- a/java/src/jni/h5pDAPLImp.c +++ b/java/src/jni/h5pDAPLImp.c @@ -314,12 +314,12 @@ H5D_append_cb(hid_t dataset_id, hsize_t *cur_dims, void *cb_data) /* * Class: hdf_hdf5lib_H5 - * Method: H5Pset_virtual_dset_use_spatial_tree + * Method: H5Pset_virtual_spatial_tree * Signature: (JZ)V */ JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pset_1virtual_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id, - jboolean use_tree) +Java_hdf_hdf5lib_H5_H5Pset_1virtual_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id, + jboolean use_tree) { bool use_tree_val; herr_t retVal = FAIL; @@ -328,27 +328,27 @@ Java_hdf_hdf5lib_H5_H5Pset_1virtual_1dset_1use_1spatial_1tree(JNIEnv *env, jclas use_tree_val = (JNI_TRUE == use_tree) ? true : false; - if ((retVal = H5Pset_virtual_dset_use_spatial_tree((hid_t)dapl_id, (bool)use_tree_val)) < 0) + if ((retVal = H5Pset_virtual_spatial_tree((hid_t)dapl_id, (bool)use_tree_val)) < 0) H5_LIBRARY_ERROR(ENVONLY); done: return; -} /* end Java_hdf_hdf5lib_H5_H5Pset_1virtual_1dset_1use_1spatial_1tree */ +} /* end Java_hdf_hdf5lib_H5_H5Pset_1virtual_1spatial_1tree */ /* * Class: hdf_hdf5lib_H5 - * Method: H5Pget_virtual_dset_use_spatial_tree + * Method: H5Pget_virtual_spatial_tree * Signature: (J)Z */ JNIEXPORT jboolean JNICALL -Java_hdf_hdf5lib_H5_H5Pget_1virtual_1dset_1use_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id) +Java_hdf_hdf5lib_H5_H5Pget_1virtual_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id) { bool use_tree = false; jboolean bval = JNI_FALSE; UNUSED(clss); - if (H5Pget_virtual_dset_use_spatial_tree((hid_t)dapl_id, (bool *)&use_tree) < 0) + if (H5Pget_virtual_spatial_tree((hid_t)dapl_id, (bool *)&use_tree) < 0) H5_LIBRARY_ERROR(ENVONLY); if (use_tree == true) @@ -356,7 +356,7 @@ Java_hdf_hdf5lib_H5_H5Pget_1virtual_1dset_1use_1spatial_1tree(JNIEnv *env, jclas done: return bval; -} /* end Java_hdf_hdf5lib_H5_H5Pget_1virtual_1dset_1use_1spatial_1tree */ +} /* end Java_hdf_hdf5lib_H5_H5Pget_1virtual_1spatial_1tree */ #ifdef __cplusplus } /* end extern "C" */ diff --git a/java/src/jni/h5pDAPLImp.h b/java/src/jni/h5pDAPLImp.h index bd6a149d15d..76292071860 100644 --- a/java/src/jni/h5pDAPLImp.h +++ b/java/src/jni/h5pDAPLImp.h @@ -91,19 +91,19 @@ JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1printf_1gap(JNIEnv /* * Class: hdf_hdf5lib_H5 - * Method: H5Pset_virtual_dset_use_spatial_tree + * Method: H5Pset_virtual_spatial_tree * Signature: (JZ)V */ -JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1virtual_1dset_1use_1spatial_1tree(JNIEnv *, jclass, jlong, - jboolean); +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1virtual_1spatial_1tree(JNIEnv *, jclass, jlong, + jboolean); /* * Class: hdf_hdf5lib_H5 - * Method: H5Pget_virtual_dset_use_spatial_tree + * Method: H5Pget_virtual_spatial_tree * Signature: (J)Z */ -JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1dset_1use_1spatial_1tree(JNIEnv *, jclass, - jlong); +JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1spatial_1tree(JNIEnv *, jclass, + jlong); #ifdef __cplusplus } /* end extern "C" */ diff --git a/release_docs/CHANGELOG.md b/release_docs/CHANGELOG.md index e02a0b03506..6c1db1e099b 100644 --- a/release_docs/CHANGELOG.md +++ b/release_docs/CHANGELOG.md @@ -480,7 +480,7 @@ Simple example programs showing how to use complex number datatypes have been ad a new Dataset Access Property List (DAPL) property to control use of the spatial tree. This property can be set or queried with the new API functions - H5Pset_virtual_dset_use_spatial_tree()/H5Pget_virtual_dset_use_spatial_tree(). + H5Pset_virtual_spatial_tree()/H5Pget_virtual_spatial_tree(). ## Parallel Library diff --git a/src/H5Pdapl.c b/src/H5Pdapl.c index a33f0e757e6..a29c07042de 100644 --- a/src/H5Pdapl.c +++ b/src/H5Pdapl.c @@ -1557,7 +1557,7 @@ H5Pget_virtual_prefix(hid_t plist_id, char *prefix /*out*/, size_t size) } /* end H5Pget_virtual_prefix() */ /*----------------------------------------------------------------------------- - * Function: H5Pget_virtual_dset_use_spatial_tree + * Function: H5Pget_virtual_spatial_tree * * Purpose: * @@ -1581,7 +1581,7 @@ H5Pget_virtual_prefix(hid_t plist_id, char *prefix /*out*/, size_t size) *----------------------------------------------------------------------------- */ herr_t -H5Pget_virtual_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) +H5Pget_virtual_spatial_tree(hid_t dcpl_id, bool *use_tree) { bool setting = false; H5P_genplist_t *plist = NULL; @@ -1603,10 +1603,10 @@ H5Pget_virtual_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) done: FUNC_LEAVE_API(ret_value) -} /* H5Pget_virtual_dset_use_spatial_tree() */ +} /* H5Pget_virtual_spatial_tree() */ /*----------------------------------------------------------------------------- - * Function: H5Pset_virtual_dset_use_spatial_tree + * Function: H5Pset_virtual_spatial_tree * * Purpose: * @@ -1630,7 +1630,7 @@ H5Pget_virtual_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree) *----------------------------------------------------------------------------- */ herr_t -H5Pset_virtual_dset_use_spatial_tree(hid_t dapl_id, bool use_tree) +H5Pset_virtual_spatial_tree(hid_t dapl_id, bool use_tree) { H5P_genplist_t *plist = NULL; bool prev_set = false; @@ -1650,4 +1650,4 @@ H5Pset_virtual_dset_use_spatial_tree(hid_t dapl_id, bool use_tree) done: FUNC_LEAVE_API(ret_value) -} /* H5Pset_virtual_dset_use_spatial_tree() */ +} /* H5Pset_virtual_spatial_tree() */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 902ce59b99e..d5b4ea1a945 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -6022,7 +6022,7 @@ H5_DLL herr_t H5Pget_dset_no_attrs_hint(hid_t dcpl_id, bool *minimize); * * \return \herr_t * - * \details H5Pget_virtual_dset_use_spatial_tree() retrieves the + * \details H5Pget_virtual_spatial_tree() retrieves the * use spatial tree flag setting for the dataset * creation property list \p dcpl_id. This setting determines * whether a dataset created with the dataset creation @@ -6043,7 +6043,7 @@ H5_DLL herr_t H5Pget_dset_no_attrs_hint(hid_t dcpl_id, bool *minimize); * \since 2.0.0 * */ -H5_DLL herr_t H5Pget_virtual_dset_use_spatial_tree(hid_t dcpl_id, bool *use_tree); +H5_DLL herr_t H5Pget_virtual_spatial_tree(hid_t dcpl_id, bool *use_tree); /** * \ingroup DCPL * @@ -6538,7 +6538,7 @@ H5_DLL herr_t H5Pset_dset_no_attrs_hint(hid_t dcpl_id, bool minimize); * * \return \herr_t * - * \details H5Pset_virtual_dset_use_spatial_tree() sets the use-tree flag + * \details H5Pset_virtual_spatial_tree() sets the use-tree flag * for the dataset creation property list \p dcpl_id. * Datasets created with the dataset creation property * list \p dcpl_id will construct a spatial tree and use @@ -6557,7 +6557,7 @@ H5_DLL herr_t H5Pset_dset_no_attrs_hint(hid_t dcpl_id, bool minimize); * \since 2.0.0 * */ -H5_DLL herr_t H5Pset_virtual_dset_use_spatial_tree(hid_t dcpl_id, bool use_tree); +H5_DLL herr_t H5Pset_virtual_spatial_tree(hid_t dcpl_id, bool use_tree); /** * \ingroup DCPL * diff --git a/test/rtree.c b/test/rtree.c index 02db3a37ddf..c47fc16e179 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -670,7 +670,7 @@ test_rtree_dapl(bool use_tree, bool read_init) FAIL_STACK_ERROR; /* Set the spatial tree property */ - if (H5Pset_virtual_dset_use_spatial_tree(dapl_id, use_tree) < 0) + if (H5Pset_virtual_spatial_tree(dapl_id, use_tree) < 0) FAIL_STACK_ERROR; if ((vdset_id = H5Dopen2(file_id, RTREE_DAPL_VDS_NAME, dapl_id)) < 0) @@ -714,7 +714,7 @@ test_rtree_dapl(bool use_tree, bool read_init) if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) FAIL_STACK_ERROR; - if (H5Pset_virtual_dset_use_spatial_tree(dapl_id, use_tree_inverse) < 0) + if (H5Pset_virtual_spatial_tree(dapl_id, use_tree_inverse) < 0) FAIL_STACK_ERROR; if ((file_id = H5Fopen(RTREE_DAPL_FILENAME, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) @@ -808,7 +808,7 @@ test_rtree_threshold(bool use_tree) FAIL_STACK_ERROR; /* Set the spatial tree property */ - if (H5Pset_virtual_dset_use_spatial_tree(dapl_id, use_tree) < 0) + if (H5Pset_virtual_spatial_tree(dapl_id, use_tree) < 0) FAIL_STACK_ERROR; /* Create virtual dataset with specified number of mappings */ @@ -929,7 +929,7 @@ test_rtree_rw(bool use_tree) FAIL_STACK_ERROR; /* Set the spatial tree property */ - if (H5Pset_virtual_dset_use_spatial_tree(dapl_id, use_tree) < 0) + if (H5Pset_virtual_spatial_tree(dapl_id, use_tree) < 0) FAIL_STACK_ERROR; /* Create virtual dataset with specified number of mappings */ From cd374d957407ab15703384eb7efdfcaf284f913e Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 3 Oct 2025 19:48:57 +0000 Subject: [PATCH 42/43] Committing clang-format changes --- java/src/hdf/hdf5lib/H5.java | 3 +-- java/src/jni/h5pDAPLImp.c | 3 +-- java/src/jni/h5pDAPLImp.h | 6 ++---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 74a84d3b537..39245bf8816 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -10725,8 +10725,7 @@ public synchronized static native boolean H5Pget_virtual_spatial_tree(long dapl_ * @exception HDF5LibraryException * Error from the HDF5 Library. **/ - public synchronized static native void H5Pset_virtual_spatial_tree(long dapl_id, - boolean use_tree) + public synchronized static native void H5Pset_virtual_spatial_tree(long dapl_id, boolean use_tree) throws HDF5LibraryException; // public synchronized static native void H5Pset_append_flush(long plist_id, int ndims, long[] boundary, diff --git a/java/src/jni/h5pDAPLImp.c b/java/src/jni/h5pDAPLImp.c index cab7c14f02c..e20a2081e81 100644 --- a/java/src/jni/h5pDAPLImp.c +++ b/java/src/jni/h5pDAPLImp.c @@ -318,8 +318,7 @@ H5D_append_cb(hid_t dataset_id, hsize_t *cur_dims, void *cb_data) * Signature: (JZ)V */ JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pset_1virtual_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id, - jboolean use_tree) +Java_hdf_hdf5lib_H5_H5Pset_1virtual_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id, jboolean use_tree) { bool use_tree_val; herr_t retVal = FAIL; diff --git a/java/src/jni/h5pDAPLImp.h b/java/src/jni/h5pDAPLImp.h index 76292071860..600f636038e 100644 --- a/java/src/jni/h5pDAPLImp.h +++ b/java/src/jni/h5pDAPLImp.h @@ -94,16 +94,14 @@ JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1printf_1gap(JNIEnv * Method: H5Pset_virtual_spatial_tree * Signature: (JZ)V */ -JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1virtual_1spatial_1tree(JNIEnv *, jclass, jlong, - jboolean); +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1virtual_1spatial_1tree(JNIEnv *, jclass, jlong, jboolean); /* * Class: hdf_hdf5lib_H5 * Method: H5Pget_virtual_spatial_tree * Signature: (J)Z */ -JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1spatial_1tree(JNIEnv *, jclass, - jlong); +JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1spatial_1tree(JNIEnv *, jclass, jlong); #ifdef __cplusplus } /* end extern "C" */ From 86624f95a4d21902f47d2b47661852c1f9e3aae7 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Mon, 6 Oct 2025 09:50:10 -0500 Subject: [PATCH 43/43] Attempt to close all mapping dataspaces even during failure --- src/H5Dvirtual.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 3d063229316..9322ec2b120 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -4329,8 +4329,9 @@ H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping) for (size_t j = mapping->sub_dset_io_start; j < mapping->sub_dset_io_end; j++) /* Close projected memory space */ if (mapping->sub_dset[j].projected_mem_space) { + /* Use HDONE_ERROR to attempt to close all spaces even after failure */ if (H5S_close(mapping->sub_dset[j].projected_mem_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); mapping->sub_dset[j].projected_mem_space = NULL; } /* end if */ } /* end if */ @@ -4338,9 +4339,9 @@ H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping) /* Close projected memory space */ if (mapping->source_dset.projected_mem_space) { if (H5S_close(mapping->source_dset.projected_mem_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); mapping->source_dset.projected_mem_space = NULL; } /* end if */ -done: + FUNC_LEAVE_NOAPI(ret_value); } /* end H5D__virtual_close_mapping() */