diff --git a/doxygen/examples/tables/propertyLists.dox b/doxygen/examples/tables/propertyLists.dox index 4480cabc64f..6323bd740a9 100644 --- a/doxygen/examples/tables/propertyLists.dox +++ b/doxygen/examples/tables/propertyLists.dox @@ -616,6 +616,10 @@ encoding for object names. #H5Pset_virtual_view/#H5Pget_virtual_view Sets/gets the view of the virtual dataset (VDS) to include or exclude missing mapped elements. + +#H5Pset_virtual_spatial_tree/#H5Pget_virtual_spatial_tree +Sets/gets the flag to use spatial trees when searching many VDS mappings + //! [dapl_table] * diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index 41529a17180..a8a0e91c193 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -4365,6 +4365,96 @@ END FUNCTION h5pget_chunk_cache_c END SUBROUTINE h5pget_chunk_cache_f +!> +!! \ingroup FH5P +!! +!! \brief Retrieves the flag for whether to use/not use a spatial tree +!! during mapping operations on a Virtual Dataset. The default value is true. +!! +!! Use of a spatial tree will accelerate the process of searching through mappings +!! to determine which contain intersections with the user's selection region. +!! With the tree disabled, all mappings will simply be iterated through and +!! checked directly. +!! +!! Certain workflows may find that tree creation overhead outweighs the time saved +!! on reads. In this case, disabling this property will lead to a performance improvement, +!! though it is expected that almost all cases will benefit from the tree on net. +!! +!! \param dapl_id Target dataset access property list identifier. +!! \param use_tree Value of the setting. +!! \param hdferr \fortran_error +!! +!! See C API: @ref H5Pget_virtual_spatial_tree() +!! + SUBROUTINE h5pget_virtual_spatial_tree_f(dapl_id, use_tree, hdferr) + IMPLICIT NONE + INTEGER(HID_T) , INTENT(IN) :: dapl_id + LOGICAL , INTENT(OUT) :: use_tree + INTEGER , INTENT(OUT) :: hdferr + LOGICAL(C_BOOL) :: c_use_tree + + INTERFACE + INTEGER(C_INT) FUNCTION H5Pget_virtual_spatial_tree_c(dapl_id, use_tree) & + BIND(C, NAME='H5Pget_virtual_spatial_tree') + IMPORT :: C_INT, HID_T, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), INTENT(IN), VALUE :: dapl_id + LOGICAL(C_BOOL), INTENT(OUT) :: use_tree + END FUNCTION H5Pget_virtual_spatial_tree_c + END INTERFACE + + hdferr = INT(H5Pget_virtual_spatial_tree_c(dapl_id, c_use_tree)) + + ! Transfer value of C C_BOOL type to Fortran LOGICAL + use_tree = c_use_tree + + END SUBROUTINE h5pget_virtual_spatial_tree_f + +!> +!! \ingroup FH5P +!! +!! \brief Sets the dapl to use/not use a spatial tree +!! during mapping operations on a Virtual Dataset. The default value is true. +!! +!! Use of a spatial tree will accelerate the process of searching through mappings +!! to determine which contain intersections with the user's selection region. +!! With the tree disabled, all mappings will simply be iterated through and +!! checked directly. +!! +!! Certain workflows may find that tree creation overhead outweighs the time saved +!! on reads. In this case, disabling this property will lead to a performance improvement, +!! though it is expected that almost all cases will benefit from the tree on net. +!! +!! \param dapl_id Target dataset access property list identifier. +!! \param use_tree Value of the setting. +!! \param hdferr \fortran_error +!! +!! See C API: @ref H5Pset_virtual_spatial_tree() +!! + SUBROUTINE h5pset_virtual_spatial_tree_f(dapl_id, use_tree, hdferr) + IMPLICIT NONE + INTEGER(HID_T) , INTENT(IN) :: dapl_id + LOGICAL , INTENT(IN) :: use_tree + INTEGER , INTENT(OUT) :: hdferr + LOGICAL(C_BOOL) :: c_use_tree + + INTERFACE + INTEGER FUNCTION h5pset_virtual_spatial_tree_c(dapl_id, use_tree) & + BIND(C, NAME='H5Pset_virtual_spatial_tree') + IMPORT :: HID_T, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), INTENT(IN), VALUE :: dapl_id + LOGICAL(C_BOOL), INTENT(IN), VALUE :: use_tree + END FUNCTION h5pset_virtual_spatial_tree_c + END INTERFACE + + ! Transfer value of Fortran LOGICAL to C C_BOOL type + c_use_tree = use_tree + + hdferr = INT(h5pset_virtual_spatial_tree_c(dapl_id, c_use_tree)) + + END SUBROUTINE h5pset_virtual_spatial_tree_f + #ifdef H5_DOXYGEN !> !! \ingroup FH5P diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index 1769264001c..8a6ff9cbe0e 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -420,6 +420,8 @@ H5P_mp_H5PGET_VIRTUAL_VSPACE_F H5P_mp_H5PGET_VIRTUAL_SRCSPACE_F H5P_mp_H5PGET_VIRTUAL_FILENAME_F H5P_mp_H5PGET_VIRTUAL_DSETNAME_F +H5P_mp_H5PGET_VIRTUAL_SPATIAL_TREE_F +H5P_mp_H5PSET_VIRTUAL_SPATIAL_TREE_F H5P_mp_H5PGET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_VOL_F diff --git a/fortran/test/tH5P.F90 b/fortran/test/tH5P.F90 index 5a317d5593c..477b43060e5 100644 --- a/fortran/test/tH5P.F90 +++ b/fortran/test/tH5P.F90 @@ -777,8 +777,10 @@ SUBROUTINE test_misc_properties(total_error) INTEGER, INTENT(INOUT) :: total_error INTEGER(hid_t) :: fapl_id = -1 ! Local fapl + INTEGER(hid_t) :: dapl_id = -1 ! Local dapl LOGICAL :: use_file_locking ! (H5Pset/get_file_locking_f) LOGICAL :: ignore_disabled_locks ! (H5Pset/get_file_locking_f) + LOGICAL :: use_spatial_tree ! (H5Pset/get_dset_use_spatial_tree_f) INTEGER :: error ! Create a default fapl @@ -826,6 +828,39 @@ SUBROUTINE test_misc_properties(total_error) CALL H5Pclose_f(fapl_id, error) CALL check("H5Pclose_f", error, total_error) + ! Create a dataset access property list + CALL H5Pcreate_f(H5P_DATASET_ACCESS_F, dapl_id, error) + CALL check("H5Pcreate_f", error, total_error) + + ! Test H5Pset/get_virtual_spatial_tree_f + ! true value + use_spatial_tree = .TRUE. + CALL h5pset_virtual_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pset_virtual_spatial_tree_f", error, total_error) + use_spatial_tree = .FALSE. + CALL h5pget_virtual_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pget_virtual_spatial_tree_f", error, total_error) + if(use_spatial_tree .neqv. .TRUE.) then + total_error = total_error + 1 + write(*,*) "Got wrong use_spatial_tree flag from h5pget_virtual_spatial_tree_f" + endif + + ! false value + use_spatial_tree = .FALSE. + CALL h5pset_virtual_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pset_virtual_spatial_tree_f", error, total_error) + use_spatial_tree = .TRUE. + CALL h5pget_virtual_spatial_tree_f(dapl_id, use_spatial_tree, error) + CALL check("h5pget_virtual_spatial_tree_f", error, total_error) + if(use_spatial_tree .neqv. .FALSE.) then + total_error = total_error + 1 + write(*,*) "Got wrong use_spatial_tree flag from h5pget_virtual_spatial_tree_f" + endif + + ! Close the dapl + CALL H5Pclose_f(dapl_id, error) + CALL check("H5Pclose_f", error, total_error) + END SUBROUTINE test_misc_properties !------------------------------------------------------------------------- diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 2fd8dcf8544..39245bf8816 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -10675,6 +10675,59 @@ public synchronized static native void H5Pset_virtual_prefix(long dapl_id, Strin public synchronized static native void H5Pset_efile_prefix(long dapl_id, String prefix) throws HDF5LibraryException, NullPointerException; + /** + * @ingroup JH5P + * + * H5Pget_virtual_spatial_tree accesses the flag for whether to use/not use a spatial tree + * during mapping operations on a Virtual Dataset. The default value is true. + * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. + * + * @param dapl_id + * IN: Dataset access property list + * + * @return true if the given dapl is set to use a spatial tree, false if not. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + **/ + public synchronized static native boolean H5Pget_virtual_spatial_tree(long dapl_id) + throws HDF5LibraryException; + + /** + * @ingroup JH5P + * + * H5Pset_virtual_spatial_tree sets the dapl to use/not use a spatial tree + * during mapping operations on a Virtual Dataset. The default value is true. + * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. + * + * @param dapl_id + * IN: Dataset access property list + * + * @param use_tree + * IN: the use_tree flag setting + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + **/ + public synchronized static native void H5Pset_virtual_spatial_tree(long dapl_id, boolean use_tree) + throws HDF5LibraryException; + // public synchronized static native void H5Pset_append_flush(long plist_id, int ndims, long[] boundary, // H5D_append_cb func, H5D_append_t udata) throws HDF5LibraryException; diff --git a/java/src/jni/h5pDAPLImp.c b/java/src/jni/h5pDAPLImp.c index b6ae1836bb9..e20a2081e81 100644 --- a/java/src/jni/h5pDAPLImp.c +++ b/java/src/jni/h5pDAPLImp.c @@ -312,6 +312,51 @@ H5D_append_cb(hid_t dataset_id, hsize_t *cur_dims, void *cb_data) return (herr_t)status; } /* end H5D_append_cb */ +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pset_virtual_spatial_tree + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Pset_1virtual_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id, jboolean use_tree) +{ + bool use_tree_val; + herr_t retVal = FAIL; + + UNUSED(clss); + + use_tree_val = (JNI_TRUE == use_tree) ? true : false; + + if ((retVal = H5Pset_virtual_spatial_tree((hid_t)dapl_id, (bool)use_tree_val)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + +done: + return; +} /* end Java_hdf_hdf5lib_H5_H5Pset_1virtual_1spatial_1tree */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_virtual_spatial_tree + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1virtual_1spatial_1tree(JNIEnv *env, jclass clss, jlong dapl_id) +{ + bool use_tree = false; + jboolean bval = JNI_FALSE; + + UNUSED(clss); + + if (H5Pget_virtual_spatial_tree((hid_t)dapl_id, (bool *)&use_tree) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + if (use_tree == true) + bval = JNI_TRUE; + +done: + return bval; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1virtual_1spatial_1tree */ + #ifdef __cplusplus } /* end extern "C" */ #endif /* __cplusplus */ diff --git a/java/src/jni/h5pDAPLImp.h b/java/src/jni/h5pDAPLImp.h index ac4828b8403..600f636038e 100644 --- a/java/src/jni/h5pDAPLImp.h +++ b/java/src/jni/h5pDAPLImp.h @@ -89,6 +89,20 @@ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1virtual_1printf_1gap(JNIEnv * */ JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1printf_1gap(JNIEnv *, jclass, jlong); +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pset_virtual_spatial_tree + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1virtual_1spatial_1tree(JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_virtual_spatial_tree + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1virtual_1spatial_1tree(JNIEnv *, jclass, jlong); + #ifdef __cplusplus } /* end extern "C" */ #endif /* __cplusplus */ diff --git a/release_docs/CHANGELOG.md b/release_docs/CHANGELOG.md index ce89a350b8f..6c1db1e099b 100644 --- a/release_docs/CHANGELOG.md +++ b/release_docs/CHANGELOG.md @@ -25,6 +25,7 @@ For releases prior to version 2.0.0, please see the release.txt file and for mor ## Performance Enhancements: +- Up to [2500% faster](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#rtree) Virtual Dataset read/write operations - [30% faster opening](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#layoutcopydelay) and [25% faster closing](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#fileformat) of virtual datasets. - [Reduced memory overhead](https://github.com/HDFGroup/hdf5/blob/develop/release_docs/CHANGELOG.md#fileformat) via shared name strings and optimized spatial search algorithms for virtual datasets. @@ -461,6 +462,26 @@ Simple example programs showing how to use complex number datatypes have been ad This layout copy is now delayed until either a user requests the DCPL, or until the start of an operation that needs to read the layout from the DCPL. +### Virtual datasets now use a spatial tree to optimize searches + + Virtual dataset operations with many (>1,000) mappings were much slower than + corresponding operations on normal datasets. This was due to the need + to iterate through every source dataset's dataspace and check for an intersection + with the user-selected region for a read/write in the virtual dataset. + + Virtual datasets with many mappings now use an r-tree (defined in H5RT.c) to + perform a spatial search. This allows the dataspaces that intersect the + user-selection to be computed with, in most cases, much fewer intersection checks, + improving the speed of VDS read/write operations. + + Virtual datasets will use the r-tree by default, since the majority of use cases, + should see improvements from use of the tree. However, because some workflows may + find that the overhead of the tree outweighs the time saved on searches, there is + a new Dataset Access Property List (DAPL) property to control use of the spatial tree. + + This property can be set or queried with the new API functions + H5Pset_virtual_spatial_tree()/H5Pget_virtual_spatial_tree(). + ## Parallel Library ### Added H5FDsubfiling_get_file_mapping() API function for subfiling VFD diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 7088443f6a2..a6175492016 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -54,6 +54,7 @@ #define H5D_ACS_VDS_PREFIX_NAME "vds_prefix" /* VDS file prefix */ #define H5D_ACS_APPEND_FLUSH_NAME "append_flush" /* Append flush actions */ #define H5D_ACS_EFILE_PREFIX_NAME "external file prefix" /* External file prefix */ +#define H5D_ACS_USE_TREE_NAME "tree" /* Whether to use spatial tree */ /* ======== Data transfer properties ======== */ #define H5D_XFER_MAX_TEMP_BUF_NAME "max_temp_buf" /* Maximum temp buffer size */ @@ -124,6 +125,9 @@ /* Default virtual dataset list size */ #define H5D_VIRTUAL_DEF_LIST_SIZE 8 +/* Threshold for use of a tree for VDS mappings */ +#define H5D_VIRTUAL_TREE_THRESHOLD 50 + #ifdef H5D_MODULE #define H5D_OBJ_ID(D) (((H5D_obj_create_t *)(D))->dcpl_id) #else /* H5D_MODULE */ diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 62e1d39cbfb..9322ec2b120 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -60,6 +60,7 @@ #include "H5MMprivate.h" /* Memory management */ #include "H5Oprivate.h" /* Object headers */ #include "H5Pprivate.h" /* Property Lists */ +#include "H5RTprivate.h" /* R-trees */ #include "H5Sprivate.h" /* Dataspaces */ #include "H5VLprivate.h" /* Virtual Object Layer */ @@ -70,6 +71,24 @@ /* Default size for sub_dset array */ #define H5D_VIRTUAL_DEF_SUB_DSET_SIZE 128 +/* Default size for not_in_tree list allocation */ +#define H5D_VIRTUAL_NOT_IN_TREE_INIT_SIZE 64 + +/* + * Determines whether a virtual dataset mapping entry should be inserted + * into the R-tree spatial index. + * + * The following mappings cannot be added to the tree: + * - Mappings with an unlimited dimension, since this would invalidate the spatial search process + * - Mappings with zero dimensions, since the tree relies on each entry having at least one dimension + * + */ + +/* Mappings with an unlimited dimension */ +#define H5D_RTREE_SHOULD_INSERT(entry) \ + !(((entry)->unlim_dim_virtual >= 0) || ((entry)->source_dset.virtual_select && \ + H5S_GET_EXTENT_NDIMS((entry)->source_dset.virtual_select) < 1)) + /******************/ /* Local Typedefs */ /******************/ @@ -101,13 +120,29 @@ static herr_t H5D__virtual_build_source_name(char char **built_name); static herr_t H5D__virtual_init_all(const H5D_t *dset); static herr_t H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, - H5S_t *file_space, H5S_t *mem_space, hsize_t *tot_nelmts); -static herr_t H5D__virtual_post_io(H5O_storage_virtual_t *storage); -static herr_t H5D__virtual_read_one(H5D_dset_io_info_t *dset_info, - H5O_storage_virtual_srcdset_t *source_dset); -static herr_t H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, - H5O_storage_virtual_srcdset_t *source_dset); - + H5S_t *file_space, H5S_t *mem_space, hsize_t *tot_nelmts, + H5RT_result_set_t *mappings); +static herr_t H5D__virtual_post_io(H5O_storage_virtual_t *storage, H5RT_result_set_t *mappings); +static herr_t H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping); +static herr_t H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, + H5O_storage_virtual_ent_t *mapping); +static herr_t H5D__virtual_read_one_src(H5D_dset_io_info_t *dset_info, + H5O_storage_virtual_srcdset_t *source_dset); +static herr_t H5D__virtual_write_one_mapping(H5D_dset_io_info_t *dset_info, + H5O_storage_virtual_ent_t *mapping); +static herr_t H5D__virtual_write_one_src(H5D_dset_io_info_t *dset_info, + H5O_storage_virtual_srcdset_t *source_dset); + +/* R-tree helper functions */ +static herr_t H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank); +static herr_t H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, + H5RT_leaf_t **leaves_out, H5O_storage_virtual_ent_t ***not_in_tree_out, + size_t *leaf_count, size_t *not_in_tree_count, + size_t *not_in_tree_nalloc); +static herr_t H5D__should_build_tree(H5O_storage_virtual_t *storage, hid_t dapl_id, bool *should_build_tree); +static herr_t H5D__virtual_not_in_tree_grow(H5O_storage_virtual_ent_t ***list, size_t *nalloc); +static herr_t H5D__virtual_not_in_tree_add(H5O_storage_virtual_ent_t ***list, size_t *nused, size_t *nalloc, + H5O_storage_virtual_ent_t *mapping); /*********************/ /* Package Variables */ /*********************/ @@ -914,13 +949,15 @@ H5D__virtual_load_layout(H5F_t *f, H5O_layout_t *layout) herr_t H5D__virtual_copy_layout(H5O_layout_t *layout) { - H5O_storage_virtual_ent_t *orig_list = NULL; - H5O_storage_virtual_t *virt = &layout->storage.u.virt; - hid_t orig_source_fapl; - hid_t orig_source_dapl; - H5P_genplist_t *plist; - size_t i; - herr_t ret_value = SUCCEED; + H5O_storage_virtual_ent_t *orig_list = NULL; + H5O_storage_virtual_ent_t **orig_not_in_tree_list = NULL; + H5O_storage_virtual_t *virt = &layout->storage.u.virt; + hid_t orig_source_fapl; + hid_t orig_source_dapl; + H5P_genplist_t *plist; + size_t i; + herr_t ret_value = SUCCEED; + H5RT_t *new_tree = NULL; FUNC_ENTER_PACKAGE @@ -935,12 +972,14 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) /* Save original entry list and top-level property lists and reset in layout * so the originals aren't closed on error */ - orig_source_fapl = virt->source_fapl; - virt->source_fapl = -1; - orig_source_dapl = virt->source_dapl; - virt->source_dapl = -1; - orig_list = virt->list; - virt->list = NULL; + orig_source_fapl = virt->source_fapl; + virt->source_fapl = -1; + orig_source_dapl = virt->source_dapl; + virt->source_dapl = -1; + orig_list = virt->list; + virt->list = NULL; + orig_not_in_tree_list = virt->not_in_tree_list; + virt->not_in_tree_list = NULL; /* Copy entry list */ if (virt->list_nused > 0) { @@ -1053,6 +1092,39 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) virt->list_nalloc = 0; } /* end else */ + /* Copy spatial tree if it exists */ + if (virt->tree) { + if ((new_tree = H5RT_copy(virt->tree)) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy spatial tree"); + virt->tree = new_tree; + + /* Copy not_in_tree_list (pointer array) */ + if (virt->not_in_tree_nused > 0) { + assert(orig_not_in_tree_list); + + /* Allocate new pointer array */ + if ((virt->not_in_tree_list = (H5O_storage_virtual_ent_t **)H5MM_calloc( + virt->not_in_tree_nused * sizeof(H5O_storage_virtual_ent_t *))) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "unable to allocate not_in_tree_list pointer array"); + + virt->not_in_tree_nalloc = virt->not_in_tree_nused; + + /* Point to corresponding entries in the new list */ + for (i = 0; i < virt->not_in_tree_nused; i++) { + ptrdiff_t offset = orig_not_in_tree_list[i] - orig_list; /* Calculate original offset */ + assert(offset >= 0 && (size_t)offset < virt->list_nused); /* Validate offset */ + virt->not_in_tree_list[i] = &virt->list[offset]; /* Point to new list entry */ + } + } + } + else { + virt->tree = NULL; + virt->not_in_tree_list = NULL; + virt->not_in_tree_nused = 0; + virt->not_in_tree_nalloc = 0; + } + /* Copy property lists */ if (orig_source_fapl >= 0) { if (NULL == (plist = (H5P_genplist_t *)H5I_object_verify(orig_source_fapl, H5I_GENPROP_LST))) @@ -1145,6 +1217,22 @@ H5D__virtual_free_layout_mappings(H5O_storage_virtual_t *virt) virt->list_nused = (size_t)0; (void)memset(virt->min_dims, 0, sizeof(virt->min_dims)); + /* Destroy the spatial tree, if it exists */ + if (virt->tree) { + if (H5RT_free(virt->tree) < 0) { + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to destroy spatial tree"); + } + virt->tree = NULL; + } + + /* Destroy the pointer array tracking which mappings are not in spatial tree */ + if (virt->not_in_tree_list) { + /* Only free the pointer array itself - the entries are owned by the main list */ + virt->not_in_tree_list = H5MM_xfree(virt->not_in_tree_list); + virt->not_in_tree_nused = 0; + virt->not_in_tree_nalloc = 0; + } + /* Note the lack of a done: label. This is because there are no HGOTO_ERROR * calls. If one is added, a done: label must also be added */ FUNC_LEAVE_NOAPI(ret_value) @@ -2777,20 +2865,20 @@ H5D__virtual_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t H5_ATTR_UNUSED * } /* end H5D__virtual_io_init() */ /*------------------------------------------------------------------------- - * Function: H5D__virtual_pre_io + * Function: H5D__virtual_pre_io_process_mapping * - * Purpose: Project all virtual mappings onto mem_space, with the - * results stored in projected_mem_space for each mapping. - * Opens all source datasets if possible. The total number - * of elements is stored in tot_nelmts. + * Purpose: Process a single virtual mapping to prepare for I/O. + * This includes projecting the virtual mapping onto mem_space + * and opening source datasets. The number of elements included + * in this mapping selection is added to tot_nelmts. * * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, H5S_t *file_space, - H5S_t *mem_space, hsize_t *tot_nelmts) +H5D__virtual_pre_io_process_mapping(H5D_dset_io_info_t *dset_info, H5S_t *file_space, H5S_t *mem_space, + hsize_t *tot_nelmts, H5O_storage_virtual_ent_t *curr_mapping) { const H5D_t *dset = dset_info->dset; /* Local pointer to dataset info */ hssize_t select_nelmts; /* Number of elements in selection */ @@ -2798,245 +2886,279 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag hsize_t bounds_end[H5S_MAX_RANK]; /* Selection bounds end */ int rank = 0; bool bounds_init = false; /* Whether bounds_start, bounds_end, and rank are valid */ - size_t i, j, k; /* Local index variables */ + size_t j, k; /* Local index variables */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE - /* Sanity check */ - assert(storage); - assert(mem_space); - assert(file_space); - assert(tot_nelmts); - - /* Initialize layout if necessary */ - if (!storage->init) - if (H5D__virtual_init_all(dset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize virtual layout"); + /* Sanity check that the virtual space has been patched by now */ + assert(curr_mapping->virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); - /* Initialize tot_nelmts */ - *tot_nelmts = 0; + /* Check for "printf" source dataset resolution */ + if (curr_mapping->psfn_nsubs || curr_mapping->psdn_nsubs) { + bool partial_block; - /* Iterate over mappings */ - for (i = 0; i < storage->list_nused; i++) { - /* Sanity check that the virtual space has been patched by now */ - assert(storage->list[i].virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); - - /* Check for "printf" source dataset resolution */ - if (storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { - bool partial_block; + assert(curr_mapping->unlim_dim_virtual >= 0); - assert(storage->list[i].unlim_dim_virtual >= 0); + /* Get selection bounds if necessary */ + if (!bounds_init) { + /* Get rank of VDS */ + if ((rank = H5S_GET_EXTENT_NDIMS(dset->shared->space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions"); - /* Get selection bounds if necessary */ - if (!bounds_init) { - /* Get rank of VDS */ - if ((rank = H5S_GET_EXTENT_NDIMS(dset->shared->space)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions"); + /* Get selection bounds */ + if (H5S_SELECT_BOUNDS(file_space, bounds_start, bounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); - /* Get selection bounds */ - if (H5S_SELECT_BOUNDS(file_space, bounds_start, bounds_end) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); + /* Adjust bounds_end to represent the extent just enclosing them + * (add 1) */ + for (j = 0; j < (size_t)rank; j++) + bounds_end[j]++; - /* Adjust bounds_end to represent the extent just enclosing them - * (add 1) */ - for (j = 0; j < (size_t)rank; j++) - bounds_end[j]++; + /* Bounds are now initialized */ + bounds_init = true; + } /* end if */ - /* Bounds are now initialized */ - bounds_init = true; - } /* end if */ + /* Get index of first block in virtual selection */ + curr_mapping->sub_dset_io_start = (size_t)H5S_hyper_get_first_inc_block( + curr_mapping->source_dset.virtual_select, bounds_start[curr_mapping->unlim_dim_virtual], NULL); + + /* Get index of first block outside of virtual selection */ + curr_mapping->sub_dset_io_end = (size_t)H5S_hyper_get_first_inc_block( + curr_mapping->source_dset.virtual_select, bounds_end[curr_mapping->unlim_dim_virtual], + &partial_block); + if (partial_block) + curr_mapping->sub_dset_io_end++; + if (curr_mapping->sub_dset_io_end > curr_mapping->sub_dset_nused) + curr_mapping->sub_dset_io_end = curr_mapping->sub_dset_nused; + + /* Iterate over sub-source dsets */ + for (j = curr_mapping->sub_dset_io_start; j < curr_mapping->sub_dset_io_end; j++) { + /* Check for clipped virtual selection */ + if (!curr_mapping->sub_dset[j].clipped_virtual_select) { + hsize_t start[H5S_MAX_RANK]; + /* This should only be NULL if this is a partial block */ + assert((j == (curr_mapping->sub_dset_io_end - 1)) && partial_block); + + /* If the source space status is not correct, we must try to + * open the source dataset to patch it */ + if (curr_mapping->source_space_status != H5O_VIRTUAL_STATUS_CORRECT) { + assert(!curr_mapping->sub_dset[j].dset); + if (H5D__virtual_open_source_dset(dset, curr_mapping, &curr_mapping->sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset"); + } /* end if */ - /* Get index of first block in virtual selection */ - storage->list[i].sub_dset_io_start = - (size_t)H5S_hyper_get_first_inc_block(storage->list[i].source_dset.virtual_select, - bounds_start[storage->list[i].unlim_dim_virtual], NULL); + /* If we obtained a valid source space, we must create + * clipped source and virtual selections, otherwise we + * cannot do this and we will leave them NULL. This doesn't + * hurt anything because we can't do I/O because the dataset + * must not have been found. */ + if (curr_mapping->source_space_status == H5O_VIRTUAL_STATUS_CORRECT) { + hsize_t tmp_dims[H5S_MAX_RANK]; + hsize_t vbounds_end[H5S_MAX_RANK]; - /* Get index of first block outside of virtual selection */ - storage->list[i].sub_dset_io_end = (size_t)H5S_hyper_get_first_inc_block( - storage->list[i].source_dset.virtual_select, bounds_end[storage->list[i].unlim_dim_virtual], - &partial_block); - if (partial_block) - storage->list[i].sub_dset_io_end++; - if (storage->list[i].sub_dset_io_end > storage->list[i].sub_dset_nused) - storage->list[i].sub_dset_io_end = storage->list[i].sub_dset_nused; + /* Get bounds of virtual selection */ + if (H5S_SELECT_BOUNDS(curr_mapping->sub_dset[j].virtual_select, tmp_dims, vbounds_end) < + 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); - /* Iterate over sub-source dsets */ - for (j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++) { - /* Check for clipped virtual selection */ - if (!storage->list[i].sub_dset[j].clipped_virtual_select) { - hsize_t start[H5S_MAX_RANK]; - /* This should only be NULL if this is a partial block */ - assert((j == (storage->list[i].sub_dset_io_end - 1)) && partial_block); - - /* If the source space status is not correct, we must try to - * open the source dataset to patch it */ - if (storage->list[i].source_space_status != H5O_VIRTUAL_STATUS_CORRECT) { - assert(!storage->list[i].sub_dset[j].dset); - if (H5D__virtual_open_source_dset(dset, &storage->list[i], - &storage->list[i].sub_dset[j]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset"); - } /* end if */ - - /* If we obtained a valid source space, we must create - * clipped source and virtual selections, otherwise we - * cannot do this and we will leave them NULL. This doesn't - * hurt anything because we can't do I/O because the dataset - * must not have been found. */ - if (storage->list[i].source_space_status == H5O_VIRTUAL_STATUS_CORRECT) { - hsize_t tmp_dims[H5S_MAX_RANK]; - hsize_t vbounds_end[H5S_MAX_RANK]; + assert(bounds_init); - /* Get bounds of virtual selection */ - if (H5S_SELECT_BOUNDS(storage->list[i].sub_dset[j].virtual_select, tmp_dims, - vbounds_end) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); + /* Convert bounds to extent (add 1) */ + for (k = 0; k < (size_t)rank; k++) + vbounds_end[k]++; - assert(bounds_init); + /* Temporarily set extent of virtual selection to bounds */ + if (H5S_set_extent(curr_mapping->sub_dset[j].virtual_select, vbounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of dataspace"); - /* Convert bounds to extent (add 1) */ - for (k = 0; k < (size_t)rank; k++) - vbounds_end[k]++; + /* Get current VDS dimensions */ + if (H5S_get_simple_extent_dims(dset->shared->space, tmp_dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get VDS dimensions"); - /* Temporarily set extent of virtual selection to bounds */ - if (H5S_set_extent(storage->list[i].sub_dset[j].virtual_select, vbounds_end) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to modify size of dataspace"); - - /* Get current VDS dimensions */ - if (H5S_get_simple_extent_dims(dset->shared->space, tmp_dims, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get VDS dimensions"); - - /* Copy virtual selection */ - if (NULL == (storage->list[i].sub_dset[j].clipped_virtual_select = - H5S_copy(storage->list[i].sub_dset[j].virtual_select, false, true))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection"); - - /* Clip virtual selection to real virtual extent */ - (void)memset(start, 0, sizeof(start)); - if (H5S_select_hyperslab(storage->list[i].sub_dset[j].clipped_virtual_select, - H5S_SELECT_AND, start, NULL, tmp_dims, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTSELECT, FAIL, "unable to clip hyperslab"); - - /* Project intersection of virtual space and clipped - * virtual space onto source space (create - * clipped_source_select) */ - if (H5S_select_project_intersection( - storage->list[i].sub_dset[j].virtual_select, storage->list[i].source_select, - storage->list[i].sub_dset[j].clipped_virtual_select, - &storage->list[i].sub_dset[j].clipped_source_select, true) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, - "can't project virtual intersection onto memory space"); + /* Copy virtual selection */ + if (NULL == (curr_mapping->sub_dset[j].clipped_virtual_select = + H5S_copy(curr_mapping->sub_dset[j].virtual_select, false, true))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection"); - /* Set extents of virtual_select and - * clipped_virtual_select to virtual extent */ - if (H5S_set_extent(storage->list[i].sub_dset[j].virtual_select, tmp_dims) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to modify size of dataspace"); - if (H5S_set_extent(storage->list[i].sub_dset[j].clipped_virtual_select, tmp_dims) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to modify size of dataspace"); - } /* end if */ - } /* end if */ + /* Clip virtual selection to real virtual extent */ + (void)memset(start, 0, sizeof(start)); + if (H5S_select_hyperslab(curr_mapping->sub_dset[j].clipped_virtual_select, H5S_SELECT_AND, + start, NULL, tmp_dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSELECT, FAIL, "unable to clip hyperslab"); - /* Only continue if we managed to obtain a - * clipped_virtual_select */ - if (storage->list[i].sub_dset[j].clipped_virtual_select) { - /* Project intersection of file space and mapping virtual space - * onto memory space */ + /* Project intersection of virtual space and clipped + * virtual space onto source space (create + * clipped_source_select) */ if (H5S_select_project_intersection( - file_space, mem_space, storage->list[i].sub_dset[j].clipped_virtual_select, - &storage->list[i].sub_dset[j].projected_mem_space, true) < 0) + curr_mapping->sub_dset[j].virtual_select, curr_mapping->source_select, + curr_mapping->sub_dset[j].clipped_virtual_select, + &curr_mapping->sub_dset[j].clipped_source_select, true) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space"); - /* Check number of elements selected */ - if ((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS( - storage->list[i].sub_dset[j].projected_mem_space)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, - "unable to get number of elements in selection"); - - /* Check if anything is selected */ - if (select_nelmts > (hssize_t)0) { - /* Open source dataset */ - if (!storage->list[i].sub_dset[j].dset) - /* Try to open dataset */ - if (H5D__virtual_open_source_dset(dset, &storage->list[i], - &storage->list[i].sub_dset[j]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, - "unable to open source dataset"); - - /* If the source dataset is not open, mark the selected - * elements as zero so projected_mem_space is freed */ - if (!storage->list[i].sub_dset[j].dset) - select_nelmts = (hssize_t)0; - } /* end if */ - - /* If there are not elements selected in this mapping, free - * projected_mem_space, otherwise update tot_nelmts */ - if (select_nelmts == (hssize_t)0) { - if (H5S_close(storage->list[i].sub_dset[j].projected_mem_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, - "can't close projected memory space"); - storage->list[i].sub_dset[j].projected_mem_space = NULL; - } /* end if */ - else - *tot_nelmts += (hsize_t)select_nelmts; + /* Set extents of virtual_select and + * clipped_virtual_select to virtual extent */ + if (H5S_set_extent(curr_mapping->sub_dset[j].virtual_select, tmp_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of dataspace"); + if (H5S_set_extent(curr_mapping->sub_dset[j].clipped_virtual_select, tmp_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of dataspace"); } /* end if */ - } /* end for */ - } /* end if */ - else { - if (storage->list[i].source_dset.clipped_virtual_select) { - /* Project intersection of file space and mapping virtual space onto - * memory space */ - if (H5S_select_project_intersection( - file_space, mem_space, storage->list[i].source_dset.clipped_virtual_select, - &storage->list[i].source_dset.projected_mem_space, true) < 0) + } /* end if */ + + /* Only continue if we managed to obtain a + * clipped_virtual_select */ + if (curr_mapping->sub_dset[j].clipped_virtual_select) { + /* Project intersection of file space and mapping virtual space + * onto memory space */ + if (H5S_select_project_intersection(file_space, mem_space, + curr_mapping->sub_dset[j].clipped_virtual_select, + &curr_mapping->sub_dset[j].projected_mem_space, true) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space"); - /* Check number of elements selected, add to tot_nelmts */ - if ((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS( - storage->list[i].source_dset.projected_mem_space)) < 0) + /* Check number of elements selected */ + if ((select_nelmts = + (hssize_t)H5S_GET_SELECT_NPOINTS(curr_mapping->sub_dset[j].projected_mem_space)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to get number of elements in selection"); /* Check if anything is selected */ if (select_nelmts > (hssize_t)0) { /* Open source dataset */ - if (!storage->list[i].source_dset.dset) + if (!curr_mapping->sub_dset[j].dset) /* Try to open dataset */ - if (H5D__virtual_open_source_dset(dset, &storage->list[i], - &storage->list[i].source_dset) < 0) + if (H5D__virtual_open_source_dset(dset, curr_mapping, &curr_mapping->sub_dset[j]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset"); - /* If the source dataset is not open, mark the selected elements - * as zero so projected_mem_space is freed */ - if (!storage->list[i].source_dset.dset) + /* If the source dataset is not open, mark the selected + * elements as zero so projected_mem_space is freed */ + if (!curr_mapping->sub_dset[j].dset) select_nelmts = (hssize_t)0; } /* end if */ /* If there are not elements selected in this mapping, free * projected_mem_space, otherwise update tot_nelmts */ if (select_nelmts == (hssize_t)0) { - if (H5S_close(storage->list[i].source_dset.projected_mem_space) < 0) + if (H5S_close(curr_mapping->sub_dset[j].projected_mem_space) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space"); - storage->list[i].source_dset.projected_mem_space = NULL; + curr_mapping->sub_dset[j].projected_mem_space = NULL; } /* end if */ else *tot_nelmts += (hsize_t)select_nelmts; } /* end if */ - else { - /* If there is no clipped_dim_virtual, this must be an unlimited - * selection whose dataset was not found in the last call to - * H5Dget_space(). Do not attempt to open it as this might - * affect the extent and we are not going to recalculate it - * here. */ - assert(storage->list[i].unlim_dim_virtual >= 0); - assert(!storage->list[i].source_dset.dset); - } /* end else */ - } /* end else */ - } /* end for */ + } /* end for */ + } /* end if */ + else { + if (curr_mapping->source_dset.clipped_virtual_select) { + /* Project intersection of file space and mapping virtual space onto + * memory space */ + if (H5S_select_project_intersection(file_space, mem_space, + curr_mapping->source_dset.clipped_virtual_select, + &curr_mapping->source_dset.projected_mem_space, true) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, + "can't project virtual intersection onto memory space"); + + /* Check number of elements selected, add to tot_nelmts */ + if ((select_nelmts = + (hssize_t)H5S_GET_SELECT_NPOINTS(curr_mapping->source_dset.projected_mem_space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, + "unable to get number of elements in selection"); + + /* Check if anything is selected */ + if (select_nelmts > (hssize_t)0) { + /* Open source dataset */ + if (!curr_mapping->source_dset.dset) + /* Try to open dataset */ + if (H5D__virtual_open_source_dset(dset, curr_mapping, &curr_mapping->source_dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset"); + + /* If the source dataset is not open, mark the selected elements + * as zero so projected_mem_space is freed */ + if (!curr_mapping->source_dset.dset) + select_nelmts = (hssize_t)0; + } /* end if */ + + /* If there are not elements selected in this mapping, free + * projected_mem_space, otherwise update tot_nelmts */ + if (select_nelmts == (hssize_t)0) { + if (H5S_close(curr_mapping->source_dset.projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space"); + curr_mapping->source_dset.projected_mem_space = NULL; + } /* end if */ + else + *tot_nelmts += (hsize_t)select_nelmts; + } /* end if */ + else { + /* If there is no clipped_dim_virtual, this must be an unlimited + * selection whose dataset was not found in the last call to + * H5Dget_space(). Do not attempt to open it as this might + * affect the extent and we are not going to recalculate it + * here. */ + assert(curr_mapping->unlim_dim_virtual >= 0); + assert(!curr_mapping->source_dset.dset); + } /* end else */ + } /* end else */ +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_pre_io_process_mapping() */ + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_pre_io + * + * Purpose: Project all virtual mappings onto mem_space, with the + * results stored in projected_mem_space for each mapping. + * Opens all source datasets if possible. The total number + * of elements is stored in tot_nelmts. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storage, H5S_t *file_space, + H5S_t *mem_space, hsize_t *tot_nelmts, H5RT_result_set_t *mappings) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + assert(storage); + assert(mem_space); + assert(file_space); + assert(tot_nelmts); + + /* Initialize tot_nelmts */ + *tot_nelmts = 0; + + /* Iterate over the mappings */ + if (mappings) { + /* First, iterate over the mappings with an intersection found via the tree */ + for (size_t i = 0; i < mappings->count; i++) { + H5RT_leaf_t *curr_leaf = mappings->results[i]; + assert(curr_leaf); + + if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, + (H5O_storage_virtual_ent_t *)curr_leaf->record) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + } + + /* Iterate over the mappings that are not stored in the tree */ + for (size_t i = 0; i < storage->not_in_tree_nused; i++) { + if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, + storage->not_in_tree_list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + } + } + else { + /* No tree - iterate over all mappings directly */ + for (size_t i = 0; i < storage->list_nused; i++) { + if (H5D__virtual_pre_io_process_mapping(dset_info, file_space, mem_space, tot_nelmts, + &storage->list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + } + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -3052,9 +3174,9 @@ H5D__virtual_pre_io(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_t *storag *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_post_io(H5O_storage_virtual_t *storage) +H5D__virtual_post_io(H5O_storage_virtual_t *storage, H5RT_result_set_t *mappings) { - size_t i, j; /* Local index variables */ + size_t i; /* Local index variables */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -3062,34 +3184,35 @@ H5D__virtual_post_io(H5O_storage_virtual_t *storage) /* Sanity check */ assert(storage); - /* Iterate over mappings */ - for (i = 0; i < storage->list_nused; i++) - /* Check for "printf" source dataset resolution */ - if (storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { - /* Iterate over sub-source dsets */ - for (j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++) - /* Close projected memory space */ - if (storage->list[i].sub_dset[j].projected_mem_space) { - if (H5S_close(storage->list[i].sub_dset[j].projected_mem_space) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); - storage->list[i].sub_dset[j].projected_mem_space = NULL; - } /* end if */ - } /* end if */ - else - /* Close projected memory space */ - if (storage->list[i].source_dset.projected_mem_space) { - if (H5S_close(storage->list[i].source_dset.projected_mem_space) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); - storage->list[i].source_dset.projected_mem_space = NULL; - } /* end if */ + if (mappings) { + /* Iterate over mappings in tree */ + for (i = 0; i < mappings->count; i++) { + H5RT_leaf_t *curr_leaf = mappings->results[i]; + assert(curr_leaf); - /* Note the lack of a done: label. This is because there are no HGOTO_ERROR - * calls. If one is added, a done: label must also be added */ + if (H5D__virtual_close_mapping(curr_leaf->record) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + } + + /* Iterate over the mappings that are not stored in the tree */ + for (i = 0; i < storage->not_in_tree_nused; i++) { + if (H5D__virtual_close_mapping(storage->not_in_tree_list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't process mapping for pre I/O"); + } + } + else { + /* Iterate over all mappings */ + for (i = 0; i < storage->list_nused; i++) + if (H5D__virtual_close_mapping(&storage->list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "failed to close mapping"); + } + +done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_post_io() */ /*------------------------------------------------------------------------- - * Function: H5D__virtual_read_one + * Function: H5D__virtual_read_one_src * * Purpose: Read from a single source dataset in a virtual dataset. * @@ -3098,7 +3221,7 @@ H5D__virtual_post_io(H5O_storage_virtual_t *storage) *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_read_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset) +H5D__virtual_read_one_src(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset) { H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ H5D_dset_io_info_t source_dinfo; /* Dataset info for source dataset read */ @@ -3151,7 +3274,7 @@ H5D__virtual_read_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__virtual_read_one() */ +} /* end H5D__virtual_read_one_src() */ /*------------------------------------------------------------------------- * Function: H5D__virtual_read @@ -3165,12 +3288,16 @@ H5D__virtual_read_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset static herr_t H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info_t *dset_info) { - H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ - hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ - H5S_t *fill_space = NULL; /* Space to fill with fill value */ - size_t nelmts; /* Number of elements to process */ - size_t i, j; /* Local index variables */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ + hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ + H5S_t *fill_space = NULL; /* Space to fill with fill value */ + size_t nelmts; /* Number of elements to process */ + size_t i, j; /* Local index variables */ + herr_t ret_value = SUCCEED; /* Return value */ + bool should_build_tree = false; /* Whether to build a spatial tree */ + H5RT_result_set_t *mappings = NULL; /* Search results from R-tree */ + hsize_t min[H5S_MAX_RANK]; + hsize_t max[H5S_MAX_RANK]; FUNC_ENTER_PACKAGE @@ -3187,33 +3314,75 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info /* Initialize nelmts */ nelmts = H5S_GET_SELECT_NPOINTS(dset_info->file_space); + memset(min, 0, sizeof(min)); + memset(max, 0, sizeof(max)); #ifdef H5_HAVE_PARALLEL /* Parallel reads are not supported (yet) */ if (H5F_HAS_FEATURE(dset_info->dset->oloc.file, H5FD_FEAT_HAS_MPI)) HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "parallel reads not supported on virtual datasets"); #endif /* H5_HAVE_PARALLEL */ + /* Initialize layout if necessary */ + if (!storage->init) + if (H5D__virtual_init_all(dset_info->dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize virtual layout"); + + if (H5D__should_build_tree(storage, dset_info->dset->shared->dapl_id, &should_build_tree) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't determine if should build VDS tree"); + + if (should_build_tree) { + int rank = 0; + + /* Get the rank of the dataset */ + if ((rank = H5S_GET_EXTENT_NDIMS(dset_info->dset->shared->space)) < 0 || rank >= H5S_MAX_RANK) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset rank"); + + if (rank == 0) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "virtual dataset has no rank"); + + if (H5D__virtual_build_tree(storage, rank) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build virtual mapping tree"); + } + + if (storage->tree) { + /* Perform a spatial tree search to get a list of mappings + * whose virtual selection intersects the IO operation */ + if (H5S_SELECT_BOUNDS(dset_info->file_space, min, max) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); + + if (H5RT_search(storage->tree, min, max, &mappings) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "R-tree search failed"); + } + /* Prepare for I/O operation */ - if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts) < 0) + if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts, + mappings) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to prepare for I/O operation"); /* Iterate over mappings */ - for (i = 0; i < storage->list_nused; i++) { - /* Sanity check that the virtual space has been patched by now */ - assert(storage->list[i].virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); + if (mappings) { + /* Iterate over intersections in tree */ + for (i = 0; i < mappings->count; i++) { + H5RT_leaf_t *curr_leaf = mappings->results[i]; + assert(curr_leaf); - /* Check for "printf" source dataset resolution */ - if (storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { - /* Iterate over sub-source dsets */ - for (j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++) - if (H5D__virtual_read_one(dset_info, &storage->list[i].sub_dset[j]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); - } /* end if */ - else - /* Read from source dataset */ - if (H5D__virtual_read_one(dset_info, &storage->list[i].source_dset) < 0) + if (H5D__virtual_read_one_mapping(dset_info, curr_leaf->record) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); - } /* end for */ + } + + /* Iterate over not-in-tree mappings */ + for (i = 0; i < storage->not_in_tree_nused; i++) { + if (H5D__virtual_read_one_mapping(dset_info, storage->not_in_tree_list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + } + } + else { + /* Iterate over all mappings */ + for (i = 0; i < storage->list_nused; i++) { + if (H5D__virtual_read_one_mapping(dset_info, &storage->list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + } /* end for */ + } /* Fill unmapped part of buffer with fill value */ if (tot_nelmts < nelmts) { @@ -3272,9 +3441,13 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info done: /* Cleanup I/O operation */ - if (H5D__virtual_post_io(storage) < 0) + if (H5D__virtual_post_io(storage, mappings) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't cleanup I/O operation"); + if (mappings) + if (H5RT_free_results(mappings) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't free R-tree search results"); + /* Close fill space */ if (fill_space) if (H5S_close(fill_space) < 0) @@ -3284,7 +3457,7 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info } /* end H5D__virtual_read() */ /*------------------------------------------------------------------------- - * Function: H5D__virtual_write_one + * Function: H5D__virtual_write_one_src * * Purpose: Write to a single source dataset in a virtual dataset. * @@ -3293,7 +3466,7 @@ H5D__virtual_read(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset) +H5D__virtual_write_one_src(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdset_t *source_dset) { H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ H5D_dset_io_info_t source_dinfo; /* Dataset info for source dataset write */ @@ -3348,7 +3521,7 @@ H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdse } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__virtual_write_one() */ +} /* end H5D__virtual_write_one_src() */ /*------------------------------------------------------------------------- * Function: H5D__virtual_write @@ -3362,11 +3535,15 @@ H5D__virtual_write_one(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_srcdse static herr_t H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_info_t *dset_info) { - H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ - hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ - size_t nelmts; /* Number of elements to process */ - size_t i, j; /* Local index variables */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ + hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ + size_t nelmts; /* Number of elements to process */ + size_t i; /* Local index variables */ + herr_t ret_value = SUCCEED; /* Return value */ + bool should_build_tree = false; /* Whether to build a spatial tree */ + H5RT_result_set_t *mappings = NULL; /* Search results from R-tree */ + hsize_t min[H5S_MAX_RANK]; + hsize_t max[H5S_MAX_RANK]; FUNC_ENTER_PACKAGE @@ -3383,14 +3560,49 @@ H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_inf /* Initialize nelmts */ nelmts = H5S_GET_SELECT_NPOINTS(dset_info->file_space); + memset(min, 0, sizeof(min)); + memset(max, 0, sizeof(max)); #ifdef H5_HAVE_PARALLEL /* Parallel writes are not supported (yet) */ if (H5F_HAS_FEATURE(dset_info->dset->oloc.file, H5FD_FEAT_HAS_MPI)) HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "parallel writes not supported on virtual datasets"); #endif /* H5_HAVE_PARALLEL */ + /* Initialize layout if necessary */ + if (!storage->init) + if (H5D__virtual_init_all(dset_info->dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize virtual layout"); + + if (H5D__should_build_tree(storage, dset_info->dset->shared->dapl_id, &should_build_tree) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't determine if should build VDS tree"); + + if (should_build_tree) { + int rank = 0; + + /* Get the rank of the dataset */ + if ((rank = H5S_GET_EXTENT_NDIMS(dset_info->dset->shared->space)) < 0 || rank >= H5S_MAX_RANK) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset rank"); + + if (rank == 0) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "virtual dataset has no rank"); + + if (H5D__virtual_build_tree(storage, rank) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build virtual mapping tree"); + } + + if (storage->tree) { + /* Perform a spatial tree search to get a list of mappings + * whose virtual selection intersects the IO operation */ + if (H5S_SELECT_BOUNDS(dset_info->file_space, min, max) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds"); + + if (H5RT_search(storage->tree, min, max, &mappings) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "R-tree search failed"); + } + /* Prepare for I/O operation */ - if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts) < 0) + if (H5D__virtual_pre_io(dset_info, storage, dset_info->file_space, dset_info->mem_space, &tot_nelmts, + mappings) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to prepare for I/O operation"); /* Fail if there are unmapped parts of the selection as they would not be @@ -3399,29 +3611,38 @@ H5D__virtual_write(H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, H5D_dset_io_inf HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "write requested to unmapped portion of virtual dataset"); - /* Iterate over mappings */ - for (i = 0; i < storage->list_nused; i++) { - /* Sanity check that virtual space has been patched by now */ - assert(storage->list[i].virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); + if (mappings) { + /* Iterate over intersections in tree */ + for (i = 0; i < mappings->count; i++) { + H5RT_leaf_t *curr_leaf = mappings->results[i]; + assert(curr_leaf); - /* Check for "printf" source dataset resolution */ - if (storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { - /* Iterate over sub-source dsets */ - for (j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++) - if (H5D__virtual_write_one(dset_info, &storage->list[i].sub_dset[j]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset"); - } /* end if */ - else - /* Write to source dataset */ - if (H5D__virtual_write_one(dset_info, &storage->list[i].source_dset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset"); - } /* end for */ + if (H5D__virtual_write_one_mapping(dset_info, curr_leaf->record) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + } + + /* Iterate over not-in-tree mappings */ + for (i = 0; i < storage->not_in_tree_nused; i++) { + if (H5D__virtual_write_one_mapping(dset_info, storage->not_in_tree_list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + } + } + else { + /* Iterate over all mappings */ + for (i = 0; i < storage->list_nused; i++) { + if (H5D__virtual_write_one_mapping(dset_info, &storage->list[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write to virtual mapping"); + } + } done: /* Cleanup I/O operation */ - if (H5D__virtual_post_io(storage) < 0) + if (H5D__virtual_post_io(storage, mappings) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't cleanup I/O operation"); + if (mappings) + H5RT_free_results(mappings); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_write() */ @@ -3672,3 +3893,455 @@ H5D__virtual_release_source_dset_files(H5D_virtual_held_file_t *head) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_release_source_dset_files() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mappings_to_leaves + * + * Purpose: Allocate leaf array and boolean array for construction of a + * spatial tree from a list of mappings + * + * Parameters: mappings : Pointer to array of mappings to be inserted + * num_mappings: Number of mappings in the array + * leaves_out: Pointer to array of leaves, one per mapping that should be inserted + * Allocated on success and must be freed by caller. + * not_in_tree_out: Pointer to array of pointers to mappings NOT in tree. + * Allocated on success and must be freed by caller. + * leaf_count: Pointer to number of leaves allocated in leaves_out. + * not_in_tree_count: Pointer to number of entries in not_in_tree_out. + * not_in_tree_nalloc: Pointer to allocated capacity of not_in_tree_out. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mappings_to_leaves(H5O_storage_virtual_ent_t *mappings, size_t num_mappings, H5RT_leaf_t **leaves_out, + H5O_storage_virtual_ent_t ***not_in_tree_out, size_t *leaf_count, + size_t *not_in_tree_count, size_t *not_in_tree_nalloc) +{ + herr_t ret_value = SUCCEED; + + H5RT_leaf_t *leaves_temp = NULL; + H5O_storage_virtual_ent_t **not_in_tree = NULL; + + H5O_storage_virtual_ent_t *curr_mapping = NULL; + H5RT_leaf_t *curr_leaf = NULL; + size_t curr_leaf_count = 0; + size_t curr_not_tree_count = 0; + size_t not_in_tree_capacity = 0; + H5S_t *curr_space = NULL; + + int rank = 0; + + FUNC_ENTER_PACKAGE + + assert(mappings); + assert(num_mappings > 0); + assert(leaf_count); + assert(leaves_out); + assert(not_in_tree_out); + assert(not_in_tree_count); + assert(not_in_tree_nalloc); + + /* Get rank from the first mapping's virtual selection */ + if ((curr_space = mappings[0].source_dset.virtual_select) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "first mapping has no virtual space"); + + if ((rank = H5S_GET_EXTENT_NDIMS(curr_space)) < 0 || rank > H5S_MAX_RANK) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get rank of dataspace"); + + if (rank == 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "mapping has zero-dimensional space"); + + /* Allocate array of leaf structures */ + if ((leaves_temp = (H5RT_leaf_t *)calloc(num_mappings, sizeof(H5RT_leaf_t))) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate leaves array"); + + /* Initialize not_in_tree list with initial capacity */ + not_in_tree_capacity = H5D_VIRTUAL_NOT_IN_TREE_INIT_SIZE; + + if (NULL == (not_in_tree = (H5O_storage_virtual_ent_t **)H5MM_malloc( + not_in_tree_capacity * sizeof(H5O_storage_virtual_ent_t *)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failed to allocate not_in_tree_list"); + + for (size_t i = 0; i < num_mappings; i++) { + curr_mapping = &mappings[i]; + + if (!(H5D_RTREE_SHOULD_INSERT(curr_mapping))) { + /* Add to not_in_tree list, growing if needed */ + if (H5D__virtual_not_in_tree_add(¬_in_tree, &curr_not_tree_count, ¬_in_tree_capacity, + curr_mapping) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failed to add to not_in_tree_list"); + continue; + } + + /* Initialize leaf with dynamic coordinate allocation */ + curr_leaf = &leaves_temp[curr_leaf_count]; + if (H5RT_leaf_init(curr_leaf, rank, (void *)curr_mapping) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't initialize R-tree leaf"); + + /* Record is already set by H5RT_leaf_init */ + assert(mappings[i].source_dset.virtual_select); + curr_space = mappings[i].source_dset.virtual_select; + + /* Get selection bounds */ + if (H5S_SELECT_BOUNDS(curr_space, curr_leaf->min, curr_leaf->max) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get selection bounds"); + + for (int d = 0; d < rank; d++) { + /* Validate bounds and compute midpoint safely */ + if (curr_leaf->min[d] > curr_leaf->max[d]) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "invalid selection bounds: min > max"); + curr_leaf->mid[d] = curr_leaf->min[d] + (curr_leaf->max[d] - curr_leaf->min[d]) / 2; + } + + curr_leaf_count++; + } + + *leaves_out = leaves_temp; + *leaf_count = curr_leaf_count; + *not_in_tree_out = not_in_tree; + *not_in_tree_count = curr_not_tree_count; + *not_in_tree_nalloc = not_in_tree_capacity; +done: + if (ret_value < 0) { + if (leaves_temp) { + /* Clean up coordinate arrays for initialized leaves */ + for (size_t j = 0; j < curr_leaf_count; j++) { + H5RT_leaf_cleanup(&leaves_temp[j]); + } + free(leaves_temp); + } + if (not_in_tree) + H5MM_free(not_in_tree); + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D__mappings_to_leaves() */ + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_build_tree + * + * Purpose: Build a spatial tree of mapping indices, and a list of + * mappings not in the tree, and store them on + * the provided virtual layout + * + * Parameters: virt: The virtual layout with the mapping to build the + * tree from. The tree will be stored at virt->tree, + * and the list of non-tree mappings will be stored at + * virt->not_in_tree_list. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_build_tree(H5O_storage_virtual_t *virt, int rank) +{ + H5O_storage_virtual_ent_t *mappings = virt->list; + size_t num_mappings = virt->list_nused; + + H5RT_leaf_t *leaves = NULL; + size_t num_leaves = 0; + H5O_storage_virtual_ent_t **not_in_tree_mappings = NULL; + size_t not_in_tree_count = 0; + size_t not_in_tree_nalloc = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + assert(virt); + + if (H5D__mappings_to_leaves(mappings, num_mappings, &leaves, ¬_in_tree_mappings, &num_leaves, + ¬_in_tree_count, ¬_in_tree_nalloc) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get leaves from mappings"); + + if (num_leaves == 0) { + /* No tree to build */ + virt->tree = NULL; + if (leaves) { + free(leaves); + leaves = NULL; + } + } + else { + /* Build the tree */ + if ((virt->tree = H5RT_create(rank, leaves, num_leaves)) == NULL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to create mapping tree"); + /* Tree takes ownership of leaves array and coordinate arrays */ + leaves = NULL; + } + + /* Store not-in-tree mappings (regardless of whether tree was built) */ + if (not_in_tree_count > 0) { + virt->not_in_tree_list = not_in_tree_mappings; + virt->not_in_tree_nused = not_in_tree_count; + virt->not_in_tree_nalloc = not_in_tree_nalloc; + not_in_tree_mappings = NULL; /* Transfer ownership to virt */ + } + else { + /* Clean up any existing allocation */ + if (virt->not_in_tree_list) { + H5MM_free(virt->not_in_tree_list); + } + virt->not_in_tree_list = NULL; + virt->not_in_tree_nused = 0; + virt->not_in_tree_nalloc = 0; + if (not_in_tree_mappings) { + H5MM_free(not_in_tree_mappings); + not_in_tree_mappings = NULL; + } + } + +done: + if (ret_value < 0) { + if (leaves) { + /* Clean up coordinate arrays and the leaf array */ + for (size_t i = 0; i < num_leaves; i++) { + H5RT_leaf_cleanup(&leaves[i]); + } + free(leaves); + } + if (not_in_tree_mappings) + H5MM_free(not_in_tree_mappings); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_build_tree() */ + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_not_in_tree_grow + * + * Purpose: Double the capacity of the not_in_tree_list buffer + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_not_in_tree_grow(H5O_storage_virtual_ent_t ***list, size_t *nalloc) +{ + size_t new_capacity = 0; + H5O_storage_virtual_ent_t **new_list = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + assert(list); + assert(*list); + assert(nalloc); + + new_capacity = *nalloc * 2; + + /* Overflow check */ + if (new_capacity < *nalloc || new_capacity > (SIZE_MAX / sizeof(H5O_storage_virtual_ent_t *))) + HGOTO_ERROR(H5E_DATASET, H5E_OVERFLOW, FAIL, "not_in_tree_list capacity overflow"); + + if (NULL == (new_list = (H5O_storage_virtual_ent_t **)H5MM_realloc( + *list, new_capacity * sizeof(H5O_storage_virtual_ent_t *)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failed to grow not_in_tree_list"); + + *list = new_list; + *nalloc = new_capacity; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_not_in_tree_grow() */ + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_not_in_tree_add + * + * Purpose: Add a mapping to the not_in_tree_list, growing if necessary + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_not_in_tree_add(H5O_storage_virtual_ent_t ***list, size_t *nused, size_t *nalloc, + H5O_storage_virtual_ent_t *mapping) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + assert(list); + assert(*list); + assert(nused); + assert(nalloc); + assert(mapping); + + /* Grow buffer if full */ + if (*nused >= *nalloc) { + if (H5D__virtual_not_in_tree_grow(list, nalloc) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failed to grow not_in_tree_list"); + } + + (*list)[*nused] = mapping; + (*nused)++; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_not_in_tree_add() */ + +/*------------------------------------------------------------------------- + * Function: H5D__should_build_tree + * + * Purpose: Determine whether to build a spatial tree of mapping indices + * for the provided dataset layout + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__should_build_tree(H5O_storage_virtual_t *storage, hid_t dapl_id, bool *should_build_tree) +{ + herr_t ret_value = SUCCEED; + H5P_genplist_t *dapl_plist = NULL; + bool tree_enabled_dapl = false; + + FUNC_ENTER_PACKAGE + + assert(storage); + assert(should_build_tree); + assert(dapl_id != H5I_INVALID_HID); + + /* Don't build if already exists */ + if (storage->tree) { + *should_build_tree = false; + HGOTO_DONE(SUCCEED); + } + + /* Don't build if too few mappings */ + if (storage->list_nused < H5D_VIRTUAL_TREE_THRESHOLD) { + *should_build_tree = false; + HGOTO_DONE(SUCCEED); + } + + /* Don't build if DAPL property has disabled the tree */ + if (NULL == (dapl_plist = (H5P_genplist_t *)H5I_object(dapl_id))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for dapl ID"); + + if (H5P_get(dapl_plist, H5D_ACS_USE_TREE_NAME, &tree_enabled_dapl) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual use tree flag"); + + if (!tree_enabled_dapl) { + *should_build_tree = false; + HGOTO_DONE(SUCCEED); + } + + *should_build_tree = true; +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D__should_build_tree() */ + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_read_one_mapping + * + * Purpose: Read from a single mapping entry in a virtual dataset + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_read_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Sanity check that the virtual space has been patched by now */ + assert(mapping->virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); + + /* Check for "printf" source dataset resolution */ + if (mapping->psfn_nsubs || mapping->psdn_nsubs) { + /* Iterate over sub-source dsets */ + for (size_t j = mapping->sub_dset_io_start; j < mapping->sub_dset_io_end; j++) + if (H5D__virtual_read_one_src(dset_info, &mapping->sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + } /* end if */ + else + /* Read from source dataset */ + if (H5D__virtual_read_one_src(dset_info, &mapping->source_dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_read_one_mapping() */ + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_write_one_mapping + * + * Purpose: Write to a single mapping entry in a virtual dataset + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_write_one_mapping(H5D_dset_io_info_t *dset_info, H5O_storage_virtual_ent_t *mapping) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Sanity check that virtual space has been patched by now */ + assert(mapping->virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); + + /* Check for "printf" source dataset resolution */ + if (mapping->psfn_nsubs || mapping->psdn_nsubs) { + /* Iterate over sub-source dsets */ + for (size_t j = mapping->sub_dset_io_start; j < mapping->sub_dset_io_end; j++) + if (H5D__virtual_write_one_src(dset_info, &mapping->sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset"); + } + else + /* Write to source dataset */ + if (H5D__virtual_write_one_src(dset_info, &mapping->source_dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_write_one_mapping() */ + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_close_mapping + * + * Purpose: Frees memory structures allocated by H5D__virtual_pre_io + * for a particular mapping + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_close_mapping(H5O_storage_virtual_ent_t *mapping) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check for "printf" source dataset resolution */ + if (mapping->psfn_nsubs || mapping->psdn_nsubs) { + /* Iterate over sub-source dsets */ + for (size_t j = mapping->sub_dset_io_start; j < mapping->sub_dset_io_end; j++) + /* Close projected memory space */ + if (mapping->sub_dset[j].projected_mem_space) { + /* Use HDONE_ERROR to attempt to close all spaces even after failure */ + if (H5S_close(mapping->sub_dset[j].projected_mem_space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); + mapping->sub_dset[j].projected_mem_space = NULL; + } /* end if */ + } /* end if */ + else + /* Close projected memory space */ + if (mapping->source_dset.projected_mem_space) { + if (H5S_close(mapping->source_dset.projected_mem_space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space"); + mapping->source_dset.projected_mem_space = NULL; + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D__virtual_close_mapping() */ diff --git a/src/H5Olayout.c b/src/H5Olayout.c index 4bf9d69ebad..9828bc39e51 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -565,13 +565,14 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU ret_value = mesg; done: - if (ret_value == NULL) + if (ret_value == NULL) { if (mesg) { if (mesg->type == H5D_VIRTUAL) if (H5D__virtual_reset_layout(mesg) < 0) HDONE_ERROR(H5E_OHDR, H5E_CANTFREE, NULL, "unable to reset virtual layout"); H5FL_FREE(H5O_layout_t, mesg); } + } FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__layout_decode() */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 74f14e01e1a..ba9901e28e8 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -42,6 +42,7 @@ typedef struct H5O_fill_t H5O_fill_t; #include "H5Tprivate.h" /* Datatype functions */ #include "H5VLprivate.h" /* Virtual Object Layer */ #include "H5Zprivate.h" /* I/O pipeline filters */ +#include "H5RTprivate.h" /* R-tree for virtual dataspaces */ /* Forward references of package typedefs */ typedef struct H5O_msg_class_t H5O_msg_class_t; @@ -603,6 +604,12 @@ typedef struct H5O_storage_virtual_t { H5O_storage_virtual_ent_t *source_dset_hash_table; /* Hash table of virtual entries sorted by source dataset name. Only the first occurrence of each source dataset name is stored. */ + H5RT_t *tree; + size_t not_in_tree_nused; /* Number of entries in not_in_tree_list */ + size_t not_in_tree_nalloc; /* Allocated size of not_in_tree_list (grows by power of 2) */ + H5O_storage_virtual_ent_t * + *not_in_tree_list; /* Array of POINTERS to mappings NOT in tree for quick access + * Some mappings cannot be stored in the tree and must be searched manually */ } H5O_storage_virtual_t; typedef struct H5O_storage_t { diff --git a/src/H5Pdapl.c b/src/H5Pdapl.c index e9c46746965..a29c07042de 100644 --- a/src/H5Pdapl.c +++ b/src/H5Pdapl.c @@ -97,6 +97,12 @@ #define H5D_ACS_EFILE_PREFIX_CMP H5P__dapl_efile_pref_cmp #define H5D_ACS_EFILE_PREFIX_CLOSE H5P__dapl_efile_pref_close +/* Definitions for use of VDS mapping spatial tree */ +#define H5D_ACS_USE_TREE_SIZE sizeof(bool) +#define H5D_ACS_USE_TREE_DEF true +#define H5D_ACS_USE_TREE_ENC H5P__encode_bool +#define H5D_ACS_USE_TREE_DEC H5P__decode_bool + /******************/ /* Local Typedefs */ /******************/ @@ -175,6 +181,7 @@ static const H5D_append_flush_t H5D_def_append_flush_g = static const char *H5D_def_efile_prefix_g = H5D_ACS_EFILE_PREFIX_DEF; /* Default external file prefix string */ static const char *H5D_def_vds_prefix_g = H5D_ACS_VDS_PREFIX_DEF; /* Default vds prefix string */ +static const bool H5D_def_tree_g = H5D_ACS_USE_TREE_DEF; /* Default use of spatial tree for VDS mappings */ /*------------------------------------------------------------------------- * Function: H5P__dacc_reg_prop @@ -247,6 +254,11 @@ H5P__dacc_reg_prop(H5P_genclass_t *pclass) H5D_ACS_EFILE_PREFIX_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class"); + /* Register the spatial tree use property */ + if (H5P__register_real(pclass, H5D_ACS_USE_TREE_NAME, H5D_ACS_USE_TREE_SIZE, &H5D_def_tree_g, NULL, NULL, + NULL, H5D_ACS_USE_TREE_ENC, H5D_ACS_USE_TREE_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class"); + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__dacc_reg_prop() */ @@ -1543,3 +1555,99 @@ H5Pget_virtual_prefix(hid_t plist_id, char *prefix /*out*/, size_t size) done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_virtual_prefix() */ + +/*----------------------------------------------------------------------------- + * Function: H5Pget_virtual_spatial_tree + * + * Purpose: + * + * Access the flag for whether or not datasets created by the given dcpl + * construct a spatial tree and use it when searching over VDS mappings + * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. + * + * Return: + * + * Failure: Negative value (FAIL) + * Success: Non-negative value (SUCCEED) + * + *----------------------------------------------------------------------------- + */ +herr_t +H5Pget_virtual_spatial_tree(hid_t dcpl_id, bool *use_tree) +{ + bool setting = false; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + + if (NULL == use_tree) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "receiving pointer cannot be NULL"); + + plist = H5P_object_verify(dcpl_id, H5P_DATASET_ACCESS, true); + if (NULL == plist) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + + if (H5P_peek(plist, H5D_ACS_USE_TREE_NAME, &setting) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get dset use spatial tree flag value"); + + *use_tree = setting; + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_virtual_spatial_tree() */ + +/*----------------------------------------------------------------------------- + * Function: H5Pset_virtual_spatial_tree + * + * Purpose: + * + * Set the DAPL to construct a spatial tree and use it when searching over + * VDS mappings + * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. + * + * Return: + * + * Failure: Negative value (FAIL) + * Success: Non-negative value (SUCCEED) + * + *----------------------------------------------------------------------------- + */ +herr_t +H5Pset_virtual_spatial_tree(hid_t dapl_id, bool use_tree) +{ + H5P_genplist_t *plist = NULL; + bool prev_set = false; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + + plist = H5P_object_verify(dapl_id, H5P_DATASET_ACCESS, false); + if (NULL == plist) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + + if (H5P_peek(plist, H5D_ACS_USE_TREE_NAME, &prev_set) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get extant dset use spatial tree flag value"); + + if (H5P_poke(plist, H5D_ACS_USE_TREE_NAME, &use_tree) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set dset use spatial tree flag value"); + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_virtual_spatial_tree() */ diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index df51df30237..0e0689e2df4 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -89,7 +89,7 @@ { \ {HADDR_UNDEF, 0}, 0, NULL, 0, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ - H5D_VDS_ERROR, HSIZE_UNDEF, -1, -1, false, NULL, NULL \ + H5D_VDS_ERROR, HSIZE_UNDEF, -1, -1, false, NULL, NULL, NULL, 0, 0, NULL, \ } #define H5D_DEF_STORAGE_COMPACT \ { \ @@ -2213,9 +2213,8 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const if (H5D_virtual_update_min_dims(&virtual_layout, virtual_layout.storage.u.virt.list_nused) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "unable to update virtual dataset minimum dimensions"); - /* Finish adding entry */ + /* Finish adding entry to list */ virtual_layout.storage.u.virt.list_nused++; - done: /* Set VDS layout information in property list */ /* (Even on failure, so there's not a mangled layout struct in the list) */ @@ -2257,7 +2256,6 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const virtual_layout.storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_xfree(virtual_layout.storage.u.virt.list); } /* end if */ - FUNC_LEAVE_API(ret_value) } /* end H5Pset_virtual() */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index e454669f6fe..d5b4ea1a945 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -6010,6 +6010,40 @@ H5_DLL herr_t H5Pget_chunk_opts(hid_t plist_id, unsigned *opts); * */ H5_DLL herr_t H5Pget_dset_no_attrs_hint(hid_t dcpl_id, bool *minimize); +/** + * \ingroup DCPL + * + * \brief Retrieves the setting for whether or not to use a spatial tree + * for VDS mappings + * + * \dcpl_id + * \param[out] use_tree Flag indicating whether the dataset will or + * will not use a spatial tree + * + * \return \herr_t + * + * \details H5Pget_virtual_spatial_tree() retrieves the + * use spatial tree flag setting for the dataset + * creation property list \p dcpl_id. This setting determines + * whether a dataset created with the dataset creation + * property list \p dcpl_id will construct a spatial tree + * and use it in an attempt to optimize certain operations. + * The setting value is returned in the boolean pointer + * \p use_tree. + * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. + * + * \since 2.0.0 + * + */ +H5_DLL herr_t H5Pget_virtual_spatial_tree(hid_t dcpl_id, bool *use_tree); /** * \ingroup DCPL * @@ -6493,6 +6527,37 @@ H5_DLL herr_t H5Pset_chunk_opts(hid_t plist_id, unsigned opts); * */ H5_DLL herr_t H5Pset_dset_no_attrs_hint(hid_t dcpl_id, bool minimize); +/** + * \ingroup DCPL + * + * \brief Sets the flag to use a spatial tree for mappings + * + * \dcpl_id + * \param[in] use_tree Flag for indicating whether or not a dataset + * should use a spatial tree for mappings + * + * \return \herr_t + * + * \details H5Pset_virtual_spatial_tree() sets the use-tree flag + * for the dataset creation property list \p dcpl_id. + * Datasets created with the dataset creation property + * list \p dcpl_id will construct a spatial tree and use + * it in an attempt to optimize intersection-check + * operations on mappings when \p use_tree is set to true. + * + * Use of a spatial tree will accelerate the process of searching through mappings + * to determine which contain intersections with the user's selection region. + * With the tree disabled, all mappings will simply be iterated through and + * checked directly. + * + * Certain workflows may find that tree creation overhead outweighs the time saved + * on reads. In this case, disabling this property will lead to a performance improvement, + * though it is expected that almost all cases will benefit from the tree on net. + * + * \since 2.0.0 + * + */ +H5_DLL herr_t H5Pset_virtual_spatial_tree(hid_t dcpl_id, bool use_tree); /** * \ingroup DCPL * diff --git a/test/dsets.c b/test/dsets.c index b9e1d342a7d..9047973c95c 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -83,6 +83,8 @@ static const char *FILENAME[] = {"dataset", /* 0 */ #define OHMIN_FILENAME_A "ohdr_min_a" +#define SPATIAL_TREE_FILENAME "spatial_tree" + #define FILENAME_BUF_SIZE 1024 #define KB 1024 diff --git a/test/rtree.c b/test/rtree.c index 4674dbeadbf..c47fc16e179 100644 --- a/test/rtree.c +++ b/test/rtree.c @@ -22,13 +22,12 @@ /* * This file needs to access private datatypes from the H5RT package. */ -#define H5RT_FRIEND /*suppress error about including H5RTpkg */ -#define H5RT_TESTING -#include "H5RTpkg.h" +#define H5RT_FRIEND /*suppress error about including H5RTpkg */ +#include "H5RTpkg.h" /* R-tree package */ -/* Other private headers */ -#include "H5CXprivate.h" /* API Contexts */ -#include "H5VLprivate.h" /* Virtual Object Layer */ +#define H5D_FRIEND /*suppress error about including H5Dpkg */ +#define H5D_TESTING +#include "H5Dpkg.h" /* Datasets */ #define RTREE_TEST_BASE_COORD 10000 #define RTREE_TEST_BASE_SIZE 1000 @@ -36,7 +35,23 @@ #define RTREE_TEST_CREATE_RANK 8 #define RTREE_TEST_CREATE_NUM_COUNTS 4 -static const size_t test_counts[RTREE_TEST_CREATE_NUM_COUNTS] = {1, 100, 500, 10000}; +#define RTREE_SRC_FILENAME "vds_src_file.h5" + +#define RTREE_DAPL_FILENAME "vds_rtree_test.h5" +#define RTREE_DAPL_SRC_FILENAME "vds_src_rtree_test.h5" +#define RTREE_DAPL_VDS_NAME "vdset" +#define RTREE_DAPL_SRC_DATASET_NAME "src_dset" + +#define RTREE_DAPL_DATASET_DIM1 10 +#define RTREE_DAPL_DATASET_DIM2 10 + +#define RTREE_THRESHOLD_FILENAME "vds_rtree_threshold_test.h5" +#define RTREE_MAX_TEST_MAPPINGS (H5D_VIRTUAL_TREE_THRESHOLD + 100) + +#define RTREE_RW_FILENAME "vds_rtree_rw.h5" + +static const size_t test_counts[RTREE_TEST_CREATE_NUM_COUNTS] = {H5D_VIRTUAL_TREE_THRESHOLD, 100, 1000, + 10000}; /* Helper function to generate leaf data */ static H5RT_leaf_t *generate_leaves(int rank, size_t leaf_count); @@ -52,6 +67,9 @@ static H5RT_leaf_t **manual_search(H5RT_leaf_t *leaves, size_t leaf_count, int r static herr_t verify_rtree_search(H5RT_result_set_t *result_set, H5RT_leaf_t *leaves, size_t leaf_count, hsize_t min[], hsize_t max[], int rank); +/* Helper to create and initialize virtual dset in a file */ +static hid_t create_virtual_dataset(hid_t file_id, hid_t dapl_id, int num_mappings); + static herr_t verify_rtree_search(H5RT_result_set_t *result_set, H5RT_leaf_t *leaves, size_t leaf_count, hsize_t min[], hsize_t max[], int rank) @@ -415,6 +433,573 @@ test_rtree_copy(void) return FAIL; } +/*------------------------------------------------------------------------- + * Function: create_virtual_dataset + * + * Purpose: Helper function to create a 1D virtual dataset with mappings + * + * Return: Success: dataset ID + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +static hid_t +create_virtual_dataset(hid_t file_id, hid_t dapl_id, int num_mappings) +{ + hid_t vspace_id = H5I_INVALID_HID; + hid_t srcspace_id = H5I_INVALID_HID; + hid_t vsel_id = H5I_INVALID_HID; + hid_t srcfile_id = H5I_INVALID_HID; + hid_t srcdset_id = H5I_INVALID_HID; + hid_t vdset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hsize_t vdims[1] = {(hsize_t)num_mappings}; + hsize_t srcdims[1] = {1}; + hsize_t start[1], count[1]; + char srcdset_name[256]; + int wdata; + int i; + + /* Create 1D virtual dataset space */ + if ((vspace_id = H5Screate_simple(1, vdims, NULL)) < 0) + goto error; + + /* Create 1D source dataset space (single element) */ + if ((srcspace_id = H5Screate_simple(1, srcdims, NULL)) < 0) + goto error; + + /* Create dataset creation property list */ + if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + /* Create source file */ + if ((srcfile_id = H5Fcreate(RTREE_SRC_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* Create multiple source dsets and add virtual mappings */ + for (i = 0; i < num_mappings; i++) { + sprintf(srcdset_name, "src_dset_%d", i); + + /* Create source dataset */ + if ((srcdset_id = H5Dcreate2(srcfile_id, srcdset_name, H5T_NATIVE_INT, srcspace_id, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* Write data to source dataset (value equals index) */ + wdata = i; + if (H5Dwrite(srcdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &wdata) < 0) + goto error; + + /* Create hyperslab selection for virtual dataset (one element at position i) */ + if ((vsel_id = H5Scopy(vspace_id)) < 0) + goto error; + + start[0] = (hsize_t)i; + count[0] = 1; + if (H5Sselect_hyperslab(vsel_id, H5S_SELECT_SET, start, NULL, count, NULL) < 0) + goto error; + + /* Add virtual mapping */ + if (H5Pset_virtual(dcpl_id, vsel_id, RTREE_SRC_FILENAME, srcdset_name, srcspace_id) < 0) + goto error; + + /* Close source dataset and selection */ + if (H5Dclose(srcdset_id) < 0) + goto error; + if (H5Sclose(vsel_id) < 0) + goto error; + + srcdset_id = H5I_INVALID_HID; + vsel_id = H5I_INVALID_HID; + } + + /* Create virtual dataset */ + if ((vdset_id = H5Dcreate2(file_id, RTREE_DAPL_VDS_NAME, H5T_NATIVE_INT, vspace_id, H5P_DEFAULT, dcpl_id, + dapl_id)) < 0) + goto error; + + /* Cleanup */ + if (H5Sclose(vspace_id) < 0) + goto error; + if (H5Sclose(srcspace_id) < 0) + goto error; + if (H5Fclose(srcfile_id) < 0) + goto error; + if (H5Pclose(dcpl_id) < 0) + goto error; + + return vdset_id; +error: + /* Cleanup */ + H5E_BEGIN_TRY + { + H5Sclose(vspace_id); + H5Sclose(srcspace_id); + H5Sclose(vsel_id); + H5Dclose(srcdset_id); + H5Fclose(srcfile_id); + H5Dclose(vdset_id); + H5Pclose(dcpl_id); + } + H5E_END_TRY; + + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: test_rtree_existence_helper + * + * Purpose: Test helper to verify that r-tree existence on a dataset + * matches what is expected + * + * Return: Success: SUCCEED + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_rtree_existence_helper(hid_t vdset_id, bool expect_tree, bool *correct_out) +{ + herr_t ret_value = SUCCEED; + H5D_t *dset = NULL; + H5O_storage_virtual_t *storage = NULL; + + assert(correct_out); + *correct_out = false; + + /* Get the dataset object - this is using internal API for testing */ + if (NULL == (dset = (H5D_t *)H5VL_object(vdset_id))) { + ret_value = FAIL; + goto done; + } + + if (dset->shared->layout.type != H5D_VIRTUAL) { + ret_value = FAIL; + goto done; + } + + /* Get the virtual storage structure */ + storage = &(dset->shared->layout.storage.u.virt); + + /* Verify tree existence matches expectation */ + if (expect_tree) { + if (storage->tree == NULL) { + puts("Expected spatial tree to exist but it was NULL"); + *correct_out = false; + goto done; + } + + if (storage->not_in_tree_nused > 0 && storage->not_in_tree_list == NULL) { + puts("Expected not_in_tree_list array to exist but it was NULL"); + *correct_out = false; + goto done; + } + } + else { + if (storage->tree != NULL) { + puts("Expected spatial tree to be NULL but it exists"); + *correct_out = false; + goto done; + } + if (storage->not_in_tree_list != NULL || storage->not_in_tree_nused > 0) { + puts("Expected not_in_tree_list to be empty but it exists"); + *correct_out = false; + goto done; + } + } + + *correct_out = true; +done: + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: test_rtree_dapl + * + * Purpose: Test R-tree options on the DAPL + * + * Return: Success: SUCCEED + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_rtree_dapl(bool use_tree, bool read_init) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t dapl_id = H5I_INVALID_HID; + hid_t vdset_id = H5I_INVALID_HID; + + int rbuf[RTREE_MAX_TEST_MAPPINGS]; + int wbuf[RTREE_MAX_TEST_MAPPINGS]; + bool tree_correct = false; + char test_str[256]; + + /* Inverse of use_tree for re-open part of test */ + bool use_tree_inverse = !use_tree; + + memset(test_str, 0, sizeof(test_str)); + + if (snprintf(test_str, sizeof(test_str), "spatial tree option %s", use_tree ? "enabled" : "disabled") < 0) + FAIL_STACK_ERROR; + + if (read_init) { + strncat(test_str, " with read initialization", sizeof(test_str) - strlen(test_str) - 1); + } + else { + strncat(test_str, " with write initialization", sizeof(test_str) - strlen(test_str) - 1); + } + + TESTING(test_str); + + memset(rbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); + memset(wbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); + + /* One-time setup */ + if ((file_id = H5Fcreate(RTREE_DAPL_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + + /* Create virtual dataset with enough mappings to use tree */ + if ((vdset_id = create_virtual_dataset(file_id, dapl_id, RTREE_MAX_TEST_MAPPINGS)) < 0) + FAIL_STACK_ERROR; + + if (H5Dclose(vdset_id) < 0) + FAIL_STACK_ERROR; + + /* Set the spatial tree property */ + if (H5Pset_virtual_spatial_tree(dapl_id, use_tree) < 0) + FAIL_STACK_ERROR; + + if ((vdset_id = H5Dopen2(file_id, RTREE_DAPL_VDS_NAME, dapl_id)) < 0) + FAIL_STACK_ERROR; + + /* Read/write the entire virtual dataset to force tree initialization */ + if (read_init) { + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + } + else { + if (H5Dwrite(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) + FAIL_STACK_ERROR; + } + + /* Verify tree existence matches expectation */ + if (test_rtree_existence_helper(vdset_id, use_tree, &tree_correct) < 0) + FAIL_STACK_ERROR; + + if (!tree_correct) + FAIL_STACK_ERROR; + + /* Close the dataset and re-open it with the opposite value set in DAPL */ + if (H5Dclose(vdset_id) < 0) + FAIL_STACK_ERROR; + + vdset_id = H5I_INVALID_HID; + + if (H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + file_id = H5I_INVALID_HID; + + if (H5Pclose(dapl_id) < 0) + FAIL_STACK_ERROR; + + dapl_id = H5I_INVALID_HID; + tree_correct = false; + memset(rbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); + + if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_virtual_spatial_tree(dapl_id, use_tree_inverse) < 0) + FAIL_STACK_ERROR; + + if ((file_id = H5Fopen(RTREE_DAPL_FILENAME, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if ((vdset_id = H5Dopen2(file_id, RTREE_DAPL_VDS_NAME, dapl_id)) < 0) + FAIL_STACK_ERROR; + + /* Read/write the entire virtual dataset to force tree initialization */ + if (read_init) { + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + } + else { + if (H5Dwrite(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) + FAIL_STACK_ERROR; + } + + /* Verify tree existence matches expectation after re-open */ + if (test_rtree_existence_helper(vdset_id, use_tree_inverse, &tree_correct) < 0) + FAIL_STACK_ERROR; + + if (!tree_correct) + FAIL_STACK_ERROR; + + /* Cleanup */ + if (H5Dclose(vdset_id) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dapl_id) < 0) + FAIL_STACK_ERROR; + if (H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Dclose(vdset_id); + H5Pclose(dapl_id); + H5Fclose(file_id); + } + H5E_END_TRY; + + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: test_rtree_threshold + * + * Purpose: Test that threshold controls r-tree usage properly + * + * Return: Success: SUCCEED + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_rtree_threshold(bool use_tree) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t dapl_id = H5I_INVALID_HID; + hid_t vdset_id = H5I_INVALID_HID; + int rbuf[RTREE_MAX_TEST_MAPPINGS]; + + /* Internal values for introspection */ + H5D_t *dset = NULL; + H5O_storage_virtual_t *storage = NULL; + + const char *test_str = + use_tree ? "threshold behavior with tree enabled" : "threshold behavior with tree disabled"; + + TESTING(test_str); + + /* Test cases: below threshold, at threshold, above threshold */ + int test_cases[3] = {H5D_VIRTUAL_TREE_THRESHOLD - 1, H5D_VIRTUAL_TREE_THRESHOLD, RTREE_MAX_TEST_MAPPINGS}; + + for (int test_idx = 0; test_idx < 3; test_idx++) { + int num_mappings = test_cases[test_idx]; + bool expect_tree; + + /* Determine expected tree behavior based on threshold and use_tree setting */ + /* Tree is created only when: tree_enabled AND num_mappings >= threshold */ + expect_tree = (use_tree && (num_mappings >= H5D_VIRTUAL_TREE_THRESHOLD)); + + if ((file_id = H5Fcreate(RTREE_THRESHOLD_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + + /* Set the spatial tree property */ + if (H5Pset_virtual_spatial_tree(dapl_id, use_tree) < 0) + FAIL_STACK_ERROR; + + /* Create virtual dataset with specified number of mappings */ + if ((vdset_id = create_virtual_dataset(file_id, dapl_id, num_mappings)) < 0) + FAIL_STACK_ERROR; + + /* Read the virtual dataset to force initialization */ + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data pattern (each element should equal its index) */ + for (int i = 0; i < num_mappings; i++) { + if (rbuf[i] != i) { + printf("%d mappings: Data mismatch at [%d]: expected %d, got %d\n", num_mappings, i, i, + rbuf[i]); + FAIL_STACK_ERROR; + } + } + + /* Get the dataset object for introspection */ + if (NULL == (dset = (H5D_t *)H5VL_object(vdset_id))) + FAIL_STACK_ERROR; + + if (dset->shared->layout.type != H5D_VIRTUAL) + FAIL_STACK_ERROR; + + /* Get the virtual storage structure */ + storage = &(dset->shared->layout.storage.u.virt); + + /* Verify tree existence matches expectation */ + if (expect_tree) { + if (storage->tree == NULL) { + printf("%d mappings: Expected spatial tree to exist but it was NULL\n", num_mappings); + FAIL_STACK_ERROR; + } + /* not_in_tree_list can be NULL if all mappings fit in tree - this is OK */ + /* Just verify consistency: if nused > 0, then list should exist */ + if (storage->not_in_tree_nused > 0 && storage->not_in_tree_list == NULL) { + printf("%d mappings: Expected not_in_tree_list array to exist but it was NULL\n", + num_mappings); + FAIL_STACK_ERROR; + } + } + else { + if (storage->tree != NULL) { + printf("%d mappings: Expected spatial tree to be NULL but it exists\n", num_mappings); + FAIL_STACK_ERROR; + } + if (storage->not_in_tree_list != NULL) { + printf("%d mappings: Expected not_in_tree_list array to be NULL but it exists\n", + num_mappings); + FAIL_STACK_ERROR; + } + } + + /* Cleanup */ + if (H5Dclose(vdset_id) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dapl_id) < 0) + FAIL_STACK_ERROR; + if (H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + vdset_id = H5I_INVALID_HID; + dapl_id = H5I_INVALID_HID; + file_id = H5I_INVALID_HID; + } + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Dclose(vdset_id); + H5Pclose(dapl_id); + H5Fclose(file_id); + } + H5E_END_TRY; + + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: test_rtree_rw + * + * Purpose: Test that dataset reads/writes produce correctly values + * with rtree on/off + * + * Return: Success: SUCCEED + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_rtree_rw(bool use_tree) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t dapl_id = H5I_INVALID_HID; + hid_t vdset_id = H5I_INVALID_HID; + hid_t space_id = H5I_INVALID_HID; + hsize_t wdims = RTREE_MAX_TEST_MAPPINGS / 2; + int rbuf[RTREE_MAX_TEST_MAPPINGS]; + int wbuf[RTREE_MAX_TEST_MAPPINGS]; + int num_mappings = RTREE_MAX_TEST_MAPPINGS; + + const char *test_str = use_tree ? "R/W behavior with tree enabled" : "R/W behavior with tree disabled"; + + TESTING(test_str); + + memset(rbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); + memset(wbuf, 0, sizeof(int) * RTREE_MAX_TEST_MAPPINGS); + + if ((file_id = H5Fcreate(RTREE_RW_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if ((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + + /* Set the spatial tree property */ + if (H5Pset_virtual_spatial_tree(dapl_id, use_tree) < 0) + FAIL_STACK_ERROR; + + /* Create virtual dataset with specified number of mappings */ + if ((vdset_id = create_virtual_dataset(file_id, dapl_id, num_mappings)) < 0) + FAIL_STACK_ERROR; + + /* Verify initial read values (each element should equal its index) */ + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + for (int i = 0; i < num_mappings; i++) { + if (rbuf[i] != i) { + printf("%d mappings: Data mismatch at [%d]: expected %d, got %d\n", num_mappings, i, i, rbuf[i]); + FAIL_STACK_ERROR; + } + } + + /* Write to first half of dataset with 2*index */ + for (int i = 0; i < num_mappings / 2; i++) + wbuf[i] = 2 * i; + + if ((space_id = H5Screate_simple(1, (const hsize_t *)&wdims, NULL)) < 0) + FAIL_STACK_ERROR; + + if (H5Sselect_hyperslab(space_id, H5S_SELECT_SET, (const hsize_t *)&(hsize_t){0}, NULL, + (const hsize_t *)&wdims, NULL) < 0) + FAIL_STACK_ERROR; + + if (H5Dwrite(vdset_id, H5T_NATIVE_INT, space_id, space_id, H5P_DEFAULT, wbuf) < 0) + FAIL_STACK_ERROR; + + /* Read back entire dataset and verify values */ + if (H5Dread(vdset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + for (int i = 0; i < num_mappings; i++) { + int expected = (i < num_mappings / 2) ? (2 * i) : i; + if (rbuf[i] != expected) { + printf("%d mappings: Post-write data mismatch at [%d]: expected %d, got %d\n", num_mappings, i, + expected, rbuf[i]); + FAIL_STACK_ERROR; + } + } + + /* Cleanup */ + if (H5Dclose(vdset_id) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dapl_id) < 0) + FAIL_STACK_ERROR; + if (H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(space_id) < 0) + FAIL_STACK_ERROR; + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(space_id); + H5Dclose(vdset_id); + H5Pclose(dapl_id); + H5Fclose(file_id); + } + H5E_END_TRY; + + return FAIL; +} + /*------------------------------------------------------------------------- * Function: main * @@ -439,9 +1024,31 @@ main(void) nerrors += test_rtree_search() < 0 ? 1 : 0; nerrors += test_rtree_copy() < 0 ? 1 : 0; + /* Test spatial tree with DAPL property enabled */ + nerrors += test_rtree_dapl(true, true) < 0 ? 1 : 0; + nerrors += test_rtree_dapl(true, false) < 0 ? 1 : 0; + nerrors += test_rtree_dapl(false, true) < 0 ? 1 : 0; + nerrors += test_rtree_dapl(false, false) < 0 ? 1 : 0; + + /* Test the mapping count threshold */ + nerrors += test_rtree_threshold(true) < 0 ? 1 : 0; + // TODO - Fix failure + nerrors += test_rtree_threshold(false) < 0 ? 1 : 0; + nerrors += test_rtree_rw(true) < 0 ? 1 : 0; + nerrors += test_rtree_rw(false) < 0 ? 1 : 0; + if (nerrors) goto error; + H5E_BEGIN_TRY + { + H5Fdelete(RTREE_SRC_FILENAME, H5P_DEFAULT); + H5Fdelete(RTREE_DAPL_FILENAME, H5P_DEFAULT); + H5Fdelete(RTREE_THRESHOLD_FILENAME, H5P_DEFAULT); + H5Fdelete(RTREE_RW_FILENAME, H5P_DEFAULT); + } + H5E_END_TRY; + printf("All R-tree tests passed.\n"); return EXIT_SUCCESS;