From f2011d5abecb6f2f5ecc8beb8dbe705e47626c44 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 23 Oct 2023 10:31:08 -0700 Subject: [PATCH] Synchronize stable with develop branch (#152) Synchronize stable with develop branch --- .devcontainer/devcontainer.Dockerfile | 95 + .devcontainer/devcontainer.json | 9 + .devcontainer/post-create.sh | 1 + .devcontainer/post-start.sh | 31 + .docker/base.Dockerfile | 90 + .docker/local.Dockerfile | 29 + .docker/local.Dockerfile.dockerignore | 2 + .github/ISSUE_TEMPLATE/bug_report.md | 35 + .github/ISSUE_TEMPLATE/feature_request.md | 17 + .github/ISSUE_TEMPLATE/install_problem.md | 27 + .github/ISSUE_TEMPLATE/question.md | 16 + .github/pull_request_template.md | 22 + .github/workflows/clang-format-fix.yml | 14 +- .github/workflows/nersc.yml | 38 + .gitignore | 8 +- .gitlab-ci.yml | 4 +- .gitmodules | 3 - .readthedocs.yaml | 22 + CMake/FindJULIA.cmake | 45 + CMake/FindMERCURY.cmake | 47 + CMake/FindUUID.cmake | 36 + CMake/pdc-config.cmake.build.in | 2 +- CMake/pdc-config.cmake.install.in | 2 +- CMakeLists.txt | 188 +- docs/readme.md | 154 +- docs/source/api.rst | 37 +- docs/source/developer-notes.rst | 278 ++- docs/source/getting_started.rst | 43 + pdc_config_sys.h.cmake | 12 + .../cori => dart_attr_dist_test}/clean.sh | 0 scripts/dart_attr_dist_test/gen_script.sh | 46 + .../cori => dart_attr_dist_test}/submit.sh | 55 +- scripts/dart_attr_dist_test/template.sh | 103 + .../cori/gen_scripts.sh | 25 - .../kvtag_add_get_benchmark/cori/template.sh | 68 - .../perlmutter/gen_scripts.sh | 5 +- .../perlmutter/template.sh | 13 +- .../kvtag_add_get_scale/cori/gen_scripts.sh | 19 - scripts/kvtag_add_get_scale/cori/template.sh | 66 - .../perlmutter/gen_scripts.sh | 3 + .../perlmutter/template.sh | 11 +- .../cori => kvtag_query_scale}/clean.sh | 0 scripts/kvtag_query_scale/gen_script.sh | 49 + .../cori => kvtag_query_scale}/submit.sh | 8 +- scripts/kvtag_query_scale/template.sh | 105 + scripts/kvtag_query_scale_mpi/clean.sh | 26 + scripts/kvtag_query_scale_mpi/gen_script.sh | 92 + scripts/kvtag_query_scale_mpi/submit.sh | 73 + scripts/kvtag_query_scale_mpi/template.sh | 108 + scripts/llsm_importer/clean.sh | 8 + scripts/llsm_importer/gen_script.sh | 46 + scripts/llsm_importer/submit.sh | 76 + scripts/llsm_importer/template.sh | 107 + spack/package.py | 48 - src/api/CMakeLists.txt | 35 +- src/api/include/pdc_client_connect.h | 184 +- src/api/pdc.c | 2 +- .../pdc_analysis_and_transforms_common.h | 3 +- src/api/pdc_analysis/include/pdc_hist_pkg.h | 1 + .../pdc_analysis_and_transforms_connect.c | 29 +- src/api/pdc_analysis/pdc_hist_pkg.c | 1 - src/api/pdc_client_connect.c | 1767 ++++++++++++++--- src/api/pdc_obj/include/pdc_cont.h | 6 +- src/api/pdc_obj/include/pdc_obj.h | 8 +- src/api/pdc_obj/include/pdc_obj_pkg.h | 3 +- src/api/pdc_obj/include/pdc_prop.h | 10 - src/api/pdc_obj/include/pdc_prop_pkg.h | 9 +- src/api/pdc_obj/pdc_cont.c | 25 +- src/api/pdc_obj/pdc_dt_conv.c | 8 +- src/api/pdc_obj/pdc_obj.c | 77 +- src/api/pdc_obj/pdc_prop.c | 20 +- src/api/pdc_region/pdc_region_transfer.c | 3 +- .../include/pdc_transforms_pkg.h | 2 + src/api/profiling/CMakeLists.txt | 120 -- src/commons/CMakeLists.txt | 241 +++ src/commons/Readme.md | 2 + src/commons/collections/Readme.md | 9 + src/commons/collections/art.c | 1096 ++++++++++ src/commons/collections/include/art.h | 246 +++ src/commons/collections/include/pdc_compare.h | 146 ++ src/commons/collections/include/pdc_deque.h | 33 + src/commons/collections/include/pdc_hash.h | 65 + .../collections/include/pdc_hash_table.h | 280 +++ src/commons/collections/include/pdc_set.h | 263 +++ src/commons/collections/include/pdc_stack.h | 25 + src/commons/collections/include/pdc_vector.h | 111 ++ src/commons/collections/pdc_compare.c | 177 ++ src/commons/collections/pdc_deque.c | 94 + src/commons/collections/pdc_hash.c | 75 + src/commons/collections/pdc_hash_table.c | 580 ++++++ src/commons/collections/pdc_set.c | 619 ++++++ src/commons/collections/pdc_stack.c | 46 + src/commons/collections/pdc_vector.c | 165 ++ src/commons/file/Readme.md | 0 src/commons/file/common_io.c | 188 ++ src/commons/file/include/common_io.h | 159 ++ src/commons/generic/include/pdc_generic.h | 215 ++ src/{api => commons}/include/pdc_public.h | 30 +- src/commons/index/Readme.md | 0 src/commons/index/dart/dart_algo.c | 212 ++ src/commons/index/dart/dart_core.c | 690 +++++++ src/commons/index/dart/dart_core_test.c | 92 + src/commons/index/dart/dart_math.c | 43 + src/commons/index/dart/dart_utils.c | 97 + src/commons/index/dart/include/dart_algo.h | 24 + src/commons/index/dart/include/dart_core.h | 198 ++ src/commons/index/dart/include/dart_math.h | 16 + src/commons/index/dart/include/dart_utils.h | 25 + .../profiling/include/pdc_hashtab.h | 0 .../profiling/include/pdc_stack_ops.h | 0 src/{api => commons}/profiling/pdc_hashtab.c | 0 .../profiling/pdc_stack_ops.c | 0 src/commons/serde/bulki.c | 115 ++ src/commons/serde/bulki_serde.c | 229 +++ src/commons/serde/bulki_serde_test.c | 84 + src/commons/serde/include/bulki.h | 101 + src/commons/serde/include/bulki_serde.h | 42 + src/commons/utils/Readme.md | 0 src/{ => commons}/utils/include/pdc_id_pkg.h | 7 +- .../utils/include/pdc_linkedlist.h | 4 +- src/{ => commons}/utils/include/pdc_malloc.h | 37 +- src/{ => commons}/utils/include/pdc_private.h | 83 +- src/{ => commons}/utils/include/pdc_timing.h | 2 + src/commons/utils/include/query_utils.h | 120 ++ src/commons/utils/include/string_utils.h | 196 ++ src/commons/utils/include/thpool.h | 178 ++ src/commons/utils/include/timer_utils.h | 77 + src/commons/utils/pdc_malloc.c | 188 ++ src/{ => commons}/utils/pdc_timing.c | 0 src/commons/utils/query_utils.c | 346 ++++ src/commons/utils/query_utils_test.c | 64 + src/commons/utils/string_utils.c | 330 +++ src/commons/utils/thpool.c | 542 +++++ src/commons/utils/timer_utils.c | 103 + src/server/CMakeLists.txt | 35 +- src/server/dablooms/pdc_dablooms.c | 9 +- src/server/include/pdc_client_server_common.h | 344 +++- src/server/include/pdc_server.h | 9 + src/server/include/pdc_server_metadata.h | 5 +- .../include/pdc_server_metadata_index.h | 84 + src/server/pdc_client_server_common.c | 193 +- src/server/pdc_server.c | 67 +- src/server/pdc_server_main.c | 7 + src/server/pdc_server_metadata.c | 133 +- src/server/pdc_server_metadata_index.c | 730 +++++++ src/server/pdc_server_metadata_index_test.c | 74 + .../include/pdc_server_data.h | 2 +- .../pdc_server_region/pdc_server_data.c | 51 +- .../pdc_server_region_cache.c | 43 +- .../pdc_server_region_request_handler.h | 4 +- src/tests/CMakeLists.txt | 80 +- src/tests/bdcats_v2.c | 8 +- src/tests/client_server.c | 14 +- src/tests/cont_add_del.c | 4 +- src/tests/cont_del.c | 5 + src/tests/cont_tags.c | 96 +- src/tests/create_obj_scale.c | 4 +- src/tests/dart_algo_sim.c | 468 +++++ src/tests/dart_attr_dist_test.c | 377 ++++ src/tests/dart_func_test.c | 110 + src/tests/dart_test.c | 590 ++++++ src/tests/data_server_read.c | 2 +- src/tests/data_server_read_multi.c | 2 +- src/tests/data_server_read_vpic_multits.c | 2 +- .../data_server_read_vpic_spatial_multits.c | 2 +- src/tests/data_server_write.c | 2 +- src/tests/data_server_write_multi.c | 2 +- src/tests/data_server_write_vpic_multits.c | 2 +- src/tests/delete_obj_scale.c | 2 +- src/tests/helper/JuliaHelper.jl | 105 + .../helper/include/julia_helper_loader.h | 134 ++ src/tests/helper/julia_helper_loader.c | 118 ++ src/tests/kvtag_add_get.c | 37 +- src/tests/kvtag_add_get_benchmark.c | 50 +- src/tests/kvtag_add_get_scale.c | 9 +- src/tests/kvtag_get.c | 15 +- src/tests/kvtag_query.c | 11 +- src/tests/kvtag_query_mpi.c | 207 ++ src/tests/kvtag_query_scale.c | 112 +- src/tests/kvtag_query_scale_col.c | 395 ++++ src/tests/list_all.c | 2 +- src/tests/obj_lock.c | 4 +- src/tests/obj_map.c | 4 +- src/tests/obj_round_robin_io_all.c | 6 +- src/tests/obj_tags.c | 23 +- src/tests/obj_transformation.c | 6 +- src/tests/pdc_transforms_lib.c | 35 +- src/tests/query_data.c | 2 +- src/tests/query_vpic.c | 6 +- src/tests/query_vpic_bin_sds1_nopreload.c | 4 +- src/tests/query_vpic_bin_sds1_preload.c | 4 +- src/tests/query_vpic_bin_sds_nopreload.c | 4 +- src/tests/query_vpic_bin_sds_preload.c | 4 +- src/tests/query_vpic_exyz_nopreload.c | 4 +- src/tests/query_vpic_exyz_preload.c | 4 +- src/tests/query_vpic_multi.c | 6 +- src/tests/query_vpic_multi_nopreload.c | 6 +- src/tests/query_vpic_multi_nopreload1.c | 6 +- src/tests/query_vpic_multi_preload.c | 6 +- src/tests/read_obj.c | 4 +- src/tests/read_obj_shared.c | 4 +- src/tests/region_transfer.c | 2 +- src/tests/region_transfer_all_append.c | 206 +- src/tests/region_transfer_query.c | 4 +- src/tests/search_obj.c | 6 +- src/tests/search_obj_scale.c | 4 +- src/tests/stat_obj.c | 4 +- src/tests/update_obj.c | 4 +- src/tests/vpicio.c | 10 +- src/tests/vpicio_mts.c | 22 +- src/tests/vpicio_v2.c | 6 +- src/tests/write_obj_shared.c | 2 +- src/tools/CMakeLists.txt | 33 + {tools => src/tools}/cjson/cJSON.c | 0 {tools => src/tools}/cjson/cJSON.h | 0 {tools => src/tools}/pdc_export.c | 59 +- {tools => src/tools}/pdc_import.c | 149 +- {tools => src/tools}/pdc_ls.c | 51 +- src/utils/pdc_interface.c | 4 +- src/utils/pdc_malloc.c | 76 - tools/CMakeLists.txt | 88 +- tools/LLSM_IMPORTER.md | 76 + tools/llsm/csvReader.c | 395 ++++ tools/llsm/csvReader.h | 190 ++ tools/llsm/parallelReadTiff.c | 818 ++++++++ tools/llsm/parallelReadTiff.h | 34 + tools/llsm/pdc_list.c | 165 ++ tools/llsm/pdc_list.h | 110 + tools/llsm_importer.c | 392 ++++ 229 files changed, 19763 insertions(+), 1936 deletions(-) create mode 100644 .devcontainer/devcontainer.Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/post-create.sh create mode 100644 .devcontainer/post-start.sh create mode 100644 .docker/base.Dockerfile create mode 100644 .docker/local.Dockerfile create mode 100644 .docker/local.Dockerfile.dockerignore create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/install_problem.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/nersc.yml delete mode 100644 .gitmodules create mode 100644 .readthedocs.yaml create mode 100644 CMake/FindJULIA.cmake create mode 100644 CMake/FindMERCURY.cmake create mode 100644 CMake/FindUUID.cmake rename scripts/{kvtag_add_get_benchmark/cori => dart_attr_dist_test}/clean.sh (100%) create mode 100755 scripts/dart_attr_dist_test/gen_script.sh rename scripts/{kvtag_add_get_benchmark/cori => dart_attr_dist_test}/submit.sh (51%) create mode 100755 scripts/dart_attr_dist_test/template.sh delete mode 100755 scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh delete mode 100755 scripts/kvtag_add_get_benchmark/cori/template.sh delete mode 100755 scripts/kvtag_add_get_scale/cori/gen_scripts.sh delete mode 100755 scripts/kvtag_add_get_scale/cori/template.sh rename scripts/{kvtag_add_get_scale/cori => kvtag_query_scale}/clean.sh (100%) create mode 100755 scripts/kvtag_query_scale/gen_script.sh rename scripts/{kvtag_add_get_scale/cori => kvtag_query_scale}/submit.sh (89%) create mode 100755 scripts/kvtag_query_scale/template.sh create mode 100755 scripts/kvtag_query_scale_mpi/clean.sh create mode 100755 scripts/kvtag_query_scale_mpi/gen_script.sh create mode 100755 scripts/kvtag_query_scale_mpi/submit.sh create mode 100755 scripts/kvtag_query_scale_mpi/template.sh create mode 100755 scripts/llsm_importer/clean.sh create mode 100755 scripts/llsm_importer/gen_script.sh create mode 100755 scripts/llsm_importer/submit.sh create mode 100755 scripts/llsm_importer/template.sh delete mode 100644 spack/package.py delete mode 100644 src/api/profiling/CMakeLists.txt create mode 100644 src/commons/CMakeLists.txt create mode 100644 src/commons/Readme.md create mode 100644 src/commons/collections/Readme.md create mode 100644 src/commons/collections/art.c create mode 100644 src/commons/collections/include/art.h create mode 100644 src/commons/collections/include/pdc_compare.h create mode 100644 src/commons/collections/include/pdc_deque.h create mode 100644 src/commons/collections/include/pdc_hash.h create mode 100644 src/commons/collections/include/pdc_hash_table.h create mode 100644 src/commons/collections/include/pdc_set.h create mode 100644 src/commons/collections/include/pdc_stack.h create mode 100644 src/commons/collections/include/pdc_vector.h create mode 100644 src/commons/collections/pdc_compare.c create mode 100644 src/commons/collections/pdc_deque.c create mode 100644 src/commons/collections/pdc_hash.c create mode 100644 src/commons/collections/pdc_hash_table.c create mode 100644 src/commons/collections/pdc_set.c create mode 100644 src/commons/collections/pdc_stack.c create mode 100644 src/commons/collections/pdc_vector.c create mode 100644 src/commons/file/Readme.md create mode 100644 src/commons/file/common_io.c create mode 100644 src/commons/file/include/common_io.h create mode 100644 src/commons/generic/include/pdc_generic.h rename src/{api => commons}/include/pdc_public.h (66%) create mode 100644 src/commons/index/Readme.md create mode 100644 src/commons/index/dart/dart_algo.c create mode 100644 src/commons/index/dart/dart_core.c create mode 100644 src/commons/index/dart/dart_core_test.c create mode 100644 src/commons/index/dart/dart_math.c create mode 100644 src/commons/index/dart/dart_utils.c create mode 100644 src/commons/index/dart/include/dart_algo.h create mode 100644 src/commons/index/dart/include/dart_core.h create mode 100644 src/commons/index/dart/include/dart_math.h create mode 100644 src/commons/index/dart/include/dart_utils.h rename src/{api => commons}/profiling/include/pdc_hashtab.h (100%) rename src/{api => commons}/profiling/include/pdc_stack_ops.h (100%) rename src/{api => commons}/profiling/pdc_hashtab.c (100%) rename src/{api => commons}/profiling/pdc_stack_ops.c (100%) create mode 100644 src/commons/serde/bulki.c create mode 100644 src/commons/serde/bulki_serde.c create mode 100644 src/commons/serde/bulki_serde_test.c create mode 100644 src/commons/serde/include/bulki.h create mode 100644 src/commons/serde/include/bulki_serde.h create mode 100644 src/commons/utils/Readme.md rename src/{ => commons}/utils/include/pdc_id_pkg.h (96%) rename src/{ => commons}/utils/include/pdc_linkedlist.h (99%) rename src/{ => commons}/utils/include/pdc_malloc.h (66%) rename src/{ => commons}/utils/include/pdc_private.h (63%) rename src/{ => commons}/utils/include/pdc_timing.h (99%) create mode 100644 src/commons/utils/include/query_utils.h create mode 100644 src/commons/utils/include/string_utils.h create mode 100644 src/commons/utils/include/thpool.h create mode 100644 src/commons/utils/include/timer_utils.h create mode 100644 src/commons/utils/pdc_malloc.c rename src/{ => commons}/utils/pdc_timing.c (100%) create mode 100644 src/commons/utils/query_utils.c create mode 100644 src/commons/utils/query_utils_test.c create mode 100644 src/commons/utils/string_utils.c create mode 100644 src/commons/utils/thpool.c create mode 100644 src/commons/utils/timer_utils.c create mode 100644 src/server/include/pdc_server_metadata_index.h create mode 100644 src/server/pdc_server_main.c create mode 100644 src/server/pdc_server_metadata_index.c create mode 100644 src/server/pdc_server_metadata_index_test.c create mode 100644 src/tests/dart_algo_sim.c create mode 100644 src/tests/dart_attr_dist_test.c create mode 100644 src/tests/dart_func_test.c create mode 100644 src/tests/dart_test.c create mode 100644 src/tests/helper/JuliaHelper.jl create mode 100644 src/tests/helper/include/julia_helper_loader.h create mode 100644 src/tests/helper/julia_helper_loader.c create mode 100644 src/tests/kvtag_query_mpi.c create mode 100644 src/tests/kvtag_query_scale_col.c create mode 100644 src/tools/CMakeLists.txt rename {tools => src/tools}/cjson/cJSON.c (100%) rename {tools => src/tools}/cjson/cJSON.h (100%) rename {tools => src/tools}/pdc_export.c (96%) rename {tools => src/tools}/pdc_import.c (89%) rename {tools => src/tools}/pdc_ls.c (96%) delete mode 100644 src/utils/pdc_malloc.c create mode 100644 tools/LLSM_IMPORTER.md create mode 100644 tools/llsm/csvReader.c create mode 100644 tools/llsm/csvReader.h create mode 100644 tools/llsm/parallelReadTiff.c create mode 100644 tools/llsm/parallelReadTiff.h create mode 100644 tools/llsm/pdc_list.c create mode 100644 tools/llsm/pdc_list.h create mode 100644 tools/llsm_importer.c diff --git a/.devcontainer/devcontainer.Dockerfile b/.devcontainer/devcontainer.Dockerfile new file mode 100644 index 000000000..2eecb45b1 --- /dev/null +++ b/.devcontainer/devcontainer.Dockerfile @@ -0,0 +1,95 @@ +# Note: Run `docker build -f .devcontainer/Dockerfile -t pdc:latest .` from the root directory of the repository to build the docker image. + +# Use Ubuntu Jammy (latest LTS) as the base image +FROM ubuntu:jammy + + + +# Install necessary tools, MPICH, UUID library and developer files +RUN apt-get update && apt-get install -y \ + build-essential \ + git \ + mpich \ + libmpich-dev \ + uuid \ + uuid-dev \ + autoconf \ + libtool \ + cmake \ + cmake-curses-gui \ + wget \ + axel \ + curl \ + vim \ + nano \ + gdb \ + cgdb \ + curl \ + valgrind + +# Set WORK_SPACE environment variable and create necessary directories +RUN mkdir -p /workspaces +ENV WORK_SPACE=/workspaces + + +# Clone the repositories +WORKDIR $WORK_SPACE/source +RUN git clone https://github.com/ofiwg/libfabric.git && \ + git clone https://github.com/mercury-hpc/mercury.git --recursive + +COPY ./ ${WORK_SPACE}/source/pdc + +ENV LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric +ENV MERCURY_SRC_DIR=$WORK_SPACE/source/mercury +ENV PDC_SRC_DIR=$WORK_SPACE/source/pdc +ENV LIBFABRIC_DIR=$WORK_SPACE/install/libfabric +ENV MERCURY_DIR=$WORK_SPACE/install/mercury +ENV PDC_DIR=$WORK_SPACE/install/pdc + +RUN mkdir -p $LIBFABRIC_SRC_DIR && \ + mkdir -p $MERCURY_SRC_DIR && \ + mkdir -p $LIBFABRIC_DIR && \ + mkdir -p $MERCURY_DIR && \ + mkdir -p $PDC_DIR + + +# Save the environment variables to a file +RUN echo "export LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric" > $WORK_SPACE/pdc_env.sh && \ + echo "export MERCURY_SRC_DIR=$WORK_SPACE/source/mercury" >> $WORK_SPACE/pdc_env.sh && \ + echo "export PDC_SRC_DIR=$WORK_SPACE/source/pdc" >> $WORK_SPACE/pdc_env.sh && \ + echo "export LIBFABRIC_DIR=$WORK_SPACE/install/libfabric" >> $WORK_SPACE/pdc_env.sh && \ + echo "export MERCURY_DIR=$WORK_SPACE/install/mercury" >> $WORK_SPACE/pdc_env.sh && \ + echo "export PDC_DIR=$WORK_SPACE/install/pdc" >> $WORK_SPACE/pdc_env.sh + + +# Build and install libfabric +WORKDIR $LIBFABRIC_SRC_DIR +RUN git checkout v1.18.0 && \ + ./autogen.sh && \ + ./configure --prefix=$LIBFABRIC_DIR CC=mpicc CFLAG="-O2" && \ + make clean && \ + make -j && make install && \ + make check + +ENV LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH" +ENV PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH" +RUN echo 'export LD_LIBRARY_PATH=$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh && \ + echo 'export PATH=$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh + + +# Build and install Mercury +WORKDIR $MERCURY_SRC_DIR +ENV MERCURY_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DNA_OFI_TESTING_PROTOCOL=tcp " +RUN git checkout v2.2.0 \ + mkdir -p build +WORKDIR ${MERCURY_SRC_DIR}/build +RUN cmake $MERCURY_CMAKE_FLAGS ../ && \ + make -j && make install && \ + ctest + +# Set the environment variables +ENV LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" +ENV PATH="$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH" +RUN echo 'export LD_LIBRARY_PATH=$MERCURY_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh \ + echo 'export PATH=$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..ac2f53e57 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,9 @@ +{ + "name": "pdc_devcontainer", + "dockerFile": "devcontainer.Dockerfile", + "forwardPorts": [ + 3000 + ], + "postCreateCommand": ".devcontainer/post-create.sh", + "postStartCommand": ".devcontainer/post-start.sh" +} \ No newline at end of file diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh new file mode 100644 index 000000000..a9bf588e2 --- /dev/null +++ b/.devcontainer/post-create.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/.devcontainer/post-start.sh b/.devcontainer/post-start.sh new file mode 100644 index 000000000..d4d6dbc5f --- /dev/null +++ b/.devcontainer/post-start.sh @@ -0,0 +1,31 @@ +#!/bin/bash + + +ln -s /workspaces/pdc /home/codespace/source/pdc +mkdir -p /workspaces/install +mkdir -p /workspaces/source +ln -s $PDC_SRC_DIR /workspaces/source/pdc +ln -s $PDC_DIR /workspaces/install/pdc + +export PDC_SRC_DIR=/workspaces/source/pdc + +# Build and install PDC +export PDC_CMAKE_FLAGS="-DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DMPI_RUN_CMD=mpiexec " + +cd $PDC_SRC_DIR +rm -rf build && mkdir -p build + + +cd ${PDC_SRC_DIR}/build +cmake $PDC_CMAKE_FLAGS ../ 2>&1 > ./cmake_config.log || echo "ignoring cmake config error and proceed" +make -j && make install + +# Set the environment variables +export LD_LIBRARY_PATH="$PDC_DIR/lib:$LD_LIBRARY_PATH" +export PATH="$PDC_DIR/include:$PDC_DIR/lib:$PATH" +echo 'export LD_LIBRARY_PATH=$PDC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh +echo 'export PATH=$PDC_DIR/include:$PDC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh + + +cd $PDC_SRC_DIR/build +# ctest \ No newline at end of file diff --git a/.docker/base.Dockerfile b/.docker/base.Dockerfile new file mode 100644 index 000000000..13af12510 --- /dev/null +++ b/.docker/base.Dockerfile @@ -0,0 +1,90 @@ +# Use Ubuntu Jammy (latest LTS) as the base image +FROM ubuntu:jammy + +# Install necessary tools, MPICH, UUID library and developer files +RUN apt-get update && apt-get install -y \ + build-essential \ + git \ + mpich \ + libmpich-dev \ + uuid \ + uuid-dev \ + autoconf \ + libtool \ + cmake \ + cmake-curses-gui \ + wget \ + axel \ + curl \ + vim \ + nano \ + gdb \ + cgdb \ + curl \ + valgrind + +# Set WORK_SPACE environment variable and create necessary directories +ENV WORK_SPACE=/home/codespace +RUN mkdir -p $WORK_SPACE + +# Clone the repositories +WORKDIR $WORK_SPACE/source +RUN git clone https://github.com/ofiwg/libfabric.git && \ + git clone https://github.com/mercury-hpc/mercury.git --recursive + +COPY ./ ${WORK_SPACE}/source/pdc + +ENV LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric +ENV MERCURY_SRC_DIR=$WORK_SPACE/source/mercury +ENV PDC_SRC_DIR=$WORK_SPACE/source/pdc +ENV LIBFABRIC_DIR=$WORK_SPACE/install/libfabric +ENV MERCURY_DIR=$WORK_SPACE/install/mercury +ENV PDC_DIR=$WORK_SPACE/install/pdc + +RUN mkdir -p $LIBFABRIC_SRC_DIR \ + mkdir -p $MERCURY_SRC_DIR \ + mkdir -p $PDC_SRC_DIR \ + mkdir -p $LIBFABRIC_DIR \ + mkdir -p $MERCURY_DIR \ + mkdir -p $PDC_DIR + + +# Save the environment variables to a file +RUN echo "export LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric" > $WORK_SPACE/pdc_env.sh && \ + echo "export MERCURY_SRC_DIR=$WORK_SPACE/source/mercury" >> $WORK_SPACE/pdc_env.sh && \ + echo "export PDC_SRC_DIR=$WORK_SPACE/source/pdc" >> $WORK_SPACE/pdc_env.sh && \ + echo "export LIBFABRIC_DIR=$WORK_SPACE/install/libfabric" >> $WORK_SPACE/pdc_env.sh && \ + echo "export MERCURY_DIR=$WORK_SPACE/install/mercury" >> $WORK_SPACE/pdc_env.sh && \ + echo "export PDC_DIR=$WORK_SPACE/install/pdc" >> $WORK_SPACE/pdc_env.sh + + +# Build and install libfabric +WORKDIR $LIBFABRIC_SRC_DIR +RUN git checkout v1.18.0 && \ + ./autogen.sh && \ + ./configure --prefix=$LIBFABRIC_DIR CC=mpicc CFLAG="-O2" && \ + make clean && \ + make -j && make install && \ + make check + +ENV LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH" +ENV PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH" +RUN echo 'export LD_LIBRARY_PATH=$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh && \ + echo 'export PATH=$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh + + +# Build and install Mercury +WORKDIR $MERCURY_SRC_DIR +ENV MERCURY_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DNA_OFI_TESTING_PROTOCOL=tcp " +RUN git checkout v2.2.0 \ + mkdir -p build +WORKDIR ${MERCURY_SRC_DIR}/build +RUN cmake $MERCURY_CMAKE_FLAGS ../ && \ + make -j && make install && \ + ctest + +# Set the environment variables +ENV LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" +ENV PATH="$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH" +RUN echo 'export LD_LIBRARY_PATH=$MERCURY_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh \ + echo 'export PATH=$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh \ No newline at end of file diff --git a/.docker/local.Dockerfile b/.docker/local.Dockerfile new file mode 100644 index 000000000..f5a0e0302 --- /dev/null +++ b/.docker/local.Dockerfile @@ -0,0 +1,29 @@ +# Note: Run `docker build -f .docker/Dockerfile -t pdc:latest .` from the root directory of the repository to build the docker image. + +# Use Ubuntu Jammy (latest LTS) as the base image +FROM zhangwei217245/pdc_dev_base:latest + +# Build and install PDC +ENV PDC_CMAKE_FLAGS="-DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DMPI_RUN_CMD=mpiexec " + + +WORKDIR $PDC_SRC_DIR +RUN rm -rf build && \ + mkdir -p build + +# COPY ../ ${PDC_SRC_DIR} +# RUN ls -l $PDC_SRC_DIR + +WORKDIR ${PDC_SRC_DIR}/build +RUN cmake $PDC_CMAKE_FLAGS ../ 2>&1 > ./cmake_config.log || echo "ignoring cmake config error and proceed" && \ + make -j && make install + +# Set the environment variables +ENV LD_LIBRARY_PATH="$PDC_DIR/lib:$LD_LIBRARY_PATH" +ENV PATH="$PDC_DIR/include:$PDC_DIR/lib:$PATH" +RUN echo 'export LD_LIBRARY_PATH=$PDC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh && \ + echo 'export PATH=$PDC_DIR/include:$PDC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh + + +# WORKDIR $PDC_SRC_DIR/build +# RUN ctest \ No newline at end of file diff --git a/.docker/local.Dockerfile.dockerignore b/.docker/local.Dockerfile.dockerignore new file mode 100644 index 000000000..c29c58f5a --- /dev/null +++ b/.docker/local.Dockerfile.dockerignore @@ -0,0 +1,2 @@ +# Exclude files and directories from the Docker build context +!/.git/ diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..f34ee0ed1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,35 @@ +**Bug Report** + +A clear and concise description of what the bug is, including module and feature. + +**To Reproduce** + +How are you building/running PDC? + +- version of PDC: [e.g., 0.3, branch, or hash] +- installed PDC using: [spack, from source] +- operating system: [name and version] +- machine: [Are you running on a supercomputer or public cluster?] +- version of Mercury: [e.g., 1.12.0] +- name and version of MPI: [e.g., OpenMPI 4.1.1] + +What did you use to build PDC (cmake command)? + +```bash +... +``` + +What is the running setup you use? + +```bash +... +``` + +**Expected Behavior** + +A clear and concise description of what you expected to happen. + +**Additional Information** + +If built from source, include the cmake options you used. +Add any other information about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..d4caa8413 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +**What does this feature solve or improve?** + +A clear and concise description of what the problem is. +Example: I always do the following workflow [...] + +**Describe the solution you'd like** + +A clear and concise description of what you want to happen. +Example: It would be fantastic if one could [...] + +**Describe alternatives you've considered** + +A clear and concise description of any alternative solutions or features you've considered. + +**Additional Information** + +Add any other information about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/install_problem.md b/.github/ISSUE_TEMPLATE/install_problem.md new file mode 100644 index 000000000..dea13744b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/install_problem.md @@ -0,0 +1,27 @@ +**Issue Description** + +This is what I have tried so far: + + +```commandline +... +``` + +This is the error I am facing during installation: + +``` +... +``` + +**Software Environment** + +- version of PDC: [e.g. 0.3] +- installed PDC using: [spack, from source] +- operating system: [name and version] +- machine: [Are you running on a supercomputer or public cluster?] +- version of Mercury: [e.g. 1.12.0] +- name and version of MPI: [e.g. OpenMPI 4.1.1] + +**Additional Information** + +Add any other information about the problem here. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 000000000..907d68eda --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,16 @@ +**Usage Case** + +A clear and concise description of what your usage case is. + +**Additional Information** + +Add any other information about your intended usage, limitation, or questions here. + +**Software Environment** + +- version of PDC: [e.g. 0.3] +- installed PDC using: [spack, from source] +- operating system: [name and version] +- machine: [Are you running on a supercomputer or public cluster?] +- version of Mercury: [e.g. 1.12.0] +- name and version of MPI: [e.g. OpenMPI 4.1.1] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..e84c24a1d --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,22 @@ +# Related Issues / Pull Requests + +List all related issues and/or pull requests if there are any. + +# Description + +Include a brief summary of the proposed changes. + +# What changes are proposed in this pull request? + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality not to work as expected; for instance, examples in this repository must be updated too) +- [ ] This change requires a documentation update + +# Checklist: + +- [ ] My code modifies existing public API, or introduces new public API, and I updated or wrote docstrings +- [ ] I have commented my code +- [ ] My code requires documentation updates, and I have made corresponding changes to the documentation +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes diff --git a/.github/workflows/clang-format-fix.yml b/.github/workflows/clang-format-fix.yml index 9145f65ca..686218650 100644 --- a/.github/workflows/clang-format-fix.yml +++ b/.github/workflows/clang-format-fix.yml @@ -1,16 +1,18 @@ -name: clang-format Check +name: clang-format Commit Chanages on: workflow_dispatch: push: jobs: formatting-check: - name: Formatting Check + name: Commit Format Changes runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, 'skip-ci')" + permissions: + contents: write # In order to allow EndBug/add-and-commit to commit changes steps: - - uses: actions/checkout@v2 - - name: Run clang-format style check for C programs. - uses: DoozyX/clang-format-lint-action@v0.11 + - uses: actions/checkout@v3 + - name: Fix C formatting issues detected by clang-format + uses: DoozyX/clang-format-lint-action@v0.13 with: source: '.' extensions: 'c,h,cpp,hpp' @@ -18,7 +20,7 @@ jobs: inplace: True style: file # exclude: './config ' - - uses: EndBug/add-and-commit@v7 + - uses: EndBug/add-and-commit@v9 with: author_name: github-actions author_email: github-actions[bot]@users.noreply.github.com diff --git a/.github/workflows/nersc.yml b/.github/workflows/nersc.yml new file mode 100644 index 000000000..e941f177b --- /dev/null +++ b/.github/workflows/nersc.yml @@ -0,0 +1,38 @@ +name: NERSC + +on: + pull_request_target + +env: + PR_NUMBER: ${{ github.event.number }} + +jobs: + authorize: + environment: + ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && 'external' || 'internal' }} + runs-on: ubuntu-latest + steps: + - run: true + + sync-with-nersc: + needs: authorize + runs-on: ubuntu-latest + timeout-minutes: 2 + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup git + run: | + git config --global user.name "PDC BOT" + git config --global user.email "pdc@example.com" + + - name: Create PR branch + run: | + PR_SHA=$(git rev-parse --short "$GITHUB_SHA") + git fetch origin pull/${PR_NUMBER}/head:PR-${PR_SHA} + git remote add gitlab https://${{ secrets.NERSC_GITLAB_TOKEN_NAME }}:${{ secrets.NERSC_GITLAB_TOKEN }}@${{ secrets.NERSC_GITLAB_URL }} + git checkout PR-${PR_SHA} + git push -f gitlab -u PR-${PR_SHA} diff --git a/.gitignore b/.gitignore index f65ea896c..860c7a60a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ # ignore swp files *.swp -# build dir - # exes *.exe @@ -10,8 +8,14 @@ src/install +# ignore vscode files .vscode +# ignore macos files +.DS_Store + +# ignore build dir build +# ignore docs build dir docs/build diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90d60224c..ddef71e19 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,8 @@ stages: perlmutter-build: stage: build + when: manual + allow_failure: false tags: - perlmutter variables: @@ -20,7 +22,7 @@ perlmutter-build: - module list - mkdir -p ${PDC_BUILD_PATH}/perlmutter - cd ${PDC_BUILD_PATH}/perlmutter - - cmake ../.. -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DPDC_SERVER_CACHE=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DMPI_RUN_CMD="srun -A m2621 --qos=debug --constraint=cpu --tasks-per-node=64" -DCMAKE_INSTALL_PREFIX=${PDC_INSTALL_PATH}/perlmutter + - cmake ../.. -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DPDC_SERVER_CACHE=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=cc -DMPI_RUN_CMD="srun -A m2621 --qos=debug --constraint=cpu --tasks-per-node=64" -DCMAKE_INSTALL_PREFIX=${PDC_INSTALL_PATH}/perlmutter - make -j - make install artifacts: diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 442a4bdb8..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "api/src/server/dablooms"] - path = api/src/server/dablooms - url = git@github.com:bitly/dablooms.git diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..863de73f7 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file diff --git a/CMake/FindJULIA.cmake b/CMake/FindJULIA.cmake new file mode 100644 index 000000000..2a42366df --- /dev/null +++ b/CMake/FindJULIA.cmake @@ -0,0 +1,45 @@ +# FindJULIA.cmake + +# Check if JULIA_HOME is set +if(NOT DEFINED ENV{JULIA_HOME}) + message(FATAL_ERROR "JULIA_HOME environment variable is not set. Please set it to the JULIA installation directory.") +endif() + +# Set JULIA installation directory +set(JULIA_INSTALL_DIR $ENV{JULIA_HOME}) + +# Find JULIA library +find_library(JULIA_LIBRARY + NAMES julia + PATHS ${JULIA_INSTALL_DIR}/lib + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH +) + +# Find JULIA include directory +find_path(JULIA_INCLUDE_DIR + NAMES julia.h + PATHS ${JULIA_INSTALL_DIR}/include/julia + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH +) + +# Check if JULIA library and include directory are found +if(NOT JULIA_LIBRARY) + message(FATAL_ERROR "JULIA library not found. Please set the JULIA_HOME environment variable to the correct JULIA installation directory.") +endif() + +if(NOT JULIA_INCLUDE_DIR) + message(FATAL_ERROR "JULIA include directory not found. Please set the JULIA_HOME environment variable to the correct JULIA installation directory.") +endif() + +# Set JULIA_FOUND variable +set(JULIA_FOUND TRUE CACHE BOOL "JULIA found") + +# Set JULIA_INCLUDE_DIRS and JULIA_LIBRARIES variables +set(JULIA_INCLUDE_DIRS ${JULIA_INCLUDE_DIR}) +set(JULIA_LIBRARIES ${JULIA_LIBRARY}) + +# Report JULIA library and include directory +message(STATUS "JULIA library found: ${JULIA_LIBRARY}") +message(STATUS "JULIA include directory found: ${JULIA_INCLUDE_DIR}") \ No newline at end of file diff --git a/CMake/FindMERCURY.cmake b/CMake/FindMERCURY.cmake new file mode 100644 index 000000000..d159efbaf --- /dev/null +++ b/CMake/FindMERCURY.cmake @@ -0,0 +1,47 @@ +# FindMERCURY.cmake + +# Find the system's MERCURY library +# This will define: +# +# MERCURY_FOUND - System has MERCURY +# MERCURY_INCLUDE_DIRS - The MERCURY include directory +# MERCURY_LIBRARIES - The libraries needed to use MERCURY + +find_package(MERCURY QUIET HINTS $ENV{MERCURY_HOME} $ENV{MERCURY_DIR} $ENV{MERCURY_ROOT} $ENV{MERCURYPATH} $ENV{MERCURY_PATH}) +if(MERCURY_FOUND) + message(STATUS "mercury dir = ${MERCURY_DIR}") + # NOTE: enable the following if you need ${MERCURY_INCLUDE_DIR} in the future + # NOTE: remember to add MERCURY_HOME to PATH or CMAKE_PREFIX_PATH if you enable the following. + find_path(MERCURY_INCLUDE_DIR mercury.h HINTS ${MERCURY_DIR}) + find_library(MERCURY_LIBRARY + NAMES + mercury + mercury_debug + HINTS ${MERCURY_DIR} + ) + + find_library(MERCURY_NA_LIBRARY + NAMES + na + na_debug + HINTS ${MERCURY_DIR} + ) + + # find_library(MERCURY_UTIL_LIBRARY + # NAMES + # mercury_util + # HINTS ${MERCURY_DIR} + # ) + + # set(MERCURY_LIBRARIES ${MERCURY_LIBRARY} ${MERCURY_NA_LIBRARY} ${MERCURY_UTIL_LIBRARY}) + set(MERCURY_LIBRARIES ${MERCURY_LIBRARY} ${MERCURY_NA_LIBRARY}) + set(MERCURY_INCLUDE_DIRS ${MERCURY_INCLUDE_DIR}) + message(STATUS "mercury include dir = ${MERCURY_INCLUDE_DIRS}") + message(STATUS "mercury lib = ${MERCURY_LIBRARIES}") + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(MERCURY DEFAULT_MSG MERCURY_LIBRARY MERCURY_INCLUDE_DIR) +else(MERCURY_FOUND) + set(MERCURY_LIBRARIES "") +endif() + +mark_as_advanced(MERCURY_INCLUDE_DIR MERCURY_LIBRARY) diff --git a/CMake/FindUUID.cmake b/CMake/FindUUID.cmake new file mode 100644 index 000000000..22d87b38f --- /dev/null +++ b/CMake/FindUUID.cmake @@ -0,0 +1,36 @@ +# FindUUID.cmake + +# Find the system's UUID library +# This will define: +# +# UUID_FOUND - System has UUID +# UUID_INCLUDE_DIRS - The UUID include directory +# UUID_LIBRARIES - The libraries needed to use UUID + +# - Try to find UUID +# Once done this will define +# UUID_FOUND - System has UUID +# UUID_INCLUDE_DIRS - The UUID include directories +# UUID_LIBRARIES - The libraries needed to use UUID + +find_package(PkgConfig) +pkg_check_modules(PC_UUID uuid) + +find_path(UUID_INCLUDE_DIR uuid/uuid.h + HINTS ${PC_UUID_INCLUDEDIR} ${PC_UUID_INCLUDE_DIRS} + PATHS /usr/local/include /usr/include) + +find_library(UUID_LIBRARY NAMES uuid + HINTS ${PC_DRC_LIBDIR} ${PC_DRC_LIBRARY_DIRS} + PATHS /usr/local/lib64 /usr/local/lib /usr/lib64 /usr/lib) + +set(UUID_INCLUDE_DIRS ${UUID_INCLUDE_DIR}) +set(UUID_LIBRARIES ${UUID_LIBRARY}) + +include(FindPackageHandleStandardArgs) +# handle the QUIETLY and REQUIRED arguments and set UUID_FOUND to TRUE +# if all listed variables are TRUE +find_package_handle_standard_args(UUID DEFAULT_MSG + UUID_INCLUDE_DIR UUID_LIBRARY) + +mark_as_advanced(UUID_INCLUDE_DIR UUID_LIBRARY) \ No newline at end of file diff --git a/CMake/pdc-config.cmake.build.in b/CMake/pdc-config.cmake.build.in index e61dd4bac..920bad2a1 100644 --- a/CMake/pdc-config.cmake.build.in +++ b/CMake/pdc-config.cmake.build.in @@ -24,5 +24,5 @@ if(NOT TARGET "pdc" AND NOT PDC_INSTALL_SKIP_TARGETS) if(NOT TARGET "mercury") include(@MERCURY_DIR@/mercury-config.cmake) endif() - include(${SELF_DIR}/api/pdc-targets.cmake) + include(@PDC_INSTALL_SHARE_DIR@/pdc-targets.cmake) endif() diff --git a/CMake/pdc-config.cmake.install.in b/CMake/pdc-config.cmake.install.in index 2e8bba491..e3ead7e94 100644 --- a/CMake/pdc-config.cmake.install.in +++ b/CMake/pdc-config.cmake.install.in @@ -24,5 +24,5 @@ if(NOT TARGET "pdc" AND NOT PDC_INSTALL_SKIP_TARGETS) if(NOT TARGET "mercury") include(@MERCURY_DIR@/mercury-config.cmake) endif() - include(${SELF_DIR}/pdc-targets.cmake) + include(@PDC_INSTALL_SHARE_DIR@/pdc-targets.cmake) endif() diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e353dbc2..975dac351 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,17 +5,17 @@ cmake_minimum_required (VERSION 3.0) # Setup cmake policies. foreach(p - CMP0012 - CMP0013 - CMP0014 - CMP0022 # CMake 2.8.12 - CMP0025 # CMake 3.0 - CMP0053 # CMake 3.1 - CMP0054 # CMake 3.1 - CMP0074 # CMake 3.12 - CMP0075 # CMake 3.12 - CMP0083 # CMake 3.14 - CMP0093 # CMake 3.15 + CMP0012 # CMake 2.6; Full Path Libraries; When set to NEW, the full path is included when specifying a link library in the target_link_libraries command. + CMP0013 # CMake 2.6; Quoted Argument Expansion; When set to NEW, variables within quoted arguments are expanded, allowing variable substitution. + CMP0014 # CMake 2.8; Relative Path Libraries; When set to NEW, relative paths in link libraries are interpreted as relative to the current source directory. + CMP0022 # CMake 2.8.12; Same SONAME Overwrite; When set to NEW, shared libraries with the same SONAME are not overwritten during installation. + CMP0025 # CMake 3.0; Empty if Statement; When set to NEW, an empty if condition is treated as false, providing better control over conditionals. + CMP0053 # CMake 3.1; Find Package Version; When set to NEW, the VERSION option in find_package is interpreted as a version requirement rather than a version hint. + CMP0054 # CMake 3.1; Quoted Empty String; When set to NEW, a quoted empty string is treated as false in the if() command. + CMP0074 # CMake 3.12; Imported Target Usage Requirements; When set to NEW, the "PRIVATE" and "PUBLIC" keywords in target_link_libraries are treated as usage requirements for imported targets. + CMP0075 # CMake 3.12; GNU-Style Installation Paths on macOS; When set to NEW, the "GNUInstallDirs" module is enabled by default on macOS for "GNU-style" installation paths. + CMP0083 # CMake 3.14; Imported Targets Visibility; When set to NEW, imported targets created by find_package with namespaced aliases are visible to other targets within the same directory scope. + CMP0093 # CMake 3.15; Compiler and Linker Flags Handling; When set to NEW, each option in a list of compiler and linker flags is interpreted separately, allowing better control over individual options. ) if(POLICY ${p}) cmake_policy(SET ${p} NEW) @@ -23,18 +23,34 @@ foreach(p endforeach() # Set a consistent MACOSX_RPATH default across all CMake versions. +# checks if the variable CMAKE_MACOSX_RPATH is not defined and sets it to 0 if it is not defined. +# The CMAKE_MACOSX_RPATH variable is used to control the behavior of the macOS-specific RPATH feature in CMake. +# RPATH is a linker option that specifies the runtime library search path for executables and shared libraries. +# By default, when CMAKE_MACOSX_RPATH is not defined, the code sets it to 0, indicating that the project does not use the macOS RPATH feature. +# This means that the runtime library search path will not be embedded in the executables or shared libraries produced by the project. +# This code snippet ensures a consistent default value for CMAKE_MACOSX_RPATH across different configurations of the project, even if the variable is not explicitly set before. # When CMake 2.8.12 is required, change this default to 1. # When CMake 3.0.0 is required, remove this block (see CMP0042). if(NOT DEFINED CMAKE_MACOSX_RPATH) set(CMAKE_MACOSX_RPATH 0) endif() +# Project Name : PDC, Programming Language : C project(PDC C) +# include CheckSymbolExists module +include(CheckSymbolExists) + +# CMake will search for any header files under the project root directory. include_directories( ${PROJECT_SOURCE_DIR} ) +# Check if MPI_RUN_CMD is not defind. If not, set it to mpiexec. +if(NOT MPI_RUN_CMD) + set(MPI_RUN_CMD mpiexec) +endif() + #------------------------------------------------------------------------------ # Version information #------------------------------------------------------------------------------ @@ -48,9 +64,26 @@ set(PDC_PACKAGE_VERSION_MAJOR "${PDC_VERSION_MAJOR}.${PDC_VERSION_MINOR}") set(PDC_PACKAGE_VERSION_MINOR "${PDC_VERSION_PATCH}") set(PDC_PACKAGE_STRING "${PDC_PACKAGE_NAME} ${PDC_PACKAGE_VERSION}") set(PDC_PACKAGE_TARNAME "${PDC_PACKAGE}") -if(NOT MPI_RUN_CMD) - set(MPI_RUN_CMD mpiexec) + + +#------------------------------------------------------------------------------ +# Setup CMake Prefix Paths for searching external libraries. +#------------------------------------------------------------------------------ +if(NOT CMAKE_PREFIX_PATH) + set(CMAKE_PREFIX_PATH ${CMAKE_INSTALL_PREFIX}) endif() + +#------------------------------------------------------------------------------ +# general cmake flags: +# -DCMAKE_INSTALL_PREFIX=/usr/local -- the prefix for installing +# -DCMAKE_BUILD_TYPE=type -- type can be Debug, Release, ... +# -DCMAKE_PREFIX_PATH=/dir -- external packages +# +# note that CMAKE_PREFIX_PATH can be a list of directories: +# -DCMAKE_PREFIX_PATH='/dir1;/dir2;/dir3' +#------------------------------------------------------------------------------ + + #------------------------------------------------------------------------------ # Setup install and output Directories #------------------------------------------------------------------------------ @@ -69,7 +102,16 @@ endif() if(NOT PDC_INSTALL_DATA_DIR) set(PDC_INSTALL_DATA_DIR ${CMAKE_INSTALL_PREFIX}/share) endif() +if(NOT PDC_INSTALL_SHARE_DIR) + set(PDC_INSTALL_SHARE_DIR ${CMAKE_INSTALL_PREFIX}/share/cmake/pdc) +endif() + +# The purpose of setting CMAKE_INSTALL_RPATH_USE_LINK_PATH to TRUE is to +# include the directories specified by the linker during the build process +# in the install RPATH, ensuring that the installed binary can locate its +# required shared libraries during runtime, regardless of the platform being used. +# # Setting this ensures that "make install" will leave rpaths to external # libraries intact on "make install". This ensures that one can install a # version of PDC on the build machine without any issues. If this not @@ -79,6 +121,19 @@ if(NOT CMAKE_INSTALL_RPATH_USE_LINK_PATH) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) endif() +#------------------------------------------------------------------------------ +# Setup CMake Prefix Paths for searching external libraries. +#------------------------------------------------------------------------------ +# note that CMAKE_PREFIX_PATH can be a list of directories: +# -DCMAKE_PREFIX_PATH='/dir1;/dir2;/dir3' + +if(NOT CMAKE_PREFIX_PATH) + set(CMAKE_PREFIX_PATH ${CMAKE_INSTALL_PREFIX}) +endif() +# MERCURY +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} $ENV{MERCURY_DIR}) +# ANY Future external package goes here... + #------------------------------------------------------------------------------ # Setup CMake Environment #------------------------------------------------------------------------------ @@ -131,6 +186,33 @@ if("${PDC_SOURCE_DIR}" STREQUAL "${PDC_BINARY_DIR}") "Please create a separate binary directory and run CMake there.") endif() +#------------------------------------------------------------------------------ +# Set whether or not to disable compiler warnings +#------------------------------------------------------------------------------ +set(SUPPRESSED_LIST "") + +# Disable warnings for potentially ignorable issues +option(SUPPRESS_IGNORABLE_WARNINGS "Disable warnings for potentially ignorable issues" ON) +if(SUPPRESS_IGNORABLE_WARNINGS) + if(APPLE) + set(SUPPRESSED_LIST "-Wno-deprecated-non-prototype" ${SUPPRESSED_LIST}) + else() + set(SUPPRESSED_LIST "-Wno-maybe-uninitialized" ${SUPPRESSED_LIST}) + endif() + set(SUPPRESSED_LIST "-Wno-sign-compare" ${SUPPRESSED_LIST}) + set(SUPPRESSED_LIST "-Wno-format" ${SUPPRESSED_LIST}) + set(SUPPRESSED_LIST "-Wno-cast-qual" ${SUPPRESSED_LIST}) + set(SUPPRESSED_LIST "-Wno-unused-parameter" ${SUPPRESSED_LIST}) + set(SUPPRESSED_LIST "-Wno-unused-variable" ${SUPPRESSED_LIST}) + set(SUPPRESSED_LIST "-Wno-unused-function" ${SUPPRESSED_LIST}) + set(SUPPRESSED_LIST "-Wno-unused-result" ${SUPPRESSED_LIST}) + set(SUPPRESSED_LIST "-Wno-unused-but-set-variable" ${SUPPRESSED_LIST}) + + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + add_compile_options(${SUPPRESSED_LIST}) + endif() +endif() + #------------------------------------------------------------------------------ # Set a default build type if none was specified #------------------------------------------------------------------------------ @@ -240,6 +322,33 @@ if(PDC_USE_CRAY_DRC) endif() endif() +#----------------------------------------------------------------------------- +# DART Suffix Tree mode +#----------------------------------------------------------------------------- +option(PDC_DART_SUFFIX_TREE_MODE "Enable DART Suffix Tree mode." ON) +if(PDC_DART_SUFFIX_TREE_MODE) + set(PDC_DART_SFX_TREE 1) + # add_compile_definitions(PDC_DART_SFX_TREE=${PDC_DART_SFX_TREE}) +endif() + +#----------------------------------------------------------------------------- +# Julia Support option +#----------------------------------------------------------------------------- +option(PDC_ENABLE_JULIA_SUPPORT "Enable Julia Support." OFF) +if(PDC_ENABLE_JULIA_SUPPORT) + find_package(JULIA REQUIRED) + if(JULIA_FOUND) + set(PDC_ENABLE_JULIA 1) + # include_directories(${JULIA_INCLUDE_DIRS}) + # set(PDC_EXT_LIB_DEPENDENCIES + # ${PDC_EXT_LIB_DEPENDENCIES} + # ${JULIA_LIBRARIES} + # ) + else() + message(FATAL_ERROR "Could not find Julia.") + endif() +endif() + #----------------------------------------------------------------------------- # MPI option #----------------------------------------------------------------------------- @@ -257,7 +366,7 @@ endif() option(PDC_ENABLE_LUSTRE "Enable Lustre." OFF) if(PDC_ENABLE_LUSTRE) set(ENABLE_LUSTRE 1) - set(PDC_LUSTRE_TOTAL_OST "248" CACHE STRING "Number of Lustre OSTs") + set(PDC_LUSTRE_TOTAL_OST "256" CACHE STRING "Number of Lustre OSTs") endif() #----------------------------------------------------------------------------- @@ -271,9 +380,13 @@ endif() #----------------------------------------------------------------------------- # SERVER CACHE option #----------------------------------------------------------------------------- -option(PDC_SERVER_CACHE "Enable timing." OFF) +option(PDC_SERVER_CACHE "Enable Server Caching." OFF) if(PDC_SERVER_CACHE) set(PDC_SERVER_CACHE 1) + set(PDC_SERVER_CACHE_MAX_GB "3" CACHE STRING "Max GB for server cache") + set(PDC_SERVER_CACHE_FLUSH_TIME "30" CACHE STRING "Flush time for server cache") + + add_compile_definitions(PDC_SERVER_CACHE_MAX_GB=${PDC_SERVER_CACHE_MAX_GB} PDC_SERVER_CACHE_FLUSH_TIME=${PDC_SERVER_CACHE_FLUSH_TIME}) endif() @@ -317,19 +430,18 @@ if(PDC_ENABLE_MULTITHREAD) endif() #----------------------------------------------------------------------------- -# PROFILING option +# Pthread support on Mac OS X #----------------------------------------------------------------------------- -option(PDC_ENABLE_PROFILING "Enable profiling." OFF) -if(PDC_ENABLE_PROFILING) - set(ENABLE_PROFILING 1) +if(APPLE) + find_package(Threads REQUIRED) endif() #----------------------------------------------------------------------------- -# CACHE option +# PROFILING option #----------------------------------------------------------------------------- -option(PDC_SERVER_CACHE "Enable server data caching with pthread." OFF) -if(PDC_SERVER_CACHE) - set(PDC_SERVER_CACHE 1) +option(PDC_ENABLE_PROFILING "Enable profiling." OFF) +if(PDC_ENABLE_PROFILING) + set(ENABLE_PROFILING 1) endif() # Not used @@ -355,6 +467,14 @@ if(PDC_ENABLE_FASTBIT) set(ENABLE_FASTBIT 1) endif() +#----------------------------------------------------------------------------- +# Check availability of symbols +#----------------------------------------------------------------------------- +check_symbol_exists(malloc_usable_size "malloc.h" HAVE_MALLOC_USABLE_SIZE) +if(HAVE_MALLOC_USABLE_SIZE) + add_definitions(-DHAVE_MALLOC_USABLE_SIZE) +endif() + #------------------------------------------------------------------------------ # Data type #------------------------------------------------------------------------------ @@ -384,9 +504,12 @@ configure_file( #----------------------------------------------------------------------------- # Source #----------------------------------------------------------------------------- +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/commons) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/api) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/server) + + #----------------------------------------------------------------------------- # Testing #----------------------------------------------------------------------------- @@ -398,6 +521,11 @@ if(NOT PDC_EXTERNALLY_CONFIGURED AND BUILD_TESTING) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/tests) endif() +option(BUILD_TOOLS "Build Tools." OFF) +if(BUILD_TOOLS) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/tools) +endif() + #----------------------------------------------------------------------------- # Build doxygen documentation. #----------------------------------------------------------------------------- @@ -455,6 +583,17 @@ install( PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) +if(BUILD_TOOLS) + install( + FILES + ${PDC_BINARY_DIR}/bin/pdc_ls + ${PDC_BINARY_DIR}/bin/pdc_export + ${PDC_BINARY_DIR}/bin/pdc_import + DESTINATION + ${CMAKE_INSTALL_PREFIX}/bin + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE +) +endif() install( DIRECTORY @@ -465,6 +604,9 @@ install( PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) +# set(ADD_EXE_PERMISSION_CMD "chmod +x ${}/test/*") +# add_custom_command(TARGET ${PROJECT_NAME} POST_INSTALL COMMAND ${add_permission_cmd}) + #install( # FILES diff --git a/docs/readme.md b/docs/readme.md index df19eba94..74be2d0e4 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -1,58 +1,67 @@ # PDC Documentations - + [PDC user APIs](#pdc-user-apis) - - [PDC general APIs](#pdc-general-apis) - - [PDC container APIs](#pdc-container-apis) - - [PDC object APIs](#pdc-object-apis) - - [PDC region APIs](#pdc-region-apis) - - [PDC property APIs](#pdc-property-apis) - - [PDC query APIs](#pdc-query-apis) - + [PDC data types](#PDC-type-categories) - - [Basic types](#basic-types) - - [Histogram structure](#histogram-structure) - - [Container info](#container-info) - - [Container life time](#container-life-time) - - [Object property](#object-property) - - [Object info](#object-info) - - [Object structure](#object-structure) - - [Region info](#region-info) - - [Access type](#access-type) - - [Transfer request status](#transfer-request-status) - - [Query operators](#query-operators) - - [Query structures](#query-structures) - - [Selection structure](#selection-structure) - + [Developers notes](#developers-notes) - - [How to implement an RPC from client to server](#how-to-implement-an-rpc-from-client-to-server) - - [PDC Server metadata overview](#pdc-server-metadata-overview) - + [PDC metadata structure](#pdc-metadata-structure) - + [Metadata operations at client side](#metadata-operations-at-client-side) - - [PDC metadata management strategy](#pdc-metadata-management-strategy) - + [Managing metadata and data by the same server](#managing-metadata-and-data-by-the-same-server) - + [Separate metadata server from data server](#separate-metadata-server-from-data-server) - + [Static object region mappings](#static-object-region-mappings) - + [Dynamic object region mappings](#dynamic-object-region-mappings) - - [PDC metadata management implementation](#pdc-metadata-management-implementation) - + [Create metadata](#create-metadata) - + [Binding metadata to object](#binding-metadata-to-object) - + [Register object metadata at metadata server](#register-object-metadata-at-metadata-server) - + [Retrieve metadata from metadata server](#retrieve-metadata-from-metadata-server) - + [Object metadata at client](#object-metadata-at-client) - + [Metadata at data server](#metadata-at-data-server) - + [Object metadata update](#object-metadata-update) - + [Object region metadata](#object-region-metadata) - + [Metadata checkpoint](#object-metadata-update) - - [Region transfer request at client](#region-transfer-request-at-client) - + [Region transfer request create and close](#region-transfer-request-create-and-close) - + [Region transfer request start](#region-transfer-request-start) - + [Region transfer request wait](#region-transfer-request-wait) - - [Region transfer request at server](#region-transfer-request-at-server) - + [Server region transfer request RPC](#server-region-transfer-request-rpc) - - [Server nonblocking control](#server-nonblocking-control) - - [Server region transfer request start](#server-region-transfer-request-start) - - [Server region transfer request wait](#server-region-transfer-request-wait) - + [Server region storage](#server-region-storage) - - [Storage by file offset](#storage-by-file-offset) - - [Storage by region](#storage-by-region) - - [Open tasks for PDC](#open-tasks-for-pdc) +- [PDC Documentations](#pdc-documentations) +- [PDC user APIs](#pdc-user-apis) + - [PDC general APIs](#pdc-general-apis) + - [PDC container APIs](#pdc-container-apis) + - [PDC object APIs](#pdc-object-apis) + - [PDC region APIs](#pdc-region-apis) + - [PDC property APIs](#pdc-property-apis) + - [PDC query APIs](#pdc-query-apis) + - [PDC hist APIs](#pdc-hist-apis) +- [PDC Data types](#pdc-data-types) + - [Basic types](#basic-types) + - [region transfer partition type](#region-transfer-partition-type) + - [Object consistency semantics type](#object-consistency-semantics-type) + - [Histogram structure](#histogram-structure) + - [Container info](#container-info) + - [Container life time](#container-life-time) + - [Object property public](#object-property-public) + - [Object property](#object-property) + - [Object info](#object-info) + - [Object structure](#object-structure) + - [Region info](#region-info) + - [Access type](#access-type) + - [Transfer request status](#transfer-request-status) + - [Query operators](#query-operators) + - [Query structures](#query-structures) + - [Selection structure](#selection-structure) +- [Developers notes](#developers-notes) + - [How to implement an RPC from client to server](#how-to-implement-an-rpc-from-client-to-server) + - [PDC Server metadata overview](#pdc-server-metadata-overview) + - [PDC metadata structure](#pdc-metadata-structure) + - [Metadata operations at client side](#metadata-operations-at-client-side) + - [PDC metadata management strategy](#pdc-metadata-management-strategy) + - [Managing metadata and data by the same server](#managing-metadata-and-data-by-the-same-server) + - [Separate metadata server from data server](#separate-metadata-server-from-data-server) + - [Static object region mappings](#static-object-region-mappings) + - [Dynamic object region mappings](#dynamic-object-region-mappings) + - [PDC metadata management implementation](#pdc-metadata-management-implementation) + - [Create metadata](#create-metadata) + - [Binding metadata to object](#binding-metadata-to-object) + - [Register object metadata at metadata server](#register-object-metadata-at-metadata-server) + - [Retrieve metadata from metadata server](#retrieve-metadata-from-metadata-server) + - [Object metadata at client](#object-metadata-at-client) + - [Metadata at data server](#metadata-at-data-server) + - [Object metadata update](#object-metadata-update) + - [Object region metadata](#object-region-metadata) + - [Metadata checkpoint](#metadata-checkpoint) + - [Region transfer request at client](#region-transfer-request-at-client) + - [Region transfer request create and close](#region-transfer-request-create-and-close) + - [Region transfer request start](#region-transfer-request-start) + - [Region transfer request wait](#region-transfer-request-wait) + - [Region transfer request at server](#region-transfer-request-at-server) + - [Server region transfer request RPC](#server-region-transfer-request-rpc) + - [Server nonblocking control](#server-nonblocking-control) + - [Server region transfer request start](#server-region-transfer-request-start) + - [Server region transfer request wait](#server-region-transfer-request-wait) + - [Server region storage](#server-region-storage) + - [Storage by file offset](#storage-by-file-offset) + - [Storage by region](#storage-by-region) + - [Open tasks for PDC](#open-tasks-for-pdc) + - [Replacing individual modules with efficient Hash table data structures](#replacing-individual-modules-with-efficient-hash-table-data-structures) + - [Restarting pdc\_server.exe with different numbers of servers](#restarting-pdc_serverexe-with-different-numbers-of-servers) + - [Fast region search mechanisms](#fast-region-search-mechanisms) + - [Merge overlapping regions](#merge-overlapping-regions) # PDC user APIs ## PDC general APIs + pdcid_t PDCinit(const char *pdc_name) @@ -683,21 +692,28 @@ ## Basic types ``` typedef enum { - PDC_UNKNOWN = -1, /* error */ - PDC_INT = 0, /* integer types */ - PDC_FLOAT = 1, /* floating-point types */ - PDC_DOUBLE = 2, /* double types */ - PDC_CHAR = 3, /* character types */ - PDC_COMPOUND = 4, /* compound types */ - PDC_ENUM = 5, /* enumeration types */ - PDC_ARRAY = 6, /* Array types */ - PDC_UINT = 7, /* unsigned integer types */ - PDC_INT64 = 8, /* 64-bit integer types */ - PDC_UINT64 = 9, /* 64-bit unsigned integer types */ - PDC_INT16 = 10, - PDC_INT8 = 11, - NCLASSES = 12 /* this must be last */ - } pdc_var_type_t; + PDC_UNKNOWN = -1, /* error */ + PDC_INT = 0, /* integer types (identical to int32_t) */ + PDC_FLOAT = 1, /* floating-point types */ + PDC_DOUBLE = 2, /* double types */ + PDC_CHAR = 3, /* character types */ + PDC_STRING = 4, /* string types */ + PDC_BOOLEAN = 5, /* boolean types */ + PDC_SHORT = 6, /* short types */ + PDC_UINT = 7, /* unsigned integer types (identical to uint32_t) */ + PDC_INT64 = 8, /* 64-bit integer types */ + PDC_UINT64 = 9, /* 64-bit unsigned integer types */ + PDC_INT16 = 10, /* 16-bit integer types */ + PDC_INT8 = 11, /* 8-bit integer types */ + PDC_UINT8 = 12, /* 8-bit unsigned integer types */ + PDC_UINT16 = 13, /* 16-bit unsigned integer types */ + PDC_INT32 = 14, /* 32-bit integer types */ + PDC_UINT32 = 15, /* 32-bit unsigned integer types */ + PDC_LONG = 16, /* long types */ + PDC_VOID_PTR = 17, /* void pointer type */ + PDC_SIZE_T = 18, /* size_t type */ + PDC_TYPE_COUNT = 19 /* this is the number of var types and has to be the last */ + } pdc_c_var_type_t; ``` ## region transfer partition type ``` diff --git a/docs/source/api.rst b/docs/source/api.rst index e9b1e6567..ab058f10a 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -471,21 +471,28 @@ Basic types .. code-block:: c typedef enum { - PDC_UNKNOWN = -1, /* error */ - PDC_INT = 0, /* integer types */ - PDC_FLOAT = 1, /* floating-point types */ - PDC_DOUBLE = 2, /* double types */ - PDC_CHAR = 3, /* character types */ - PDC_COMPOUND = 4, /* compound types */ - PDC_ENUM = 5, /* enumeration types */ - PDC_ARRAY = 6, /* Array types */ - PDC_UINT = 7, /* unsigned integer types */ - PDC_INT64 = 8, /* 64-bit integer types */ - PDC_UINT64 = 9, /* 64-bit unsigned integer types */ - PDC_INT16 = 10, - PDC_INT8 = 11, - NCLASSES = 12 /* this must be last */ - } pdc_var_type_t; + PDC_UNKNOWN = -1, /* error */ + PDC_INT = 0, /* integer types (identical to int32_t) */ + PDC_FLOAT = 1, /* floating-point types */ + PDC_DOUBLE = 2, /* double types */ + PDC_CHAR = 3, /* character types */ + PDC_STRING = 4, /* string types */ + PDC_BOOLEAN = 5, /* boolean types */ + PDC_SHORT = 6, /* short types */ + PDC_UINT = 7, /* unsigned integer types (identical to uint32_t) */ + PDC_INT64 = 8, /* 64-bit integer types */ + PDC_UINT64 = 9, /* 64-bit unsigned integer types */ + PDC_INT16 = 10, /* 16-bit integer types */ + PDC_INT8 = 11, /* 8-bit integer types */ + PDC_UINT8 = 12, /* 8-bit unsigned integer types */ + PDC_UINT16 = 13, /* 16-bit unsigned integer types */ + PDC_INT32 = 14, /* 32-bit integer types */ + PDC_UINT32 = 15, /* 32-bit unsigned integer types */ + PDC_LONG = 16, /* long types */ + PDC_VOID_PTR = 17, /* void pointer type */ + PDC_SIZE_T = 18, /* size_t type */ + PDC_TYPE_COUNT = 19 /* this is the number of var types and has to be the last */ + } pdc_c_var_type_t; diff --git a/docs/source/developer-notes.rst b/docs/source/developer-notes.rst index 3dfab28e6..f9d9cdd28 100644 --- a/docs/source/developer-notes.rst +++ b/docs/source/developer-notes.rst @@ -2,30 +2,9 @@ Developer Notes ================================ -+++++++++++++++++++++++++++++++++++++++++++++ -How to implement an RPC? -+++++++++++++++++++++++++++++++++++++++++++++ - -This section covers how to implement a simple RPC from client to server. If you call an RPC on the client side, the server should be able to get the argument you passed from the client and execute the corresponding server RPC function. - -A concrete example is ``PDC_region_transfer_wait_all``. Mercury transfers at the client side are implemented in ``pdc_client_connect.c``. The name of the function we are using in this example is ``transfer_request_wait_all``. For each component mentioned next, replace ``transfer_request_wait_all`` with your function name. This section will not discuss the design of ``transfer_request_wait_all`` but rather point out where the Mercury components are and how they interact. - -Firstly, in ``pdc_client_connect.c``, search for ``transfer_request_wait_all_register_id_g``. Create another variable by replacing ``transfer_request_wait_all`` with your function name. Secondly, search for ``client_send_transfer_request_wait_all_rpc_cb``, and do the same text copy and replacement. This is the callback function on the client side when the RPC is finished on the server side. For most cases, this function loads the server return arguments to a structure and returns the values to the client RPC function. There is also some error checking. Then, search for ``PDC_transfer_request_wait_all_register(*hg_class)`` and ``PDC_Client_transfer_request_wait_all``, and do text copy and replacement for both. This function is the entry point of the mercury RPC call. It contains argument loading, which has the variable name ``in``' This RPC creates a mercury bulk transfer inside it. ``HG_Create`` and ``HG_Bulk_create`` are unnecessary if your mercury transfer does not transfer variable-sized data. ``HG_Forward`` has an argument ``client_send_transfer_request_wait_all_rpc_cb``. The return values from the callback function are placed in ``transfer_args``. - -In file ``pdc_client_connect.h``, search for ``_pdc_transfer_request_wait_all_args``, do the text copy and replacement. This structure is the structure for returning values from client call back function ``client_send_transfer_request_wait_all_rpc_cb`` to client RPC function ``PDC_Client_transfer_request_wait_all``. For most cases, an error code is sufficient. For other cases, like creating some object IDs, you must define the structure accordingly. Do not forget to load data in ``_pdc_transfer_request_wait_all_args``. Search for ``PDC_Client_transfer_request_wait_all``, and make sure you register your client connect entry function in the same way. - -In file ``pdc_server.c``, search for ``PDC_transfer_request_wait_all_register(hg_class_g);``, make a copy, and replace the ``transfer_request_wait_all`` part with your function name (your function name has to be defined and used consistently throughout all these copy and replacement). -In the file ``pdc_client_server_common.h``, search for ``typedef struct transfer_request_wait_all_in_t``. This is the structure used by a client passing its argument to the server side. You can define whatever you want that is fixed-sized inside this structure. If you have variable-sized data, it can be passed through mercury bulk transfer. The handle is ``hg_bulk_t local_bulk_handle``. ``typedef struct transfer_request_wait_all_out_t`` is the return argument from the server to the client after the server RPC is finished. Next, search for ``hg_proc_transfer_request_wait_all_in_t``. This function defines how arguments are transferred through mercury. -Similarly, ``hg_proc_transfer_request_wait_all_in_t`` is the other way around. Next, search for ``struct transfer_request_wait_all_local_bulk_args``. This structure is useful when a bulk transfer is used. Using this function, the server passes its variables from the RPC call to the bulk transfer callback function. Finally, search for ``PDC_transfer_request_wait_all_register``. For all these structures and functions, you should copy and replace ``transfer_request_wait_all`` with your own function name. - -In file ``pdc_client_server_common.c``, search for ``PDC_FUNC_DECLARE_REGISTER(transfer_request_wait_all)`` and ``HG_TEST_THREAD_CB(transfer_request_wait_all)``, do text copy and function name replacement. ``pdc_server_region_request_handler.h`` is included directly in ``pdc_client_server_common.c``. The server RPC of ``transfer_request_wait_all`` is implemented in ``pdc_server_region_request_handler.h``. However, it is possible to put it directly in the ``pdc_client_server_common.c``. - -Let us open ``pdc_server_region_request_handler.h``. Search for ``HG_TEST_RPC_CB(transfer_request_wait_all, handle)``. This function is the entry point for the server RPC function call. ``transfer_request_wait_all_in_t`` contains the arguments you loaded previously from the client side. If you want to add more arguments, return to ``pdc_client_server_common.h`` and modify it correctly. ``HG_Bulk_create`` and ``HG_Bulk_transfer`` are the mercury bulk function calls. When the bulk transfer is finished, ``transfer_request_wait_all_bulk_transfer_cb`` is called. - -After a walk-through of ``transfer_request_wait_all``, you should have learned where different components of a mercury RPC should be placed and how they interact with each other. You can trace other RPC by searching their function names. If you miss things that are not optional, the program will likely hang there forever or run into segmentation faults. +++++++++++++++++++++++++++++++++++++++++++++ -PDC Server Metadata Overview +PDC Server Metadata Management +++++++++++++++++++++++++++++++++++++++++++++ PDC metadata servers, a subset of PDC servers, store metadata for PDC classes such as objects and containers. PDC data server, also a subset of PDC servers (potentially overlapping with PDC metadata server), manages data from users. Such management includes server local caching and I/O to the file system. Both PDC metadata and data servers have some local metadata. @@ -50,13 +29,13 @@ Metadata Operations at Client Side In general, PDC object metadata is initialized when an object is created. The metadata stored at the metadata server is permanent. When clients create the objects, a PDC property is used as one of the arguments for the object creation function. Metadata for the object is set by using PDC property APIs. Most of the metadata are not subject to any changes. Currently, we support setting/getting object dimensions using object API. -+++++++++++++++++++++++++++++++++++++++++++++ +--------------------------------------------- PDC Metadata Management Strategy -+++++++++++++++++++++++++++++++++++++++++++++ +--------------------------------------------- This section discusses the metadata management approaches of PDC. First, we briefly summarize how PDC managed metadata in the past. Then, we propose new infrastructures for metadata management. ---------------------------------------------- + Managing Metadata and Data by the Same Server --------------------------------------------- @@ -64,7 +43,7 @@ Historically, a PDC server manages both metadata and data for objects it is resp However, this design has two potential drawbacks. The first disadvantage is supporting general I/O access. For clients served by different PDC servers, accessing overlapping regions is infeasible. Therefore, this design is specialized in applications with a non-overlapping I/O pattern. The second disadvantage is a lack of dynamic load-balancing mechanisms. For example, some applications use a subset of processes for processing I/O. A subset of servers may stay idle because the clients mapped to them are not sending I/O requests. ---------------------------------------------- + Separate Metadata Server from Data Server --------------------------------------------- @@ -76,54 +55,6 @@ This approach's main advantage is that the object regions' assignment to data se When a client accesses regions of an object, the metadata server informs the client of the corresponding data servers it should transfer its I/O requests. Metadata servers can map object regions to data servers in a few different methods. ---------------------------------------------- -Static Object Region Mappings ---------------------------------------------- - -A metadata server can partition the object space evenly among all data servers. For high-dimensional objects, it is possible to define block partitioning methods similar to HDF5s's chunking strategies. - -The static object region partitioning can theoretically achieve optimal parallel performance for applications with a balanced workload. In addition, static partitioning determines the mapping from object regions to data servers at object create/open time. No additional metadata management is required. - ---------------------------------------------- -Dynamic Object Region Mappings ---------------------------------------------- - -For applications that access a subset of regions for different objects, some data servers can stay idle while the rest are busy fetching or storing data for these regions concentrated around coordinates of interest. Dynamic object partitioning allows metadata servers to balance data server workloads in runtime. The mapping from object regions to the data server is determined at the time of starting region transfer request time. -Partitioning object regions dynamically increases the complexity of metadata management. For example, a read from one client 0 after a write from another client 1 on overlapping regions demands metadata support. Client 0 has to locate the data server to which client 1 writes the region data using information from the metadata server. As a result, metadata servers must maintain up-to-date metadata of the objects they manage. There are a few options we can implement this feature. - -*Option 1*: When a client attempts to modify object regions, the client can also send the metadata of this transfer request to the metadata server. Consequently, the metadata server serving for the modified objects always has the most up-to-date metadata. - -Advantage: No need to perform communications between the servers (current strategy) -Disadvantage: The metadata server can be a bottleneck because the number of clients accessing the server may scale up quickly. - -*Option 2*: When a data server receives region transfer requests from any client, the data server forwards the corresponding metadata to the metadata server of the object. - -Advantage: The number of servers is less than the number of clients, so we are reducing the chance of communication contention -Disadvantage: Server-to-server RPC infrastructures need to be put in place. - -*Option 3*: Similar to Option 2, but the data server will modify a metadata file. Later, a metadata server always checks the metadata file for metadata information updates. - -Advantage: No communications are required if a metadata file is used. -Disadvantage: Reading metadata files may take some time. If multiple servers are modifying the same metadata file, how should we proceed? - -The following table summarizes the communication of the three mapping methods from clients to types of PDC servers when different PDC functions are called. - -+-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ -| | Static Object Mapping | Dynamic Object Mapping & Static Region Mapping | Dynamic Object Mapping & Dynamic Region Mapping | -+===============================+=============================================+===================================================+===================================================+ -| ``PDC_obj_create`` | Client - Metadata Server | Client - Metadata Server | Client - Metadata Server | -+-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ -| ``PDC_obj_open`` | Client - Metadata Server | Client - Metadata Server | Client - Metadata Server | -+-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ -| ``PDC_region_transfer_start`` | Client - Data Server | Client - Data Server | Client - Data Server | -+-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ -| ``PDC_region_transfer_start`` | Client - Data Server | Client - Data Server | Client - Metadata Server (Option 1) | -+-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ -| ``PDC_region_transfer_start`` | Client - Data Server | Client - Data Server | Data Server - Metadata Server (Option 2) | -+-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ -| ``PDC_region_transfer_wait`` | Data Server - Client (PDC_READ) | Data Server - Client (PDC_READ) | Data Server - Client (PDC_READ) | -+-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ - --------------------------------------------- PDC Metadata Management Implementation --------------------------------------------- @@ -191,9 +122,89 @@ There are four categories of metadata to be checkpointed. One category is concat Region metadata checkpoint is placed at the end of the server checkpoint file, right after the last byte of data server region. Function ``transfer_request_metadata_query_checkpoint(char **checkpoint, uint64_t *checkpoint_size)`` in ``pdc_server_region_transfer_metadata_query.c`` file handles the wrapping of region metadata. --------------------------------------------- -Region Transfer Request at Client +Metadata Search and Its Implementation +--------------------------------------------- + +For Metadata search, we current provide no-index approaches and index-facilitated approaches. +For either of these approaches, we consider two types of communication model : point-to-point and collective. + +Point-to-point communication model is for distributed applications where each single client may not follow the exact same workflows, and the timing for them to trigger a metadata search function call can be really random. In this case, each client contacts one or more metadata servers and get the complete result. +Collective communication model applies when a typical application is running. In such an application, each rank follows the exact same workflow and they may trigger a metadata search function call at the same time and the metadata search requests are sent from these clients collectively. In this case, each rank contacts one metadata server and retrieves partial result. Then these clients have to communicate with each other to get the complete result. + +No-index Approach +--------------------------------------------- + +For No-index approach, here are the APIs you can call for different communication models: + * PDC_Client_query_kvtag (point-to-point) + * PDC_Client_query_kvtag_mpi (collective) + +Index-facilitated Approach +--------------------------------------------- + +For index-facilitated approach, here are the APIs you can call for different communication models: + * PDC_Client_search_obj_ref_through_dart (point-to-point) + * PDC_Client_search_obj_ref_through_dart_mpi (collective) + +Before using these APIs, you need to create your index first, so please remember to call `PDC_Client_insert_obj_ref_into_dart` right after a successful function call of `PDCobj_put_tag`. + ++++++++++++++++++++++++++++++++++++++++++++++ +Object and Region Management ++++++++++++++++++++++++++++++++++++++++++++++ + +This section discusses how PDC manages objects and regions. + +--------------------------------------------- +Static Object Region Mappings +--------------------------------------------- + +A metadata server can partition the object space evenly among all data servers. For high-dimensional objects, it is possible to define block partitioning methods similar to HDF5s's chunking strategies. + +The static object region partitioning can theoretically achieve optimal parallel performance for applications with a balanced workload. In addition, static partitioning determines the mapping from object regions to data servers at object create/open time. No additional metadata management is required. + +--------------------------------------------- +Dynamic Object Region Mappings --------------------------------------------- +For applications that access a subset of regions for different objects, some data servers can stay idle while the rest are busy fetching or storing data for these regions concentrated around coordinates of interest. Dynamic object partitioning allows metadata servers to balance data server workloads in runtime. The mapping from object regions to the data server is determined at the time of starting region transfer request time. +Partitioning object regions dynamically increases the complexity of metadata management. For example, a read from one client 0 after a write from another client 1 on overlapping regions demands metadata support. Client 0 has to locate the data server to which client 1 writes the region data using information from the metadata server. As a result, metadata servers must maintain up-to-date metadata of the objects they manage. There are a few options we can implement this feature. + +*Option 1*: When a client attempts to modify object regions, the client can also send the metadata of this transfer request to the metadata server. Consequently, the metadata server serving for the modified objects always has the most up-to-date metadata. + +Advantage: No need to perform communications between the servers (current strategy) +Disadvantage: The metadata server can be a bottleneck because the number of clients accessing the server may scale up quickly. + +*Option 2*: When a data server receives region transfer requests from any client, the data server forwards the corresponding metadata to the metadata server of the object. + +Advantage: The number of servers is less than the number of clients, so we are reducing the chance of communication contention +Disadvantage: Server-to-server RPC infrastructures need to be put in place. + +*Option 3*: Similar to Option 2, but the data server will modify a metadata file. Later, a metadata server always checks the metadata file for metadata information updates. + +Advantage: No communications are required if a metadata file is used. +Disadvantage: Reading metadata files may take some time. If multiple servers are modifying the same metadata file, how should we proceed? + +The following table summarizes the communication of the three mapping methods from clients to types of PDC servers when different PDC functions are called. + ++-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| | Static Object Mapping | Dynamic Object Mapping & Static Region Mapping | Dynamic Object Mapping & Dynamic Region Mapping | ++===============================+=============================================+===================================================+===================================================+ +| ``PDC_obj_create`` | Client - Metadata Server | Client - Metadata Server | Client - Metadata Server | ++-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| ``PDC_obj_open`` | Client - Metadata Server | Client - Metadata Server | Client - Metadata Server | ++-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| ``PDC_region_transfer_start`` | Client - Data Server | Client - Data Server | Client - Data Server | ++-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| ``PDC_region_transfer_start`` | Client - Data Server | Client - Data Server | Client - Metadata Server (Option 1) | ++-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| ``PDC_region_transfer_start`` | Client - Data Server | Client - Data Server | Data Server - Metadata Server (Option 2) | ++-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| ``PDC_region_transfer_wait`` | Data Server - Client (PDC_READ) | Data Server - Client (PDC_READ) | Data Server - Client (PDC_READ) | ++-------------------------------+---------------------------------------------+---------------------------------------------------+---------------------------------------------------+ + + +--------------------------------------------- +Region Transfer Request at Client +--------------------------------------------- !!!!! This section describes how the region transfer request module in PDC works. The region transfer request module is the core of PDC I/O. From the client's point of view, some data is written to regions of objects through transfer request APIs. PDC region transfer request module arranges how data is transferred from clients to servers and how data is stored at servers. @@ -305,3 +316,120 @@ However, when a new region is written to an object, it is necessary to scan all I/O by region will store repeated bytes when write requests contain overlapping parts. In addition, the region update mechanism generates extra I/O operations. This is one of its disadvantages. Optimization for region search (as R trees) in the future can relieve this problem. ++++++++++++++++++++++++++++++++++++++++++++++ +Contributing to PDC project ++++++++++++++++++++++++++++++++++++++++++++++ + +In this section, we will offer you some helpful technical guidance on how to contribute to the PDC project. These 'HowTos' will help you when implementing new features or fixing bugs. + + +--------------------------------------------- +How to set up code formatter for PDC on Mac? +--------------------------------------------- + +1. PDC project uses clang-format v10 for code formatting and style check. + 1. However, on MacOS, the only available clang-format versions are v8 and v11 if you try to install it via Homebrew. + 2. To install v10, you need to download it from: https://releases.llvm.org/download.html (https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.1/llvm-project-10.0.1.tar.xz) + 3. Then follow instruction here to install clang-format: https://clang.llvm.org/get_started.html. I would suggest you do the following (suppose if you already have homebrew installed) + + .. code-block:: Bash + cd $LLVM_SRC_ROOT + mkdir build + cd build + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX=/opt/llvm/v10 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DLLVM_ENABLE_PROJECTS=clang ../llvm + make -j 128 + sudo make install + sudo ln -s /opt/llvm/v10/bin/clang-format /opt/homebrew/bin/clang-format-v10 + + + 1. To format all your source code, do the following + + .. code-block:: Bash + cd pdc + clang-format-v10 -i -style=file src/* + find src -iname *.h -o -iname *.c | xargs clang-format-v10 -i -style=file + + + 1. You can also configure clang-format to be your default C/C++ formatting tool in VSCode, and the automatic code formatter is really convenient to use. + +--------------------------------------------- +How to implement an RPC? +--------------------------------------------- + +This section covers how to implement a simple RPC from client to server. If you call an RPC on the client side, the server should be able to get the argument you passed from the client and execute the corresponding server RPC function. + +A concrete example is ``PDC_region_transfer_wait_all``. Mercury transfers at the client side are implemented in ``pdc_client_connect.c``. The name of the function we are using in this example is ``transfer_request_wait_all``. For each component mentioned next, replace ``transfer_request_wait_all`` with your function name. This section will not discuss the design of ``transfer_request_wait_all`` but rather point out where the Mercury components are and how they interact. + +Firstly, in ``pdc_client_connect.c``, search for ``transfer_request_wait_all_register_id_g``. Create another variable by replacing ``transfer_request_wait_all`` with your function name. Secondly, search for ``client_send_transfer_request_wait_all_rpc_cb``, and do the same text copy and replacement. This is the callback function on the client side when the RPC is finished on the server side. For most cases, this function loads the server return arguments to a structure and returns the values to the client RPC function. There is also some error checking. Then, search for ``PDC_transfer_request_wait_all_register(*hg_class)`` and ``PDC_Client_transfer_request_wait_all``, and do text copy and replacement for both. This function is the entry point of the mercury RPC call. It contains argument loading, which has the variable name ``in``' This RPC creates a mercury bulk transfer inside it. ``HG_Create`` and ``HG_Bulk_create`` are unnecessary if your mercury transfer does not transfer variable-sized data. ``HG_Forward`` has an argument ``client_send_transfer_request_wait_all_rpc_cb``. The return values from the callback function are placed in ``transfer_args``. + +In file ``pdc_client_connect.h``, search for ``_pdc_transfer_request_wait_all_args``, do the text copy and replacement. This structure is the structure for returning values from client call back function ``client_send_transfer_request_wait_all_rpc_cb`` to client RPC function ``PDC_Client_transfer_request_wait_all``. For most cases, an error code is sufficient. For other cases, like creating some object IDs, you must define the structure accordingly. Do not forget to load data in ``_pdc_transfer_request_wait_all_args``. Search for ``PDC_Client_transfer_request_wait_all``, and make sure you register your client connect entry function in the same way. + +In file ``pdc_server.c``, search for ``PDC_transfer_request_wait_all_register(hg_class_g);``, make a copy, and replace the ``transfer_request_wait_all`` part with your function name (your function name has to be defined and used consistently throughout all these copy and replacement). +In the file ``pdc_client_server_common.h``, search for ``typedef struct transfer_request_wait_all_in_t``. This is the structure used by a client passing its argument to the server side. You can define whatever you want that is fixed-sized inside this structure. If you have variable-sized data, it can be passed through mercury bulk transfer. The handle is ``hg_bulk_t local_bulk_handle``. ``typedef struct transfer_request_wait_all_out_t`` is the return argument from the server to the client after the server RPC is finished. Next, search for ``hg_proc_transfer_request_wait_all_in_t``. This function defines how arguments are transferred through mercury. +Similarly, ``hg_proc_transfer_request_wait_all_in_t`` is the other way around. Next, search for ``struct transfer_request_wait_all_local_bulk_args``. This structure is useful when a bulk transfer is used. Using this function, the server passes its variables from the RPC call to the bulk transfer callback function. Finally, search for ``PDC_transfer_request_wait_all_register``. For all these structures and functions, you should copy and replace ``transfer_request_wait_all`` with your own function name. + +In file ``pdc_client_server_common.c``, search for ``PDC_FUNC_DECLARE_REGISTER(transfer_request_wait_all)`` and ``HG_TEST_THREAD_CB(transfer_request_wait_all)``, do text copy and function name replacement. ``pdc_server_region_request_handler.h`` is included directly in ``pdc_client_server_common.c``. The server RPC of ``transfer_request_wait_all`` is implemented in ``pdc_server_region_request_handler.h``. However, it is possible to put it directly in the ``pdc_client_server_common.c``. + +Let us open ``pdc_server_region_request_handler.h``. Search for ``HG_TEST_RPC_CB(transfer_request_wait_all, handle)``. This function is the entry point for the server RPC function call. ``transfer_request_wait_all_in_t`` contains the arguments you loaded previously from the client side. If you want to add more arguments, return to ``pdc_client_server_common.h`` and modify it correctly. ``HG_Bulk_create`` and ``HG_Bulk_transfer`` are the mercury bulk function calls. When the bulk transfer is finished, ``transfer_request_wait_all_bulk_transfer_cb`` is called. + +After a walk-through of ``transfer_request_wait_all``, you should have learned where different components of a mercury RPC should be placed and how they interact with each other. You can trace other RPC by searching their function names. If you miss things that are not optional, the program will likely hang there forever or run into segmentation faults. + + +--------------------------------------------- +Julia Support for tests +--------------------------------------------- +Currently, we add all Julia helper functions to `src/tests/helper/JuliaHelper.jl` + +Once you implement your own Julia function, you can use the bridging functions (named with prefix `run_jl_*`) defined in `src/tests/helper/include/julia_helper_loader.h` to call your Julia functions. If the current bridging functions are not sufficient for interacting with your Julia functions, you can add your own bridging functions in `src/tests/helper/include/julia_helper_loader.h` and implement it in `src/tests/helper/include/julia_helper_loader.c`. + +When calling your bridging functions, the best example you can follow is `src/tests/dart_attr_dist_test.c`. + +Remember, you must include all your bridging function calls inside the following code blocks, so that the process can have its own Julia runtime loaded. + +.. code-block:: C + jl_module_list_t modules = {.julia_modules = (char *[]){JULIA_HELPER_NAME}, .num_modules = 1}; + init_julia(&modules); + ...... + ... call your bridging functions + ...... + close_julia(); + +Also, to make sure your code with Julia function calls doesn't get compiled when Julia support is not there, you can add your new test to the list of `ENHANCED_PROGRAMS` in `src/tests/CMakeLists.txt`. + +For more info on embedded Julia support, please visit: `Embedded Julia https://docs.julialang.org/en/v1/manual/embedding/`_. + + + +--------------------------------------------- +Github Codespace Support +--------------------------------------------- + +This is a feature current in progress. +We are trying to make PDC work with Github Codespace. + +Currently, with `.devcontainer/devcontainer.json` and `.devcontainer/Dockerfile`, you can build a docker image that contains all the dependencies for PDC. +However, when the Codespace is created, the predefined directories in the docker file will disappear. + + +------------------------------------------------------------ +Tracking your memory consumption with each memory allocation +------------------------------------------------------------ + +Now, you can use the APIs in `src/commons/utils/pdc_malloc.c` to allocate memory when needed. +Using these APIs and macros will allow you to track your memory consumption with each memory allocation. +You can get the total memory consumption anytime by calling `PDC_get_global_mem_usage()`. + +Also, the root CMakeLists.txt file will automatically detect if HAVE_MALLOC_USABLE_SIZE is available. +If so, the memory consumption will be more accurate (summation of both allocation and freeing). Otherwise, it will be less accurate but still usable (only measure the total memory ever allocated). + + +------------------------------------------------------------ +DART Suffix Tree Mode +------------------------------------------------------------ + +In DART, to support efficient infix search, we can enable the suffix tree mode, +where suffix search becomes an exact search and infix search becomes a prefix search, +at the cost of indexing every possible suffix of indexed keywords. + +To enable the suffix tree mode, you can turn on/off this switch in CMakeLists.txt: +`PDC_DART_SUFFIX_TREE_MODE` diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 508faf17b..2769c487a 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -172,6 +172,8 @@ Install PDC .. note:: ``-DCMAKE_C_COMPILER=mpicc -DMPI_RUN_CMD=mpiexec`` may need to be changed to ``-DCMAKE_C_COMPILER=cc -DMPI_RUN_CMD=srun`` depending on your system environment. +.. note:: + If you are trying to compile PDC on your Mac, ``LibUUID`` needs to be installed on your MacOS first. Simple use ``brew install ossp-uuid`` to install it. Test Your PDC Installation -------------------------- @@ -185,6 +187,47 @@ PDC's ``ctest`` contains both sequential and parallel/MPI tests, and can be run If you are using PDC on an HPC system, e.g. Perlmutter@NERSC, ``ctest`` should be run on a compute node, you can submit an interactive job on Perlmutter: ``salloc --nodes 1 --qos interactive --time 01:00:00 --constraint cpu --account=mxxxx`` +Install/Configure Julia Support for PDC +--------------------------------------- +Currently, in ``src/tests``, we experimentally added support for Julia programming language. With this support, you can either hand over the data to a Julia-based function to process, or you can use the result returned by a Julia-based function in which a complex logic is implemented (such as complex algebra operations or connecting to databases/cloud storages, etc). +To know how to add a Julia function, please see Developer Notes. + +But before adding any of your function, we need to enable the Julia support first, and here is how: + +Prerequisite +^^^^^^^^^^^^ + +Make sure you have Julia-lang installed. You can check with your system administrator to see if you already have Julia-lang installed. If not, you can either ask your system administrator to install it for you or you can install it yourself if permitted. On macOS, the best way to install Julia is via `Homebrew https://brew.sh`_. You may also refer to `Julia Download Page https://julialang.org/downloads/`_ for instructions on installing Julia. +Once you installed Julia, you can set `JULIA_HOME` to be where Julia-lang is installed. + +.. code-block:: Bash + export JULIA_HOME=/path/to/julia/install/directory + + +.. note:: Note on perlmutter: + You can easily perform `module load julia` to load the Julia-lang environment. Then, you can do the following to set `$JULIA_HOME`: + + .. code-block:: Bash + export JULIA_HOME=$(dirname $(dirname $(which julia))) + + +Enabling Julia Support for PDC Tests +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Once the Prerequisite is satisfied, you can enable Julia support by adding `--DPDC_ENABLE_JULIA_SUPPORT=ON` to your cmake command and re-run it. +Then you can compile your PDC project with Julia support. + +Now, see Developer Notes to know how you can add your own Julia functions to enhance your test cases in PDC. + + +Build PDC Docker Image and Run PDC Docker Container +--------------------------------------------------- +We provide a Dockerfile to build a PDC Docker image. The Dockerfile is located at `--$PDC_ROOT/.docker/local.Dockerfile` +To build the PDC Docker image, you can run the following command from `$PDC_ROOT`: +`docker build -t pdc_dev_base:latest -f $PDC_ROOT/.docker/base.Dockerfile .` + +To run the PDC Docker container, you can run the following command: +`docker run -it --rm --name pdc -v $PDC_ROOT:/home/codespace/source/pdc pdc_dev_base:latest /bin/bash` + --------------------------- Running PDC --------------------------- diff --git a/pdc_config_sys.h.cmake b/pdc_config_sys.h.cmake index 3e5438008..297c48eb4 100644 --- a/pdc_config_sys.h.cmake +++ b/pdc_config_sys.h.cmake @@ -12,12 +12,24 @@ /* Define if you want to enable server cache */ #cmakedefine PDC_SERVER_CACHE +/* Define the max region cache size in GB */ +#cmakedefine PDC_SERVER_CACHE_MAX_GB @PDC_SERVER_CACHE_MAX_GB@ + +/* Define region cache flush time interval */ +#cmakedefine PDC_SERVER_CACHE_FLUSH_TIME @PDC_SERVER_CACHE_FLUSH_TIME@ + /* Define if you want to enable checkpoint */ #cmakedefine PDC_ENABLE_CHECKPOINT /* Define if you want to enable profiling */ #cmakedefine ENABLE_PROFILING +/* Define if you want to enable DART_SUFFIX_TREE mode */ +#cmakedefine PDC_DART_SFX_TREE + +/* Define if you want to enable Julia */ +#cmakedefine PDC_ENABLE_JULIA + /* Define if you want to enable multithread */ #cmakedefine ENABLE_MULTITHREAD diff --git a/scripts/kvtag_add_get_benchmark/cori/clean.sh b/scripts/dart_attr_dist_test/clean.sh similarity index 100% rename from scripts/kvtag_add_get_benchmark/cori/clean.sh rename to scripts/dart_attr_dist_test/clean.sh diff --git a/scripts/dart_attr_dist_test/gen_script.sh b/scripts/dart_attr_dist_test/gen_script.sh new file mode 100755 index 000000000..cd9a18a50 --- /dev/null +++ b/scripts/dart_attr_dist_test/gen_script.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +PROJECT_NAME=$1 + +# Per node configuration of your HPC system. +MAX_PYSICAL_CORE=128 +MAX_HYPERTHREADING=2 + +# Designated number of threads per process on each node +# (this should be associated with -c option in srun) +NUM_THREAD_PER_SERVER_PROC=2 +NUM_THREAD_PER_CLIENT_PROC=2 + + +# Designated number of processes for server anc client on each node +# (this should be associated with -n option in srun) +NUM_SERVER_PROC_PER_NODE=1 +NUM_CLIENT_PROC_PER_NODE=64 + + +MAX_NODE=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +PROG_BASENAME=dart_attr_dist_test + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + mkdir -p $i + JOBNAME=${PROG_BASENAME}_${i} + TARGET=./$i/$JOBNAME.sbatch + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + sed -i "s/MPHYSICALCORE/${MAX_PYSICAL_CORE}/g" $TARGET + sed -i "s/MHYPERTHREADING/${MAX_HYPERTHREADING}/g" $TARGET + sed -i "s/N_SERVER_PROC/${NUM_SERVER_PROC_PER_NODE}/g" $TARGET + sed -i "s/N_CLIENT_PROC/${NUM_CLIENT_PROC_PER_NODE}/g" $TARGET + sed -i "s/NTHREAD_PER_SPROC/${NUM_THREAD_PER_SERVER_PROC}/g" $TARGET + sed -i "s/NTHREAD_PER_CPROC/${NUM_THREAD_PER_CLIENT_PROC}/g" $TARGET + sed -i "s/PROJNAME/${PROJECT_NAME}/g" $TARGET + if [[ "$i" -gt "4" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi +done diff --git a/scripts/kvtag_add_get_benchmark/cori/submit.sh b/scripts/dart_attr_dist_test/submit.sh similarity index 51% rename from scripts/kvtag_add_get_benchmark/cori/submit.sh rename to scripts/dart_attr_dist_test/submit.sh index 2ca6badf8..79d2d04b4 100755 --- a/scripts/kvtag_add_get_benchmark/cori/submit.sh +++ b/scripts/dart_attr_dist_test/submit.sh @@ -7,41 +7,40 @@ MAX_PROC=512 MAX_ATTR=1024 MAX_ATTRLEN=1000 +PROG_BASENAME=llsm_importer + curdir=$(pwd) first_submit=1 + for (( i = 1; i <= $MAX_PROC; i*=2 )); do mkdir -p $i - for (( j = 1; j <= $MAX_ATTR; j*=4 )); do - for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do - JOBNAME=kvtag_bench_${i}_${j}_${k} - TARGET=./$i/JOBNAME.sh - - njob=`squeue -u $USER | grep kvtag_bench | wc -l` - echo $njob - while [ $njob -ge 4 ] - do - sleeptime=$[ ( $RANDOM % 1000 ) ] - sleep $sleeptime - njob=`squeue -u $USER | grep kvtag_bench | wc -l` - echo $njob - done - - if [[ $first_submit == 1 ]]; then - # Submit first job w/o dependency - echo "Submitting $TARGET" - job=`sbatch $TARGET` - first_submit=0 - else - echo "Submitting $TARGET after ${job: -8}" - job=`sbatch -d afterany:${job: -8} $TARGET` - fi - - sleeptime=$[ ( $RANDOM % 5 ) ] - sleep $sleeptime - done + JOBNAME=${PROG_BASENAME}_${i} + TARGET=./$i/JOBNAME.sh + + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` + echo $njob + while [ $njob -ge 4 ] + do + sleeptime=$[ ( $RANDOM % 1000 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` + echo $njob done + + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi + + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime done diff --git a/scripts/dart_attr_dist_test/template.sh b/scripts/dart_attr_dist_test/template.sh new file mode 100755 index 000000000..76e535476 --- /dev/null +++ b/scripts/dart_attr_dist_test/template.sh @@ -0,0 +1,103 @@ +#!/bin/bash -l + +#REGSBATCH -q regular +#DBGSBATCH -q debug +#SBATCH -N NODENUM +#REGSBATCH -t 1:00:00 +#DBGSBATCH -t 0:30:00 +#SBATCH -C cpu +#SBATCH -J JOBNAME +#SBATCH -A PROJNAME +#SBATCH -o o%j.JOBNAME.out +#SBATCH -e o%j.JOBNAME.out + +# export PDC_DEBUG=0 + +# This is a script for running PDC in shared mode on Perlmutter +# When running in Shared mode, the client processes and server processes are running on the same node. +# By alternating the number of server processes and the number client processes, you should be able to change the C/S ratio. +# You can simply set the number of server processes, and let the script to calculate the number of client processes. + +# Per node configuration of your HPC system. +MAX_PYSICAL_CORE=MPHYSICALCORE +MAX_HYPERTHREADING=MHYPERTHREADING + +# Designated number of threads per process on each node +# (this should be associated with -c option in srun) +NUM_THREAD_PER_SERVER_PROC=NTHREAD_PER_SPROC +NUM_THREAD_PER_CLIENT_PROC=NTHREAD_PER_CPROC + + +# Designated number of processes for server anc client on each node +# (this should be associated with -n option in srun) +NUM_SERVER_PROC_PER_NODE=N_SERVER_PROC +NUM_CLIENT_PROC_PER_NODE=N_CLIENT_PROC + +# test if the number of threads is no larger than the total number of logical cores +TOTAL_NUM_PROC_PER_NODE=$((NUM_THREAD_PER_SERVER_PROC * NUM_SERVER_PROC_PER_NODE + NUM_THREAD_PER_CLIENT_PROC * NUM_CLIENT_PROC_PER_NODE)) +TOTAL_NUM_LOGICAL_CORE_PER_NODE=$((MAX_PYSICAL_CORE * MAX_HYPERTHREADING)) +if [[ "$TOTAL_NUM_PROC_PER_NODE" -gt "$TOTAL_NUM_LOGICAL_CORE_PER_NODE" ]]; then + echo "Error: TOTAL_NUM_PROC_PER_NODE is larger than TOTAL_NUM_LOGICAL_CORE_PER_NODE" + TOTAL_AVAILABLE_CORE=$((TOTAL_NUM_LOGICAL_CORE_PER_NODE - NUM_THREAD_PER_SERVER_PROC * NUM_SERVER_PROC_PER_NODE)) + NUM_CLIENT_PROC_PER_NODE=$(( TOTAL_AVAILABLE_CORE / NUM_THREAD_PER_CLIENT_PROC)) + echo "fixing the number of client processes to $NUM_CLIENT_PROC_PER_NODE" +fi + +# Set the number of times the test should be repeated. +REPEAT=1 + +# calculate the number of total processes for both server side and client side. +N_NODE=NODENUM +NCLIENT=$((NUM_CLIENT_PROC_PER_NODE * N_NODE)) +NSERVER=$((NUM_SERVER_PROC_PER_NODE * N_NODE)) + +# clean up the PDC tmp directory +export PDC_TMPDIR=$SCRATCH/data/pdc/conf +rm -rf $PDC_TMPDIR/* +export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE +mkdir -p $PDC_TMPDIR + +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin +TOOLPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin +SERVER=$EXECPATH/pdc_server.exe +CLIENT=$TOOLPATH/dart_attr_dist_test +CLOSE=$EXECPATH/close_server + +chmod +x $EXECPATH/* +chmod +x $TOOLPATH/* + +date + +# OpenMP settings: +# set the OPENMP thread number to the smaller number between $NUM_THREAD_PER_SERVER_PROC and $NUM_THREAD_PER_CLIENT_PROC +export OMP_NUM_THREADS=$((NUM_THREAD_PER_SERVER_PROC < NUM_THREAD_PER_CLIENT_PROC ? NUM_THREAD_PER_SERVER_PROC : NUM_THREAD_PER_CLIENT_PROC)) +export OMP_PLACES=threads +export OMP_PROC_BIND=close + +echo "OMP_NUM_THREADS=$OMP_NUM_THREADS" +echo "NSERVER=$NSERVER" +echo "NUM_THREAD_PER_SERVER_PROC=$NUM_THREAD_PER_SERVER_PROC" +echo "NCLIENT=$NCLIENT" +echo "NUM_THREAD_PER_CLIENT_PROC=$NUM_THREAD_PER_CLIENT_PROC" + + +echo "" +echo "=============" +echo "$i Init server" +echo "=============" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $NSERVER -c $NUM_THREAD_PER_SERVER_PROC --cpu_bind=cores $SERVER & +sleep 5 + + +echo "============================================" +echo "KVTAGS with $N_NODE nodes" +echo "============================================" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $NCLIENT -c $NUM_THREAD_PER_CLIENT_PROC --cpu_bind=cores $CLIENT + +echo "" +echo "=================" +echo "$i Closing server" +echo "=================" +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE + +date diff --git a/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh b/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh deleted file mode 100755 index 62eb1a2b2..000000000 --- a/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -N_THREAD=NO -MAX_NODE=512 -MAX_ATTR=1024 -MAX_ATTRLEN=1000 - -for (( i = 1; i <= $MAX_NODE; i*=2 )); do - mkdir -p $i - for (( j = 1; j <= $MAX_ATTR; j*=4 )); do - for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do - JOBNAME=kvtag_bench_${i}_${j}_${k} - TARGET=./$i/$JOBNAME.sbatch - cp template.sh $TARGET - sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET - sed -i "s/NODENUM/${i}/g" $TARGET - sed -i "s/ATTRNUM/${j}/g" $TARGET - sed -i "s/ATTRLEN/${k}/g" $TARGET - if [[ "$i" -gt "16" ]]; then - sed -i "s/REG//g" $TARGET - else - sed -i "s/DBG//g" $TARGET - fi - done - done -done diff --git a/scripts/kvtag_add_get_benchmark/cori/template.sh b/scripts/kvtag_add_get_benchmark/cori/template.sh deleted file mode 100755 index 1db2938f3..000000000 --- a/scripts/kvtag_add_get_benchmark/cori/template.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -l - -#REGSBATCH -q regular -#DBGSBATCH -q debug -#SBATCH -N NODENUM -#REGSBATCH -t 4:00:00 -#DBGSBATCH -t 0:30:00 -#SBATCH --gres=craynetwork:2 -#SBATCH -L SCRATCH -#SBATCH -C haswell -#SBATCH -J JOBNAME -#SBATCH -A m2621 -#SBATCH -o o%j.JOBNAME.out -#SBATCH -e o%j.JOBNAME.out - - -# export PDC_DEBUG=0 - -export PDC_TMPDIR=$SCRATCH/data/pdc/conf - -rm -rf $PDC_TMPDIR/* - -REPEAT=1 - -N_NODE=NODENUM -NCLIENT=31 - -export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE -mkdir -p $PDC_TMPDIR - -let TOTALPROC=$NCLIENT*$N_NODE - -EXECPATH=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/share/test/bin -SERVER=$EXECPATH/pdc_server.exe -CLIENT=$EXECPATH/kvtag_add_get_benchmark -CLOSE=$EXECPATH/close_server - -chmod +x $EXECPATH/* - -MAX_OBJ_COUNT=$((1024*1024*1024)) -OBJ_INCR=$((MAX_OBJ_COUNT/1024)) -ATTR_COUNT=ATTRNUM -ATTR_LENGTH=ATTRLEN -QUERY_COUNT=$((OBJ_INCR)) - -date - - -echo "" -echo "=============" -echo "$i Init server" -echo "=============" -srun -N $N_NODE -n $N_NODE -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap stdbuf -i0 -o0 -e0 $SERVER & -sleep 5 - - -echo "============================================" -echo "KVTAGS with $N_NODE nodes" -echo "============================================" -srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap stdbuf -i0 -o0 -e0 $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE - -echo "" -echo "=================" -echo "$i Closing server" -echo "=================" -srun -N 1 -n 1 -c 2 --mem=25600 --gres=craynetwork:1 --cpu_bind=cores --overlap stdbuf -i0 -o0 -e0 $CLOSE - -date diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh b/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh index 62eb1a2b2..c84c2f10f 100755 --- a/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh +++ b/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh @@ -2,7 +2,9 @@ N_THREAD=NO MAX_NODE=512 MAX_ATTR=1024 -MAX_ATTRLEN=1000 +MAX_ATTRLEN=1000000 + +PROJECT_NAME=$1 for (( i = 1; i <= $MAX_NODE; i*=2 )); do mkdir -p $i @@ -15,6 +17,7 @@ for (( i = 1; i <= $MAX_NODE; i*=2 )); do sed -i "s/NODENUM/${i}/g" $TARGET sed -i "s/ATTRNUM/${j}/g" $TARGET sed -i "s/ATTRLEN/${k}/g" $TARGET + sed -i "s/PROJNAME/${PROJECT_NAME}/g" $TARGET if [[ "$i" -gt "16" ]]; then sed -i "s/REG//g" $TARGET else diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh index 9c69b9872..cccb986ca 100755 --- a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh +++ b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh @@ -3,11 +3,11 @@ #REGSBATCH -q regular #DBGSBATCH -q debug #SBATCH -N NODENUM -#REGSBATCH -t 4:00:00 +#REGSBATCH -t 1:00:00 #DBGSBATCH -t 0:30:00 #SBATCH -C cpu #SBATCH -J JOBNAME -#SBATCH -A m2621 +#SBATCH -A PROJNAME #SBATCH -o o%j.JOBNAME.out #SBATCH -e o%j.JOBNAME.out @@ -21,6 +21,7 @@ rm -rf $PDC_TMPDIR/* REPEAT=1 N_NODE=NODENUM +# NCLIENT=127 NCLIENT=31 export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE @@ -35,7 +36,7 @@ CLOSE=$EXECPATH/close_server chmod +x $EXECPATH/* -MAX_OBJ_COUNT=$((1024*1024*1024)) +MAX_OBJ_COUNT=$((1024*1024)) OBJ_INCR=$((MAX_OBJ_COUNT/1024)) ATTR_COUNT=ATTRNUM ATTR_LENGTH=ATTRLEN @@ -48,19 +49,19 @@ echo "" echo "=============" echo "$i Init server" echo "=============" -srun -N $N_NODE -n $N_NODE -c 2 --mem=100000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $SERVER & +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*1)) -c 2 --cpu_bind=cores $SERVER & sleep 5 echo "============================================" echo "KVTAGS with $N_NODE nodes" echo "============================================" -srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $TOTALPROC -c 2 --cpu_bind=cores $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE echo "" echo "=================" echo "$i Closing server" echo "=================" -srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLOSE +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE date diff --git a/scripts/kvtag_add_get_scale/cori/gen_scripts.sh b/scripts/kvtag_add_get_scale/cori/gen_scripts.sh deleted file mode 100755 index e8ee7b11d..000000000 --- a/scripts/kvtag_add_get_scale/cori/gen_scripts.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -N_THREAD=NO -MAX_NODE=512 -MAX_ATTR=1024 -MAX_ATTRLEN=1000 - -for (( i = 1; i <= $MAX_NODE; i*=2 )); do - mkdir -p $i - JOBNAME=kvtag_scale${i} - TARGET=./$i/$JOBNAME.sbatch - cp template.sh $TARGET - sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET - sed -i "s/NODENUM/${i}/g" $TARGET - if [[ "$i" -gt "16" ]]; then - sed -i "s/REG//g" $TARGET - else - sed -i "s/DBG//g" $TARGET - fi -done diff --git a/scripts/kvtag_add_get_scale/cori/template.sh b/scripts/kvtag_add_get_scale/cori/template.sh deleted file mode 100755 index 40a4dca34..000000000 --- a/scripts/kvtag_add_get_scale/cori/template.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -l - -#REGSBATCH -q regular -#DBGSBATCH -q debug -#SBATCH -N NODENUM -#REGSBATCH -t 4:00:00 -#DBGSBATCH -t 0:30:00 -#SBATCH --gres=craynetwork:2 -#SBATCH -L SCRATCH -#SBATCH -C haswell -#SBATCH -J JOBNAME -#SBATCH -A m2621 -#SBATCH -o o%j.JOBNAME.out -#SBATCH -e o%j.JOBNAME.out - - -# export PDC_DEBUG=0 - -export PDC_TMPDIR=$SCRATCH/data/pdc/conf - -rm -rf $PDC_TMPDIR/* - -REPEAT=1 - -N_NODE=NODENUM -NCLIENT=31 - -export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE -mkdir -p $PDC_TMPDIR - -let TOTALPROC=$NCLIENT*$N_NODE - -EXECPATH=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/share/test/bin -SERVER=$EXECPATH/pdc_server.exe -CLIENT=$EXECPATH/kvtag_add_get_scale -CLOSE=$EXECPATH/close_server - -chmod +x $EXECPATH/* - -NUM_OBJ=$((1024*1024*100)) -NUM_TAGS=$NUM_OBJ -NUM_QUERY=$((NUM_OBJ)) - -date - - -echo "" -echo "=============" -echo "$i Init server" -echo "=============" -srun -N $N_NODE -n $N_NODE -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap stdbuf -i0 -o0 -e0 $SERVER & -sleep 5 - - -echo "============================================" -echo "KVTAGS with $N_NODE nodes" -echo "============================================" -srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap stdbuf -i0 -o0 -e0 $CLIENT $NUM_OBJ $NUM_TAGS $NUM_QUERY - -echo "" -echo "=================" -echo "$i Closing server" -echo "=================" -srun -N 1 -n 1 -c 2 --mem=25600 --gres=craynetwork:1 --cpu_bind=cores --overlap stdbuf -i0 -o0 -e0 $CLOSE - -date diff --git a/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh b/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh index 4af771821..12c9b2d4f 100755 --- a/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh +++ b/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh @@ -4,6 +4,8 @@ MAX_NODE=512 MAX_ATTR=1024 MAX_ATTRLEN=1000 +PROJECT_NAME=$1 + for (( i = 1; i <= $MAX_NODE; i*=2 )); do mkdir -p $i JOBNAME=kvtag_scale_${i} @@ -11,6 +13,7 @@ for (( i = 1; i <= $MAX_NODE; i*=2 )); do cp template.sh $TARGET sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET sed -i "s/NODENUM/${i}/g" $TARGET + sed -i "s/PROJNAME/${PROJECT_NAME}/g" $TARGET if [[ "$i" -gt "16" ]]; then sed -i "s/REG//g" $TARGET else diff --git a/scripts/kvtag_add_get_scale/perlmutter/template.sh b/scripts/kvtag_add_get_scale/perlmutter/template.sh index d48cb29f6..3690a94ff 100755 --- a/scripts/kvtag_add_get_scale/perlmutter/template.sh +++ b/scripts/kvtag_add_get_scale/perlmutter/template.sh @@ -3,11 +3,11 @@ #REGSBATCH -q regular #DBGSBATCH -q debug #SBATCH -N NODENUM -#REGSBATCH -t 4:00:00 +#REGSBATCH -t 1:00:00 #DBGSBATCH -t 0:30:00 #SBATCH -C cpu #SBATCH -J JOBNAME -#SBATCH -A m2621 +#SBATCH -A PROJNAME #SBATCH -o o%j.JOBNAME.out #SBATCH -e o%j.JOBNAME.out @@ -21,6 +21,7 @@ REPEAT=1 N_NODE=NODENUM NCLIENT=31 +# NCLIENT=126 export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE mkdir -p $PDC_TMPDIR @@ -45,19 +46,19 @@ echo "" echo "=============" echo "$i Init server" echo "=============" -srun -N $N_NODE -n $N_NODE -c 2 --mem=128000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $SERVER & +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*1)) -c 2 --cpu_bind=cores $SERVER & sleep 5 echo "============================================" echo "KVTAGS with $N_NODE nodes" echo "============================================" -srun -N $N_NODE -n $TOTALPROC -c 2 --mem=256000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLIENT $NUM_OBJ $NUM_TAGS $NUM_QUERY +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $TOTALPROC -c 2 --cpu_bind=cores $CLIENT $NUM_OBJ $NUM_TAGS $NUM_QUERY echo "" echo "=================" echo "$i Closing server" echo "=================" -srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLOSE +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE date diff --git a/scripts/kvtag_add_get_scale/cori/clean.sh b/scripts/kvtag_query_scale/clean.sh similarity index 100% rename from scripts/kvtag_add_get_scale/cori/clean.sh rename to scripts/kvtag_query_scale/clean.sh diff --git a/scripts/kvtag_query_scale/gen_script.sh b/scripts/kvtag_query_scale/gen_script.sh new file mode 100755 index 000000000..9daad526c --- /dev/null +++ b/scripts/kvtag_query_scale/gen_script.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +PROJECT_NAME=$1 + +# Per node configuration of your HPC system. +MAX_PYSICAL_CORE=128 +MAX_HYPERTHREADING=2 + +# Designated number of threads per process on each node +# (this should be associated with -c option in srun) +NUM_THREAD_PER_SERVER_PROC=2 +NUM_THREAD_PER_CLIENT_PROC=2 + + +# Designated number of processes for server anc client on each node +# (this should be associated with -n option in srun) +NUM_SERVER_PROC_PER_NODE=1 +NUM_CLIENT_PROC_PER_NODE=1 + + +MAX_NODE=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +PROG_BASENAME=kvtag_query_scale + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + for (( j = 0; j <= 1; j+=1 )); do + mkdir -p $i + JOBNAME=${PROG_BASENAME}_${i} + TARGET=./$i/$JOBNAME.sbatch.${j} + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + sed -i "s/MPHYSICALCORE/${MAX_PYSICAL_CORE}/g" $TARGET + sed -i "s/MHYPERTHREADING/${MAX_HYPERTHREADING}/g" $TARGET + sed -i "s/N_SERVER_PROC/${NUM_SERVER_PROC_PER_NODE}/g" $TARGET + sed -i "s/N_CLIENT_PROC/${NUM_CLIENT_PROC_PER_NODE}/g" $TARGET + sed -i "s/NTHREAD_PER_SPROC/${NUM_THREAD_PER_SERVER_PROC}/g" $TARGET + sed -i "s/NTHREAD_PER_CPROC/${NUM_THREAD_PER_CLIENT_PROC}/g" $TARGET + sed -i "s/PROJNAME/${PROJECT_NAME}/g" $TARGET + sed -i "s/USING_DART/${j}/g" $TARGET + if [[ "$i" -gt "4" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi + done +done diff --git a/scripts/kvtag_add_get_scale/cori/submit.sh b/scripts/kvtag_query_scale/submit.sh similarity index 89% rename from scripts/kvtag_add_get_scale/cori/submit.sh rename to scripts/kvtag_query_scale/submit.sh index 192522e31..b9019d149 100755 --- a/scripts/kvtag_add_get_scale/cori/submit.sh +++ b/scripts/kvtag_query_scale/submit.sh @@ -7,22 +7,24 @@ MAX_PROC=512 MAX_ATTR=1024 MAX_ATTRLEN=1000 +PROG_BASENAME=llsm_importer + curdir=$(pwd) first_submit=1 for (( i = 1; i <= $MAX_PROC; i*=2 )); do mkdir -p $i - JOBNAME=kvtag_scale_${i} + JOBNAME=${PROG_BASENAME}_${i} TARGET=./$i/JOBNAME.sh - njob=`squeue -u $USER | grep kvtag_scale | wc -l` + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` echo $njob while [ $njob -ge 4 ] do sleeptime=$[ ( $RANDOM % 1000 ) ] sleep $sleeptime - njob=`squeue -u $USER | grep kvtag_scale | wc -l` + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` echo $njob done diff --git a/scripts/kvtag_query_scale/template.sh b/scripts/kvtag_query_scale/template.sh new file mode 100755 index 000000000..b9bad3bdf --- /dev/null +++ b/scripts/kvtag_query_scale/template.sh @@ -0,0 +1,105 @@ +#!/bin/bash -l + +#REGSBATCH -q regular +#DBGSBATCH -q debug +#SBATCH -N NODENUM +#REGSBATCH -t 1:00:00 +#DBGSBATCH -t 0:30:00 +#SBATCH -C cpu +#SBATCH -J JOBNAME +#SBATCH -A PROJNAME +#SBATCH -o o%j.JOBNAME.out +#SBATCH -e o%j.JOBNAME.out + +# export PDC_DEBUG=0 + +# This is a script for running PDC in shared mode on Perlmutter +# When running in Shared mode, the client processes and server processes are running on the same node. +# By alternating the number of server processes and the number client processes, you should be able to change the C/S ratio. +# You can simply set the number of server processes, and let the script to calculate the number of client processes. + +# Per node configuration of your HPC system. +MAX_PYSICAL_CORE=MPHYSICALCORE +MAX_HYPERTHREADING=MHYPERTHREADING + +# Designated number of threads per process on each node +# (this should be associated with -c option in srun) +NUM_THREAD_PER_SERVER_PROC=NTHREAD_PER_SPROC +NUM_THREAD_PER_CLIENT_PROC=NTHREAD_PER_CPROC + + +# Designated number of processes for server anc client on each node +# (this should be associated with -n option in srun) +NUM_SERVER_PROC_PER_NODE=N_SERVER_PROC +NUM_CLIENT_PROC_PER_NODE=N_CLIENT_PROC + +# test if the number of threads is no larger than the total number of logical cores +TOTAL_NUM_PROC_PER_NODE=$((NUM_THREAD_PER_SERVER_PROC * NUM_SERVER_PROC_PER_NODE + NUM_THREAD_PER_CLIENT_PROC * NUM_CLIENT_PROC_PER_NODE)) +TOTAL_NUM_LOGICAL_CORE_PER_NODE=$((MAX_PYSICAL_CORE * MAX_HYPERTHREADING)) +if [[ "$TOTAL_NUM_PROC_PER_NODE" -gt "$TOTAL_NUM_LOGICAL_CORE_PER_NODE" ]]; then + echo "Error: TOTAL_NUM_PROC_PER_NODE is larger than TOTAL_NUM_LOGICAL_CORE_PER_NODE" + TOTAL_AVAILABLE_CORE=$((TOTAL_NUM_LOGICAL_CORE_PER_NODE - NUM_THREAD_PER_SERVER_PROC * NUM_SERVER_PROC_PER_NODE)) + NUM_CLIENT_PROC_PER_NODE=$(( TOTAL_AVAILABLE_CORE / NUM_THREAD_PER_CLIENT_PROC)) + echo "fixing the number of client processes to $NUM_CLIENT_PROC_PER_NODE" +fi + +# Set the number of times the test should be repeated. +REPEAT=1 + +# calculate the number of total processes for both server side and client side. +N_NODE=NODENUM +NCLIENT=$((NUM_CLIENT_PROC_PER_NODE * N_NODE)) +NSERVER=$((NUM_SERVER_PROC_PER_NODE * N_NODE)) + +USE_DART=USING_DART + +# clean up the PDC tmp directory +export PDC_TMPDIR=$SCRATCH/data/pdc/conf +rm -rf $PDC_TMPDIR/* +export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE/$USE_DART +mkdir -p $PDC_TMPDIR + +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin +TOOLPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin +SERVER=$EXECPATH/pdc_server.exe +CLIENT=$TOOLPATH/kvtag_query_scale +CLOSE=$EXECPATH/close_server + +chmod +x $EXECPATH/* +chmod +x $TOOLPATH/* + +date + +# OpenMP settings: +# set the OPENMP thread number to the smaller number between $NUM_THREAD_PER_SERVER_PROC and $NUM_THREAD_PER_CLIENT_PROC +export OMP_NUM_THREADS=$((NUM_THREAD_PER_SERVER_PROC < NUM_THREAD_PER_CLIENT_PROC ? NUM_THREAD_PER_SERVER_PROC : NUM_THREAD_PER_CLIENT_PROC)) +export OMP_PLACES=threads +export OMP_PROC_BIND=close + +echo "OMP_NUM_THREADS=$OMP_NUM_THREADS" +echo "NSERVER=$NSERVER" +echo "NUM_THREAD_PER_SERVER_PROC=$NUM_THREAD_PER_SERVER_PROC" +echo "NCLIENT=$NCLIENT" +echo "NUM_THREAD_PER_CLIENT_PROC=$NUM_THREAD_PER_CLIENT_PROC" + + +echo "" +echo "=============" +echo "$i Init server" +echo "=============" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $NSERVER -c $NUM_THREAD_PER_SERVER_PROC --cpu_bind=cores $SERVER & +sleep 5 + + +echo "============================================" +echo "KVTAGS with $N_NODE nodes" +echo "============================================" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $NCLIENT -c $NUM_THREAD_PER_CLIENT_PROC --cpu_bind=cores $CLIENT 1000000 100 10 $USE_DART + +echo "" +echo "=================" +echo "$i Closing server" +echo "=================" +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE + +date diff --git a/scripts/kvtag_query_scale_mpi/clean.sh b/scripts/kvtag_query_scale_mpi/clean.sh new file mode 100755 index 000000000..0108aceb6 --- /dev/null +++ b/scripts/kvtag_query_scale_mpi/clean.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +if [[ "$#" -ne 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +# test if $1 is a single digit number between 0 and 1 (inclusive) +if [[ "$1" =~ ^[0-1]$ ]]; then + echo "Error: clean_dir should be a single digit number between 0 and 1 (inclusive). 1 means clean the directories named with numbers, 0 means clean the sbatch script only." + exit 1 +fi + +CLEAN_DIR=$1 +MAX_NODE=512 + +# if CLEAN_DIR is set to '1', then clean all the directories named with numbers, otherwise, clean the sbatch script only +find ./ -name "*.sbatch*" -delete + +if [[ "$CLEAN_DIR" -eq "1" ]]; then + for (( i = 1; i <= $MAX_NODE; i*=2 )); do + + rm -rf $i/* + + done +fi \ No newline at end of file diff --git a/scripts/kvtag_query_scale_mpi/gen_script.sh b/scripts/kvtag_query_scale_mpi/gen_script.sh new file mode 100755 index 000000000..3a1765c02 --- /dev/null +++ b/scripts/kvtag_query_scale_mpi/gen_script.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +if [[ "$#" -ne 2 ]]; then + echo "Usage: $0 " + exit 1 +fi + +# test if $1 is a number +re='^[0-9]+$' +if ! [[ $1 =~ $re ]] ; then + echo "Error: min_node is not a number" >&2; exit 1 +fi + +if [[ "$1" -lt "1" ]]; then + echo "Error: min_node should be larger than 0" + exit 1 +fi + +if [[ "$1" -gt "512" ]]; then + echo "Error: min_node should be smaller than 512" + exit 1 +fi + +MIN_NODE=$1 +MAX_NODE=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + + + +# Per node configuration of your HPC system. +MAX_PYSICAL_CORE=128 +MAX_HYPERTHREADING=2 + +# Designated number of threads per process on each node +# (this should be associated with -c option in srun) +NUM_THREAD_PER_SERVER_PROC=2 +NUM_THREAD_PER_CLIENT_PROC=2 + +# test if $2 is a string starting with a letter, and followed by a 4-digit number +re='^[a-zA-Z][0-9]{4}$' +if ! [[ $2 =~ $re ]] ; then + echo "Error: proj_name should be a string starting with a letter, and followed by a 4-digit number, e.g. m2021" >&2; exit 1 +fi + +PROJECT_NAME=$2 + + +# Designated number of processes for server anc client on each node +# (this should be associated with -n option in srun) +TOTAL_NUM_CLIENT_PROC=$((128 * MIN_NODE)) +NUM_SERVER_PROC_PER_NODE=1 +NUM_CLIENT_PROC_PER_NODE=$((TOTAL_NUM_CLIENT_PROC)) + + +PROG_BASENAME=kvqry + +for (( i = $MIN_NODE; i <= $MAX_NODE; i*=2 )); do + mkdir -p $i + NUM_CLIENT_PROC_PER_NODE=$((TOTAL_NUM_CLIENT_PROC/i)) + AVAIL_CLIENT_THREAD_CORES=$((MAX_PYSICAL_CORE * MAX_HYPERTHREADING - NUM_SERVER_PROC_PER_NODE * NUM_THREAD_PER_SERVER_PROC)) + AVAIL_CLIENT_PHYSICAL_CORES=$((AVAIL_CLIENT_THREAD_CORES / NUM_THREAD_PER_CLIENT_PROC)) + if [[ $(( NUM_CLIENT_PROC_PER_NODE > AVAIL_CLIENT_PHYSICAL_CORES )) -eq 1 ]]; then + NUM_CLIENT_PROC_PER_NODE=$((AVAIL_CLIENT_PHYSICAL_CORES - 2)) + fi + for (( j = 0; j <= 1; j+=1 )); do + for (( q = 0; q < 4; q+=1 )); do + for (( c = 0; c < 2; c+=1 )); do + JOBNAME=${PROG_BASENAME}_${i}_${j}_${q}_${c} + TARGET=./$i/$JOBNAME.sbatch + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + sed -i "s/MPHYSICALCORE/${MAX_PYSICAL_CORE}/g" $TARGET + sed -i "s/MHYPERTHREADING/${MAX_HYPERTHREADING}/g" $TARGET + sed -i "s/N_SERVER_PROC/${NUM_SERVER_PROC_PER_NODE}/g" $TARGET + sed -i "s/N_CLIENT_PROC/${NUM_CLIENT_PROC_PER_NODE}/g" $TARGET + sed -i "s/NTHREAD_PER_SPROC/${NUM_THREAD_PER_SERVER_PROC}/g" $TARGET + sed -i "s/NTHREAD_PER_CPROC/${NUM_THREAD_PER_CLIENT_PROC}/g" $TARGET + sed -i "s/PROJNAME/${PROJECT_NAME}/g" $TARGET + sed -i "s/USING_DART/${j}/g" $TARGET + sed -i "s/QUERY_TYPE/${q}/g" $TARGET + sed -i "s/COMMUNICATION_TYPE/${c}/g" $TARGET + if [[ "$i" -gt "4" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi + done + done + done +done diff --git a/scripts/kvtag_query_scale_mpi/submit.sh b/scripts/kvtag_query_scale_mpi/submit.sh new file mode 100755 index 000000000..978110349 --- /dev/null +++ b/scripts/kvtag_query_scale_mpi/submit.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +MIN_PROC=16 +MAX_PROC=128 + +PROG_BASENAME=kvqry + +curdir=$(pwd) + +first_submit=1 + +if [[ "$#" -ne 3 ]]; then + echo "Usage: $0 " + exit 1 +fi + +# test if $1 is a single digit number between 0 and 1 (inclusive) +re='^[0-1]$' +if ! [[ "$1" =~ $re ]]; then + echo "Error: i_type should be a single digit number between 0 and 1 (inclusive), 0 means not using index, 1 means using index" + exit 1 +fi + +# test if $2 is a single digit number between 0 and 3 (inclusive) +re='^[0-3]$' +if ! [[ "$2" =~ $re ]]; then + echo "Error: q_type should be a single digit number between 0 and 3 (inclusive), 0: exact query, 1: prefix query, 2: suffix query, 3: infix query" + exit 1 +fi + +# test if $3 is a single digit number between 0 and 1 (inclusive) +re='^[0-1]$' +if ! [[ "$3" =~ $re ]]; then + echo "Error: c_type should be a single digit number between 0 and 1 (inclusive), 0 means using non-collective mode, 1 means using collective mode" + exit 1 +fi + + +i_type=$1 +q_type=$2 +c_type=$3 + +for (( i = $MIN_PROC; i <= $MAX_PROC; i*=2 )); do + + cd $curdir/$i + + JOBNAME=${PROG_BASENAME}_${i}_${i_type}_${q_type}_${c_type} + TARGET=$JOBNAME.sbatch + + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` + echo $njob + while [ $njob -ge 16 ] + do + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` + echo $njob + done + + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi + + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime + +done diff --git a/scripts/kvtag_query_scale_mpi/template.sh b/scripts/kvtag_query_scale_mpi/template.sh new file mode 100755 index 000000000..a6b57511d --- /dev/null +++ b/scripts/kvtag_query_scale_mpi/template.sh @@ -0,0 +1,108 @@ +#!/bin/bash -l + +#REGSBATCH -q regular +#DBGSBATCH -q debug +#SBATCH -N NODENUM +#REGSBATCH -t 2:00:00 +#DBGSBATCH -t 0:30:00 +#SBATCH -C cpu +#SBATCH -J JOBNAME +#SBATCH -A PROJNAME +#SBATCH -o o%j.JOBNAME.out +#SBATCH -e o%j.JOBNAME.out + +# export PDC_DEBUG=0 + +# This is a script for running PDC in shared mode on Perlmutter +# When running in Shared mode, the client processes and server processes are running on the same node. +# By alternating the number of server processes and the number client processes, you should be able to change the C/S ratio. +# You can simply set the number of server processes, and let the script to calculate the number of client processes. + +# Per node configuration of your HPC system. +MAX_PYSICAL_CORE=MPHYSICALCORE +MAX_HYPERTHREADING=MHYPERTHREADING + +# Designated number of threads per process on each node +# (this should be associated with -c option in srun) +NUM_THREAD_PER_SERVER_PROC=NTHREAD_PER_SPROC +NUM_THREAD_PER_CLIENT_PROC=NTHREAD_PER_CPROC + + +# Designated number of processes for server anc client on each node +# (this should be associated with -n option in srun) +NUM_SERVER_PROC_PER_NODE=N_SERVER_PROC +NUM_CLIENT_PROC_PER_NODE=N_CLIENT_PROC + +# test if the number of threads is no larger than the total number of logical cores +TOTAL_NUM_PROC_PER_NODE=$((NUM_THREAD_PER_SERVER_PROC * NUM_SERVER_PROC_PER_NODE + NUM_THREAD_PER_CLIENT_PROC * NUM_CLIENT_PROC_PER_NODE)) +TOTAL_NUM_LOGICAL_CORE_PER_NODE=$((MAX_PYSICAL_CORE * MAX_HYPERTHREADING)) +if [[ "$TOTAL_NUM_PROC_PER_NODE" -gt "$TOTAL_NUM_LOGICAL_CORE_PER_NODE" ]]; then + echo "Error: TOTAL_NUM_PROC_PER_NODE is larger than TOTAL_NUM_LOGICAL_CORE_PER_NODE" + TOTAL_AVAILABLE_CORE=$((TOTAL_NUM_LOGICAL_CORE_PER_NODE - NUM_THREAD_PER_SERVER_PROC * NUM_SERVER_PROC_PER_NODE)) + NUM_CLIENT_PROC_PER_NODE=$(( TOTAL_AVAILABLE_CORE / NUM_THREAD_PER_CLIENT_PROC)) + echo "fixing the number of client processes to $NUM_CLIENT_PROC_PER_NODE" +fi + +# Set the number of times the test should be repeated. +REPEAT=1 + +# calculate the number of total processes for both server side and client side. +N_NODE=NODENUM +NCLIENT=$((NUM_CLIENT_PROC_PER_NODE * N_NODE)) +NSERVER=$((NUM_SERVER_PROC_PER_NODE * N_NODE)) + +USE_DART=USING_DART +Q_TYPE=QUERY_TYPE +COM_TYPE=COMMUNICATION_TYPE + +# clean up the PDC tmp directory +export PDC_TMPDIR=$SCRATCH/data/pdc/conf +export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE/$USE_DART/$Q_TYPE/$COM_TYPE +rm -rf $PDC_TMPDIR/* +mkdir -p $PDC_TMPDIR + +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin +TOOLPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin +SERVER=$EXECPATH/pdc_server.exe +CLIENT=$TOOLPATH/kvtag_query_scale_col +CLOSE=$EXECPATH/close_server + +chmod +x $EXECPATH/* +chmod +x $TOOLPATH/* + +date + +# OpenMP settings: +# set the OPENMP thread number to the smaller number between $NUM_THREAD_PER_SERVER_PROC and $NUM_THREAD_PER_CLIENT_PROC +export OMP_NUM_THREADS=$((NUM_THREAD_PER_SERVER_PROC < NUM_THREAD_PER_CLIENT_PROC ? NUM_THREAD_PER_SERVER_PROC : NUM_THREAD_PER_CLIENT_PROC)) +export OMP_PLACES=threads +export OMP_PROC_BIND=close + +echo "OMP_NUM_THREADS=$OMP_NUM_THREADS" +echo "NSERVER=$NSERVER" +echo "NUM_THREAD_PER_SERVER_PROC=$NUM_THREAD_PER_SERVER_PROC" +echo "NCLIENT=$NCLIENT" +echo "NUM_THREAD_PER_CLIENT_PROC=$NUM_THREAD_PER_CLIENT_PROC" + +export ATP_ENABLED=1 + +echo "" +echo "=============" +echo "$i Init server" +echo "=============" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $NSERVER -c $NUM_THREAD_PER_SERVER_PROC --cpu_bind=cores $SERVER & +sleep 5 + + +echo "============================================" +echo "KVTAGS with $N_NODE nodes" +echo "============================================" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $NCLIENT -c $NUM_THREAD_PER_CLIENT_PROC --cpu_bind=cores $CLIENT 1000000 100 10 $USE_DART $Q_TYPE $COM_TYPE + +echo "" +echo "=================" +echo "$i Closing server" +echo "=================" +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE + +date diff --git a/scripts/llsm_importer/clean.sh b/scripts/llsm_importer/clean.sh new file mode 100755 index 000000000..d6476962e --- /dev/null +++ b/scripts/llsm_importer/clean.sh @@ -0,0 +1,8 @@ +#!/bin/bash +MAX_NODE=512 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + + rm -rf $i/* + +done \ No newline at end of file diff --git a/scripts/llsm_importer/gen_script.sh b/scripts/llsm_importer/gen_script.sh new file mode 100755 index 000000000..7ae9069c5 --- /dev/null +++ b/scripts/llsm_importer/gen_script.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +PROJECT_NAME=$1 + +# Per node configuration of your HPC system. +MAX_PYSICAL_CORE=128 +MAX_HYPERTHREADING=2 + +# Designated number of threads per process on each node +# (this should be associated with -c option in srun) +NUM_THREAD_PER_SERVER_PROC=64 +NUM_THREAD_PER_CLIENT_PROC=64 + + +# Designated number of processes for server anc client on each node +# (this should be associated with -n option in srun) +NUM_SERVER_PROC_PER_NODE=1 +NUM_CLIENT_PROC_PER_NODE=1 + + +MAX_NODE=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +PROG_BASENAME=llsm_importer + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + mkdir -p $i + JOBNAME=${PROG_BASENAME}_${i} + TARGET=./$i/$JOBNAME.sbatch + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + sed -i "s/MPHYSICALCORE/${MAX_PYSICAL_CORE}/g" $TARGET + sed -i "s/MHYPERTHREADING/${MAX_HYPERTHREADING}/g" $TARGET + sed -i "s/N_SERVER_PROC/${NUM_SERVER_PROC_PER_NODE}/g" $TARGET + sed -i "s/N_CLIENT_PROC/${NUM_CLIENT_PROC_PER_NODE}/g" $TARGET + sed -i "s/NTHREAD_PER_SPROC/${NUM_THREAD_PER_SERVER_PROC}/g" $TARGET + sed -i "s/NTHREAD_PER_CPROC/${NUM_THREAD_PER_CLIENT_PROC}/g" $TARGET + sed -i "s/PROJNAME/${PROJECT_NAME}/g" $TARGET + if [[ "$i" -gt "4" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi +done diff --git a/scripts/llsm_importer/submit.sh b/scripts/llsm_importer/submit.sh new file mode 100755 index 000000000..b9019d149 --- /dev/null +++ b/scripts/llsm_importer/submit.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# MIN_PROC=4 +# MAX_PROC=128 +MIN_PROC=1 +MAX_PROC=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +PROG_BASENAME=llsm_importer + +curdir=$(pwd) + +first_submit=1 + +for (( i = 1; i <= $MAX_PROC; i*=2 )); do + mkdir -p $i + JOBNAME=${PROG_BASENAME}_${i} + TARGET=./$i/JOBNAME.sh + + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` + echo $njob + while [ $njob -ge 4 ] + do + sleeptime=$[ ( $RANDOM % 1000 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` + echo $njob + done + + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi + + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime +done + + +# for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do + +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# while [ $njob -ge 4 ] +# do +# sleeptime=$[ ( $RANDOM % 1000 ) ] +# sleep $sleeptime +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# done + + +# cd $curdir/$j +# for filename in ./*.sh ; do + +# if [[ $first_submit == 1 ]]; then +# # Submit first job w/o dependency +# echo "Submitting $filename" +# job=`sbatch $filename` +# first_submit=0 +# else +# echo "Submitting $filename after ${job: -8}" +# job=`sbatch -d afterany:${job: -8} $filename` +# fi + +# sleeptime=$[ ( $RANDOM % 5 ) ] +# sleep $sleeptime + +# done +# done diff --git a/scripts/llsm_importer/template.sh b/scripts/llsm_importer/template.sh new file mode 100755 index 000000000..fae7ee493 --- /dev/null +++ b/scripts/llsm_importer/template.sh @@ -0,0 +1,107 @@ +#!/bin/bash -l + +#REGSBATCH -q regular +#DBGSBATCH -q debug +#SBATCH -N NODENUM +#REGSBATCH -t 1:00:00 +#DBGSBATCH -t 0:30:00 +#SBATCH -C cpu +#SBATCH -J JOBNAME +#SBATCH -A PROJNAME +#SBATCH -o o%j.JOBNAME.out +#SBATCH -e o%j.JOBNAME.out + +# export PDC_DEBUG=0 + +# This is a script for running PDC in shared mode on Perlmutter +# When running in Shared mode, the client processes and server processes are running on the same node. +# By alternating the number of server processes and the number client processes, you should be able to change the C/S ratio. +# You can simply set the number of server processes, and let the script to calculate the number of client processes. + +# Per node configuration of your HPC system. +MAX_PYSICAL_CORE=MPHYSICALCORE +MAX_HYPERTHREADING=MHYPERTHREADING + +# Designated number of threads per process on each node +# (this should be associated with -c option in srun) +NUM_THREAD_PER_SERVER_PROC=NTHREAD_PER_SPROC +NUM_THREAD_PER_CLIENT_PROC=NTHREAD_PER_CPROC + + +# Designated number of processes for server anc client on each node +# (this should be associated with -n option in srun) +NUM_SERVER_PROC_PER_NODE=N_SERVER_PROC +NUM_CLIENT_PROC_PER_NODE=N_CLIENT_PROC + +# test if the number of threads is no larger than the total number of logical cores +TOTAL_NUM_PROC_PER_NODE=$((NUM_THREAD_PER_SERVER_PROC * NUM_SERVER_PROC_PER_NODE + NUM_THREAD_PER_CLIENT_PROC * NUM_CLIENT_PROC_PER_NODE)) +TOTAL_NUM_LOGICAL_CORE_PER_NODE=$((MAX_PYSICAL_CORE * MAX_HYPERTHREADING)) +if [[ "$TOTAL_NUM_PROC_PER_NODE" -gt "$TOTAL_NUM_LOGICAL_CORE_PER_NODE" ]]; then + echo "Error: TOTAL_NUM_PROC_PER_NODE is larger than TOTAL_NUM_LOGICAL_CORE_PER_NODE" + TOTAL_AVAILABLE_CORE=$((TOTAL_NUM_LOGICAL_CORE_PER_NODE - NUM_THREAD_PER_SERVER_PROC * NUM_SERVER_PROC_PER_NODE)) + NUM_CLIENT_PROC_PER_NODE=$(( TOTAL_AVAILABLE_CORE / NUM_THREAD_PER_CLIENT_PROC)) + echo "fixing the number of client processes to $NUM_CLIENT_PROC_PER_NODE" +fi + +# Set the number of times the test should be repeated. +REPEAT=1 + +# calculate the number of total processes for both server side and client side. +N_NODE=NODENUM +NCLIENT=$((NUM_CLIENT_PROC_PER_NODE * N_NODE)) +NSERVER=$((NUM_SERVER_PROC_PER_NODE * N_NODE)) + +# clean up the PDC tmp directory +export PDC_TMPDIR=$SCRATCH/data/pdc/conf +rm -rf $PDC_TMPDIR/* +export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE +mkdir -p $PDC_TMPDIR + +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin +TOOLPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/source/pdc/tools/build +SERVER=$EXECPATH/pdc_server.exe +CLIENT=$TOOLPATH/llsm_importer +CLOSE=$EXECPATH/close_server + +chmod +x $EXECPATH/* +chmod +x $TOOLPATH/llsm_importer + +LLSM_DATA_PATH=/pscratch/sd/w/wzhang5/data/llsm/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1 +# LLSM_DATA_PATH=/global/cfs/cdirs/m2621/wzhang5/data/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1 +IMGLIST_PATH=${LLSM_DATA_PATH}/ImageList_from_encoder.csv + +date + +# OpenMP settings: +# set the OPENMP thread number to the smaller number between $NUM_THREAD_PER_SERVER_PROC and $NUM_THREAD_PER_CLIENT_PROC +export OMP_NUM_THREADS=$((NUM_THREAD_PER_SERVER_PROC < NUM_THREAD_PER_CLIENT_PROC ? NUM_THREAD_PER_SERVER_PROC : NUM_THREAD_PER_CLIENT_PROC)) +export OMP_PLACES=threads +export OMP_PROC_BIND=close + +echo "OMP_NUM_THREADS=$OMP_NUM_THREADS" +echo "NSERVER=$NSERVER" +echo "NUM_THREAD_PER_SERVER_PROC=$NUM_THREAD_PER_SERVER_PROC" +echo "NCLIENT=$NCLIENT" +echo "NUM_THREAD_PER_CLIENT_PROC=$NUM_THREAD_PER_CLIENT_PROC" + + +echo "" +echo "=============" +echo "$i Init server" +echo "=============" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $NSERVER -c $NUM_THREAD_PER_SERVER_PROC --cpu_bind=cores $SERVER & +sleep 5 + + +echo "============================================" +echo "KVTAGS with $N_NODE nodes" +echo "============================================" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $NCLIENT -c $NUM_THREAD_PER_CLIENT_PROC --cpu_bind=cores $CLIENT -f $IMGLIST_PATH + +echo "" +echo "=================" +echo "$i Closing server" +echo "=================" +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE + +date diff --git a/spack/package.py b/spack/package.py deleted file mode 100644 index 17b9a41eb..000000000 --- a/spack/package.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * - - -class Pdc(CMakePackage): - """Proactive Data Containers (PDC) software provides an object-centric - API and a runtime system with a set of data object management services. - These services allow placing data in the memory and storage hierarchy, - performing data movement asynchronously, and providing scalable - metadata operations to find data objects.""" - - homepage = "https://pdc.readthedocs.io/en/latest/" - url = "https://github.com/hpc-io/pdc/archive/refs/tags/0.3.tar.gz" - git = "https://github.com/hpc-io/pdc.git" - - maintainers = ['houjun', 'sbyna'] - - version('0.3', sha256='14a3abd5e1e604f9527105709fca545bcdebe51abd2b89884db74d48a38b5443') - version('0.2', sha256='2829e74da227913a1a8e3e4f64e8f422ab9c0a049f8d73ff7b6ca12463959f8b') - version('0.1', sha256='01b4207ecf71594a7f339c315f2869b3fa8fbd34b085963dc4c1bdc5b66bb93e') - - version('stable', branch='stable') - version('develop', branch='develop') - - conflicts('%clang') - depends_on('libfabric') - depends_on('mercury') - depends_on('mpi') - - root_cmakelists_dir = 'src' - - def cmake_args(self): - args = [ - self.define('MPI_C_COMPILER', self.spec['mpi'].mpicc), - self.define('BUILD_MPI_TESTING', 'ON'), - self.define('BUILD_SHARED_LIBS', 'ON'), - self.define('BUILD_TESTING', 'ON'), - self.define('PDC_ENABLE_MPI', 'ON'), - self.define('CMAKE_C_COMPILER', self.spec['mpi'].mpicc) - ] - - if self.spec.satisfies('platform=cray'): - args.append("-DRANKSTR_LINK_STATIC=ON") - return args \ No newline at end of file diff --git a/src/api/CMakeLists.txt b/src/api/CMakeLists.txt index 6d4e0b722..7b9a356fb 100644 --- a/src/api/CMakeLists.txt +++ b/src/api/CMakeLists.txt @@ -1,7 +1,7 @@ - #------------------------------------------------------------------------------ +#------------------------------------------------------------------------------ # Include source and build directories #------------------------------------------------------------------------------ -set( LOCAL_INCLUDE_DIR +set(LOCAL_INCLUDE_DIR ${PDC_INCLUDES_BUILD_TIME} ${PROJECT_BINARY_DIR} ${PDC_SOURCE_DIR} @@ -32,14 +32,13 @@ include_directories( # External dependencies #------------------------------------------------------------------------------ # profiling -#set(PDC_EXT_LIB_DEPENDENCIES pdcprof ${PDC_EXT_LIB_DEPENDENCIES}) -set(PDC_EXT_INCLUDE_DEPENDENCIES ${CMAKE_CURRENT_SOURCE_DIR}/profiling) -set(PDC_EXPORTED_LIBS pdcprof) +# set(PDC_EXT_LIB_DEPENDENCIES pdcprof ${PDC_EXT_LIB_DEPENDENCIES}) +# set(PDC_EXT_INCLUDE_DEPENDENCIES ${CMAKE_CURRENT_SOURCE_DIR}/profiling) +# set(PDC_EXPORTED_LIBS pdcprof) # Mercury find_package(MERCURY REQUIRED) if(MERCURY_FOUND) - message(STATUS "mercury dir ${MERCURY_DIR}") set(PDC_EXT_INCLUDE_DEPENDENCIES ${MERCURY_INCLUDE_DIR} ${PDC_EXT_INCLUDE_DEPENDENCIES} ) @@ -87,22 +86,24 @@ set(PDC_SRCS ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_cache.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c - ${PDC_SOURCE_DIR}/src/utils/pdc_timing.c - ${PDC_SOURCE_DIR}/src/utils/pdc_malloc.c ${PDC_SOURCE_DIR}/src/utils/pdc_interface.c ${PDC_SOURCE_DIR}/src/utils/pdc_region_utils.c ) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/profiling) + set(PDC_COMMON_INCLUDE_DIRS ${PDC_COMMON_INCLUDE_DIRS}) + # add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/profiling) #------------------------------------------------------------------------------ # Libraries #------------------------------------------------------------------------------ # PDC set(PDC_BUILD_INCLUDE_DEPENDENCIES + ${PDC_COMMON_INCLUDE_DIRS} ${LOCAL_INCLUDE_DIR} ) +message(STATUS "PDC_BUILD_INCLUDE_DEPENDENCIES: ${PDC_BUILD_INCLUDE_DEPENDENCIES}") + add_library(pdc ${PDC_SRCS}) target_include_directories(pdc @@ -110,7 +111,12 @@ target_include_directories(pdc $ ) +message(STATUS "PDC_EXPORTED_LIBS: ${PDC_EXPORTED_LIBS}") +message(STATUS "PDC_EXT_LIB_DEPENDENCIES: ${PDC_EXT_LIB_DEPENDENCIES}") +message(STATUS "PDC_COMMONS_LIBRARIES: ${PDC_COMMONS_LIBRARIES}") + target_link_libraries(pdc + ${PDC_COMMONS_LIBRARIES} ${PDC_EXPORTED_LIBS} ${PDC_EXT_LIB_DEPENDENCIES} -ldl @@ -122,7 +128,8 @@ set(PDC_EXPORTED_LIBS pdc ${PDC_EXPORTED_LIBS}) add_executable(close_server close_server.c ) -target_link_libraries(close_server pdc) + +target_link_libraries(close_server pdc ${PDC_COMMON_LIBRARIES}) install( TARGETS @@ -135,13 +142,15 @@ install( #----------------------------------------------------------------------------- set(PDC_HEADERS ${PDC_SOURCE_DIR}/src/api/include/pdc.h - ${PDC_SOURCE_DIR}/src/api/include/pdc_public.h ${PDC_SOURCE_DIR}/src/api/pdc_analysis/include/pdc_analysis.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_cont.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_mpi.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_obj.h ${PDC_SOURCE_DIR}/src/api/pdc_region/include/pdc_region.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_prop.h + ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_cont_pkg.h + ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_obj_pkg.h + ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_prop_pkg.h ${PDC_SOURCE_DIR}/src/api/pdc_query/include/pdc_query.h ${PDC_SOURCE_DIR}/src/api/pdc_region/include/pdc_region.h ${PDC_SOURCE_DIR}/src/api/pdc_transform/include/pdc_transform.h @@ -173,9 +182,10 @@ install( #----------------------------------------------------------------------------- # Add Target(s) to CMake Install #----------------------------------------------------------------------------- + install( TARGETS - pdc + pdc EXPORT ${PDC_EXPORTED_TARGETS} LIBRARY DESTINATION ${PDC_INSTALL_LIB_DIR} @@ -204,6 +214,7 @@ EXPORT ( ${PDC_EXPORTED_LIBS} FILE ${PDC_EXPORTED_TARGETS}.cmake + APPEND ) endif() diff --git a/src/api/include/pdc_client_connect.h b/src/api/include/pdc_client_connect.h index 4b0816996..acf8fa7f0 100644 --- a/src/api/include/pdc_client_connect.h +++ b/src/api/include/pdc_client_connect.h @@ -32,6 +32,11 @@ #include "mercury_proc_string.h" #include "mercury_request.h" #include "pdc_region.h" +#include "dart_core.h" +#include "query_utils.h" +#include "pdc_set.h" +#include "pdc_compare.h" +#include "pdc_hash.h" extern int pdc_server_num_g; extern int pdc_client_mpi_rank_g; @@ -54,6 +59,7 @@ struct _pdc_client_lookup_args { uint32_t server_id; uint32_t client_id; int ret; + int is_init; char * ret_string; char * client_addr; @@ -151,6 +157,40 @@ struct _pdc_query_result_list { struct _pdc_query_result_list *next; }; +struct client_genetic_lookup_args { + char *char_value1; + char *char_value2; + char *char_value3; + char *char_value4; + + int int_value1; + int int_value2; + int int_value3; + int int_value4; + + uint32_t uint32_value1; + uint32_t uint32_value2; + uint32_t uint32_value3; + uint32_t uint32_value4; + + uint64_t uint64_value1; + uint64_t uint64_value2; + uint64_t uint64_value3; + uint64_t uint64_value4; + + int64_t int64_value1; + int64_t int64_value2; + int64_t int64_value3; + int64_t int64_value4; +}; + +struct _dart_perform_one_thread_param { + int server_id; + dart_perform_one_server_in_t *dart_in; + uint64_t ** dart_out; + size_t * dart_out_size; +}; + #define PDC_CLIENT_DATA_SERVER() ((pdc_client_mpi_rank_g / pdc_nclient_per_server_g) % pdc_server_num_g) uint32_t PDC_get_client_data_server(); @@ -476,6 +516,49 @@ perr_t PDC_partial_query(int is_list_all, int user_id, const char *app_name, con */ perr_t PDC_Client_write_wait_notify(pdc_metadata_t *meta, struct pdc_region_info *region, void *buf); +/** + * Insert data into index with key value pair + * + * \param attr_name [IN] Name of the attribute + * \param attr_value [IN] Value of the attribute + * \param data [IN] Associated value along with the key-value pair. + * + * \return Non-negative on success/Negative on failure + */ +// perr_t PDC_Client_insert_into_index( int index_type, char *attr_name, char *attr_value, void *data); + +/** + * Search through index with key value pair. + * + * if the value is not specified, we just retrieve all the indexed data + * on the secondary index associated with the primary index + * specified by attr_name; + * + * \param query_string [IN] Name of the attribute + * \param n_res [OUT] Number of object IDs + * \param out [OUT] Object IDs + * + * \return Non-negative on success/Negative on failure + */ +// perr_t PDC_Client_search_through_index(char *query_string, int index_type, int *n_res, uint64_t ***out); + +/** + * Delete the inverted mapping between value and data. + * + * \param attr_name [IN] Name of the attribute + * \param attr_value [IN] Value of the attribute + * \param data [IN] Associated value along with the key-value pair. + * + * \return Non-negative on success/Negative on failure + */ +// perr_t PDC_Client_delete_index_mapping( int index_type, char *attr_name, char *attr_value, void *data); + +/** + * To lookup the server just in case. + * + */ +perr_t server_lookup_connection(int serverId, int retry_times); + /** * Client request server to check IO status of a previous IO request * @@ -593,16 +676,19 @@ perr_t PDC_Client_create_cont_id_mpi(const char *cont_name, pdcid_t cont_create_ */ perr_t PDC_Client_query_kvtag(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids); +#ifdef ENABLE_MPI /** - * Client sends query requests to server (used by MPI mode) + * Client sends query requests to server (used by MPI mode), all clients get the same aggregated + * query results, currently assumes MPI_COMM_WORLD * - * \param kvtag [IN] ********* - * \param n_res [IN] ********** - * \param pdc_ids [OUT] ********* + * \param kvtag [IN] kvtag + * \param n_res [OUT] number of hits + * \param pdc_ids [OUT] object ids of hits, unordered * * \return Non-negative on success/Negative on failure */ -perr_t PDC_Client_query_kvtag_col(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids); +perr_t PDC_Client_query_kvtag_mpi(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids, MPI_Comm comm); +#endif /** * Client sends query requests to server (used by MPI mode) @@ -687,7 +773,7 @@ hg_return_t PDC_Client_get_data_from_server_shm_cb(const struct hg_cb_info *call * * \return Non-negative on success/Negative on failure */ -perr_t PDC_Client_lookup_server(int server_id); +perr_t PDC_Client_lookup_server(int server_id, int is_init); /** * ******** @@ -928,6 +1014,92 @@ int PDC_get_nproc_per_node(); */ perr_t PDC_free_kvtag(pdc_kvtag_t **kvtag); +/** + * @brief delete the metadata of the specified PDC object or container + * @param id [IN] ID of the object or container + * @param is_cont [IN] 1 if the id is a container ID, 0 if the id is an object ID + * @return Non-negative on success/Negative on failure + */ perr_t PDC_Client_del_metadata(pdcid_t id, int is_cont); +/** + * Return the global dart_g from client + * + */ +DART *get_dart_g(); + +/** + * Return the abstract of the server by server ID + * + * + */ +dart_server dart_retrieve_server_info_cb(uint32_t serverId); + +/** + * Search through dart index with key-value pair. + * Each calling client will send the request and get the complete result. + * if the value is not specified, we just retrieve all the indexed data + * on the secondary index associated with the primary index + * specified by attr_name; + * + * \param hash_algo [IN] name of the hashing algorithm + * \param query_string [IN] Name of the attribute + * \param n_res [OUT] Number of object IDs + * \param out [OUT] Object IDs + */ +perr_t PDC_Client_search_obj_ref_through_dart(dart_hash_algo_t hash_algo, char *query_string, + dart_object_ref_type_t ref_type, int *n_res, uint64_t **out); + +#ifdef ENABLE_MPI +/** + * Search through dart index with key-value pair. + * This is an MPI version of the search function. Multiple ranks can call this function, but only one rank + * sends the request and gets the result and broadcasts the result to all other ranks. + * if the value is not + * specified, we just retrieve all the indexed data on the secondary index associated with the primary index + * specified by attr_name; + * + * \param hash_algo [IN] name of the hashing algorithm + * \param query_string [IN] Name of the attribute + * \param n_res [OUT] Number of object IDs + * \param out [OUT] Object IDs + */ +perr_t PDC_Client_search_obj_ref_through_dart_mpi(dart_hash_algo_t hash_algo, char *query_string, + dart_object_ref_type_t ref_type, int *n_res, uint64_t **out, + MPI_Comm comm); +#endif + +/** + * Delete the inverted mapping between value and data. + * + * \param hash_algo [IN] name of the hashing algorithm + * \param attr_key [IN] Name of the attribute + * \param attr_value [IN] Value of the attribute + * \param ref_type [IN] The reference type of the object, e.g. PRIMARY_ID, SECONDARY_ID, SERVER_ID + * \param data [IN] Associated value along with the key-value pair. + * + * \return Non-negative on success/Negative on failure + */ +perr_t PDC_Client_delete_obj_ref_from_dart(dart_hash_algo_t hash_algo, char *attr_key, char *attr_val, + dart_object_ref_type_t ref_type, uint64_t data); + +/** + * Insert data into index with key value pair + * + * \param hash_algo [IN] name of the hashing algorithm + * \param attr_key [IN] Name of the attribute + * \param attr_value [IN] Value of the attribute + * \param ref_type [IN] The reference type of the object, e.g. PRIMARY_ID, SECONDARY_ID, SERVER_ID + * \param data [IN] Associated value along with the key-value pair. + * + * \return Non-negative on success/Negative on failure + */ +perr_t PDC_Client_insert_obj_ref_into_dart(dart_hash_algo_t hash_algo, char *attr_key, char *attr_val, + dart_object_ref_type_t ref_type, uint64_t data); + +/** + * Report the average profiling time of the server if the info is available. + */ +void report_avg_server_profiling_rst(); + #endif /* PDC_CLIENT_CONNECT_H */ diff --git a/src/api/pdc.c b/src/api/pdc.c index 3f5dd2384..6010552b3 100644 --- a/src/api/pdc.c +++ b/src/api/pdc.c @@ -89,7 +89,7 @@ PDCinit(const char *pdc_name) FUNC_ENTER(NULL); - if (NULL == (pdc_id_list_g = PDC_CALLOC(struct pdc_id_list))) + if (NULL == (pdc_id_list_g = PDC_CALLOC(1, struct pdc_id_list))) PGOTO_ERROR(FAIL, "PDC global id list: memory allocation failed"); if (PDC_class_init() < 0) diff --git a/src/api/pdc_analysis/include/pdc_analysis_and_transforms_common.h b/src/api/pdc_analysis/include/pdc_analysis_and_transforms_common.h index 082a3dd02..79ccbf3db 100644 --- a/src/api/pdc_analysis/include/pdc_analysis_and_transforms_common.h +++ b/src/api/pdc_analysis/include/pdc_analysis_and_transforms_common.h @@ -24,7 +24,8 @@ #ifndef PDC_OBJ_ANALYSIS_H #define PDC_OBJ_ANALYSIS_H -#include "pdc_private.h" +#include "pdc_prop_pkg.h" +#include "pdc_transforms_pkg.h" #include "mercury_proc_string.h" #include "mercury_atomic.h" #include diff --git a/src/api/pdc_analysis/include/pdc_hist_pkg.h b/src/api/pdc_analysis/include/pdc_hist_pkg.h index 3697743c3..6be68af7f 100644 --- a/src/api/pdc_analysis/include/pdc_hist_pkg.h +++ b/src/api/pdc_analysis/include/pdc_hist_pkg.h @@ -25,6 +25,7 @@ #ifndef PDC_HIST_H #define PDC_HIST_H +#include "pdc_private.h" #include "pdc_public.h" #include "math.h" #include diff --git a/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c b/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c index ac957b394..4fd664e4c 100644 --- a/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c +++ b/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c @@ -27,8 +27,8 @@ #include "pdc_analysis_pkg.h" #include "pdc_transforms_common.h" -static hg_context_t *send_context_g = NULL; -static int work_todo_g = 0; +static hg_context_t * send_context_g = NULL; +static hg_atomic_int32_t *atomic_work_todo_g = NULL; /* Forward References:: */ // Analysis and Transformations @@ -59,6 +59,9 @@ PDC_Client_send_iter_recv_id(pdcid_t iter_id, pdcid_t *meta_id) struct _pdc_obj_info * object_info; FUNC_ENTER(NULL); + if (atomic_work_todo_g == NULL) { + hg_atomic_init32(atomic_work_todo_g, 0); + } my_rpc_state_p = (struct _pdc_my_rpc_state *)calloc(1, sizeof(struct _pdc_my_rpc_state)); if (my_rpc_state_p == NULL) @@ -101,7 +104,7 @@ PDC_Client_send_iter_recv_id(pdcid_t iter_id, pdcid_t *meta_id) while (pdc_server_info_g[server_id].addr_valid != 1) { if (n_retry > 0) break; - if (PDC_Client_lookup_server(server_id) != SUCCEED) + if (PDC_Client_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_lookup_server", pdc_client_mpi_rank_g); n_retry++; @@ -114,7 +117,7 @@ PDC_Client_send_iter_recv_id(pdcid_t iter_id, pdcid_t *meta_id) if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "PDC_client_send_iter_recv_id(): Could not start HG_Forward()"); - work_todo_g = 1; + hg_atomic_incr32(atomic_work_todo_g); PDC_Client_check_response(&send_context_g); if (my_rpc_state_p->value == 0) { @@ -154,7 +157,7 @@ client_register_iterator_rpc_cb(const struct hg_cb_info *info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(atomic_work_todo_g); HG_Free_output(info->info.forward.handle, &output); FUNC_LEAVE(ret_value); @@ -176,6 +179,9 @@ PDC_Client_register_obj_analysis(struct _pdc_region_analysis_ftn_info *thisFtn, struct _pdc_obj_info * obj_prop; FUNC_ENTER(NULL); + if (atomic_work_todo_g == NULL) { + hg_atomic_init32(atomic_work_todo_g, 0); + } my_rpc_state_p = (struct _pdc_my_rpc_state *)calloc(1, sizeof(struct _pdc_my_rpc_state)); if (my_rpc_state_p == NULL) @@ -212,7 +218,7 @@ PDC_Client_register_obj_analysis(struct _pdc_region_analysis_ftn_info *thisFtn, while (pdc_server_info_g[server_id].addr_valid != 1) { if (n_retry > 0) break; - if (PDC_Client_lookup_server(server_id) != SUCCEED) + if (PDC_Client_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_lookup_server", pdc_client_mpi_rank_g); n_retry++; @@ -232,7 +238,7 @@ PDC_Client_register_obj_analysis(struct _pdc_region_analysis_ftn_info *thisFtn, if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "PDC_Client_register_obj_analysis(): Could not start HG_Forward()"); - work_todo_g = 1; + hg_atomic_incr32(atomic_work_todo_g); PDC_Client_check_response(&send_context_g); if (my_rpc_state_p->value < 0) { @@ -271,7 +277,7 @@ client_register_analysis_rpc_cb(const struct hg_cb_info *info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(atomic_work_todo_g); HG_Free_output(info->info.forward.handle, &output); FUNC_LEAVE(ret_value); @@ -291,6 +297,9 @@ PDC_Client_register_region_transform(const char *func, const char *loadpath, struct _pdc_my_rpc_state *my_rpc_state_p; FUNC_ENTER(NULL); + if (atomic_work_todo_g == NULL) { + hg_atomic_init32(atomic_work_todo_g, 0); + } my_rpc_state_p = (struct _pdc_my_rpc_state *)calloc(1, sizeof(struct _pdc_my_rpc_state)); if (my_rpc_state_p == NULL) @@ -323,7 +332,7 @@ PDC_Client_register_region_transform(const char *func, const char *loadpath, if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "Could not start HG_Forward()"); - work_todo_g = 1; + hg_atomic_incr32(atomic_work_todo_g); PDC_Client_check_response(&send_context_g); if (my_rpc_state_p->value < 0) { @@ -361,7 +370,7 @@ client_register_transform_rpc_cb(const struct hg_cb_info *info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(atomic_work_todo_g); HG_Free_output(info->info.forward.handle, &output); FUNC_LEAVE(ret_value); diff --git a/src/api/pdc_analysis/pdc_hist_pkg.c b/src/api/pdc_analysis/pdc_hist_pkg.c index 47c93a94d..75343a144 100644 --- a/src/api/pdc_analysis/pdc_hist_pkg.c +++ b/src/api/pdc_analysis/pdc_hist_pkg.c @@ -1,5 +1,4 @@ #include "pdc_hist_pkg.h" -#include "pdc_private.h" #include #include diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index 9079e04f9..44c1786e6 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -33,6 +33,7 @@ #include "pdc_utlist.h" #include "pdc_id_pkg.h" +#include "pdc_cont_pkg.h" #include "pdc_prop_pkg.h" #include "pdc_obj_pkg.h" #include "pdc_cont.h" @@ -45,6 +46,11 @@ #include "mercury.h" #include "mercury_macros.h" +#include "string_utils.h" +#include "dart_core.h" +#include "timer_utils.h" +#include "query_utils.h" + #include #include #include @@ -65,6 +71,13 @@ pdc_server_selection_t pdc_server_selection_g = PDC_SERVER_DEFAULT; int pdc_client_mpi_rank_g = 0; int pdc_client_mpi_size_g = 1; +// FIXME: this is a temporary solution for printing out debug info, like memory usage. +int memory_debug_g = 0; // when it is no longer 0, stop printing debug info. + +int64_t *server_time_total_g; +int64_t *server_call_count_g; +int64_t *server_mem_usage_g; + #ifdef ENABLE_MPI MPI_Comm PDC_SAME_NODE_COMM_g; MPI_Comm PDC_CLIENT_COMM_WORLD_g; @@ -96,9 +109,24 @@ double read_bb_size_g = 0.0; static int mercury_has_init_g = 0; static hg_class_t * send_class_g = NULL; static hg_context_t *send_context_g = NULL; -static int work_todo_g = 0; int query_id_g = 0; +// flags for RPC request and Bulk transfer. +// When a work is put in the queue, increase todo_g by 1 +// When a work is done and popped from the queue, decrease (todo_g) by 1. +static hg_atomic_int32_t atomic_work_todo_g; +hg_atomic_int32_t bulk_todo_g; +// When a work is initialized, set done_g flag to 0. +// When a work is done, set done_g flag to 1 using atomic cas operation. +static hg_atomic_int32_t response_done_g; +hg_atomic_int32_t bulk_transfer_done_g; + +// global variables for DART +static DART * dart_g; +static dart_hash_algo_t dart_hash_algo_g = DART_HASH; +static dart_object_ref_type_t dart_obj_ref_type_g = REF_PRIMARY_ID; + +// global variables for Mercury RPC registration static hg_id_t client_test_connect_register_id_g; static hg_id_t gen_obj_register_id_g; static hg_id_t gen_cont_register_id_g; @@ -128,10 +156,8 @@ static hg_id_t server_checkpoint_rpc_register_id_g; static hg_id_t send_shm_register_id_g; // bulk -static hg_id_t query_partial_register_id_g; -static hg_id_t query_kvtag_register_id_g; -static int bulk_todo_g = 0; -hg_atomic_int32_t bulk_transfer_done_g; +static hg_id_t query_partial_register_id_g; +static hg_id_t query_kvtag_register_id_g; static hg_id_t transfer_request_register_id_g; static hg_id_t transfer_request_all_register_id_g; @@ -153,11 +179,17 @@ static hg_id_t send_region_storage_meta_shm_bulk_rpc_register_id_g; static hg_id_t send_data_query_register_id_g; static hg_id_t get_sel_data_register_id_g; +// DART index +static hg_id_t dart_get_server_info_g; +static hg_id_t dart_perform_one_server_g; + int cache_percentage_g = 0; int cache_count_g = 0; int cache_total_g = 0; pdc_data_server_io_list_t *client_cache_list_head_g = NULL; +static uint64_t object_selection_query_counter_g = 0; + /* * * Client Functions @@ -213,14 +245,14 @@ pdc_client_check_int_ret_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); } // Check if all work has been processed -// Using global variable $mercury_work_todo_g +// Using global variable $atomic_work_todo_g perr_t PDC_Client_check_response(hg_context_t **hg_context) { @@ -236,7 +268,7 @@ PDC_Client_check_response(hg_context_t **hg_context) } while ((hg_ret == HG_SUCCESS) && actual_count); /* Do not try to make progress anymore if we're done */ - if (work_todo_g <= 0) + if (hg_atomic_get32(&atomic_work_todo_g) <= 0) break; hg_ret = HG_Progress(*hg_context, HG_MAX_IDLE_TIME); @@ -244,7 +276,6 @@ PDC_Client_check_response(hg_context_t **hg_context) ret_value = SUCCEED; - fflush(stdout); FUNC_LEAVE(ret_value); } @@ -354,7 +385,7 @@ client_send_flush_obj_all_rpc_cb(const struct hg_cb_info *callback_info) done: // printf("client close RPC is finished here, return value = %d\n", output.ret); fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -382,7 +413,7 @@ obj_reset_dims_rpc_cb(const struct hg_cb_info *callback_info) region_transfer_args->ret = output.ret; done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -411,7 +442,7 @@ client_send_flush_obj_rpc_cb(const struct hg_cb_info *callback_info) done: // printf("client close RPC is finished here, return value = %d\n", output.ret); fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -440,7 +471,7 @@ client_send_close_all_server_rpc_cb(const struct hg_cb_info *callback_info) done: // printf("client close RPC is finished here, return value = %d\n", output.ret); fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -472,7 +503,7 @@ client_send_transfer_request_metadata_query_rpc_cb(const struct hg_cb_info *call region_transfer_args->ret = output.ret; done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -503,7 +534,7 @@ client_send_transfer_request_metadata_query2_rpc_cb(const struct hg_cb_info *cal region_transfer_args->ret = output.ret; done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -534,7 +565,7 @@ client_send_transfer_request_all_rpc_cb(const struct hg_cb_info *callback_info) region_transfer_args->metadata_id = output.metadata_id; done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -563,7 +594,7 @@ client_send_transfer_request_wait_all_rpc_cb(const struct hg_cb_info *callback_i region_transfer_args->ret = output.ret; done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -594,7 +625,7 @@ client_send_transfer_request_rpc_cb(const struct hg_cb_info *callback_info) region_transfer_args->metadata_id = output.metadata_id; done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -626,7 +657,7 @@ client_send_transfer_request_status_rpc_cb(const struct hg_cb_info *callback_inf done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -659,7 +690,7 @@ client_send_transfer_request_wait_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -693,7 +724,7 @@ client_send_buf_unmap_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -725,7 +756,7 @@ client_send_buf_map_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g = 0; + hg_atomic_set32(&atomic_work_todo_g, 0); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -754,7 +785,7 @@ client_test_connect_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g = 0; + hg_atomic_set32(&atomic_work_todo_g, 0); HG_Free_output(handle, &output); HG_Destroy(callback_info->info.forward.handle); @@ -786,6 +817,7 @@ client_test_connect_lookup_cb(const struct hg_cb_info *callback_info) in.client_id = pdc_client_mpi_rank_g; in.nclient = pdc_client_mpi_size_g; in.client_addr = client_lookup_args->client_addr; + in.is_init = client_lookup_args->is_init; ret_value = HG_Forward(client_test_handle, client_test_connect_rpc_cb, client_lookup_args, &in); if (ret_value != HG_SUCCESS) @@ -797,7 +829,7 @@ client_test_connect_lookup_cb(const struct hg_cb_info *callback_info) } perr_t -PDC_Client_lookup_server(int server_id) +PDC_Client_lookup_server(int server_id, int is_init) { perr_t ret_value = SUCCEED; hg_return_t hg_ret; @@ -818,6 +850,7 @@ PDC_Client_lookup_server(int server_id) lookup_args.client_id = pdc_client_mpi_rank_g; lookup_args.server_id = server_id; lookup_args.client_addr = self_addr; + lookup_args.is_init = is_init; target_addr_string = pdc_server_info_g[lookup_args.server_id].addr_string; if (is_client_debug_g == 1) { @@ -832,7 +865,7 @@ PDC_Client_lookup_server(int server_id) PGOTO_ERROR(FAIL, "==PDC_CLIENT: Connection to server FAILED!"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (is_client_debug_g == 1) { @@ -847,7 +880,7 @@ PDC_Client_lookup_server(int server_id) } perr_t -PDC_Client_try_lookup_server(int server_id) +PDC_Client_try_lookup_server(int server_id, int is_init) { perr_t ret_value = SUCCEED; int n_retry = 1; @@ -860,7 +893,7 @@ PDC_Client_try_lookup_server(int server_id) while (pdc_server_info_g[server_id].addr_valid != 1) { if (n_retry > PDC_MAX_TRIAL_NUM) break; - ret_value = PDC_Client_lookup_server(server_id); + ret_value = PDC_Client_lookup_server(server_id, is_init); if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_lookup_server", pdc_client_mpi_rank_g); n_retry++; @@ -897,7 +930,7 @@ client_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -927,7 +960,7 @@ client_region_lock_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -955,7 +988,7 @@ client_region_release_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -993,7 +1026,7 @@ client_region_release_with_transform_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -1009,10 +1042,14 @@ hg_test_bulk_transfer_cb(const struct hg_cb_info *hg_cb_info) hg_bulk_t local_bulk_handle; uint32_t i; void * buf = NULL; + void ** ids_buf; uint32_t n_meta; uint64_t buf_sizes[2] = {0, 0}; + uint64_t * ids_buf_sizes; uint32_t actual_cnt; pdc_metadata_t * meta_ptr; + uint64_t * u64_arr_ptr; + uint32_t bulk_sgnum; FUNC_ENTER(NULL); @@ -1027,28 +1064,46 @@ hg_test_bulk_transfer_cb(const struct hg_cb_info *hg_cb_info) n_meta = bulk_args->n_meta; if (hg_cb_info->ret == HG_SUCCESS) { - HG_Bulk_access(local_bulk_handle, 0, bulk_args->nbytes, HG_BULK_READWRITE, 1, &buf, buf_sizes, - &actual_cnt); - - meta_ptr = (pdc_metadata_t *)(buf); - bulk_args->meta_arr = (pdc_metadata_t **)calloc(sizeof(pdc_metadata_t *), n_meta); - for (i = 0; i < n_meta; i++) { - bulk_args->meta_arr[i] = meta_ptr; - meta_ptr++; + if (bulk_args->is_id == 1) { + bulk_sgnum = HG_Bulk_get_segment_count(local_bulk_handle); + ids_buf = (void **)calloc(sizeof(void *), bulk_sgnum); + ids_buf_sizes = (uint64_t *)calloc(sizeof(uint64_t), bulk_sgnum); + HG_Bulk_access(local_bulk_handle, 0, bulk_args->nbytes, HG_BULK_READWRITE, bulk_sgnum, ids_buf, + ids_buf_sizes, &actual_cnt); + + u64_arr_ptr = ((uint64_t **)(ids_buf))[0]; + bulk_args->obj_ids = (uint64_t *)calloc(sizeof(uint64_t), n_meta); + for (i = 0; i < n_meta; i++) { + bulk_args->obj_ids[i] = *u64_arr_ptr; + u64_arr_ptr++; + } + } + else { + HG_Bulk_access(local_bulk_handle, 0, bulk_args->nbytes, HG_BULK_READWRITE, 1, &buf, buf_sizes, + &actual_cnt); + meta_ptr = (pdc_metadata_t *)(buf); + bulk_args->meta_arr = (pdc_metadata_t **)calloc(sizeof(pdc_metadata_t *), n_meta); + for (i = 0; i < n_meta; i++) { + bulk_args->meta_arr[i] = meta_ptr; + meta_ptr++; + } } } - bulk_todo_g--; - hg_atomic_set32(&bulk_transfer_done_g, 1); - // Free block handle ret_value = HG_Bulk_free(local_bulk_handle); if (ret_value != HG_SUCCESS) PGOTO_ERROR(ret_value, "Could not free HG bulk handle"); done: + hg_atomic_decr32(&bulk_todo_g); + // checking the following flag will make all bulk transfers globally sequential. + hg_atomic_cas32(&bulk_transfer_done_g, 0, 1); + // checking the following flag will make sure the current bulk transfer is done. + hg_atomic_cas32(&(bulk_args->bulk_done_flag), 0, 1); + fflush(stdout); - free(bulk_args); + // free(bulk_args); FUNC_LEAVE(ret_value); } @@ -1072,7 +1127,7 @@ PDC_Client_check_bulk(hg_context_t *hg_context) } while ((hg_ret == HG_SUCCESS) && actual_count); /* Do not try to make progress anymore if we're done */ - if (bulk_todo_g <= 0) + if (hg_atomic_get32(&bulk_todo_g) <= 0) break; hg_ret = HG_Progress(hg_context, HG_MAX_IDLE_TIME); @@ -1124,8 +1179,8 @@ perr_t PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int port) { perr_t ret_value = SUCCEED; - char na_info_string[PATH_MAX]; - char hostname[ADDR_MAX]; + char na_info_string[NA_STRING_INFO_LEN]; + char hostname[HOSTNAME_LEN]; int local_server_id; /* Set the default mercury transport * but enable overriding that to any of: @@ -1279,6 +1334,10 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po send_data_query_register_id_g = PDC_send_data_query_rpc_register(*hg_class); get_sel_data_register_id_g = PDC_get_sel_data_rpc_register(*hg_class); + // DART Index + dart_get_server_info_g = PDC_dart_get_server_info_register(*hg_class); + dart_perform_one_server_g = PDC_dart_perform_one_server_register(*hg_class); + #ifdef ENABLE_MULTITHREAD /* Mutex initialization for the client versions of these... */ /* The Server versions gets initialized in pdc_server.c */ @@ -1293,7 +1352,7 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po // Each client connect to its node local server only at start time local_server_id = PDC_get_local_server_id(pdc_client_mpi_rank_g, pdc_nclient_per_server_g, pdc_server_num_g); - if (PDC_Client_try_lookup_server(local_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(local_server_id, 1) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR lookup server %d\n", pdc_client_mpi_rank_g, local_server_id); } @@ -1303,7 +1362,7 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po for (local_server_id = 0; local_server_id < pdc_server_num_g; local_server_id++) { if (pdc_client_mpi_size_g > 1000) PDC_msleep(pdc_client_mpi_rank_g % 300); - if (PDC_Client_try_lookup_server(local_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(local_server_id, 1) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR lookup server %d", pdc_client_mpi_rank_g, local_server_id); } @@ -1413,9 +1472,14 @@ PDC_Client_init() if (mercury_has_init_g == 0) { // Init Mercury network connection ret_value = PDC_Client_mercury_init(&send_class_g, &send_context_g, port); - if (ret_value != SUCCEED || send_class_g == NULL || send_context_g == NULL) + if (ret_value != SUCCEED || send_class_g == NULL || send_context_g == NULL) { PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Error with Mercury Init, exiting...", pdc_client_mpi_rank_g); + } + hg_atomic_init32(&atomic_work_todo_g, 0); + hg_atomic_init32(&response_done_g, 0); + hg_atomic_init32(&bulk_todo_g, 0); + hg_atomic_init32(&bulk_transfer_done_g, 0); mercury_has_init_g = 1; } @@ -1424,7 +1488,21 @@ PDC_Client_init() pdc_client_tmp_dir_g, pdc_nclient_per_server_g); } - srand(time(NULL)); + if (mercury_has_init_g) { + srand(time(NULL)); + + /* Initialize DART space */ + dart_g = (DART *)calloc(1, sizeof(DART)); + int extra_tree_height = 0; + int replication_factor = pdc_server_num_g / 10; + replication_factor = replication_factor > 0 ? replication_factor : 2; + dart_space_init(dart_g, pdc_client_mpi_size_g, pdc_server_num_g, DART_ALPHABET_SIZE, + extra_tree_height, replication_factor); + + server_time_total_g = (int64_t *)calloc(pdc_server_num_g, sizeof(int64_t)); + server_call_count_g = (int64_t *)calloc(pdc_server_num_g, sizeof(int64_t)); + server_mem_usage_g = (int64_t *)calloc(pdc_server_num_g, sizeof(int64_t)); + } done: fflush(stdout); @@ -1559,14 +1637,14 @@ metadata_query_bulk_cb(const struct hg_cb_info *callback_info) PGOTO_ERROR(ret_value, "Could not read bulk data"); // loop - bulk_todo_g = 1; + hg_atomic_incr32(&bulk_todo_g); PDC_Client_check_bulk(send_context_g); client_lookup_args->meta_arr = bulk_args->meta_arr; done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -1644,7 +1722,7 @@ PDC_partial_query(int is_list_all, int user_id, const char *app_name, const char } for (server_id = my_server_start; server_id < my_server_end; server_id++) { - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, query_partial_register_id_g, @@ -1660,7 +1738,7 @@ PDC_partial_query(int is_list_all, int user_id, const char *app_name, const char hg_atomic_set32(&bulk_transfer_done_g, 0); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (lookup_args.n_meta == 0) @@ -1728,7 +1806,7 @@ PDC_Client_query_tag(const char *tags, int *n_res, pdc_metadata_t ***out) *n_res = 0; for (server_id = 0; server_id < pdc_server_num_g; server_id++) { - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, query_partial_register_id_g, @@ -1744,7 +1822,7 @@ PDC_Client_query_tag(const char *tags, int *n_res, pdc_metadata_t ***out) hg_atomic_set32(&bulk_transfer_done_g, 0); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if ((lookup_args.n_meta) == 0) @@ -1817,7 +1895,7 @@ metadata_query_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -1849,7 +1927,7 @@ metadata_delete_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -1879,7 +1957,7 @@ metadata_delete_by_id_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -1909,7 +1987,7 @@ metadata_add_tag_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -1939,7 +2017,7 @@ PDC_Client_add_tag(pdcid_t obj_id, const char *tag) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_add_tag_register_id_g, @@ -1955,7 +2033,7 @@ PDC_Client_add_tag(pdcid_t obj_id, const char *tag) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: - HG_Forward Error!", pdc_client_mpi_rank_g); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (lookup_args.ret != 1) @@ -1995,7 +2073,7 @@ metadata_update_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -2024,7 +2102,7 @@ PDC_Client_update_metadata(pdc_metadata_t *old, pdc_metadata_t *new) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_update_register_id_g, @@ -2078,7 +2156,7 @@ PDC_Client_update_metadata(pdc_metadata_t *old, pdc_metadata_t *new) PGOTO_ERROR(FAIL, "PDC_Client_update_metadata_with_name(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (lookup_args.ret != 1) @@ -2113,7 +2191,7 @@ PDC_Client_delete_metadata_by_id(uint64_t obj_id) else debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_delete_by_id_register_id_g, @@ -2126,7 +2204,7 @@ PDC_Client_delete_metadata_by_id(uint64_t obj_id) PGOTO_ERROR(FAIL, "PDC_Client_delete_by_id_metadata_with_name(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (lookup_args.ret < 0) @@ -2170,7 +2248,7 @@ PDC_Client_delete_metadata(char *delete_name, pdcid_t obj_delete_prop) else debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_delete_register_id_g, @@ -2180,7 +2258,7 @@ PDC_Client_delete_metadata(char *delete_name, pdcid_t obj_delete_prop) PGOTO_ERROR(FAIL, "PDC_Client_delete_metadata_with_name(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (lookup_args.ret != 1) @@ -2225,7 +2303,7 @@ PDC_Client_query_metadata_name_only(const char *obj_name, pdc_metadata_t **out) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_query_register_id_g, @@ -2238,7 +2316,7 @@ PDC_Client_query_metadata_name_only(const char *obj_name, pdc_metadata_t **out) } // Wait for response from server - work_todo_g = pdc_server_num_g; + hg_atomic_set32(&atomic_work_todo_g, pdc_server_num_g); PDC_Client_check_response(&send_context_g); for (i = 0; i < (uint32_t)pdc_server_num_g; i++) { @@ -2282,7 +2360,7 @@ PDC_Client_query_metadata_name_timestep(const char *obj_name, int time_step, pdc // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_query_register_id_g, @@ -2305,7 +2383,7 @@ PDC_Client_query_metadata_name_timestep(const char *obj_name, int time_step, pdc pdc_client_mpi_rank_g); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); *out = lookup_args.data; // printf("rank = %d, PDC_Client_query_metadata_name_timestep = %u\n", pdc_client_mpi_rank_g, @@ -2359,21 +2437,22 @@ PDC_Client_query_metadata_name_timestep_agg(const char *obj_name, int time_step, FUNC_ENTER(NULL); #ifdef ENABLE_MPI - if (pdc_client_mpi_rank_g == 0) { + if (pdc_client_mpi_rank_g == 0) ret_value = PDC_Client_query_metadata_name_timestep(obj_name, time_step, out, metadata_server_id); - if (ret_value != SUCCEED || NULL == *out) { - *out = (pdc_metadata_t *)calloc(1, sizeof(pdc_metadata_t)); - PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: - ERROR with query [%s]", pdc_client_mpi_rank_g, obj_name); - } - } - else + + MPI_Bcast(&ret_value, 1, MPI_INT, 0, PDC_CLIENT_COMM_WORLD_g); + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: - ERROR with query [%s]", pdc_client_mpi_rank_g, obj_name); + + if (pdc_client_mpi_rank_g != 0) *out = (pdc_metadata_t *)calloc(1, sizeof(pdc_metadata_t)); MPI_Bcast(*out, sizeof(pdc_metadata_t), MPI_CHAR, 0, PDC_CLIENT_COMM_WORLD_g); - MPI_Bcast(metadata_server_id, 1, MPI_UINT32_T, 0, PDC_CLIENT_COMM_WORLD_g); #else ret_value = PDC_Client_query_metadata_name_timestep(obj_name, time_step, out, metadata_server_id); + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: - ERROR with query [%s]", pdc_client_mpi_rank_g, obj_name); #endif done: @@ -2417,7 +2496,7 @@ PDC_Client_create_cont_id(const char *cont_name, pdcid_t cont_create_prop ATTRIB // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = @@ -2428,7 +2507,7 @@ PDC_Client_create_cont_id(const char *cont_name, pdcid_t cont_create_prop ATTRIB PGOTO_ERROR(FAIL, "PDC_Client_send_name_to_server(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); // Now we have obj id stored in lookup_args.obj_id @@ -2492,7 +2571,7 @@ PDC_Client_obj_reset_dims(const char *obj_name, int time_step, int ndim, uint64_ // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, obj_reset_dims_register_id_g, @@ -2520,7 +2599,7 @@ PDC_Client_obj_reset_dims(const char *obj_name, int time_step, int ndim, uint64_ pdc_client_mpi_rank_g); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (lookup_args.ret == 2) { *reset = 1; @@ -2622,7 +2701,7 @@ PDC_Client_send_name_recv_id(const char *obj_name, uint64_t cont_id, pdcid_t obj server_id); fflush(stdout); } - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); // We have already filled in the pdc_server_info_g[server_id].addr in previous @@ -2634,7 +2713,7 @@ PDC_Client_send_name_recv_id(const char *obj_name, uint64_t cont_id, pdcid_t obj PGOTO_ERROR(FAIL, "PDC_Client_send_name_to_server(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); // Now we have obj id stored in lookup_args.obj_id @@ -2683,7 +2762,7 @@ PDC_Client_close_all_server() if (pdc_client_mpi_size_g >= pdc_server_num_g) { if (pdc_client_mpi_rank_g < pdc_server_num_g) { server_id = pdc_client_mpi_rank_g; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); @@ -2697,7 +2776,7 @@ PDC_Client_close_all_server() PGOTO_ERROR(FAIL, "PDC_Client_close_all_server(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); hg_ret = HG_Destroy(close_server_handle); @@ -2709,7 +2788,7 @@ PDC_Client_close_all_server() if (pdc_client_mpi_rank_g == 0) { for (i = 0; i < (uint32_t)pdc_server_num_g; i++) { server_id = pdc_server_num_g - 1 - i; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); @@ -2725,7 +2804,7 @@ PDC_Client_close_all_server() // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); hg_ret = HG_Destroy(close_server_handle); @@ -2777,7 +2856,7 @@ PDC_Client_buf_unmap(pdcid_t remote_obj_id, pdcid_t remote_reg_id, struct pdc_re // Debug statistics for counting number of messages sent to each server. debug_server_id_count[data_server_id]++; - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[data_server_id].addr, buf_unmap_register_id_g, @@ -2789,7 +2868,7 @@ PDC_Client_buf_unmap(pdcid_t remote_obj_id, pdcid_t remote_reg_id, struct pdc_re pdc_timings.PDCbuf_obj_unmap_rpc += MPI_Wtime() - start; #endif // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); #ifdef PDC_TIMING start = MPI_Wtime(); #endif @@ -2858,7 +2937,7 @@ PDC_Client_flush_obj(uint64_t obj_id) FUNC_ENTER(NULL); for (i = 0; i < (uint32_t)pdc_server_num_g; i++) { server_id = pdc_server_num_g - 1 - i; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, flush_obj_register_id_g, @@ -2872,7 +2951,7 @@ PDC_Client_flush_obj(uint64_t obj_id) // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); hg_ret = HG_Destroy(flush_obj_handle); @@ -2898,7 +2977,7 @@ PDC_Client_flush_obj_all() FUNC_ENTER(NULL); for (i = 0; i < (uint32_t)pdc_server_num_g; i++) { server_id = pdc_server_num_g - 1 - i; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, flush_obj_all_register_id_g, @@ -2912,7 +2991,7 @@ PDC_Client_flush_obj_all() // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); hg_ret = HG_Destroy(flush_obj_all_handle); @@ -2957,7 +3036,7 @@ PDC_Client_transfer_request_all(int n_objs, pdc_access_t access_type, uint32_t d hg_class = HG_Context_get_class(send_context_g); - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); @@ -2987,7 +3066,7 @@ PDC_Client_transfer_request_all(int n_objs, pdc_access_t access_type, uint32_t d if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "PDC_Client_send_transfer_request_all(): Could not start HG_Forward() @ line %d\n", __LINE__); - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); #ifdef PDC_TIMING @@ -3043,7 +3122,7 @@ PDC_Client_transfer_request_metadata_query2(char *buf, uint64_t total_buf_size, hg_class = HG_Context_get_class(send_context_g); - if (PDC_Client_try_lookup_server(metadata_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(metadata_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); hg_ret = HG_Create(send_context_g, pdc_server_info_g[metadata_server_id].addr, @@ -3068,7 +3147,7 @@ PDC_Client_transfer_request_metadata_query2(char *buf, uint64_t total_buf_size, FAIL, "PDC_Client_send_transfer_request_metadata_query2(): Could not start HG_Forward() @ line %d\n", __LINE__); - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (transfer_args.ret != 1) @@ -3114,7 +3193,7 @@ PDC_Client_transfer_request_metadata_query(char *buf, uint64_t total_buf_size, i hg_class = HG_Context_get_class(send_context_g); - if (PDC_Client_try_lookup_server(metadata_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(metadata_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); hg_ret = HG_Create(send_context_g, pdc_server_info_g[metadata_server_id].addr, @@ -3139,7 +3218,7 @@ PDC_Client_transfer_request_metadata_query(char *buf, uint64_t total_buf_size, i FAIL, "PDC_Client_send_transfer_request_metadata_query(): Could not start HG_Forward() @ line %d\n", __LINE__); - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (transfer_args.ret != 1) PGOTO_ERROR(FAIL, "PDC_CLIENT: transfer_request_metadata_query failed... @ line %d\n", __LINE__); @@ -3185,7 +3264,7 @@ PDC_Client_transfer_request_wait_all(int n_objs, pdcid_t *transfer_request_id, u hg_class = HG_Context_get_class(send_context_g); - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); hg_ret = @@ -3213,7 +3292,7 @@ PDC_Client_transfer_request_wait_all(int n_objs, pdcid_t *transfer_request_id, u PGOTO_ERROR(FAIL, "PDC_Client_send_transfer_request_wait_all(): Could not start HG_Forward() @ line %d\n", __LINE__); - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); /* @@ -3297,7 +3376,7 @@ PDC_Client_transfer_request(void *buf, pdcid_t obj_id, uint32_t data_server_id, pack_region_metadata(remote_ndim, remote_offset, remote_size, &(in.remote_region)); - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); @@ -3329,7 +3408,7 @@ PDC_Client_transfer_request(void *buf, pdcid_t obj_id, uint32_t data_server_id, if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "PDC_Client_send_transfer_request(): Could not start HG_Forward() @ line %d\n", __LINE__); - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); #ifdef PDC_TIMING @@ -3367,7 +3446,7 @@ PDC_Client_transfer_request_status(pdcid_t transfer_request_id, uint32_t data_se in.transfer_request_id = transfer_request_id; - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); @@ -3385,7 +3464,7 @@ PDC_Client_transfer_request_status(pdcid_t transfer_request_id, uint32_t data_se if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "PDC_Client_send_transfer_request(): Could not start HG_Forward() @ line %d\n", __LINE__); - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (transfer_args.ret != 1) @@ -3419,7 +3498,7 @@ PDC_Client_transfer_request_wait(pdcid_t transfer_request_id, uint32_t data_serv in.transfer_request_id = transfer_request_id; in.access_type = access_type; - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); @@ -3447,7 +3526,7 @@ PDC_Client_transfer_request_wait(pdcid_t transfer_request_id, uint32_t data_serv if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "PDC_Client_send_transfer_request(): Could not start HG_Forward() @ line %d\n", __LINE__); - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); #ifdef PDC_TIMING @@ -3600,7 +3679,7 @@ PDC_Client_buf_map(pdcid_t local_region_id, pdcid_t remote_obj_id, size_t ndim, else PGOTO_ERROR(FAIL, "mapping for array of dimension greater than 4 is not supproted"); - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[data_server_id].addr, buf_map_register_id_g, @@ -3620,7 +3699,7 @@ PDC_Client_buf_map(pdcid_t local_region_id, pdcid_t remote_obj_id, size_t ndim, PGOTO_ERROR(FAIL, "PDC_Client_send_buf_map(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); #ifdef PDC_TIMING start = MPI_Wtime(); #endif @@ -3683,7 +3762,7 @@ PDC_Client_region_lock(pdcid_t remote_obj_id, struct _pdc_obj_info *object_info, in.data_unit = PDC_get_var_type_size(data_type); PDC_region_info_t_to_transfer_unit(region_info, &(in.region), in.data_unit); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, region_lock_register_id_g, @@ -3703,7 +3782,7 @@ PDC_Client_region_lock(pdcid_t remote_obj_id, struct _pdc_obj_info *object_info, PGOTO_ERROR(FAIL, "PDC_Client_send_name_to_server(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); #ifdef PDC_TIMING start = MPI_Wtime(); #endif @@ -3794,7 +3873,7 @@ pdc_region_release_with_server_transform(struct _pdc_obj_info * object_info, unit = type_extent; PDC_region_info_t_to_transfer_unit(region_info, &(in.region), unit); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); // Create a bulk handle for the temp buffer used by the transform @@ -3811,7 +3890,7 @@ pdc_region_release_with_server_transform(struct _pdc_obj_info * object_info, PGOTO_ERROR(FAIL, "PDC_Client_send_name_to_server(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1) ; PDC_Client_check_response(&send_context_g); // RPC is complete @@ -3903,7 +3982,7 @@ pdc_region_release_with_server_analysis(struct _pdc_obj_info * object_info, in.output_obj_id = obj_prop->obj_prop_pub->obj_prop_id; in.output_iter = outputIter->meta_id; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, region_analysis_release_register_id_g, @@ -3914,7 +3993,7 @@ pdc_region_release_with_server_analysis(struct _pdc_obj_info * object_info, PGOTO_ERROR(FAIL, "PDC_Client_send_name_to_server(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1) ; PDC_Client_check_response(&send_context_g); // Now the return value is stored in lookup_args.ret @@ -4003,7 +4082,7 @@ pdc_region_release_with_client_transform(struct _pdc_obj_info * object_info, unit = type_extent; PDC_region_info_t_to_transfer_unit(region_info, &(in.region), unit); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); transform_args.data = data_ptrs[0]; @@ -4028,7 +4107,7 @@ pdc_region_release_with_client_transform(struct _pdc_obj_info * object_info, PGOTO_ERROR(FAIL, "PDC_Client_send_name_to_server(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1) ; PDC_Client_check_response(&send_context_g); ret_value = HG_Bulk_free(in.local_bulk_handle); @@ -4335,7 +4414,7 @@ PDC_Client_region_release(pdcid_t remote_obj_id, struct _pdc_obj_info *object_in in.data_unit = PDC_get_var_type_size(data_type); PDC_region_info_t_to_transfer_unit(region_info, &(in.region), in.data_unit); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, region_release_register_id_g, @@ -4357,7 +4436,7 @@ PDC_Client_region_release(pdcid_t remote_obj_id, struct _pdc_obj_info *object_in PGOTO_ERROR(FAIL, "PDC_Client_send_name_to_server(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); #ifdef PDC_TIMING end = MPI_Wtime(); @@ -4422,7 +4501,7 @@ data_server_read_check_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -4515,7 +4594,7 @@ PDC_Client_get_data_from_server_shm_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); FUNC_LEAVE(ret_value); } @@ -4558,7 +4637,7 @@ PDC_Client_data_server_read_check(int server_id, uint32_t client_id, pdc_metadat read_size *= region->size[i]; } - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, data_server_read_check_register_id_g, @@ -4569,7 +4648,7 @@ PDC_Client_data_server_read_check(int server_id, uint32_t client_id, pdc_metadat PGOTO_ERROR(FAIL, "== Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); *status = lookup_args.ret; @@ -4654,7 +4733,7 @@ data_server_read_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -4705,7 +4784,7 @@ PDC_Client_data_server_read(struct pdc_request *request) PDC_metadata_t_to_transfer_t(meta, &in.meta); PDC_region_info_t_to_transfer(region, &in.region); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, data_server_read_register_id_g, @@ -4716,7 +4795,7 @@ PDC_Client_data_server_read(struct pdc_request *request) PGOTO_ERROR(FAIL, "Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (lookup_args.ret != 1) @@ -4787,7 +4866,7 @@ data_server_write_check_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -4825,7 +4904,7 @@ PDC_Client_data_server_write_check(struct pdc_request *request, int *status) write_size *= region->size[i]; } - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, data_server_write_check_register_id_g, @@ -4836,7 +4915,7 @@ PDC_Client_data_server_write_check(struct pdc_request *request, int *status) PGOTO_ERROR(FAIL, "Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); *status = lookup_args.ret; @@ -4886,7 +4965,7 @@ data_server_write_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -4990,7 +5069,7 @@ PDC_Client_data_server_write(struct pdc_request *request) PDC_metadata_t_to_transfer_t(meta, &in.meta); PDC_region_info_t_to_transfer(region, &in.region); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, data_server_write_register_id_g, @@ -5005,7 +5084,7 @@ PDC_Client_data_server_write(struct pdc_request *request) } // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); HG_Destroy(data_server_write_handle); @@ -5138,7 +5217,7 @@ hg_return_t PDC_Client_work_done_cb(const struct hg_cb_info *callback_info ATTRIBUTE(unused)) { // server_lookup_client_out_t *validate = callback_info->arg; - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); return HG_SUCCESS; } @@ -5273,7 +5352,7 @@ PDC_Client_write_wait_notify(pdc_metadata_t *meta, struct pdc_region_info *regio pdc_client_mpi_rank_g); fflush(stdout); - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); printf("==PDC_CLIENT[%d]: received write finish notification\n", pdc_client_mpi_rank_g); @@ -5298,7 +5377,7 @@ PDC_Client_read_wait_notify(pdc_metadata_t *meta, struct pdc_region_info *region DL_PREPEND(pdc_io_request_list_g, request); - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); done: @@ -5335,7 +5414,7 @@ PDC_Client_add_del_objects_to_container_cb(const struct hg_cb_info *callback_inf done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); FUNC_LEAVE(ret_value); } @@ -5361,7 +5440,7 @@ PDC_Client_add_del_objects_to_container(int nobj, uint64_t *obj_ids, uint64_t co // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); // Send the bulk handle to the target with RPC @@ -5392,7 +5471,7 @@ PDC_Client_add_del_objects_to_container(int nobj, uint64_t *obj_ids, uint64_t co if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "Could not forward call"); - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); done: @@ -5479,7 +5558,7 @@ PDC_Client_add_tags_to_container(pdcid_t cont_id, char *tags) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); // Send the bulk handle to the target with RPC @@ -5497,7 +5576,7 @@ PDC_Client_add_tags_to_container(pdcid_t cont_id, char *tags) if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "Could not forward call"); - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); ret_value = SUCCEED; @@ -5530,7 +5609,7 @@ container_query_rpc_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -5556,7 +5635,7 @@ PDC_Client_query_container_name(const char *cont_name, uint64_t *cont_meta_id) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, container_query_register_id_g, @@ -5575,7 +5654,7 @@ PDC_Client_query_container_name(const char *cont_name, uint64_t *cont_meta_id) pdc_client_mpi_rank_g); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); *cont_meta_id = lookup_args.cont_id; @@ -5604,8 +5683,8 @@ PDC_Client_query_container_name_col(const char *cont_name, uint64_t *cont_meta_i MPI_Bcast(cont_meta_id, 1, MPI_LONG_LONG, 0, PDC_CLIENT_COMM_WORLD_g); #else - printf("==PDC_CLIENT[%d]: Calling MPI collective operation without enabling MPI!\n", - pdc_client_mpi_rank_g); + PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Calling MPI collective operation without enabling MPI!", + pdc_client_mpi_rank_g); #endif done: @@ -5644,7 +5723,7 @@ PDC_Client_query_read_obj_name_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); FUNC_LEAVE(ret_value); } @@ -5733,7 +5812,7 @@ PDC_Client_query_name_read_entire_obj(int nobj, char **obj_names, void ***out_bu // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); // Send the bulk handle to the target with RPC @@ -5779,11 +5858,11 @@ PDC_Client_query_name_read_entire_obj(int nobj, char **obj_names, void ***out_bu PGOTO_ERROR(FAIL, "Could not forward call"); // Wait for RPC response - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); // Wait for server to complete all reads - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); done: @@ -5894,7 +5973,7 @@ PDC_Client_query_read_complete(char *shm_addrs, int size, int n_shm, int seq_id) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); FUNC_LEAVE(ret_value); } @@ -5914,7 +5993,7 @@ PDC_Client_server_checkpoint(uint32_t server_id) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, server_checkpoint_rpc_register_id_g, @@ -5928,7 +6007,7 @@ PDC_Client_server_checkpoint(uint32_t server_id) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Could not start forward to server", pdc_client_mpi_rank_g); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); done: @@ -6006,7 +6085,7 @@ PDC_Client_send_client_shm_info(uint32_t server_id, char *shm_addr, uint64_t siz // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = @@ -6024,7 +6103,7 @@ PDC_Client_send_client_shm_info(uint32_t server_id, char *shm_addr, uint64_t siz PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Could not forward to server", pdc_client_mpi_rank_g); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); done: @@ -6121,7 +6200,7 @@ PDC_send_region_storage_meta_shm(uint32_t server_id, int n, region_storage_meta_ // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, @@ -6148,7 +6227,7 @@ PDC_send_region_storage_meta_shm(uint32_t server_id, int n, region_storage_meta_ PGOTO_ERROR(FAIL, "Could not forward call"); // Wait for RPC response - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); done: @@ -6429,7 +6508,7 @@ PDC_Client_query_multi_storage_info(int nobj, char **obj_names, region_storage_m send_n_request++; debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); @@ -6476,11 +6555,11 @@ PDC_Client_query_multi_storage_info(int nobj, char **obj_names, region_storage_m PGOTO_ERROR(FAIL, "Could not forward call"); // Wait for RPC response - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); // Wait for server initiated bulk xfer - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); } // End for each meta server @@ -6772,7 +6851,7 @@ PDC_Client_recv_bulk_storage_meta(process_bulk_storage_meta_args_t *process_args done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); FUNC_LEAVE(ret_value); } @@ -7006,7 +7085,7 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_add_kvtag_register_id_g, @@ -7017,6 +7096,7 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) if (kvtag != NULL && kvtag != NULL && kvtag->size != 0) { in.kvtag.name = kvtag->name; in.kvtag.value = kvtag->value; + in.kvtag.type = kvtag->type; in.kvtag.size = kvtag->size; } else @@ -7027,7 +7107,7 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) PGOTO_ERROR(FAIL, "PDC_Client_add_kvtag_metadata_with_name(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (lookup_args.ret != 1) @@ -7054,19 +7134,24 @@ metadata_get_kvtag_rpc_cb(const struct hg_cb_info *callback_info) ret_value = HG_Get_output(handle, &output); if (ret_value != HG_SUCCESS) { client_lookup_args->ret = -1; - PGOTO_ERROR(ret_value, "==PDC_CLIENT[%d]: metadata_add_tag_rpc_cb error with HG_Get_output", - pdc_client_mpi_rank_g); + PGOTO_ERROR(ret_value, "==PDC_CLIENT[%d]: %s error with HG_Get_output", pdc_client_mpi_rank_g, + __func__); + } + client_lookup_args->ret = output.ret; + if (output.kvtag.name) + client_lookup_args->kvtag->name = strdup(output.kvtag.name); + client_lookup_args->kvtag->size = output.kvtag.size; + client_lookup_args->kvtag->type = output.kvtag.type; + if (output.kvtag.size > 0) { + client_lookup_args->kvtag->value = malloc(output.kvtag.size); + memcpy(client_lookup_args->kvtag->value, output.kvtag.value, output.kvtag.size); } - client_lookup_args->ret = output.ret; - client_lookup_args->kvtag->name = strdup(output.kvtag.name); - client_lookup_args->kvtag->size = output.kvtag.size; - client_lookup_args->kvtag->value = malloc(output.kvtag.size); - memcpy(client_lookup_args->kvtag->value, output.kvtag.value, output.kvtag.size); - /* PDC_kvtag_dup(&(output.kvtag), &client_lookup_args->kvtag); */ + else + client_lookup_args->kvtag->value = NULL; done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -7103,7 +7188,7 @@ PDC_get_kvtag(pdcid_t obj_id, char *tag_name, pdc_kvtag_t **kvtag, int is_cont) server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_get_kvtag_register_id_g, @@ -7113,16 +7198,16 @@ PDC_get_kvtag(pdcid_t obj_id, char *tag_name, pdc_kvtag_t **kvtag, int is_cont) in.key = tag_name; } else - PGOTO_ERROR(FAIL, "==PDC_Client_get_kvtag(): invalid tag content!"); + PGOTO_ERROR(FAIL, "PDC_get_kvtag: invalid tag content!"); *kvtag = (pdc_kvtag_t *)malloc(sizeof(pdc_kvtag_t)); lookup_args.kvtag = *kvtag; hg_ret = HG_Forward(metadata_get_kvtag_handle, metadata_get_kvtag_rpc_cb, &lookup_args, &in); if (hg_ret != HG_SUCCESS) - PGOTO_ERROR(FAIL, "PDC_Client_get_kvtag_metadata_with_name(): Could not start HG_Forward()"); + PGOTO_ERROR(FAIL, "PDC_get_kvtag: Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (lookup_args.ret != 1) @@ -7135,55 +7220,6 @@ PDC_get_kvtag(pdcid_t obj_id, char *tag_name, pdc_kvtag_t **kvtag, int is_cont) FUNC_LEAVE(ret_value); } -perr_t -PDCtag_delete(pdcid_t obj_id, char *tag_name) -{ - perr_t ret_value = SUCCEED; - hg_return_t hg_ret = 0; - uint64_t meta_id; - uint32_t server_id; - hg_handle_t metadata_del_kvtag_handle; - metadata_get_kvtag_in_t in; - struct _pdc_obj_info * obj_prop; - struct _pdc_client_lookup_args lookup_args; - - FUNC_ENTER(NULL); - - obj_prop = PDC_obj_get_info(obj_id); - meta_id = obj_prop->obj_info_pub->meta_id; - server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); - - debug_server_id_count[server_id]++; - - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) - PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); - - HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_del_kvtag_register_id_g, - &metadata_del_kvtag_handle); - - // Fill input structure - in.obj_id = meta_id; - in.hash_value = PDC_get_hash_by_name(obj_prop->obj_info_pub->name); - in.key = tag_name; - - hg_ret = HG_Forward(metadata_del_kvtag_handle, metadata_add_tag_rpc_cb /*reuse*/, &lookup_args, &in); - if (hg_ret != HG_SUCCESS) - PGOTO_ERROR(FAIL, "PDC_Client_del_kvtag_metadata_with_name(): Could not start HG_Forward()"); - - // Wait for response from server - work_todo_g = 1; - PDC_Client_check_response(&send_context_g); - - if (lookup_args.ret != 1) - printf("PDC_CLIENT: del kvtag NOT successful ... ret_value = %d\n", lookup_args.ret); - -done: - fflush(stdout); - HG_Destroy(metadata_del_kvtag_handle); - - FUNC_LEAVE(ret_value); -} - static hg_return_t kvtag_query_bulk_cb(const struct hg_cb_info *hg_cb_info) { @@ -7191,9 +7227,13 @@ kvtag_query_bulk_cb(const struct hg_cb_info *hg_cb_info) struct bulk_args_t *bulk_args; hg_bulk_t origin_bulk_handle = hg_cb_info->info.bulk.origin_handle; hg_bulk_t local_bulk_handle = hg_cb_info->info.bulk.local_handle; - void * buf = NULL; uint32_t n_meta, actual_cnt; + void * buf = NULL; uint64_t buf_sizes[1]; + uint32_t bulk_sgnum; + uint64_t * ids_buf_sizes; + void ** ids_buf; + uint64_t * u64_arr_ptr; FUNC_ENTER(NULL); @@ -7202,18 +7242,28 @@ kvtag_query_bulk_cb(const struct hg_cb_info *hg_cb_info) n_meta = bulk_args->n_meta; if (hg_cb_info->ret == HG_SUCCESS) { - HG_Bulk_access(local_bulk_handle, 0, bulk_args->nbytes, HG_BULK_READWRITE, 1, &buf, buf_sizes, - &actual_cnt); + bulk_sgnum = HG_Bulk_get_segment_count(local_bulk_handle); + ids_buf = (void **)calloc(sizeof(void *), bulk_sgnum); + ids_buf_sizes = (uint64_t *)calloc(sizeof(uint64_t), bulk_sgnum); + HG_Bulk_access(local_bulk_handle, 0, bulk_args->nbytes, HG_BULK_READWRITE, bulk_sgnum, ids_buf, + ids_buf_sizes, &actual_cnt); + u64_arr_ptr = ((uint64_t **)(ids_buf))[0]; bulk_args->obj_ids = (uint64_t *)calloc(sizeof(uint64_t), n_meta); - memcpy(bulk_args->obj_ids, buf, sizeof(uint64_t) * n_meta); + for (int i = 0; i < n_meta; i++) { + bulk_args->obj_ids[i] = *u64_arr_ptr; + u64_arr_ptr++; + } + + // HG_Bulk_access(local_bulk_handle, 0, bulk_args->nbytes, HG_BULK_READWRITE, 1, &buf, buf_sizes, + // &actual_cnt); + + // bulk_args->obj_ids = (uint64_t *)calloc(sizeof(uint64_t), n_meta); + // memcpy(bulk_args->obj_ids, buf, sizeof(uint64_t) * n_meta); } else PGOTO_ERROR(HG_PROTOCOL_ERROR, "==PDC_CLIENT[%d]: Error with bulk handle", pdc_client_mpi_rank_g); - bulk_todo_g--; - hg_atomic_set32(&bulk_transfer_done_g, 1); - // Free local bulk handle ret_value = HG_Bulk_free(local_bulk_handle); if (ret_value != HG_SUCCESS) @@ -7224,6 +7274,8 @@ kvtag_query_bulk_cb(const struct hg_cb_info *hg_cb_info) PGOTO_ERROR(ret_value, "Could not free HG bulk handle"); done: + hg_atomic_decr32(&bulk_todo_g); + hg_atomic_cas32(&bulk_transfer_done_g, 0, 1); fflush(stdout); HG_Destroy(bulk_args->handle); @@ -7253,9 +7305,12 @@ kvtag_query_forward_cb(const struct hg_cb_info *callback_info) if (ret_value != HG_SUCCESS) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: error HG_Get_output", pdc_client_mpi_rank_g); + bulk_arg->server_time_elapsed = output.server_time_elapsed; + bulk_arg->server_memory_consumption = output.server_memory_consumption; + if (output.bulk_handle == HG_BULK_NULL || output.ret == 0) { - bulk_todo_g = 0; - work_todo_g--; + hg_atomic_decr32(&bulk_todo_g); + // hg_atomic_decr32(&atomic_work_todo_g); bulk_arg->n_meta = 0; bulk_arg->obj_ids = NULL; HG_Free_output(handle, &output); @@ -7287,7 +7342,7 @@ kvtag_query_forward_cb(const struct hg_cb_info *callback_info) done: fflush(stdout); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); HG_Free_output(handle, &output); FUNC_LEAVE(ret_value); @@ -7304,8 +7359,12 @@ PDC_Client_query_kvtag_server(uint32_t server_id, const pdc_kvtag_t *kvtag, int FUNC_ENTER(NULL); - if (kvtag == NULL || n_res == NULL || out == NULL) - PGOTO_ERROR(FAIL, "==CLIENT[%d]: input is NULL!", pdc_client_mpi_rank_g); + if (kvtag == NULL) + PGOTO_ERROR(FAIL, "==CLIENT[%d]: %s - kvtag is NULL!", pdc_client_mpi_rank_g, __func__); + if (n_res == NULL) + PGOTO_ERROR(FAIL, "==CLIENT[%d]: %s - n_res is NULL!", pdc_client_mpi_rank_g, __func__); + if (out == NULL) + PGOTO_ERROR(FAIL, "==CLIENT[%d]: %s - out is NULL!", pdc_client_mpi_rank_g, __func__); if (kvtag->name == NULL) in.name = " "; @@ -7314,17 +7373,19 @@ PDC_Client_query_kvtag_server(uint32_t server_id, const pdc_kvtag_t *kvtag, int if (kvtag->value == NULL) { in.value = " "; + in.type = PDC_STRING; in.size = 1; } else { in.value = kvtag->value; + in.type = kvtag->type; in.size = kvtag->size; } *out = NULL; *n_res = 0; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, query_kvtag_register_id_g, @@ -7341,11 +7402,16 @@ PDC_Client_query_kvtag_server(uint32_t server_id, const pdc_kvtag_t *kvtag, int hg_atomic_set32(&bulk_transfer_done_g, 0); // Wait for response from server - bulk_todo_g = 1; + hg_atomic_incr32(&bulk_todo_g); PDC_Client_check_bulk(send_context_g); + server_call_count_g[server_id]++; + server_time_total_g[server_id] += bulk_arg->server_time_elapsed; + server_mem_usage_g[server_id] = bulk_arg->server_memory_consumption; + *n_res = bulk_arg->n_meta; - *out = bulk_arg->obj_ids; + if (*n_res > 0) + *out = bulk_arg->obj_ids; free(bulk_arg); // TODO: need to be careful when freeing the lookup_args, as it include the results returned to user @@ -7358,93 +7424,103 @@ PDC_Client_query_kvtag_server(uint32_t server_id, const pdc_kvtag_t *kvtag, int perr_t PDC_Client_query_kvtag(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids) { - perr_t ret_value = SUCCEED; - int32_t i; - int nmeta = 0; + perr_t ret_value = SUCCEED; + int32_t i; + int nmeta = 0; + uint64_t *temp_ids = NULL; + uint32_t server_id; FUNC_ENTER(NULL); - *n_res = 0; + *n_res = 0; + *pdc_ids = NULL; + for (i = 0; i < pdc_server_num_g; i++) { - ret_value = PDC_Client_query_kvtag_server((uint32_t)i, kvtag, &nmeta, pdc_ids); - if (ret_value != SUCCEED) + // TODO: when there are multiple clients issuing different queries concurrently, try to balance the + // server workload by having different clients sending queries with a different order + // server_id = (pdc_client_mpi_rank_g + i) % pdc_server_num_g; + // ret_value = PDC_Client_query_kvtag_server(server_id, kvtag, &nmeta, &temp_ids); + ret_value = PDC_Client_query_kvtag_server((uint32_t)i, kvtag, &nmeta, &temp_ids); + if (ret_value != SUCCEED) { PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: error with PDC_Client_query_kvtag_server to server %d", pdc_client_mpi_rank_g, i); + } + if (i == 0) { + *pdc_ids = temp_ids; + } + else if (nmeta > 0) { + *pdc_ids = (uint64_t *)realloc(*pdc_ids, sizeof(uint64_t) * (*n_res + nmeta)); + memcpy(*pdc_ids + (*n_res), temp_ids, nmeta * sizeof(uint64_t)); + free(temp_ids); + } + *n_res = *n_res + nmeta; } - - *n_res = nmeta; - done: + memory_debug_g = 1; fflush(stdout); FUNC_LEAVE(ret_value); } -void -PDC_assign_server(uint32_t *my_server_start, uint32_t *my_server_end, uint32_t *my_server_count) +// Delete a tag specified by a name, and whether it is from a container or an object +static perr_t +PDCtag_delete(pdcid_t obj_id, char *tag_name, int is_cont) { + perr_t ret_value = SUCCEED; + hg_return_t hg_ret = 0; + uint64_t meta_id; + uint32_t server_id; + hg_handle_t metadata_del_kvtag_handle; + metadata_get_kvtag_in_t in; + struct _pdc_obj_info * obj_prop; + struct _pdc_cont_info * cont_prop; + struct _pdc_client_lookup_args lookup_args; + FUNC_ENTER(NULL); - if (pdc_server_num_g > pdc_client_mpi_size_g) { - *my_server_count = pdc_server_num_g / pdc_client_mpi_size_g; - *my_server_start = pdc_client_mpi_rank_g * (*my_server_count); - *my_server_end = *my_server_start + (*my_server_count); - if (pdc_client_mpi_rank_g == pdc_client_mpi_size_g - 1) { - (*my_server_end) += pdc_server_num_g % pdc_client_mpi_size_g; - } + if (is_cont) { + cont_prop = PDC_cont_get_info(obj_id); + meta_id = cont_prop->cont_info_pub->meta_id; } else { - *my_server_start = pdc_client_mpi_rank_g; - *my_server_end = *my_server_start + 1; - if (pdc_client_mpi_rank_g >= pdc_server_num_g) { - *my_server_end = 0; - } + obj_prop = PDC_obj_get_info(obj_id); + meta_id = obj_prop->obj_info_pub->meta_id; } - FUNC_LEAVE_VOID; -} + server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); -// All clients collectively query all servers -perr_t -PDC_Client_query_kvtag_col(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids) -{ - perr_t ret_value = SUCCEED; - int32_t my_server_start, my_server_end, my_server_count; - int32_t i; - int nmeta = 0; + debug_server_id_count[server_id]++; - FUNC_ENTER(NULL); + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) + PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); - if (pdc_server_num_g > pdc_client_mpi_size_g) { - my_server_count = pdc_server_num_g / pdc_client_mpi_size_g; - my_server_start = pdc_client_mpi_rank_g * my_server_count; - my_server_end = my_server_start + my_server_count; - if (pdc_client_mpi_rank_g == pdc_client_mpi_size_g - 1) { - my_server_end += pdc_server_num_g % pdc_client_mpi_size_g; - } - } - else { - my_server_start = pdc_client_mpi_rank_g; - my_server_end = my_server_start + 1; - if (pdc_client_mpi_rank_g >= pdc_server_num_g) { - my_server_end = 0; - } - } + HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_del_kvtag_register_id_g, + &metadata_del_kvtag_handle); - *n_res = 0; - for (i = my_server_start; i < my_server_end; i++) { - if (i >= pdc_server_num_g) { - break; - } - ret_value = PDC_Client_query_kvtag_server((uint32_t)i, kvtag, &nmeta, pdc_ids); - if (ret_value != SUCCEED) - PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: error with PDC_Client_query_kvtag_server to server %u", - pdc_client_mpi_rank_g, i); - } + // Fill input structure + in.obj_id = meta_id; - *n_res = nmeta; + if (is_cont) + in.hash_value = PDC_get_hash_by_name(cont_prop->cont_info_pub->name); + else + in.hash_value = PDC_get_hash_by_name(obj_prop->obj_info_pub->name); + in.key = tag_name; + + // reuse metadata_add_tag_rpc_cb here since it only checks the return value + hg_ret = HG_Forward(metadata_del_kvtag_handle, metadata_add_tag_rpc_cb /*reuse*/, &lookup_args, &in); + if (hg_ret != HG_SUCCESS) + PGOTO_ERROR(FAIL, "PDC_Client_del_kvtag_metadata_with_name(): Could not start HG_Forward()"); + + // Wait for response from server + hg_atomic_set32(&atomic_work_todo_g, 1); + PDC_Client_check_response(&send_context_g); + + if (lookup_args.ret != 1) + printf("PDC_CLIENT: del kvtag NOT successful ... ret_value = %d\n", lookup_args.ret); done: fflush(stdout); + HG_Destroy(metadata_del_kvtag_handle); + FUNC_LEAVE(ret_value); } @@ -7477,7 +7553,6 @@ PDCcont_put(const char *cont_name, pdcid_t pdc) } pdcid_t - PDCcont_get_id(const char *cont_name, pdcid_t pdc_id) { pdcid_t cont_id; @@ -7575,7 +7650,8 @@ PDCcont_get_objids(pdcid_t cont_id ATTRIBUTE(unused), int *nobj ATTRIBUTE(unused } perr_t -PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_size) +PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, + psize_t value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t kvtag; @@ -7584,6 +7660,7 @@ PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_ kvtag.name = tag_name; kvtag.value = (void *)tag_value; + kvtag.type = value_type; kvtag.size = (uint64_t)value_size; ret_value = PDC_add_kvtag(cont_id, &kvtag, 1); @@ -7597,7 +7674,8 @@ PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_ } perr_t -PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, psize_t *value_size) +PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t *kvtag = NULL; @@ -7609,6 +7687,7 @@ PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, psize_t *valu PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Error with PDC_get_kvtag", pdc_client_mpi_rank_g); *tag_value = kvtag->value; + *value_type = kvtag->type; *value_size = kvtag->size; done: @@ -7623,9 +7702,9 @@ PDCcont_del_tag(pdcid_t cont_id, char *tag_name) FUNC_ENTER(NULL); - ret_value = PDCobj_del_tag(cont_id, tag_name); + ret_value = PDCtag_delete(cont_id, tag_name, 1); if (ret_value != SUCCEED) - PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: error with PDCobj_del_tag", pdc_client_mpi_rank_g); + PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: error with PDCtag_delete", pdc_client_mpi_rank_g); done: fflush(stdout); @@ -7771,7 +7850,7 @@ PDC_Client_del_metadata(pdcid_t obj_id, int is_cont) } perr_t -PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_size) +PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, psize_t value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t kvtag; @@ -7780,6 +7859,7 @@ PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_si kvtag.name = tag_name; kvtag.value = (void *)tag_value; + kvtag.type = value_type; kvtag.size = (uint64_t)value_size; ret_value = PDC_add_kvtag(obj_id, &kvtag, 0); @@ -7792,7 +7872,8 @@ PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_si } perr_t -PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, psize_t *value_size) +PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t *kvtag = NULL; @@ -7804,6 +7885,7 @@ PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, psize_t *value_ PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Error with PDC_get_kvtag", pdc_client_mpi_rank_g); *tag_value = kvtag->value; + *value_type = kvtag->type; *value_size = kvtag->size; done: @@ -7818,7 +7900,7 @@ PDCobj_del_tag(pdcid_t obj_id, char *tag_name) FUNC_ENTER(NULL); - ret_value = PDCtag_delete(obj_id, tag_name); + ret_value = PDCtag_delete(obj_id, tag_name, 0); if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Error with PDC_del_kvtag", pdc_client_mpi_rank_g); @@ -7892,7 +7974,7 @@ PDC_recv_nhits(const struct hg_cb_info *callback_info) } } - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); free(in); fflush(stdout); @@ -7949,7 +8031,7 @@ PDC_send_data_query(pdc_query_t *query, pdc_query_get_op_t get_op, uint64_t *nhi query_xfer->next_server_id = next_server; query_xfer->prev_server_id = prev_server; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, send_data_query_register_id_g, &handle); @@ -7959,7 +8041,7 @@ PDC_send_data_query(pdc_query_t *query, pdc_query_get_op_t get_op, uint64_t *nhi PGOTO_ERROR(FAIL, "PDC_Client_del_kvtag_metadata_with_name(): Could not start HG_Forward()"); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (lookup_args.ret != 1) @@ -7970,7 +8052,7 @@ PDC_send_data_query(pdc_query_t *query, pdc_query_get_op_t get_op, uint64_t *nhi } // Wait for server to send query result - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (nhits) @@ -8042,7 +8124,7 @@ PDC_recv_coords(const struct hg_cb_info *callback_info) } // End else done: - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); if (nhits > 0) { ret_value = HG_Bulk_free(local_bulk_handle); if (ret_value != HG_SUCCESS) @@ -8103,7 +8185,7 @@ PDC_Client_get_sel_data(pdcid_t obj_id, pdc_selection_t *sel, void *data) server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, get_sel_data_register_id_g, &handle); @@ -8113,7 +8195,7 @@ PDC_Client_get_sel_data(pdcid_t obj_id, pdc_selection_t *sel, void *data) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with HG_Forward", pdc_client_mpi_rank_g); // Wait for response from server - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); if (lookup_args.ret != 1) { @@ -8122,7 +8204,7 @@ PDC_Client_get_sel_data(pdcid_t obj_id, pdc_selection_t *sel, void *data) } // Wait for server to send data - work_todo_g = 1; + hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); // Copy the result to user's buffer @@ -8199,12 +8281,12 @@ PDC_recv_read_coords_data(const struct hg_cb_info *callback_info) PGOTO_ERROR(HG_OTHER_ERROR, "==PDC_CLIENT[%d]: Invalid task ID!", pdc_client_mpi_rank_g); if (result_elt->recv_data_nhits == result_elt->nhits) { - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); } else if (result_elt->recv_data_nhits > result_elt->nhits) { PGOTO_ERROR(HG_OTHER_ERROR, "==PDC_CLIENT[%d]: received more results data than expected!", pdc_client_mpi_rank_g); - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); } } // End else @@ -8225,3 +8307,1046 @@ PDC_recv_read_coords_data(const struct hg_cb_info *callback_info) FUNC_LEAVE(ret_value); } + +void +report_avg_server_profiling_rst() +{ + for (int i = 0; i < pdc_server_num_g; i++) { + + double avg_srv_time = server_call_count_g[i] > 0 + ? (double)(server_time_total_g[i]) / (double)(server_call_count_g[i]) + : 0.0; + double srv_mem_usage = server_mem_usage_g[i] / 1024.0 / 1024.0; + printf("==PDC_CLIENT[%d]: server %d, avg profiling time: %.4f ms, memory usage: %.4f MB\n", + pdc_client_mpi_rank_g, i, avg_srv_time / 1000.0, srv_mem_usage); + } +} + +/******************** METADATA INDEX BEGINS *******************************/ + +// metadata_index_create callback +// static hg_return_t +// client_metadata_index_create_cb(const struct hg_cb_info *callback_info) +// { +// hg_return_t ret_value = HG_SUCCESS; +// hg_handle_t handle; +// struct client_lookup_args *client_lookup_args; + +// metadata_index_create_out_t output; + +// FUNC_ENTER(NULL); + +// /* printf("Entered client_rpc_cb()"); */ +// client_lookup_args = (struct client_lookup_args*) callback_info->arg; +// handle = callback_info->info.forward.handle; + +// /* Get output from server*/ +// ret_value = HG_Get_output(handle, &output); +// /* printf("Return value=%llu\n", output.ret); */ +// client_lookup_args->obj_id = output.ret; + +// hg_atomic_decr32(&atomic_work_todo_g); + +// done: +// HG_Destroy(handle); +// FUNC_LEAVE(ret_value); +// } + +// metadata_index_delete callback +// static hg_return_t +// client_metadata_index_delete_cb(const struct hg_cb_info *callback_info) +// { +// hg_return_t ret_value = HG_SUCCESS; +// hg_handle_t handle; +// struct client_lookup_args *client_lookup_args; + +// metadata_index_delete_out_t output; + +// FUNC_ENTER(NULL); + +// /* printf("Entered client_rpc_cb()"); */ +// client_lookup_args = (struct client_lookup_args*) callback_info->arg; +// handle = callback_info->info.forward.handle; + +// /* Get output from server*/ +// ret_value = HG_Get_output(handle, &output); +// /* printf("Return value=%llu\n", output.ret); */ +// client_lookup_args->obj_id = output.ret; + +// hg_atomic_decr32(&atomic_work_todo_g); + +// done: +// HG_Destroy(handle); +// FUNC_LEAVE(ret_value); +// } + +// get_server_info_callback +static hg_return_t +client_dart_get_server_info_cb(const struct hg_cb_info *callback_info) +{ + hg_return_t ret_value = HG_SUCCESS; + hg_handle_t handle; + struct client_genetic_lookup_args *client_lookup_args; + + dart_get_server_info_out_t output; + + FUNC_ENTER(NULL); + + /* printf("Entered client_rpc_cb()"); */ + client_lookup_args = (struct client_genetic_lookup_args *)callback_info->arg; + handle = callback_info->info.forward.handle; + + /* Get output from server*/ + ret_value = HG_Get_output(handle, &output); + /* printf("Return value=%llu\n", output.ret); */ + client_lookup_args->int64_value1 = output.indexed_word_count; + client_lookup_args->int64_value2 = output.request_count; + + hg_atomic_decr32(&atomic_work_todo_g); + HG_Destroy(handle); + + FUNC_LEAVE(ret_value); +} + +dart_server +dart_retrieve_server_info_cb(uint32_t serverId) +{ + dart_server ret; + + perr_t srv_lookup_rst = PDC_Client_try_lookup_server(serverId, 0); + if (srv_lookup_rst == FAIL) { + println("the server %d cannot be connected. ", serverId); + goto done; + } + + // Mercury comm here. + hg_handle_t dart_get_server_info_handle; + HG_Create(send_context_g, pdc_server_info_g[serverId].addr, dart_get_server_info_g, + &dart_get_server_info_handle); + dart_get_server_info_in_t in; + in.serverId = serverId; + struct client_genetic_lookup_args lookup_args; + hg_return_t hg_ret = + HG_Forward(dart_get_server_info_handle, client_dart_get_server_info_cb, &lookup_args, &in); + if (hg_ret != HG_SUCCESS) { + fprintf(stderr, + "dart_get_server_info_g(): Could not start HG_Forward() on serverId = %ld with host = %s\n", + serverId, pdc_server_info_g[serverId].addr_string); + HG_Destroy(dart_get_server_info_handle); + return ret; + } + + // Wait for response from server + hg_atomic_set32(&atomic_work_todo_g, 1); + PDC_Client_check_response(&send_context_g); + + ret.id = serverId; + ret.indexed_word_count = lookup_args.int64_value1; + ret.request_count = lookup_args.int64_value2; +done: + HG_Destroy(dart_get_server_info_handle); + + return ret; +} + +DART * +get_dart_g() +{ + return dart_g; +} + +// Bulk +static hg_return_t +dart_perform_one_server_on_receive_cb(const struct hg_cb_info *callback_info) +{ + hg_return_t ret_value; + struct bulk_args_t * client_lookup_args; + hg_handle_t handle; + dart_perform_one_server_out_t output; + uint32_t n_meta; + hg_op_id_t hg_bulk_op_id; + + hg_bulk_t local_bulk_handle = HG_BULK_NULL; + hg_bulk_t origin_bulk_handle = HG_BULK_NULL; + const struct hg_info *hg_info = NULL; + struct bulk_args_t * bulk_args; + void * recv_meta; + + FUNC_ENTER(NULL); + + // println("[Client_Side_Bulk] Entering dart_perform_one_server_on_receive_cb. rank = %d", + // pdc_client_mpi_rank_g); + client_lookup_args = (struct bulk_args_t *)callback_info->arg; + // (client_lookup_args->n_meta) = (uint32_t *)malloc(sizeof(uint32_t)); + handle = callback_info->info.forward.handle; + // println("[Client_Side_Bulk] before get output. rank = %d", pdc_client_mpi_rank_g); + // Get output from server + ret_value = HG_Get_output(handle, &output); + if (ret_value != HG_SUCCESS) { + printf("==PDC_CLIENT[%d]: dart_perform_one_server_on_receive_cb - error HG_Get_output\n", + pdc_client_mpi_rank_g); + client_lookup_args->n_meta = 0; + goto done; + } + + client_lookup_args->server_time_elapsed = output.server_time_elapsed; + client_lookup_args->server_memory_consumption = output.server_memory_consumption; + // printf("lookup_args.op_type = %d and output.op_type = %d\n", client_lookup_args->op_type, + // output.op_type); + + if (ret_value == HG_SUCCESS && output.has_bulk == 0) { + // printf("=== NO Bulk data should be taken care of. \n"); + client_lookup_args->n_meta = 0; + goto done; + } + + // println("[Client_Side_Bulk] before copy n_meta. rank = %d", pdc_client_mpi_rank_g); + // printf("==PDC_CLIENT: Received response from server with bulk handle, n_buf=%d\n", output.ret); + n_meta = output.n_items; + + client_lookup_args->n_meta = n_meta; + + // printf("*(client_lookup_args->n_meta) = %ld\n", *(client_lookup_args->n_meta)); + + // println("[Client_Side_Bulk] before determining size. rank = %d", pdc_client_mpi_rank_g); + if (n_meta == 0) { + client_lookup_args->obj_ids = NULL; + client_lookup_args->n_meta = 0; + goto done; + } + + // Prepare to receive BULK data. + origin_bulk_handle = output.bulk_handle; + hg_info = HG_Get_info(handle); + + client_lookup_args->handle = handle; + client_lookup_args->nbytes = HG_Bulk_get_size(origin_bulk_handle); + + /* printf("nbytes=%u\n", bulk_args->nbytes); */ + + if (client_lookup_args->is_id == 1) { + recv_meta = (void *)calloc(n_meta, sizeof(uint64_t)); + } + else { + // throw an error + printf("==PDC_CLIENT[%d]: ERROR - DART queries can only retrieve object IDs. Please check " + "client_lookup_args->is_id\n", + pdc_client_mpi_rank_g); + goto done; + } + + /* Create a new bulk handle to read the data */ + HG_Bulk_create(hg_info->hg_class, 1, (void **)&recv_meta, (hg_size_t *)&client_lookup_args->nbytes, + HG_BULK_READWRITE, &local_bulk_handle); + + // println("[Client_Side_Bulk] after bulk create. rank = %d", pdc_client_mpi_rank_g); + + /* Pull bulk data */ + ret_value = HG_Bulk_transfer(hg_info->context, hg_test_bulk_transfer_cb, client_lookup_args, HG_BULK_PULL, + hg_info->addr, origin_bulk_handle, 0, local_bulk_handle, 0, + client_lookup_args->nbytes, &hg_bulk_op_id); + + // println("[Client_Side_Bulk] after bulk transfer. rank = %d", pdc_client_mpi_rank_g); + + if (ret_value != HG_SUCCESS) { + fprintf(stderr, "Could not read bulk data\n"); + client_lookup_args->n_meta = 0; + goto done; + } + + hg_atomic_init32(&(client_lookup_args->bulk_done_flag), 0); + hg_atomic_incr32(&bulk_todo_g); + + // hg_atomic_set32(&bulk_transfer_done_g, 0); + // loop + PDC_Client_check_bulk(send_context_g); + // println("[Client_Side_Bulk] after check bulk. rank = %d", pdc_client_mpi_rank_g); + + while (1) { + if (hg_atomic_get32(&(client_lookup_args->bulk_done_flag)) == 1) { + break; + } + } + +done: + // println("[Client_Side_Bulk] finish bulk. rank = %d", pdc_client_mpi_rank_g); + hg_atomic_decr32(&atomic_work_todo_g); + HG_Free_output(handle, &output); + // we destroy the handle here. There will be no need to clean it up in the loop from the request + // initiator function + HG_Destroy(handle); + FUNC_LEAVE(ret_value); +} + +perr_t +_dart_send_request_to_one_server(int server_id, dart_perform_one_server_in_t *dart_in, + struct bulk_args_t *lookup_args_ptr, hg_handle_t *handle) +{ + hg_return_t hg_ret; + HG_Create(send_context_g, pdc_server_info_g[server_id].addr, dart_perform_one_server_g, handle); + if (handle == NULL) { + printf("==CLIENT[%d]: Error with _dart_send_request_to_one_server\n", pdc_client_mpi_rank_g); + return FAIL; + } + + lookup_args_ptr->server_id = server_id; + + // println("dart_in->attr_key: %s", dart_in->attr_key); + hg_ret = HG_Forward(*handle, dart_perform_one_server_on_receive_cb, lookup_args_ptr, dart_in); + hg_atomic_incr32(&atomic_work_todo_g); + if (hg_ret != HG_SUCCESS) { + printf("==CLIENT[%d]: _dart_send_request_to_one_server(): Could not start HG_Forward()\n", + pdc_client_mpi_rank_g); + hg_atomic_decr32(&atomic_work_todo_g); + return FAIL; + } + // waiting for response and get the results if any. + // Wait for response from server + // hg_atomic_cas32(&atomic_work_todo_g, 0, num_requests); + PDC_Client_check_response(&send_context_g); // This will block until all requests are done. + + return SUCCEED; +} + +int +_aggregate_dart_results_from_all_servers(struct bulk_args_t *lookup_args, Set *output_set, int num_requests) +{ + int total_num_results = 0; + for (int i = 0; i < num_requests; i++) { + // aggregate result only for query operations + if (lookup_args[i].n_meta == 0) { + continue; + } + if (lookup_args[i].is_id == 1) { + int n_meta = lookup_args[i].n_meta; + for (int k = 0; k < n_meta; k++) { + uint64_t *id = (uint64_t *)malloc(sizeof(uint64_t)); + *id = lookup_args[i].obj_ids[k]; + set_insert(output_set, id); + } + total_num_results += n_meta; + } + } + return total_num_results; +} + +uint64_t +dart_perform_on_servers(index_hash_result_t **hash_result, int num_servers, + dart_perform_one_server_in_t *dart_in, Set *output_set) +{ + struct bulk_args_t *lookup_args = (struct bulk_args_t *)calloc(num_servers, sizeof(struct bulk_args_t)); + uint64_t ret_value = 0; + hg_handle_t * dart_request_handles = (hg_handle_t *)calloc(num_servers, sizeof(hg_handle_t)); + int num_requests = 0; + uint32_t total_n_meta = 0; + dart_op_type_t op_type = dart_in->op_type; + + FUNC_ENTER(NULL); + + stopwatch_t timer; + timer_start(&timer); + // send the requests to the required servers. + for (int i = 0; i < num_servers; i++) { + int server_id = (*hash_result)[i].server_id; + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) + PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); + + lookup_args[i].is_id = 1; + lookup_args[i].op_type = op_type; + + if (is_index_write_op(op_type)) { + dart_in->attr_key = strdup((*hash_result)[i].key); + } + + _dart_send_request_to_one_server(server_id, dart_in, &(lookup_args[i]), &(dart_request_handles[i])); + + num_requests++; + } + + // aggregate results when executing queries. + if ((!is_index_write_op(op_type)) && output_set != NULL) { + total_n_meta = _aggregate_dart_results_from_all_servers(lookup_args, output_set, num_servers); + ret_value = total_n_meta; + } + timer_pause(&timer); + + if (!is_index_write_op(op_type)) { + for (int i = 0; i < num_servers; i++) { + int srv_id = lookup_args[i].server_id; + server_time_total_g[srv_id] += lookup_args[i].server_time_elapsed; + server_call_count_g[srv_id] += 1; + server_mem_usage_g[srv_id] = lookup_args[i].server_memory_consumption; + } + // println("[CLIENT %d] (dart_perform_on_servers) %s on %d servers and get %d results, time : " + // "%.4f ms. server_time_elapsed: %.4f ms", + // pdc_client_mpi_rank_g, is_index_write_op(op_type) ? "write dart index" : "read dart index", + // num_servers, total_n_meta, timer_delta_ms(&timer), total_server_elapsed / 1000.0); + // if (memory_debug_g == 0) { + // for (int i = 0; i < num_servers; i++) { + // println("[SERVER %d] memory_usage: %" PRId64 "", i, + // lookup_args[i].server_memory_consumption); + // } + // } + } + // free(dart_request_handles); +done: + FUNC_LEAVE(ret_value); +} + +perr_t +PDC_Client_search_obj_ref_through_dart(dart_hash_algo_t hash_algo, char *query_string, + dart_object_ref_type_t ref_type, int *n_res, uint64_t **out) +{ + perr_t ret_value = SUCCEED; + + if (n_res == NULL || out == NULL) { + return ret_value; + } + + stopwatch_t timer; + timer_start(&timer); + + char * k_query = get_key(query_string, '='); + char * v_query = get_value(query_string, '='); + char * tok = NULL; + char * affix = NULL; + dart_op_type_t dart_op; + + pattern_type_t dart_query_type = determine_pattern_type(k_query); + switch (dart_query_type) { + case PATTERN_EXACT: + tok = strdup(k_query); + dart_op = OP_EXACT_QUERY; + break; + case PATTERN_PREFIX: + affix = subrstr(k_query, strlen(k_query) - 1); + tok = strdup(affix); + dart_op = OP_PREFIX_QUERY; + break; + case PATTERN_SUFFIX: + affix = substr(k_query, 1); +#ifndef PDC_DART_SFX_TREE + tok = reverse_str(affix); +#else + tok = strdup(affix); +#endif + dart_op = OP_SUFFIX_QUERY; + break; + case PATTERN_MIDDLE: + // tok = (char *)calloc(strlen(k_query)-2, sizeof(char)); + // strncpy(tok, &k_query[1], strlen(k_query)-2); + affix = substring(k_query, 1, strlen(k_query) - 1); + tok = strdup(affix); + dart_op = OP_INFIX_QUERY; + break; + default: + break; + } + if (tok == NULL) { + printf("==PDC_CLIENT[%d]: Error with tok\n", pdc_client_mpi_rank_g); + ret_value = FAIL; + return ret_value; + } + + out[0] = NULL; + + dart_perform_one_server_in_t input_param; + input_param.op_type = dart_query_type; + input_param.hash_algo = hash_algo; + input_param.attr_key = query_string; + input_param.attr_val = v_query; + input_param.obj_ref_type = ref_type; + + // TODO: see if timestamp can help + // input_param.timestamp = get_timestamp_us(); + input_param.timestamp = 1; + + index_hash_result_t *hash_result = NULL; + int num_servers = 0; + + if (hash_algo == DART_HASH) { + num_servers = DART_hash(dart_g, tok, dart_op, NULL, &hash_result); + } + else if (hash_algo == DHT_FULL_HASH) { + num_servers = DHT_hash(dart_g, strlen(tok), tok, dart_op, &hash_result); + } + else if (hash_algo == DHT_INITIAL_HASH) { + num_servers = DHT_hash(dart_g, 1, tok, dart_op, &hash_result); + } + + // Prepare the hashset for collecting deduplicated result if needed. + int i = 0; + Set *result_set = NULL; + if (!is_index_write_op(input_param.op_type)) { + result_set = set_new(ui64_hash, ui64_equal); + set_register_free_function(result_set, free); + } + + uint64_t total_count = dart_perform_on_servers(&hash_result, num_servers, &input_param, result_set); + + // Pick deduplicated result. + *n_res = set_num_entries(result_set); + // println("num_ids = %d", num_ids); + if (*n_res > 0) { + *out = (uint64_t *)calloc(*n_res, sizeof(uint64_t)); + uint64_t **set_arr = (uint64_t **)set_to_array(result_set); + for (i = 0; i < *n_res; i++) { + (*out)[i] = set_arr[i][0]; + } + free(set_arr); + } + set_free(result_set); + + // done: + free(k_query); + free(v_query); + if (affix != NULL) + free(affix); + if (tok != NULL) + free(tok); + + timer_pause(&timer); + // printf("perform search [ %s ] on %d servers from rank %d, total_count %" PRIu64 + // ", n_res %d, duration: %.4f ms\n", + // query_string, num_servers, pdc_client_mpi_rank_g, total_count, *n_res, + // timer_delta_ms(&timer) / 1000.0); + memory_debug_g = 1; + return ret_value; +} + +perr_t +PDC_Client_delete_obj_ref_from_dart(dart_hash_algo_t hash_algo, char *attr_key, char *attr_val, + dart_object_ref_type_t ref_type, uint64_t data) +{ + + perr_t ret_value = SUCCEED; + dart_perform_one_server_in_t input_param; + input_param.op_type = OP_DELETE; + input_param.hash_algo = hash_algo; + input_param.attr_key = attr_key; + input_param.attr_val = attr_val; + input_param.obj_ref_type = ref_type; + // FIXME: temporarily ugly implementation here, some assignment can be ignored + // and save some bytes for data transfer. + input_param.obj_primary_ref = data; + input_param.obj_secondary_ref = data; + input_param.obj_server_ref = data; + // TODO: see if timestamp can help + // input_param.timestamp = get_timestamp_us(); + input_param.timestamp = 1; + + int num_servers = 0; + index_hash_result_t *hash_result = NULL; + if (hash_algo == DART_HASH) { + num_servers = DART_hash(dart_g, attr_key, OP_DELETE, NULL, &hash_result); + } + else if (hash_algo == DHT_FULL_HASH) { + num_servers = DHT_hash(dart_g, strlen(attr_key), attr_key, OP_DELETE, &hash_result); + } + else if (hash_algo == DHT_INITIAL_HASH) { + num_servers = DHT_hash(dart_g, 1, attr_key, OP_DELETE, &hash_result); + } + + dart_perform_on_servers(&hash_result, num_servers, &input_param, NULL); + + // done: + return ret_value; +} + +perr_t +PDC_Client_insert_obj_ref_into_dart(dart_hash_algo_t hash_algo, char *attr_key, char *attr_val, + dart_object_ref_type_t ref_type, uint64_t data) +{ + // println("input: attr_key = %s, attr_val = %s", attr_key, attr_val); + perr_t ret_value = SUCCEED; + dart_perform_one_server_in_t input_param; + input_param.op_type = OP_INSERT; + input_param.hash_algo = hash_algo; + input_param.attr_key = attr_key; + input_param.attr_val = attr_val; + input_param.obj_ref_type = ref_type; + // FIXME: temporarily ugly implementation here, some assignment can be ignored + // and save some bytes for data transfer. + input_param.obj_primary_ref = data; + input_param.obj_secondary_ref = data; + input_param.obj_server_ref = data; + // TODO: see if timestamp can help + // input_param.timestamp = get_timestamp_us(); + input_param.timestamp = 1; + + int num_servers = 0; + index_hash_result_t *hash_result = NULL; + if (hash_algo == DART_HASH) { + num_servers = DART_hash(dart_g, attr_key, OP_INSERT, dart_retrieve_server_info_cb, &hash_result); + } + else if (hash_algo == DHT_FULL_HASH) { + num_servers = DHT_hash(dart_g, strlen(attr_key), attr_key, OP_INSERT, &hash_result); + } + else if (hash_algo == DHT_INITIAL_HASH) { + num_servers = DHT_hash(dart_g, 1, attr_key, OP_INSERT, &hash_result); + } + + dart_perform_on_servers(&hash_result, num_servers, &input_param, NULL); + + // done: + return ret_value; +} + +/******************** METADATA INDEX ENDS ***********************************/ + +/******************** Collective Object Selection Query Starts *******************************/ + +// #define ENABLE_MPI +#ifdef ENABLE_MPI + +void +get_first_sender(int *first_sender_global_rank, int *num_groups, int *sender_group_id, int *rank_in_group, + int *sender_group_size, int *prefer_custom_exchange) +{ + if (pdc_client_mpi_size_g >= pdc_server_num_g) { + *sender_group_size = pdc_server_num_g; + *num_groups = pdc_client_mpi_size_g / (*sender_group_size); + *rank_in_group = object_selection_query_counter_g % (*sender_group_size); + *sender_group_id = 0; + *first_sender_global_rank = (*rank_in_group); + *prefer_custom_exchange = 1; + } + else { + *num_groups = 1; + *first_sender_global_rank = object_selection_query_counter_g % pdc_client_mpi_size_g; + *sender_group_id = 0; + *rank_in_group = (*first_sender_global_rank); + *sender_group_size = pdc_client_mpi_size_g; + *prefer_custom_exchange = 0; + } +} + +void +PDC_assign_server(int32_t *my_server_start, int32_t *my_server_end, int32_t *my_server_count) +{ + FUNC_ENTER(NULL); + + if (pdc_server_num_g > pdc_client_mpi_size_g) { + *my_server_count = pdc_server_num_g / pdc_client_mpi_size_g; + *my_server_start = pdc_client_mpi_rank_g * (*my_server_count); + *my_server_end = *my_server_start + (*my_server_count); + if (pdc_client_mpi_rank_g == pdc_client_mpi_size_g - 1) { + (*my_server_end) += pdc_server_num_g % pdc_client_mpi_size_g; + } + } + else { + *my_server_start = pdc_client_mpi_rank_g; + *my_server_end = *my_server_start + 1; + if (pdc_client_mpi_rank_g >= pdc_server_num_g) { + *my_server_end = 0; + } + } + + FUNC_LEAVE_VOID; +} + +// All clients collectively query all servers, each client gets partial results +perr_t +PDC_Client_query_kvtag_col(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids, int *query_sent) +{ + perr_t ret_value = SUCCEED; + int32_t my_server_start, my_server_end, my_server_count; + int32_t i; + int nmeta = 0; + uint64_t *temp_ids = NULL; + + FUNC_ENTER(NULL); + + PDC_assign_server(&my_server_start, &my_server_end, &my_server_count); + + *n_res = 0; + *pdc_ids = NULL; + *query_sent = 1; + for (i = my_server_start; i < my_server_end; i++) { + if (i >= pdc_server_num_g) { + *query_sent = 0; + break; + } + /* printf("==PDC_CLIENT[%d]: querying server %u\n", pdc_client_mpi_rank_g, i); */ + temp_ids = NULL; + ret_value = PDC_Client_query_kvtag_server((uint32_t)i, kvtag, &nmeta, &temp_ids); + if (ret_value != SUCCEED) { + PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: error in %s querying server %u", pdc_client_mpi_rank_g, + __func__, i); + } + if (i == my_server_start) { + *pdc_ids = temp_ids; + } + else if (nmeta > 0) { + *pdc_ids = (uint64_t *)realloc(*pdc_ids, sizeof(uint64_t) * (*n_res + nmeta)); + memcpy(*pdc_ids + (*n_res), temp_ids, nmeta * sizeof(uint64_t)); + free(temp_ids); + } + *n_res = *n_res + nmeta; + /* printf("==PDC_CLIENT[%d]: server %u returned %d res \n", pdc_client_mpi_rank_g, i, *n_res); */ + *query_sent = 1; + } + +done: + memory_debug_g = 1; + fflush(stdout); + FUNC_LEAVE(ret_value); +} + +void +_standard_all_gather_result(int query_sent, int *n_res, uint64_t **pdc_ids, MPI_Comm world_comm) +{ + int i = 0, ntotal = 0, *disp = NULL; + double stime = 0.0, duration = 0.0; + stime = MPI_Wtime(); + + int *all_nmeta_array = (int *)calloc(pdc_client_mpi_size_g, sizeof(int)); + MPI_Allgather(n_res, 1, MPI_INT, all_nmeta_array, 1, MPI_INT, world_comm); + + duration = MPI_Wtime() - stime; + + if (pdc_client_mpi_rank_g == 0) { + println("==PDC Client[%d]: Time for MPI_Allgather for Syncing ID count: %.4f ms", + pdc_client_mpi_rank_g, duration * 1000.0); + } + + disp = (int *)calloc(pdc_client_mpi_size_g, sizeof(int)); + ntotal = 0; + for (i = 0; i < pdc_client_mpi_size_g; i++) { + disp[i] = ntotal; + ntotal += all_nmeta_array[i]; + } + + uint64_t *all_ids = (uint64_t *)malloc(ntotal * sizeof(uint64_t)); + MPI_Allgatherv(*pdc_ids, *n_res, MPI_UINT64_T, all_ids, all_nmeta_array, disp, MPI_UINT64_T, world_comm); + + *n_res = ntotal; + *pdc_ids = all_ids; +} + +void +_customized_all_gather_result(int query_sent, int *n_res, uint64_t **pdc_ids, MPI_Comm world_comm) +{ + int i = 0, *all_nmeta = NULL, ntotal = 0, *disp = NULL; + double stime = 0.0, duration = 0.0; + + stime = MPI_Wtime(); + // First, let's get the number of results from each client. + + // In the case where the total number of clients is far larger than the total number of servers, + // say 20x larger, since not all ranks are participating in the query, we need to limit the MPI + // communication to those ranks only. This can help reduce the communication overhead, especially + // when the number of ranks is far larger than the number of servers. + + int sub_comm_color = query_sent == 1 ? 1 : 0; + MPI_Comm sub_comm; + MPI_Comm_split(world_comm, sub_comm_color, pdc_client_mpi_rank_g, &sub_comm); + int sub_comm_rank, sub_comm_size; + MPI_Comm_rank(sub_comm, &sub_comm_rank); + MPI_Comm_size(sub_comm, &sub_comm_size); + // println("World rank %d is rank %d in the new communicator of size %d", pdc_client_mpi_rank_g, + // sub_comm_rank, sub_comm_size); + + int n_sent_ranks = sub_comm_color == 1 ? sub_comm_size : pdc_client_mpi_size_g - sub_comm_size; + int sub_n_obj_len = n_sent_ranks + 1; // the last element is the first rank who sent the query. + int *sub_n_obj_arr = (int *)calloc(sub_n_obj_len, sizeof(int)); + // FIXME: how to get the global rank number of the first rank who sent the query? + // currently, we use 0, since each time when PDC_Client_query_kvtag_col runs, it is always using the + // first N ranks to send the query, where N is the number of servers. + sub_n_obj_arr[sub_n_obj_len - 1] = 0; + + if (sub_comm_color == 1) { + // the result is first gathered among the ranks who sent the requests. + MPI_Allgather(n_res, 1, MPI_INT, sub_n_obj_arr, 1, MPI_INT, sub_comm); // get the number of results + } + + MPI_Barrier(world_comm); + // FIXME: check root for MPI_Bcast accociated with the world_comm. + int root = sub_n_obj_arr[sub_n_obj_len - 1]; + MPI_Bcast(sub_n_obj_arr, sub_n_obj_len, MPI_INT, root, world_comm); + // now all ranks in the world_comm should know about the number of results from each rank who sent the + // query. + duration = MPI_Wtime() - stime; + if (pdc_client_mpi_rank_g == 0) { + println("==PDC Client[%d]: Time for MPI_Allgather for Syncing ID count: %.4f ms", + pdc_client_mpi_rank_g, duration * 1000.0); + } + + // Okay, now each rank of the WORLD_COMM knows about the number of results from the clients who sent + // queries. + // Let's calculate the total number of results, and the displacement for each client. + MPI_Barrier(world_comm); + + all_nmeta = (int *)calloc(pdc_client_mpi_size_g, sizeof(int)); + disp = (int *)calloc(pdc_client_mpi_size_g, sizeof(int)); + ntotal = 0; + for (i = 0; i < sub_n_obj_len - 1; i++) { + all_nmeta[i] = sub_n_obj_arr[i]; + disp[i] = ntotal; + ntotal += all_nmeta[i]; + } + + // Finally, let's gather all the results. Since each client is getting a partial result which can be of + // different size, we need to use MPI_Allgatherv for gathering variable-size arrays from different + // clients. + uint64_t *all_ids = (uint64_t *)malloc(ntotal * sizeof(uint64_t)); + + MPI_Allgatherv(*pdc_ids, *n_res, MPI_UINT64_T, all_ids, all_nmeta, disp, MPI_UINT64_T, world_comm); + + MPI_Comm_free(&sub_comm); + + // Now, let's return the result to the caller + *pdc_ids = all_ids; + *n_res = ntotal; +} + +// All clients collectively query all servers, all clients get all results +perr_t +PDC_Client_query_kvtag_mpi(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids, MPI_Comm world_comm) +{ + int local_increment = 1; + MPI_Scan(&local_increment, &object_selection_query_counter_g, 1, MPI_INT, MPI_SUM, world_comm); + perr_t ret_value = SUCCEED; + int i, query_sent = 0; + double stime = 0.0, duration = 0.0; + + FUNC_ENTER(NULL); + + MPI_Barrier(world_comm); + stime = MPI_Wtime(); + + ret_value = PDC_Client_query_kvtag_col(kvtag, n_res, pdc_ids, &query_sent); + + MPI_Barrier(world_comm); + duration = MPI_Wtime() - stime; + + if (pdc_client_mpi_rank_g == 0) { + println("==PDC Client[%d]: Time for C/S communication: %.4f ms", pdc_client_mpi_rank_g, + duration * 1000.0); + } + + if (*n_res <= 0) { + *n_res = 0; + *pdc_ids = (uint64_t *)malloc(0); + } + else { + // print the pdc ids returned by this client, along with the client id + // printf("==PDC_CLIENT == COLLECTIVE [%d]: ", pdc_client_mpi_rank_g); + // for (i = 0; i < *n_res; i++) + // printf("%llu ", (*pdc_ids)[i]); + // printf("\n"); + } + + if (pdc_client_mpi_size_g == 1) { + goto done; + } + + MPI_Barrier(world_comm); + stime = MPI_Wtime(); + // perform all gather to get the complete result. + _standard_all_gather_result(query_sent, n_res, pdc_ids, world_comm); + + duration = MPI_Wtime() - stime; + if (pdc_client_mpi_rank_g == 0) { + println("==PDC Client[%d]: Time for MPI_Allgatherv for Syncing ID array: %.4f ms", + pdc_client_mpi_rank_g, duration * 1000.0); + } + + // deducplicating result with a Set. + Set *result_set = set_new(ui64_hash, ui64_equal); + set_register_free_function(result_set, free); + for (i = 0; i < *n_res; i++) { + uint64_t *id = (uint64_t *)malloc(sizeof(uint64_t)); + *id = (*pdc_ids)[i]; + set_insert(result_set, id); + } + free(*pdc_ids); + // Pick deduplicated result. + *n_res = set_num_entries(result_set); + // println("num_ids = %d", num_ids); + if (*n_res > 0) { + *pdc_ids = (uint64_t *)calloc(*n_res, sizeof(uint64_t)); + uint64_t **set_arr = (uint64_t **)set_to_array(result_set); + for (i = 0; i < *n_res; i++) { + (*pdc_ids)[i] = set_arr[i][0]; + } + free(set_arr); + } + set_free(result_set); + + // print the pdc ids returned after gathering all the results + if (pdc_client_mpi_rank_g == 0) { + // printf("==PDC_CLIENT == GATHERED [%d]: ", pdc_client_mpi_rank_g); + // for (i = 0; i < *n_res; i++) + // printf("%llu ", (*pdc_ids)[i]); + // printf("\n"); + } + +done: + fflush(stdout); + FUNC_LEAVE(ret_value); +} + +void +_standard_bcast_result(int root, int *n_res, uint64_t **out, MPI_Comm world_comm) +{ + + double stime = 0.0, duration = 0.0; + + stime = MPI_Wtime(); + // broadcast n_res to all other ranks from root + MPI_Bcast(n_res, 1, MPI_INT, root, world_comm); + + duration = MPI_Wtime() - stime; + + if (pdc_client_mpi_rank_g == 0) { + println("==PDC Client[%d]: Time for MPI_Bcast for Syncing ID count: %.4f ms", pdc_client_mpi_rank_g, + duration * 1000.0); + } + + if (pdc_client_mpi_rank_g != root) { + *out = (uint64_t *)calloc(*n_res, sizeof(uint64_t)); + } + + // broadcast the result to all other ranks + MPI_Bcast(*out, *n_res, MPI_UINT64_T, root, world_comm); +} + +void +_customized_bcast_result(int first_sender_global_rank, int num_groups, int sender_group_id, int rank_in_group, + int sender_group_size, int *n_res, uint64_t **out, MPI_Comm world_comm) +{ + double stime = 0.0, duration = 0.0; + int group_head_comm_color; + + stime = MPI_Wtime(); + + // FIXME: needs to be examined and fixed. + + if (num_groups > 1) { + group_head_comm_color = pdc_client_mpi_rank_g % sender_group_size == rank_in_group; + } + else { + group_head_comm_color = 0; + } + + // Note: we should set comm to be MPI_COMM_WORLD since all assumptions are + // made with the total number of client ranks. let's select n ranks to be the + // sender ranks, where n is the number of servers. + MPI_Comm group_head_comm; + MPI_Comm_split(world_comm, group_head_comm_color, pdc_client_mpi_rank_g, &group_head_comm); + int group_head_comm_rank, group_head_comm_size; + MPI_Comm_rank(group_head_comm, &group_head_comm_rank); + MPI_Comm_size(group_head_comm, &group_head_comm_size); + // println("World rank %d is rank %d in the 'group_head_comm' of size %d", pdc_client_mpi_rank_g, + // group_head_comm_rank, + // group_head_comm_size); + + // broadcast result size among group_head_comm + if (group_head_comm_color == 1) { + MPI_Bcast(n_res, 1, MPI_INT, rank_in_group, group_head_comm); + } + + // now, all the n sender ranks has the result. Let's broadcast the result to all other ranks. + // suppose number of servers is 16, then the groups can be + // 0-15, 16-31, 32-47, 48-63, 64-68... + // rank/16 = 0, 1, 2, 3(including 48-63 and 64-68), ..., + // this means we can divide all client ranks into rank/#server groups. + // within each group, we can perform a BCAST, using the first rank as the root. + int group_color = pdc_client_mpi_rank_g / pdc_server_num_g; + MPI_Comm group_comm; + MPI_Comm_split(world_comm, group_color, pdc_client_mpi_rank_g, &group_comm); + int group_rank, group_size; + MPI_Comm_rank(group_comm, &group_rank); + MPI_Comm_size(group_comm, &group_size); + // println("World rank %d is rank %d in the 'group_comm' of size %d", pdc_client_mpi_rank_g, + // group_rank, + // group_size); + + MPI_Bcast(n_res, 1, MPI_INT, rank_in_group, group_comm); + + MPI_Barrier(world_comm); + duration = MPI_Wtime() - stime; + + if (pdc_client_mpi_rank_g == 0) { + println("==PDC Client[%d]: Time for MPI_Bcast for Syncing ID count: %.4f ms", pdc_client_mpi_rank_g, + duration * 1000.0); + } + + // Okay, now each rank of the WORLD_COMM knows about the number of results from the sender ranks, and the + // root for WORLD_COMM. Let's perform BCAST for the array data. for those ranks that are not the root, + // allocate memory for the object IDs. + if (*out == NULL) { + *out = (uint64_t *)calloc(*n_res, sizeof(uint64_t)); + } + + MPI_Bcast(*out, *n_res, MPI_UINT64_T, first_sender_global_rank, world_comm); + + MPI_Comm_free(&group_head_comm); + MPI_Comm_free(&group_comm); +} + +perr_t +PDC_Client_search_obj_ref_through_dart_mpi(dart_hash_algo_t hash_algo, char *query_string, + dart_object_ref_type_t ref_type, int *n_res, uint64_t **out, + MPI_Comm world_comm) +{ + int local_increment = 1; + MPI_Scan(&local_increment, &object_selection_query_counter_g, 1, MPI_INT, MPI_SUM, world_comm); + perr_t ret_value = SUCCEED; + + if (n_res == NULL || out == NULL) { + ret_value = FAIL; + return ret_value; + } + + int n_obj = 0; + uint64_t *dart_out; + double stime = 0.0, duration = 0.0; + + // Let's calcualte an approprate root. + // FIXME: needs to be examined and fixed. Currently, first_sender_global_rank is different in different + // ranks. this is not correct. + int first_sender_global_rank, num_groups, sender_group_id, rank_in_group, sender_group_size, + prefer_custom_exchange; + + get_first_sender(&first_sender_global_rank, &num_groups, &sender_group_id, &rank_in_group, + &sender_group_size, &prefer_custom_exchange); + + // broadcast first_sender_global_rank to all other ranks + MPI_Bcast(&first_sender_global_rank, 1, MPI_INT, 0, world_comm); + + MPI_Barrier(world_comm); + stime = MPI_Wtime(); + + // let the root send the query + if (pdc_client_mpi_rank_g == first_sender_global_rank) { + PDC_Client_search_obj_ref_through_dart(hash_algo, query_string, ref_type, &n_obj, &dart_out); + } + + duration = MPI_Wtime() - stime; + if (pdc_client_mpi_rank_g == first_sender_global_rank) { + println("==PDC Client[%d]: Time for C/S communication: %.4f ms", pdc_client_mpi_rank_g, + duration * 1000.0); + } + + MPI_Barrier(world_comm); + stime = MPI_Wtime(); + + // Now, let's broadcast the result to all other ranks. + _standard_bcast_result(first_sender_global_rank, &n_obj, &dart_out, world_comm); + + duration = MPI_Wtime() - stime; + + if (pdc_client_mpi_rank_g == first_sender_global_rank) { + println("==PDC Client[%d]: Time for MPI_Bcast for Syncing ID array: %.4f ms", pdc_client_mpi_rank_g, + duration * 1000.0); + } + + *n_res = n_obj; + *out = dart_out; + return ret_value; +} +#endif + +/******************** Collective Object Selection Query Ends *******************************/ \ No newline at end of file diff --git a/src/api/pdc_obj/include/pdc_cont.h b/src/api/pdc_obj/include/pdc_cont.h index 844b15425..33e924c14 100644 --- a/src/api/pdc_obj/include/pdc_cont.h +++ b/src/api/pdc_obj/include/pdc_cont.h @@ -191,7 +191,8 @@ perr_t PDCcont_del(pdcid_t cont_id); * * \return Non-negative on success/Negative on failure */ -perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_size); +perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, + psize_t value_size); /** * *********** @@ -203,7 +204,8 @@ perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t * * \return Non-negative on success/Negative on failure */ -perr_t PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, psize_t *value_size); +perr_t PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size); /** * Deleta a tag from a container diff --git a/src/api/pdc_obj/include/pdc_obj.h b/src/api/pdc_obj/include/pdc_obj.h index f678adf7f..4ce3d6bb5 100644 --- a/src/api/pdc_obj/include/pdc_obj.h +++ b/src/api/pdc_obj/include/pdc_obj.h @@ -228,7 +228,7 @@ perr_t PDCprop_set_obj_tags(pdcid_t obj_prop, char *tags); * * \param obj_prop [IN] ID of object property, returned by PDCprop_create(PDC_OBJ_CREATE) * \param ndim [IN] Number of dimensions - * \param dims [IN] Size of each dimension + * \param dims [IN] Size of each dimension, positive value, can be PDC_SIZE_UNLIMITED * * \return Non-negative on success/Negative on failure */ @@ -409,7 +409,8 @@ perr_t PDCobj_del(pdcid_t obj_id); * * \return Non-negative on success/Negative on failure */ -perr_t PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_size); +perr_t PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, + psize_t value_size); /** * Get tag information @@ -421,7 +422,8 @@ perr_t PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t v * * \return Non-negative on success/Negative on failure */ -perr_t PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, psize_t *value_size); +perr_t PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size); /** * Delete a tag from the object diff --git a/src/api/pdc_obj/include/pdc_obj_pkg.h b/src/api/pdc_obj/include/pdc_obj_pkg.h index f08343a70..11effda8e 100644 --- a/src/api/pdc_obj/include/pdc_obj_pkg.h +++ b/src/api/pdc_obj/include/pdc_obj_pkg.h @@ -24,8 +24,7 @@ #ifndef PDC_OBJ_PKG_H #define PDC_OBJ_PKG_H - -#include "pdc_private.h" +#include "pdc_public.h" /****************************/ /* Library Private Typedefs */ diff --git a/src/api/pdc_obj/include/pdc_prop.h b/src/api/pdc_obj/include/pdc_prop.h index 37ff599eb..0459e0283 100644 --- a/src/api/pdc_obj/include/pdc_prop.h +++ b/src/api/pdc_obj/include/pdc_prop.h @@ -108,16 +108,6 @@ struct pdc_obj_prop *PDCobj_prop_get_info(pdcid_t prop_id); */ perr_t PDCprop_update(pdcid_t obj_id, pdcid_t prop_id); -/** - * Delete a tag with a specific name and value - * - * \param obj_id[IN] Object ID - * \param tag_name [IN] Metadta field name - * - * \return Non-negative on success/Negative on failure - */ -perr_t PDCtag_delete(pdcid_t obj_id, char *tag_name); - /** * ********** * diff --git a/src/api/pdc_obj/include/pdc_prop_pkg.h b/src/api/pdc_obj/include/pdc_prop_pkg.h index db73120ac..16524314b 100644 --- a/src/api/pdc_obj/include/pdc_prop_pkg.h +++ b/src/api/pdc_obj/include/pdc_prop_pkg.h @@ -24,8 +24,7 @@ #ifndef PDC_PROP_PKG_H #define PDC_PROP_PKG_H - -#include "pdc_private.h" +#include "pdc_public.h" /*******************/ /* Private Typedefs */ @@ -36,11 +35,7 @@ struct _pdc_cont_prop { pdc_lifetime_t cont_life; }; -typedef struct pdc_kvtag_t { - char * name; - uint32_t size; - void * value; -} pdc_kvtag_t; +typedef enum { ROW_major, COL_major } _pdc_major_type_t; struct _pdc_transform_state { _pdc_major_type_t storage_order; diff --git a/src/api/pdc_obj/pdc_cont.c b/src/api/pdc_obj/pdc_cont.c index 444596729..d0c896816 100644 --- a/src/api/pdc_obj/pdc_cont.c +++ b/src/api/pdc_obj/pdc_cont.c @@ -73,12 +73,12 @@ PDCcont_create(const char *cont_name, pdcid_t cont_prop_id) id_info = PDC_find_id(cont_prop_id); cont_prop = (struct _pdc_cont_prop *)(id_info->obj_ptr); - p->cont_pt = PDC_CALLOC(struct _pdc_cont_prop); + p->cont_pt = PDC_CALLOC(1, struct _pdc_cont_prop); if (!p->cont_pt) PGOTO_ERROR(0, "PDC container prop memory allocation failed"); memcpy(p->cont_pt, cont_prop, sizeof(struct _pdc_cont_prop)); - p->cont_pt->pdc = PDC_CALLOC(struct _pdc_class); + p->cont_pt->pdc = PDC_CALLOC(1, struct _pdc_class); if (!p->cont_pt->pdc) PGOTO_ERROR(0, "PDC container pdc class memory allocation failed"); if (cont_prop->pdc->name) @@ -121,12 +121,12 @@ PDCcont_create_col(const char *cont_name, pdcid_t cont_prop_id) id_info = PDC_find_id(cont_prop_id); cont_prop = (struct _pdc_cont_prop *)(id_info->obj_ptr); - p->cont_pt = PDC_CALLOC(struct _pdc_cont_prop); + p->cont_pt = PDC_CALLOC(1, struct _pdc_cont_prop); if (!p->cont_pt) PGOTO_ERROR(0, "PDC container prop memory allocation failed"); memcpy(p->cont_pt, cont_prop, sizeof(struct _pdc_cont_prop)); - p->cont_pt->pdc = PDC_CALLOC(struct _pdc_class); + p->cont_pt->pdc = PDC_CALLOC(1, struct _pdc_class); if (!p->cont_pt->pdc) PGOTO_ERROR(0, "PDC container pdc class memory allocation failed"); if (cont_prop->pdc->name) @@ -170,12 +170,12 @@ PDC_cont_create_local(pdcid_t pdc, const char *cont_name, uint64_t cont_meta_id) id_info = PDC_find_id(cont_prop_id); cont_prop = (struct _pdc_cont_prop *)(id_info->obj_ptr); - p->cont_pt = PDC_CALLOC(struct _pdc_cont_prop); + p->cont_pt = PDC_CALLOC(1, struct _pdc_cont_prop); if (!p->cont_pt) PGOTO_ERROR(0, "PDC container prop memory allocation failed"); memcpy(p->cont_pt, cont_prop, sizeof(struct _pdc_cont_prop)); - p->cont_pt->pdc = PDC_CALLOC(struct _pdc_class); + p->cont_pt->pdc = PDC_CALLOC(1, struct _pdc_class); if (!p->cont_pt->pdc) PGOTO_ERROR(0, "PDC container pdc class memory allocation failed"); @@ -274,6 +274,9 @@ PDCcont_open(const char *cont_name, pdcid_t pdc) ret = PDC_Client_query_container_name(cont_name, &cont_meta_id); if (ret == FAIL) PGOTO_ERROR(0, "query container name failed"); + if (cont_meta_id == 0) + PGOTO_ERROR(0, "query container not found"); + cont_id = PDC_cont_create_local(pdc, cont_name, cont_meta_id); ret_value = cont_id; @@ -315,13 +318,13 @@ PDC_cont_get_info(pdcid_t cont_id) id_info = PDC_find_id(cont_id); info = (struct _pdc_cont_info *)(id_info->obj_ptr); - ret_value = PDC_CALLOC(struct _pdc_cont_info); + ret_value = PDC_CALLOC(1, struct _pdc_cont_info); if (ret_value) memcpy(ret_value, info, sizeof(struct _pdc_cont_info)); else PGOTO_ERROR(NULL, "cannot allocate ret_value"); - ret_value->cont_info_pub = PDC_CALLOC(struct pdc_cont_info); + ret_value->cont_info_pub = PDC_CALLOC(1, struct pdc_cont_info); if (ret_value->cont_info_pub) memcpy(ret_value, info, sizeof(struct pdc_cont_info)); else @@ -334,7 +337,7 @@ PDC_cont_get_info(pdcid_t cont_id) memcpy(ret_value->cont_pt, info->cont_pt, sizeof(struct _pdc_cont_prop)); else PGOTO_ERROR(NULL, "cannot allocate ret_value->cont_pt"); - ret_value->cont_pt->pdc = PDC_CALLOC(struct _pdc_class); + ret_value->cont_pt->pdc = PDC_CALLOC(1, struct _pdc_class); if (ret_value->cont_pt->pdc) { ret_value->cont_pt->pdc->local_id = info->cont_pt->pdc->local_id; if (info->cont_pt->pdc->name) @@ -361,7 +364,7 @@ PDCcont_get_info(const char *cont_name) tmp = PDC_cont_get_info(cont_id); - ret_value = PDC_CALLOC(struct pdc_cont_info); + ret_value = PDC_CALLOC(1, struct pdc_cont_info); if (!ret_value) PGOTO_ERROR(NULL, "cannot allocate memory"); @@ -435,7 +438,7 @@ PDCcont_iter_get_info(cont_handle *chandle) if (info == NULL) PGOTO_ERROR(NULL, "PDC container info memory allocation failed"); - ret_value = PDC_CALLOC(struct pdc_cont_info); + ret_value = PDC_CALLOC(1, struct pdc_cont_info); if (!ret_value) PGOTO_ERROR(NULL, "failed to allocate memory"); diff --git a/src/api/pdc_obj/pdc_dt_conv.c b/src/api/pdc_obj/pdc_dt_conv.c index 0806919ee..f18e4aa43 100644 --- a/src/api/pdc_obj/pdc_dt_conv.c +++ b/src/api/pdc_obj/pdc_dt_conv.c @@ -34,10 +34,10 @@ PDC_UNKNOWN = -1, PDC_INT = 0, PDC_FLOAT = 1, PDC_DOUBLE = 2, -PDC_STRING = 3, -PDC_COMPOUND = 4, -PDC_ENUM = 5, -PDC_ARRAY = 6, +PDC_CHAR = 3, +PDC_STRING = 4, +PDC_BOOLEAN = 5, +PDC_SHORT = 6, */ /* Called if overflow is possible */ diff --git a/src/api/pdc_obj/pdc_obj.c b/src/api/pdc_obj/pdc_obj.c index 073ece24c..95687826c 100644 --- a/src/api/pdc_obj/pdc_obj.c +++ b/src/api/pdc_obj/pdc_obj.c @@ -27,6 +27,7 @@ #include "pdc_malloc.h" #include "pdc_id_pkg.h" #include "pdc_cont.h" +#include "pdc_cont_pkg.h" #include "pdc_prop_pkg.h" #include "pdc_obj_pkg.h" #include "pdc_obj.h" @@ -85,24 +86,24 @@ PDCobj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id) cont_info = (struct _pdc_cont_info *)(id_info->obj_ptr); /* struct _pdc_cont_info field */ - p->cont = PDC_CALLOC(struct _pdc_cont_info); + p->cont = PDC_CALLOC(1,struct _pdc_cont_info); if (!p->cont) PGOTO_ERROR(0, "PDC object container memory allocation failed"); memcpy(p->cont, cont_info, sizeof(struct _pdc_cont_info)); - p->cont->cont_info_pub = PDC_CALLOC(struct pdc_cont_info); + p->cont->cont_info_pub = PDC_CALLOC(1,struct pdc_cont_info); if (!p->cont->cont_info_pub) PGOTO_ERROR(0, "PDC object pub container memory allocation failed"); memcpy(p->cont->cont_info_pub, cont_info->cont_info_pub, sizeof(struct pdc_cont_info)); if (cont_info->cont_info_pub->name) p->cont->cont_info_pub->name = strdup(cont_info->cont_info_pub->name); - p->cont->cont_pt = PDC_CALLOC(struct _pdc_cont_prop); + p->cont->cont_pt = PDC_CALLOC(1,struct _pdc_cont_prop); if (!p->cont->cont_pt) PGOTO_ERROR(0, "PDC object container property memory allocation failed"); memcpy(p->cont->cont_pt, cont_info->cont_pt, sizeof(struct _pdc_cont_prop)); - p->cont->cont_pt->pdc = PDC_CALLOC(struct _pdc_class); + p->cont->cont_pt->pdc = PDC_CALLOC(1,struct _pdc_class); if (!p->cont->cont_pt->pdc) PGOTO_ERROR(0, "PDC object container property pdc memory allocation failed"); p->cont->cont_pt->pdc->name = strdup(cont_info->cont_pt->pdc->name); @@ -114,20 +115,20 @@ PDCobj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id) obj_prop = (struct _pdc_obj_prop *)(id_info->obj_ptr); /* struct _pdc_obj_prop field */ - p->obj_pt = PDC_CALLOC(struct _pdc_obj_prop); + p->obj_pt = PDC_CALLOC(1,struct _pdc_obj_prop); if (!p->obj_pt) PGOTO_ERROR(0, "PDC object property memory allocation failed"); memcpy(p->obj_pt, obj_prop, sizeof(struct _pdc_obj_prop)); if (obj_prop->app_name) p->obj_pt->app_name = strdup(obj_prop->app_name); - p->obj_pt->pdc = PDC_CALLOC(struct _pdc_class); + p->obj_pt->pdc = PDC_CALLOC(1,struct _pdc_class); if (!p->obj_pt->pdc) PGOTO_ERROR(0, "cannot allocate ret_value->pdc"); p->obj_pt->pdc->name = strdup(obj_prop->pdc->name); p->obj_pt->pdc->local_id = obj_prop->pdc->local_id; /* struct pdc_obj_prop field */ - p->obj_pt->obj_prop_pub = PDC_CALLOC(struct pdc_obj_prop); + p->obj_pt->obj_prop_pub = PDC_CALLOC(1,struct pdc_obj_prop); if (!p->obj_pt->obj_prop_pub) PGOTO_ERROR(0, "cannot allocate ret_value->obj_pt->obj_prop_pub"); p->obj_pt->obj_prop_pub->ndim = obj_prop->obj_prop_pub->ndim; @@ -155,7 +156,7 @@ PDCobj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id) if (ret == FAIL) PGOTO_ERROR(0, "Unable to create object on server!"); - p->obj_info_pub->obj_pt = PDC_CALLOC(struct pdc_obj_prop); + p->obj_info_pub->obj_pt = PDC_CALLOC(1,struct pdc_obj_prop); if (!p->obj_info_pub->obj_pt) PGOTO_ERROR(0, "PDC object prop memory allocation failed"); memcpy(p->obj_info_pub->obj_pt, p->obj_pt->obj_prop_pub, sizeof(struct pdc_obj_prop)); @@ -253,13 +254,13 @@ PDC_obj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id, _pdc_ /* struct _pdc_cont_info field */ cont_info = (struct _pdc_cont_info *)(id_info->obj_ptr); - p->cont = PDC_CALLOC(struct _pdc_cont_info); + p->cont = PDC_CALLOC(1, struct _pdc_cont_info); if (!p->cont) PGOTO_ERROR(0, "PDC object container memory allocation failed"); memcpy(p->cont, cont_info, sizeof(struct _pdc_cont_info)); /* struct pdc_cont_info field */ - p->cont->cont_info_pub = PDC_CALLOC(struct pdc_cont_info); + p->cont->cont_info_pub = PDC_CALLOC(1, struct pdc_cont_info); if (!p->cont->cont_info_pub) PGOTO_ERROR(0, "PDC object pub container memory allocation failed"); memcpy(p->cont->cont_info_pub, cont_info->cont_info_pub, sizeof(struct pdc_cont_info)); @@ -267,13 +268,13 @@ PDC_obj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id, _pdc_ p->cont->cont_info_pub->name = strdup(cont_info->cont_info_pub->name); /* struct _pdc_cont_prop field */ - p->cont->cont_pt = PDC_CALLOC(struct _pdc_cont_prop); + p->cont->cont_pt = PDC_CALLOC(1, struct _pdc_cont_prop); if (!p->cont->cont_pt) PGOTO_ERROR(0, "PDC object container property memory allocation failed"); memcpy(p->cont->cont_pt, cont_info->cont_pt, sizeof(struct _pdc_cont_prop)); /* struct _pdc_class field */ - p->cont->cont_pt->pdc = PDC_CALLOC(struct _pdc_class); + p->cont->cont_pt->pdc = PDC_CALLOC(1, struct _pdc_class); if (!p->cont->cont_pt->pdc) PGOTO_ERROR(0, "PDC object container property pdc memory allocation failed"); if (cont_info->cont_pt->pdc->name) @@ -286,7 +287,7 @@ PDC_obj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id, _pdc_ obj_prop = (struct _pdc_obj_prop *)(id_info->obj_ptr); /* struct _pdc_obj_prop field */ - p->obj_pt = PDC_CALLOC(struct _pdc_obj_prop); + p->obj_pt = PDC_CALLOC(1, struct _pdc_obj_prop); if (!p->obj_pt) PGOTO_ERROR(0, "PDC object property memory allocation failed"); memcpy(p->obj_pt, obj_prop, sizeof(struct _pdc_obj_prop)); @@ -298,7 +299,7 @@ PDC_obj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id, _pdc_ p->obj_pt->tags = strdup(obj_prop->tags); p->obj_pt->locus = PDC_get_execution_locus(); - p->obj_pt->pdc = PDC_CALLOC(struct _pdc_class); + p->obj_pt->pdc = PDC_CALLOC(1, struct _pdc_class); if (!p->obj_pt->pdc) PGOTO_ERROR(0, "cannot allocate ret_value->pdc"); if (obj_prop->pdc->name) @@ -306,7 +307,7 @@ PDC_obj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id, _pdc_ p->obj_pt->pdc->local_id = obj_prop->pdc->local_id; /* struct pdc_obj_prop field */ - p->obj_pt->obj_prop_pub = PDC_CALLOC(struct pdc_obj_prop); + p->obj_pt->obj_prop_pub = PDC_CALLOC(1, struct pdc_obj_prop); if (!p->obj_pt->obj_prop_pub) PGOTO_ERROR(0, "cannot allocate ret_value->obj_pt->obj_prop_pub"); p->obj_pt->obj_prop_pub->ndim = obj_prop->obj_prop_pub->ndim; @@ -348,7 +349,7 @@ PDC_obj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id, _pdc_ p->obj_pt->obj_prop_pub->region_partition, p->obj_pt->obj_prop_pub->consistency, p); - p->obj_info_pub->obj_pt = PDC_CALLOC(struct pdc_obj_prop); + p->obj_info_pub->obj_pt = PDC_CALLOC(1, struct pdc_obj_prop); if (!p->obj_info_pub->obj_pt) PGOTO_ERROR(0, "PDC object prop memory allocation failed"); memcpy(p->obj_info_pub->obj_pt, p->obj_pt->obj_prop_pub, sizeof(struct pdc_obj_prop)); @@ -539,31 +540,31 @@ PDCobj_open_common(const char *obj_name, pdcid_t pdc, int is_col) p = PDC_MALLOC(struct _pdc_obj_info); if (!p) PGOTO_ERROR(0, "PDC object memory allocation failed"); - p->cont = PDC_CALLOC(struct _pdc_cont_info); + p->cont = PDC_CALLOC(1, struct _pdc_cont_info); if (!p->cont) PGOTO_ERROR(0, "PDC object container memory allocation failed"); - p->cont->cont_info_pub = PDC_CALLOC(struct pdc_cont_info); + p->cont->cont_info_pub = PDC_CALLOC(1, struct pdc_cont_info); if (!p->cont->cont_info_pub) PGOTO_ERROR(0, "PDC object pub container memory allocation failed"); - p->cont->cont_pt = PDC_CALLOC(struct _pdc_cont_prop); + p->cont->cont_pt = PDC_CALLOC(1, struct _pdc_cont_prop); if (!p->cont->cont_pt) PGOTO_ERROR(0, "PDC object container property memory allocation failed"); - p->cont->cont_pt->pdc = PDC_CALLOC(struct _pdc_class); + p->cont->cont_pt->pdc = PDC_CALLOC(1, struct _pdc_class); if (!p->cont->cont_pt->pdc) PGOTO_ERROR(0, "PDC object container property pdc memory allocation failed"); - p->obj_pt = PDC_CALLOC(struct _pdc_obj_prop); + p->obj_pt = PDC_CALLOC(1, struct _pdc_obj_prop); if (!p->obj_pt) PGOTO_ERROR(0, "PDC object property memory allocation failed"); - p->obj_pt->obj_prop_pub = PDC_CALLOC(struct pdc_obj_prop); + p->obj_pt->obj_prop_pub = PDC_CALLOC(1, struct pdc_obj_prop); if (!p->obj_pt->obj_prop_pub) PGOTO_ERROR(0, "PDC object property memory allocation failed"); p->obj_info_pub = PDC_MALLOC(struct pdc_obj_info); if (!p->obj_info_pub) PGOTO_ERROR(0, "PDC pub object memory allocation failed"); - p->obj_info_pub->obj_pt = PDC_CALLOC(struct pdc_obj_prop); + p->obj_info_pub->obj_pt = PDC_CALLOC(1, struct pdc_obj_prop); if (!p->obj_info_pub->obj_pt) PGOTO_ERROR(0, "PDC object prop memory allocation failed"); - p->obj_pt->pdc = PDC_CALLOC(struct _pdc_class); + p->obj_pt->pdc = PDC_CALLOC(1, struct _pdc_class); if (!p->obj_pt->pdc) PGOTO_ERROR(0, "cannot allocate ret_value->pdc"); @@ -737,12 +738,12 @@ PDCobj_iter_get_info(obj_handle *ohandle) if (info == NULL) PGOTO_ERROR(NULL, "PDC container info memory allocation failed"); - ret_value = PDC_CALLOC(struct pdc_obj_info); + ret_value = PDC_CALLOC(1, struct pdc_obj_info); if (!ret_value) PGOTO_ERROR(NULL, "failed to allocate memory"); memcpy(ret_value, info->obj_info_pub, sizeof(struct pdc_obj_info)); - ret_value->obj_pt = PDC_CALLOC(struct pdc_obj_prop); + ret_value->obj_pt = PDC_CALLOC(1, struct pdc_obj_prop); if (!ret_value->obj_pt) PGOTO_ERROR(NULL, "failed to allocate memory"); memcpy(ret_value->obj_pt, info->obj_info_pub->obj_pt, sizeof(struct pdc_obj_prop)); @@ -1057,14 +1058,14 @@ PDC_obj_get_info(pdcid_t obj_id) PGOTO_ERROR(NULL, "cannot locate object"); info = (struct _pdc_obj_info *)(obj->obj_ptr); - ret_value = PDC_CALLOC(struct _pdc_obj_info); + ret_value = PDC_CALLOC(1, struct _pdc_obj_info); if (ret_value) memcpy(ret_value, info, sizeof(struct _pdc_obj_info)); else PGOTO_ERROR(NULL, "cannot allocate ret_value"); /* struct pdc_obj_info field */ - ret_value->obj_info_pub = PDC_CALLOC(struct pdc_obj_info); + ret_value->obj_info_pub = PDC_CALLOC(1, struct pdc_obj_info); if (ret_value->obj_info_pub) memcpy(ret_value->obj_info_pub, info->obj_info_pub, sizeof(struct pdc_obj_info)); else @@ -1074,7 +1075,7 @@ PDC_obj_get_info(pdcid_t obj_id) else ret_value->obj_info_pub->name = NULL; - ret_value->obj_info_pub->obj_pt = PDC_CALLOC(struct pdc_obj_prop); + ret_value->obj_info_pub->obj_pt = PDC_CALLOC(1, struct pdc_obj_prop); if (!ret_value->obj_info_pub->obj_pt) PGOTO_ERROR(NULL, "failed to allocate memory"); memcpy(ret_value->obj_info_pub->obj_pt, info->obj_info_pub->obj_pt, sizeof(struct pdc_obj_prop)); @@ -1088,13 +1089,13 @@ PDC_obj_get_info(pdcid_t obj_id) ret_value->metadata = NULL; // fill in struct _pdc_cont_info field in ret_value->cont - ret_value->cont = PDC_CALLOC(struct _pdc_cont_info); + ret_value->cont = PDC_CALLOC(1, struct _pdc_cont_info); if (ret_value->cont) memcpy(ret_value->cont, info->cont, sizeof(struct _pdc_cont_info)); else PGOTO_ERROR(NULL, "cannot allocate ret_value->cont"); - ret_value->cont->cont_info_pub = PDC_CALLOC(struct pdc_cont_info); + ret_value->cont->cont_info_pub = PDC_CALLOC(1, struct pdc_cont_info); if (ret_value->cont->cont_info_pub) memcpy(ret_value->cont->cont_info_pub, info->cont->cont_info_pub, sizeof(struct pdc_cont_info)); else @@ -1104,12 +1105,12 @@ PDC_obj_get_info(pdcid_t obj_id) else ret_value->cont->cont_info_pub->name = NULL; - ret_value->cont->cont_pt = PDC_CALLOC(struct _pdc_cont_prop); + ret_value->cont->cont_pt = PDC_CALLOC(1, struct _pdc_cont_prop); if (ret_value->cont->cont_pt) memcpy(ret_value->cont->cont_pt, info->cont->cont_pt, sizeof(struct _pdc_cont_prop)); else PGOTO_ERROR(NULL, "cannot allocate ret_value->cont->cont_pt"); - ret_value->cont->cont_pt->pdc = PDC_CALLOC(struct _pdc_class); + ret_value->cont->cont_pt->pdc = PDC_CALLOC(1, struct _pdc_class); if (ret_value->cont->cont_pt->pdc) { ret_value->cont->cont_pt->pdc->local_id = info->cont->cont_pt->pdc->local_id; if (info->cont->cont_pt->pdc->name) @@ -1121,12 +1122,12 @@ PDC_obj_get_info(pdcid_t obj_id) PGOTO_ERROR(NULL, "cannot allocate ret_value->cont->cont_pt->pdc"); // fill in struct _pdc_obj_prop field in ret_value->obj_pt - ret_value->obj_pt = PDC_CALLOC(struct _pdc_obj_prop); + ret_value->obj_pt = PDC_CALLOC(1, struct _pdc_obj_prop); if (ret_value->obj_pt) memcpy(ret_value->obj_pt, info->obj_pt, sizeof(struct _pdc_obj_prop)); else PGOTO_ERROR(NULL, "cannot allocate ret_value->obj_pt"); - ret_value->obj_pt->pdc = PDC_CALLOC(struct _pdc_class); + ret_value->obj_pt->pdc = PDC_CALLOC(1, struct _pdc_class); if (ret_value->obj_pt->pdc) { ret_value->obj_pt->pdc->local_id = info->obj_pt->pdc->local_id; if (info->obj_pt->pdc->name) @@ -1137,7 +1138,7 @@ PDC_obj_get_info(pdcid_t obj_id) else PGOTO_ERROR(NULL, "cannot allocate ret_value->obj_pt->pdc"); - ret_value->obj_pt->obj_prop_pub = PDC_CALLOC(struct pdc_obj_prop); + ret_value->obj_pt->obj_prop_pub = PDC_CALLOC(1, struct pdc_obj_prop); if (ret_value->obj_pt->obj_prop_pub) memcpy(ret_value->obj_pt->obj_prop_pub, info->obj_pt->obj_prop_pub, sizeof(struct pdc_obj_prop)); else @@ -1235,9 +1236,7 @@ PDCobj_get_info(pdcid_t obj_id) /* obj_id = PDC_find_byname(PDC_OBJ, obj_name); */ tmp = PDC_obj_get_info(obj_id); - - ret_value = PDC_CALLOC(struct pdc_obj_info); - if (!ret_value) + if (NULL == tmp) PGOTO_ERROR(NULL, "failed to allocate memory"); ret_value = tmp->obj_info_pub; diff --git a/src/api/pdc_obj/pdc_prop.c b/src/api/pdc_obj/pdc_prop.c index 4d58f5321..e0632b733 100644 --- a/src/api/pdc_obj/pdc_prop.c +++ b/src/api/pdc_obj/pdc_prop.c @@ -75,7 +75,7 @@ PDCprop_create(pdc_prop_type_t type, pdcid_t pdcid) p->cont_prop_id = new_id_c; id_info = PDC_find_id(pdcid); pdc_class = (struct _pdc_class *)(id_info->obj_ptr); - p->pdc = PDC_CALLOC(struct _pdc_class); + p->pdc = PDC_CALLOC(1, struct _pdc_class); if (p->pdc == NULL) PGOTO_ERROR(0, "PDC class allocation failed"); if (pdc_class->name) @@ -105,7 +105,7 @@ PDCprop_create(pdc_prop_type_t type, pdcid_t pdcid) q->obj_prop_pub->obj_prop_id = new_id_o; id_info = PDC_find_id(pdcid); pdc_class = (struct _pdc_class *)(id_info->obj_ptr); - q->pdc = PDC_CALLOC(struct _pdc_class); + q->pdc = PDC_CALLOC(1, struct _pdc_class); if (q->pdc == NULL) PGOTO_ERROR(0, "PDC class allocation failed"); if (pdc_class->name) @@ -141,7 +141,7 @@ PDCprop_obj_dup(pdcid_t prop_id) PGOTO_ERROR(0, "cannot locate object property"); info = (struct _pdc_obj_prop *)(prop->obj_ptr); - q = PDC_CALLOC(struct _pdc_obj_prop); + q = PDC_CALLOC(1, struct _pdc_obj_prop); if (!q) PGOTO_ERROR(0, "PDC object property memory allocation failed"); if (info->app_name) @@ -166,7 +166,7 @@ PDCprop_obj_dup(pdcid_t prop_id) (q->obj_prop_pub->dims)[i] = (info->obj_prop_pub->dims)[i]; /* struct _pdc_class field */ - q->pdc = PDC_CALLOC(struct _pdc_class); + q->pdc = PDC_CALLOC(1, struct _pdc_class); if (!q->pdc) PGOTO_ERROR(0, "PDC class memory allocation failed"); if (info->pdc->name) @@ -303,13 +303,13 @@ PDCcont_prop_get_info(pdcid_t cont_prop) PGOTO_ERROR(NULL, "cannot allocate container property"); info = (struct _pdc_cont_prop *)(prop->obj_ptr); - ret_value = PDC_CALLOC(struct _pdc_cont_prop); + ret_value = PDC_CALLOC(1, struct _pdc_cont_prop); if (!ret_value) PGOTO_ERROR(NULL, "PDC container property memory allocation failed"); ret_value->cont_life = info->cont_life; ret_value->cont_prop_id = info->cont_prop_id; - ret_value->pdc = PDC_CALLOC(struct _pdc_class); + ret_value->pdc = PDC_CALLOC(1, struct _pdc_class); if (!ret_value->pdc) PGOTO_ERROR(NULL, "cannot allocate ret_value->pdc"); if (info->pdc->name) @@ -336,7 +336,7 @@ PDCobj_prop_get_info(pdcid_t obj_prop) PGOTO_ERROR(NULL, "cannot locate object property"); info = (struct _pdc_obj_prop *)(prop->obj_ptr); - ret_value = PDC_CALLOC(struct pdc_obj_prop); + ret_value = PDC_CALLOC(1, struct pdc_obj_prop); if (ret_value == NULL) PGOTO_ERROR(NULL, "PDC object property memory allocation failed"); memcpy(ret_value, info->obj_prop_pub, sizeof(struct pdc_obj_prop)); @@ -367,7 +367,7 @@ PDC_obj_prop_get_info(pdcid_t obj_prop) PGOTO_ERROR(NULL, "cannot locate object property"); info = (struct _pdc_obj_prop *)(prop->obj_ptr); - ret_value = PDC_CALLOC(struct _pdc_obj_prop); + ret_value = PDC_CALLOC(1, struct _pdc_obj_prop); if (ret_value == NULL) PGOTO_ERROR(NULL, "PDC object property memory allocation failed"); memcpy(ret_value, info, sizeof(struct _pdc_obj_prop)); @@ -381,7 +381,7 @@ PDC_obj_prop_get_info(pdcid_t obj_prop) ret_value->tags = strdup(info->tags); /* struct _pdc_class field */ - ret_value->pdc = PDC_CALLOC(struct _pdc_class); + ret_value->pdc = PDC_CALLOC(1, struct _pdc_class); if (ret_value->pdc == NULL) PGOTO_ERROR(NULL, "cannot allocate ret_value->pdc"); if (info->pdc->name) @@ -389,7 +389,7 @@ PDC_obj_prop_get_info(pdcid_t obj_prop) ret_value->pdc->local_id = info->pdc->local_id; /* struct pdc_obj_prop field */ - ret_value->obj_prop_pub = PDC_CALLOC(struct pdc_obj_prop); + ret_value->obj_prop_pub = PDC_CALLOC(1, struct pdc_obj_prop); if (ret_value->obj_prop_pub == NULL) PGOTO_ERROR(NULL, "PDC object pub property memory allocation failed"); memcpy(ret_value->obj_prop_pub, info->obj_prop_pub, sizeof(struct pdc_obj_prop)); diff --git a/src/api/pdc_region/pdc_region_transfer.c b/src/api/pdc_region/pdc_region_transfer.c index 775947f5d..87f9f3625 100644 --- a/src/api/pdc_region/pdc_region_transfer.c +++ b/src/api/pdc_region/pdc_region_transfer.c @@ -1446,8 +1446,7 @@ PDCregion_transfer_start(pdcid_t transfer_request_id) // Pack local region to a contiguous memory buffer unit = transfer_request->unit; - // Convert user buf into a contiguous buffer called new_buf, which is determined by the shape of local - // objects. + // Convert user buf into a contiguous buffer called , which is determined by the shape of local objects. pack_region_buffer(transfer_request->buf, transfer_request->obj_dims, transfer_request->total_data_size, transfer_request->local_region_ndim, transfer_request->local_region_offset, transfer_request->local_region_size, unit, transfer_request->access_type, diff --git a/src/api/pdc_transform/include/pdc_transforms_pkg.h b/src/api/pdc_transform/include/pdc_transforms_pkg.h index 3b4153152..4929f1f12 100644 --- a/src/api/pdc_transform/include/pdc_transforms_pkg.h +++ b/src/api/pdc_transform/include/pdc_transforms_pkg.h @@ -32,6 +32,8 @@ /***************************/ /* Library Private Structs */ /***************************/ +typedef enum { C_lang = 0, FORTRAN_lang, PYTHON_lang, JULIA_lang, N_LANGUAGES } _pdc_analysis_language_t; + struct _pdc_region_transform_ftn_info { pdcid_t object_id; pdcid_t region_id; diff --git a/src/api/profiling/CMakeLists.txt b/src/api/profiling/CMakeLists.txt deleted file mode 100644 index 1b2ee8254..000000000 --- a/src/api/profiling/CMakeLists.txt +++ /dev/null @@ -1,120 +0,0 @@ -#------------------------------------------------------------------------------ -# Include source and build directories -#------------------------------------------------------------------------------ -include_directories( - ${PDC_INCLUDES_BUILD_TIME} - ${PROJECT_SOURCE_DIR} - ${PROJECT_BINARY_DIR} - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ${PDC_SOURCE_DIR}/src/server/include - ${PDC_SOURCE_DIR}/src/server/pdc_server_region/include - ${PDC_SOURCE_DIR}/src/server/dablooms - ${PDC_SOURCE_DIR}/src/api/include - ${PDC_SOURCE_DIR}/src/api/pdc_obj/include - ${PDC_SOURCE_DIR}/src/api/pdc_region/include - ${PDC_SOURCE_DIR}/src/api/pdc_query/include - ${PDC_SOURCE_DIR}/src/api/pdc_transform/include - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/include - ${PDC_SOURCE_DIR}/src/api/profiling/include - ${PDC_SOURCE_DIR}/src/utils/include - ${MERCURY_INCLUDE_DIR} - ${FASTBIT_INCLUDE_DIR} -) - -install( - FILES - ${CMAKE_BINARY_DIR}/pdc_config.h - DESTINATION - ${PDC_INSTALL_INCLUDE_DIR} - COMPONENT - headers -) - -#------------------------------------------------------------------------------ -# Options -#------------------------------------------------------------------------------ -#add_definitions(-DPDC_ENABLE_MPI=1) -#add_definitions(-DPDC_TIMING=1) -#add_definitions(-DPDC_ENABLE_CHECKPOINT=1) -#add_definitions(-DENABLE_MULTITHREAD=1) - -#------------------------------------------------------------------------------ -# Configure module header files -#------------------------------------------------------------------------------ -# Set unique vars used in the autogenerated config file (symbol import/export) -if(BUILD_SHARED_LIBS) - set(PDC_BUILD_SHARED_LIBS 1) - set(PDC_LIBTYPE SHARED) -else() - set(PDC_BUILD_SHARED_LIBS 0) - set(PDC_LIBTYPE STATIC) -endif() - -#------------------------------------------------------------------------------ -# Set sources -#------------------------------------------------------------------------------ -set(PDC_PROF_SRCS - ${CMAKE_CURRENT_SOURCE_DIR}/pdc_hashtab.c - ${CMAKE_CURRENT_SOURCE_DIR}/pdc_stack_ops.c - ) - -#------------------------------------------------------------------------------ -# Libraries -#------------------------------------------------------------------------------ -# PDCPROF -add_library(pdcprof ${PDC_PROF_SRCS}) -pdc_set_lib_options(pdcprof "pdcprof" ${PDC_LIBTYPE}) - -set(PDC_EXPORTED_LIBS pdcprof ${PDC_EXPORTED_LIBS}) - -#----------------------------------------------------------------------------- -# Specify project header files to be installed -#----------------------------------------------------------------------------- -set(PDC_PROF_HEADERS - ${CMAKE_CURRENT_SOURCE_DIR}/include/pdc_hashtab.h - ${CMAKE_CURRENT_SOURCE_DIR}/include/pdc_stack_ops.h - ) - -#----------------------------------------------------------------------------- -# Add file(s) to CMake Install -#----------------------------------------------------------------------------- -install( - FILES - ${PDC_PROF_HEADERS} - DESTINATION - ${PDC_INSTALL_INCLUDE_DIR} - COMPONENT - headers -) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install -#----------------------------------------------------------------------------- -install( - TARGETS - pdcprof - EXPORT - ${PDC_EXPORTED_TARGETS} - LIBRARY DESTINATION ${PDC_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${PDC_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${PDC_INSTALL_BIN_DIR} -) - -#------------------------------------------------------------------------------ -# Set variables for parent scope -#------------------------------------------------------------------------------ -# Used by config.cmake.build.in and Testing -set(PDC_INCLUDES_BUILD_TIME - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ${PDC_EXT_INCLUDE_DEPENDENCIES} - PARENT_SCOPE -) - -# Used by config.cmake.install.in -set(PDC_INCLUDES_INSTALL_TIME - ${PDC_INSTALL_INCLUDE_DIR} - ${PDC_EXT_INCLUDE_DEPENDENCIES} - PARENT_SCOPE -) diff --git a/src/commons/CMakeLists.txt b/src/commons/CMakeLists.txt new file mode 100644 index 000000000..9a98dcc9f --- /dev/null +++ b/src/commons/CMakeLists.txt @@ -0,0 +1,241 @@ +#------------------------------------------------------------------------------ +# PDC Commons +#------------------------------------------------------------------------------ + +set(PDC_COMMON_LIBRARY_NAME pdc_commons CACHE INTERNAL "") +set(PDC_COMMON_SERVER_LIB pdc_commons_server CACHE INTERNAL "") +set(PDC_COMMON_CLIENT_LIB pdc_commons_client CACHE INTERNAL "") +#------------------------------------------------------------------------------ +# External dependencies +#------------------------------------------------------------------------------ + +# #set(PDC_EXT_LIB_DEPENDENCIES ${PDC_COMMON_LIBRARY_NAME} ${PDC_EXT_LIB_DEPENDENCIES}) +# set(PDC_EXT_INCLUDE_DEPENDENCIES ${CMAKE_CURRENT_SOURCE_DIR}/profiling) +# set(PDC_EXPORTED_LIBS pdcprof) + +if(THREADS_HAVE_PTHREAD_ARG) + set_property(TARGET ${PDC_COMMON_LIBRARY_NAME} PROPERTY COMPILE_OPTIONS "-pthread") + set_property(TARGET ${PDC_COMMON_LIBRARY_NAME} PROPERTY INTERFACE_COMPILE_OPTIONS "-pthread") +endif() + +# Math library + +find_library(MATH_LIBRARY m) +if(MATH_LIBRARY) + set(PDC_EXT_LIB_DEPENDENCIES ${MATH_LIBRARY} ${PDC_EXT_LIB_DEPENDENCIES}) +endif() + +# Mercury +find_package(MERCURY REQUIRED) +if(MERCURY_FOUND) + set(PDC_EXT_INCLUDE_DEPENDENCIES ${MERCURY_INCLUDE_DIRS} + ${PDC_EXT_INCLUDE_DEPENDENCIES} + ) + set(PDC_EXT_LIB_DEPENDENCIES ${MERCURY_LIBRARIES} ${PDC_EXT_LIB_DEPENDENCIES}) +endif() + +include_directories(${PDC_EXT_INCLUDE_DEPENDENCIES}) + +#------------------------------------------------------------------------------ +# Include directories +#------------------------------------------------------------------------------ + +# Get a list of all directories that contain header files +file(GLOB_RECURSE LOCAL_INCLUDE_DIRS "*.h") + +# Remove specified directories from the list +# list(FILTER LOCAL_INCLUDE_DIRS EXCLUDE REGEX "${CMAKE_CURRENT_SOURCE_DIR}/comm") + +# Remove the /filename.h at the end of each directory +list(TRANSFORM LOCAL_INCLUDE_DIRS REPLACE "/[^/]*$" "") + +# Remove duplicates +list(REMOVE_DUPLICATES LOCAL_INCLUDE_DIRS) + +set(PDC_COMMONS_INCLUDE_DIRS + ${LOCAL_INCLUDE_DIRS} + ${PDC_INCLUDES_BUILD_TIME} + ${PROJECT_BINARY_DIR} + ${PDC_SOURCE_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} +) + +include_directories( + ${PDC_COMMONS_INCLUDE_DIRS} +) + +message(STATUS "===MERCURY_INCLUDE_DIRS: ${MERCURY_INCLUDE_DIRS}") +message(STATUS "===PDC_EXT_INCLUDE_DEPENDENCIES: ${PDC_EXT_INCLUDE_DEPENDENCIES}") +message(STATUS "===PDC_COMMONS_INCLUDE_DIRS: ${PDC_COMMONS_INCLUDE_DIRS}") + +install( + FILES + ${CMAKE_BINARY_DIR}/pdc_config.h + ${PDC_SOURCE_DIR}/src/commons/utils/include/pdc_id_pkg.h + ${PDC_SOURCE_DIR}/src/commons/utils/include/pdc_malloc.h + ${PDC_SOURCE_DIR}/src/commons/utils/include/pdc_linkedlist.h + DESTINATION + ${PDC_INSTALL_INCLUDE_DIR} + COMPONENT + headers +) + +#------------------------------------------------------------------------------ +# Configure module header files +#------------------------------------------------------------------------------ +# Set unique vars used in the autogenerated config file (symbol import/export) +if(BUILD_SHARED_LIBS) + set(PDC_BUILD_SHARED_LIBS 1) + set(PDC_LIBTYPE SHARED) +else() + set(PDC_BUILD_SHARED_LIBS 0) + set(PDC_LIBTYPE STATIC) +endif() + +if(PDC_ENABLE_TIMING) + add_definitions(-DPDC_TIMING=1) +endif() + +#------------------------------------------------------------------------------ +# Set sources +#------------------------------------------------------------------------------ + +# Collect all source files +file(GLOB_RECURSE PDC_COMMONS_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.c) +file(GLOB_RECURSE PDC_COMMONS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.h) +list(FILTER PDC_COMMONS_SOURCES EXCLUDE REGEX "${CMAKE_CURRENT_SOURCE_DIR}/serde/.*") +list(FILTER PDC_COMMONS_HEADERS EXCLUDE REGEX "${CMAKE_CURRENT_SOURCE_DIR}/serde/.*") +list(FILTER PDC_COMMONS_SOURCES EXCLUDE REGEX "${CMAKE_CURRENT_SOURCE_DIR}/.*_test.c") + +message(STATUS "===PDC_COMMONS_SOURCES: ${PDC_COMMONS_SOURCES}") + +file(GLOB_RECURSE PDC_COMMONS_TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*_test.c) +list(FILTER PDC_COMMONS_TEST_SRC EXCLUDE REGEX "${CMAKE_CURRENT_SOURCE_DIR}/serde/.*") +message(STATUS "===PDC_COMMONS_TEST_SRC: ${PDC_COMMONS_TEST_SRC}") + +#------------------------------------------------------------------------------ +# Libraries +#------------------------------------------------------------------------------ +# PDC COMMONS + + +add_library(${PDC_COMMON_LIBRARY_NAME} ${PDC_LIBTYPE} ${PDC_COMMONS_SOURCES} ${PDC_COMMONS_HEADERS}) + +target_include_directories(${PDC_COMMON_LIBRARY_NAME} + PUBLIC "$" + $ +) + +target_link_libraries(${PDC_COMMON_LIBRARY_NAME} INTERFACE + ${PDC_EXT_LIB_DEPENDENCIES} +) + +set(PDC_EXPORTED_LIBS ${PDC_COMMON_LIBRARY_NAME} ${PDC_EXPORTED_LIBS}) + +#------------------------------------------------------------------------------ +# Tests +#------------------------------------------------------------------------------ + +# for each test source in $PDC_COMMONS_TEST_SRC, create an executable and link with PDC_COMMON_LIBRARY_NAME +foreach(_test_src ${PDC_COMMONS_TEST_SRC}) + get_filename_component(_test_name ${_test_src} NAME_WE) + add_executable(${_test_name} ${_test_src}) + message(STATUS "===_test_name: ${_test_name}") + target_link_libraries(${_test_name} ${PDC_COMMON_LIBRARY_NAME}) + # add_test(${_test_name} ${_test_name}) +endforeach() + +#----------------------------------------------------------------------------- +# Specify project header files to be installed +#----------------------------------------------------------------------------- + + +set(PDC_PUBLIC_HEADERS "") +set(PUBLIC_HEADER_DIR_LIST + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/generic/include + ${CMAKE_CURRENT_SOURCE_DIR}/profiling/include +) + +foreach(_header_dir ${PUBLIC_HEADER_DIR_LIST}) + file(GLOB_RECURSE _dir_headers ${_header_dir}/*.h) + list(APPEND PDC_PUBLIC_HEADERS ${_dir_headers}) +endforeach() + +set(PDC_COMMONS_HEADERS + ${PDC_PUBLIC_HEADERS} + ${PROJECT_BINARY_DIR}/pdc_config_sys.h + ${PROJECT_BINARY_DIR}/pdc_config.h + ) + +#----------------------------------------------------------------------------- +# Add file(s) to CMake Install +#----------------------------------------------------------------------------- +install( + FILES + ${PDC_COMMONS_HEADERS} + DESTINATION + ${PDC_INSTALL_INCLUDE_DIR} + COMPONENT + headers +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install +#----------------------------------------------------------------------------- + +install( + TARGETS + ${PDC_COMMON_LIBRARY_NAME} + EXPORT + ${PDC_EXPORTED_TARGETS} + LIBRARY DESTINATION ${PDC_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${PDC_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${PDC_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${PDC_EXPORTED_TARGETS} + DESTINATION + ${PDC_INSTALL_DATA_DIR}/cmake/pdc + FILE + ${PDC_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +if(NOT PDC_EXTERNALLY_CONFIGURED) +EXPORT ( + TARGETS + ${PDC_EXPORTED_LIBS} + FILE + ${PDC_EXPORTED_TARGETS}.cmake +) +endif() + +#------------------------------------------------------------------------------ +# Set variables for parent scope +#------------------------------------------------------------------------------ +# Used by config.cmake.build.in and Testing +set(PDC_INCLUDES_BUILD_TIME + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} + ${PDC_COMMONS_INCLUDE_DIRS} + PARENT_SCOPE +) + +# Used by config.cmake.install.in +set(PDC_INCLUDES_INSTALL_TIME + ${PDC_COMMONS_INCLUDE_DIRS} + ${PDC_INSTALL_INCLUDE_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} + PARENT_SCOPE +) + +set(PDC_COMMONS_LIBRARIES ${PDC_COMMON_LIBRARY_NAME} PARENT_SCOPE) \ No newline at end of file diff --git a/src/commons/Readme.md b/src/commons/Readme.md new file mode 100644 index 000000000..f69d7a8bf --- /dev/null +++ b/src/commons/Readme.md @@ -0,0 +1,2 @@ +Commons directory should contain a collection of code that will be shared by both Server and Client-side APIs + diff --git a/src/commons/collections/Readme.md b/src/commons/collections/Readme.md new file mode 100644 index 000000000..38b7427fd --- /dev/null +++ b/src/commons/collections/Readme.md @@ -0,0 +1,9 @@ +Collections directory should contain a bunch of generic and basic data structures for PDC to use. + +At least, this directory should contain: + +1. Vector/List Implementation +2. Hash Table Implementation +3. Hash Set Implementation +4. Art Implementation +5. BST Implementation \ No newline at end of file diff --git a/src/commons/collections/art.c b/src/commons/collections/art.c new file mode 100644 index 000000000..6d15e1da3 --- /dev/null +++ b/src/commons/collections/art.c @@ -0,0 +1,1096 @@ +// +// * Copyright (c) 2012, Armon Dadgar +// * All rights reserved. +// * +// * Redistribution and use in source and binary forms, with or without +// * modification, are permitted provided that the following conditions are met: +// * - Redistributions of source code must retain the above copyright +// * notice, this list of conditions and the following disclaimer. +// * - Redistributions in binary form must reproduce the above copyright +// * notice, this list of conditions and the following disclaimer in the +// * documentation and/or other materials provided with the distribution. +// * - Neither the name of the organization nor the +// * names of its contributors may be used to endorse or promote products +// * derived from this software without specific prior written permission. +// * +// * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// * DISCLAIMED. IN NO EVENT SHALL ARMON DADGAR BE LIABLE FOR ANY +// * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include +#include +#include +#include +#include +#include "art.h" +#include "pdc_malloc.h" + +#ifdef __i386__ +#include +#else +#ifdef __amd64__ +#include +#endif +#endif + +/** + * Macros to manipulate pointer tags + */ +#define IS_LEAF(x) (((uintptr_t)x & 1)) +#define SET_LEAF(x) ((void *)((uintptr_t)x | 1)) +#define LEAF_RAW(x) ((art_leaf *)((void *)((uintptr_t)x & ~1))) + +/** + * Allocates a node of the given type, + * initializes to zero and sets the type. + */ +static art_node * +alloc_node(uint8_t type) +{ + art_node *n; + switch (type) { + case NODE4: + n = (art_node *)PDC_calloc(1, sizeof(art_node4)); + break; + case NODE16: + n = (art_node *)PDC_calloc(1, sizeof(art_node16)); + break; + case NODE48: + n = (art_node *)PDC_calloc(1, sizeof(art_node48)); + break; + case NODE256: + n = (art_node *)PDC_calloc(1, sizeof(art_node256)); + break; + default: + abort(); + } + n->type = type; + return n; +} + +/** + * Initializes an ART tree + * @return 0 on success. + */ +int +art_tree_init(art_tree *t) +{ + t->root = NULL; + t->size = 0; + return 0; +} + +// Recursively destroys the tree +static void +destroy_node(art_node *n) +{ + // Break if null + if (!n) + return; + + // Special case leafs + if (IS_LEAF(n)) { + free(LEAF_RAW(n)); + return; + } + + // Handle each node type + int i, idx; + union { + art_node4 * p1; + art_node16 * p2; + art_node48 * p3; + art_node256 *p4; + } p; + switch (n->type) { + case NODE4: + p.p1 = (art_node4 *)n; + for (i = 0; i < n->num_children; i++) { + destroy_node(p.p1->children[i]); + } + break; + + case NODE16: + p.p2 = (art_node16 *)n; + for (i = 0; i < n->num_children; i++) { + destroy_node(p.p2->children[i]); + } + break; + + case NODE48: + p.p3 = (art_node48 *)n; + for (i = 0; i < 256; i++) { + idx = ((art_node48 *)n)->keys[i]; + if (!idx) + continue; + destroy_node(p.p3->children[idx - 1]); + } + break; + + case NODE256: + p.p4 = (art_node256 *)n; + for (i = 0; i < 256; i++) { + if (p.p4->children[i]) + destroy_node(p.p4->children[i]); + } + break; + + default: + abort(); + } + + // Free ourself on the way up + free(n); +} + +/** + * Destroys an ART tree + * @return 0 on success. + */ +int +art_tree_destroy(art_tree *t) +{ + destroy_node(t->root); + return 0; +} + +/** + * Returns the size of the ART tree. + */ + +#ifndef BROKEN_GCC_C99_INLINE +extern inline uint64_t art_size(art_tree *t); +#endif + +static art_node ** +find_child(art_node *n, unsigned char c) +{ + int i, mask, bitfield; + union { + art_node4 * p1; + art_node16 * p2; + art_node48 * p3; + art_node256 *p4; + } p; + switch (n->type) { + case NODE4: + p.p1 = (art_node4 *)n; + for (i = 0; i < n->num_children; i++) { + /* this cast works around a bug in gcc 5.1 when unrolling loops + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 + */ + if (((unsigned char *)p.p1->keys)[i] == c) + return &p.p1->children[i]; + } + break; + + { + case NODE16: + p.p2 = (art_node16 *)n; + +// support non-86 architectures +#ifdef __i386__ + // Compare the key to all 16 stored keys + __m128i cmp; + cmp = _mm_cmpeq_epi8(_mm_set1_epi8(c), _mm_loadu_si128((__m128i *)p.p2->keys)); + + // Use a mask to ignore children that don't exist + mask = (1 << n->num_children) - 1; + bitfield = _mm_movemask_epi8(cmp) & mask; +#else +#ifdef __amd64__ + // Compare the key to all 16 stored keys + __m128i cmp; + cmp = _mm_cmpeq_epi8(_mm_set1_epi8(c), _mm_loadu_si128((__m128i *)p.p2->keys)); + + // Use a mask to ignore children that don't exist + mask = (1 << n->num_children) - 1; + bitfield = _mm_movemask_epi8(cmp) & mask; +#else + // Compare the key to all 16 stored keys + bitfield = 0; + for (i = 0; i < 16; ++i) { + if (p.p2->keys[i] == c) + bitfield |= (1 << i); + } + + // Use a mask to ignore children that don't exist + mask = (1 << n->num_children) - 1; + bitfield &= mask; +#endif +#endif + + /* + * If we have a match (any bit set) then we can + * return the pointer match using ctz to get + * the index. + */ + if (bitfield) + return &p.p2->children[__builtin_ctz(bitfield)]; + break; + } + + case NODE48: + p.p3 = (art_node48 *)n; + i = p.p3->keys[c]; + if (i) + return &p.p3->children[i - 1]; + break; + + case NODE256: + p.p4 = (art_node256 *)n; + if (p.p4->children[c]) + return &p.p4->children[c]; + break; + + default: + abort(); + } + return NULL; +} + +// Simple inlined if +static inline int +min(int a, int b) +{ + return (a < b) ? a : b; +} + +/** + * Returns the number of prefix characters shared between + * the key and node. + */ +static int +check_prefix(const art_node *n, const unsigned char *key, int key_len, int depth) +{ + int max_cmp = min(min(n->partial_len, MAX_PREFIX_LEN), key_len - depth); + int idx; + for (idx = 0; idx < max_cmp; idx++) { + if (n->partial[idx] != key[depth + idx]) + return idx; + } + return idx; +} + +/** + * Checks if a leaf matches + * @return 0 on success. + */ +static int +leaf_matches(const art_leaf *n, const unsigned char *key, int key_len, int depth) +{ + (void)depth; + // Fail if the key lengths are different + if (n->key_len != (uint32_t)key_len) + return 1; + + // Compare the keys starting at the depth + return memcmp(n->key, key, key_len); +} + +/** + * Searches for a value in the ART tree + * @arg t The tree + * @arg key The key + * @arg key_len The length of the key + * @return NULL if the item was not found, otherwise + * the value pointer is returned. + */ +void * +art_search(const art_tree *t, const unsigned char *key, int key_len) +{ + art_node **child; + art_node * n = t->root; + int prefix_len, depth = 0; + while (n) { + // Might be a leaf + if (IS_LEAF(n)) { + n = (art_node *)LEAF_RAW(n); + // Check if the expanded path matches + if (!leaf_matches((art_leaf *)n, key, key_len, depth)) { + return ((art_leaf *)n)->value; + } + return NULL; + } + + // Bail if the prefix does not match + if (n->partial_len) { + prefix_len = check_prefix(n, key, key_len, depth); + if (prefix_len != min(MAX_PREFIX_LEN, n->partial_len)) + return NULL; + depth = depth + n->partial_len; + } + + // Recursively search + child = find_child(n, key[depth]); + n = (child) ? *child : NULL; + depth++; + } + return NULL; +} + +// Find the minimum leaf under a node +static art_leaf * +minimum(const art_node *n) +{ + // Handle base cases + if (!n) + return NULL; + if (IS_LEAF(n)) + return LEAF_RAW(n); + + int idx; + switch (n->type) { + case NODE4: + return minimum(((const art_node4 *)n)->children[0]); + case NODE16: + return minimum(((const art_node16 *)n)->children[0]); + case NODE48: + idx = 0; + while (!((const art_node48 *)n)->keys[idx]) + idx++; + idx = ((const art_node48 *)n)->keys[idx] - 1; + return minimum(((const art_node48 *)n)->children[idx]); + case NODE256: + idx = 0; + while (!((const art_node256 *)n)->children[idx]) + idx++; + return minimum(((const art_node256 *)n)->children[idx]); + default: + abort(); + } +} + +// Find the maximum leaf under a node +static art_leaf * +maximum(const art_node *n) +{ + // Handle base cases + if (!n) + return NULL; + if (IS_LEAF(n)) + return LEAF_RAW(n); + + int idx; + switch (n->type) { + case NODE4: + return maximum(((const art_node4 *)n)->children[n->num_children - 1]); + case NODE16: + return maximum(((const art_node16 *)n)->children[n->num_children - 1]); + case NODE48: + idx = 255; + while (!((const art_node48 *)n)->keys[idx]) + idx--; + idx = ((const art_node48 *)n)->keys[idx] - 1; + return maximum(((const art_node48 *)n)->children[idx]); + case NODE256: + idx = 255; + while (!((const art_node256 *)n)->children[idx]) + idx--; + return maximum(((const art_node256 *)n)->children[idx]); + default: + abort(); + } +} + +/** + * Returns the minimum valued leaf + */ +art_leaf * +art_minimum(art_tree *t) +{ + return minimum((art_node *)t->root); +} + +/** + * Returns the maximum valued leaf + */ +art_leaf * +art_maximum(art_tree *t) +{ + return maximum((art_node *)t->root); +} + +static art_leaf * +make_leaf(const unsigned char *key, int key_len, void *value) +{ + art_leaf *l = (art_leaf *)PDC_calloc(1, sizeof(art_leaf) + key_len); + l->value = value; + l->key_len = key_len; + memcpy(l->key, key, key_len); + return l; +} + +static int +longest_common_prefix(art_leaf *l1, art_leaf *l2, int depth) +{ + int max_cmp = min(l1->key_len, l2->key_len) - depth; + int idx; + for (idx = 0; idx < max_cmp; idx++) { + if (l1->key[depth + idx] != l2->key[depth + idx]) + return idx; + } + return idx; +} + +static void +copy_header(art_node *dest, art_node *src) +{ + dest->num_children = src->num_children; + dest->partial_len = src->partial_len; + memcpy(dest->partial, src->partial, min(MAX_PREFIX_LEN, src->partial_len)); +} + +static void +add_child256(art_node256 *n, art_node **ref, unsigned char c, void *child) +{ + (void)ref; + n->n.num_children++; + n->children[c] = (art_node *)child; +} + +static void +add_child48(art_node48 *n, art_node **ref, unsigned char c, void *child) +{ + if (n->n.num_children < 48) { + int pos = 0; + while (n->children[pos]) + pos++; + n->children[pos] = (art_node *)child; + n->keys[c] = pos + 1; + n->n.num_children++; + } + else { + art_node256 *new_node = (art_node256 *)alloc_node(NODE256); + for (int i = 0; i < 256; i++) { + if (n->keys[i]) { + new_node->children[i] = n->children[n->keys[i] - 1]; + } + } + copy_header((art_node *)new_node, (art_node *)n); + *ref = (art_node *)new_node; + free(n); + add_child256(new_node, ref, c, child); + } +} + +static void +add_child16(art_node16 *n, art_node **ref, unsigned char c, void *child) +{ + if (n->n.num_children < 16) { + unsigned mask = (1 << n->n.num_children) - 1; + +// support non-x86 architectures +#ifdef __i386__ + __m128i cmp; + + // Compare the key to all 16 stored keys + cmp = _mm_cmplt_epi8(_mm_set1_epi8(c), _mm_loadu_si128((__m128i *)n->keys)); + + // Use a mask to ignore children that don't exist + unsigned bitfield = _mm_movemask_epi8(cmp) & mask; +#else +#ifdef __amd64__ + __m128i cmp; + + // Compare the key to all 16 stored keys + cmp = _mm_cmplt_epi8(_mm_set1_epi8(c), _mm_loadu_si128((__m128i *)n->keys)); + + // Use a mask to ignore children that don't exist + unsigned bitfield = _mm_movemask_epi8(cmp) & mask; +#else + // Compare the key to all 16 stored keys + unsigned bitfield = 0; + for (short i = 0; i < 16; ++i) { + if (c < n->keys[i]) + bitfield |= (1 << i); + } + + // Use a mask to ignore children that don't exist + bitfield &= mask; +#endif +#endif + + // Check if less than any + unsigned idx; + if (bitfield) { + idx = __builtin_ctz(bitfield); + memmove(n->keys + idx + 1, n->keys + idx, n->n.num_children - idx); + memmove(n->children + idx + 1, n->children + idx, (n->n.num_children - idx) * sizeof(void *)); + } + else + idx = n->n.num_children; + + // Set the child + n->keys[idx] = c; + n->children[idx] = (art_node *)child; + n->n.num_children++; + } + else { + art_node48 *new_node = (art_node48 *)alloc_node(NODE48); + + // Copy the child pointers and populate the key map + memcpy(new_node->children, n->children, sizeof(void *) * n->n.num_children); + for (int i = 0; i < n->n.num_children; i++) { + new_node->keys[n->keys[i]] = i + 1; + } + copy_header((art_node *)new_node, (art_node *)n); + *ref = (art_node *)new_node; + free(n); + add_child48(new_node, ref, c, child); + } +} + +static void +add_child4(art_node4 *n, art_node **ref, unsigned char c, void *child) +{ + if (n->n.num_children < 4) { + int idx; + for (idx = 0; idx < n->n.num_children; idx++) { + if (c < n->keys[idx]) + break; + } + + // Shift to make room + memmove(n->keys + idx + 1, n->keys + idx, n->n.num_children - idx); + memmove(n->children + idx + 1, n->children + idx, (n->n.num_children - idx) * sizeof(void *)); + + // Insert element + n->keys[idx] = c; + n->children[idx] = (art_node *)child; + n->n.num_children++; + } + else { + art_node16 *new_node = (art_node16 *)alloc_node(NODE16); + + // Copy the child pointers and the key map + memcpy(new_node->children, n->children, sizeof(void *) * n->n.num_children); + memcpy(new_node->keys, n->keys, sizeof(unsigned char) * n->n.num_children); + copy_header((art_node *)new_node, (art_node *)n); + *ref = (art_node *)new_node; + free(n); + add_child16(new_node, ref, c, child); + } +} + +static void +add_child(art_node *n, art_node **ref, unsigned char c, void *child) +{ + switch (n->type) { + case NODE4: + return add_child4((art_node4 *)n, ref, c, child); + case NODE16: + return add_child16((art_node16 *)n, ref, c, child); + case NODE48: + return add_child48((art_node48 *)n, ref, c, child); + case NODE256: + return add_child256((art_node256 *)n, ref, c, child); + default: + abort(); + } +} + +/** + * Calculates the index at which the prefixes mismatch + */ +static int +prefix_mismatch(const art_node *n, const unsigned char *key, int key_len, int depth) +{ + int max_cmp = min(min(MAX_PREFIX_LEN, n->partial_len), key_len - depth); + int idx; + for (idx = 0; idx < max_cmp; idx++) { + if (n->partial[idx] != key[depth + idx]) + return idx; + } + + // If the prefix is short we can avoid finding a leaf + if (n->partial_len > MAX_PREFIX_LEN) { + // Prefix is longer than what we've checked, find a leaf + art_leaf *l = minimum(n); + max_cmp = min(l->key_len, key_len) - depth; + for (; idx < max_cmp; idx++) { + if (l->key[idx + depth] != key[depth + idx]) + return idx; + } + } + return idx; +} + +static void * +recursive_insert(art_node *n, art_node **ref, const unsigned char *key, int key_len, void *value, int depth, + int *old, int replace) +{ + // If we are at a NULL node, inject a leaf + if (!n) { + *ref = (art_node *)SET_LEAF(make_leaf(key, key_len, value)); + return NULL; + } + + // If we are at a leaf, we need to replace it with a node + if (IS_LEAF(n)) { + art_leaf *l = LEAF_RAW(n); + + // Check if we are updating an existing value + if (!leaf_matches(l, key, key_len, depth)) { + *old = 1; + void *old_val = l->value; + if (replace) + l->value = value; + return old_val; + } + + // New value, we must split the leaf into a node4 + art_node4 *new_node = (art_node4 *)alloc_node(NODE4); + + // Create a new leaf + art_leaf *l2 = make_leaf(key, key_len, value); + + // Determine longest prefix + int longest_prefix = longest_common_prefix(l, l2, depth); + new_node->n.partial_len = longest_prefix; + memcpy(new_node->n.partial, key + depth, min(MAX_PREFIX_LEN, longest_prefix)); + // Add the leafs to the new node4 + *ref = (art_node *)new_node; + add_child4(new_node, ref, l->key[depth + longest_prefix], SET_LEAF(l)); + add_child4(new_node, ref, l2->key[depth + longest_prefix], SET_LEAF(l2)); + return NULL; + } + + // Check if given node has a prefix + if (n->partial_len) { + // Determine if the prefixes differ, since we need to split + int prefix_diff = prefix_mismatch(n, key, key_len, depth); + if ((uint32_t)prefix_diff >= n->partial_len) { + depth += n->partial_len; + goto RECURSE_SEARCH; + } + + // Create a new node + art_node4 *new_node = (art_node4 *)alloc_node(NODE4); + *ref = (art_node *)new_node; + new_node->n.partial_len = prefix_diff; + memcpy(new_node->n.partial, n->partial, min(MAX_PREFIX_LEN, prefix_diff)); + + // Adjust the prefix of the old node + if (n->partial_len <= MAX_PREFIX_LEN) { + add_child4(new_node, ref, n->partial[prefix_diff], n); + n->partial_len -= (prefix_diff + 1); + memmove(n->partial, n->partial + prefix_diff + 1, min(MAX_PREFIX_LEN, n->partial_len)); + } + else { + n->partial_len -= (prefix_diff + 1); + art_leaf *l = minimum(n); + add_child4(new_node, ref, l->key[depth + prefix_diff], n); + memcpy(n->partial, l->key + depth + prefix_diff + 1, min(MAX_PREFIX_LEN, n->partial_len)); + } + + // Insert the new leaf + art_leaf *l = make_leaf(key, key_len, value); + add_child4(new_node, ref, key[depth + prefix_diff], SET_LEAF(l)); + return NULL; + } + +RECURSE_SEARCH:; + + // Find a child to recurse to + art_node **child = find_child(n, key[depth]); + if (child) { + return recursive_insert(*child, child, key, key_len, value, depth + 1, old, replace); + } + + // No child, node goes within us + art_leaf *l = make_leaf(key, key_len, value); + add_child(n, ref, key[depth], SET_LEAF(l)); + return NULL; +} + +/** + * inserts a new value into the art tree + * @arg t the tree + * @arg key the key + * @arg key_len the length of the key + * @arg value opaque value. + * @return null if the item was newly inserted, otherwise + * the old value pointer is returned. + */ +void * +art_insert(art_tree *t, const unsigned char *key, int key_len, void *value) +{ + int old_val = 0; + void *old = recursive_insert(t->root, &t->root, key, key_len, value, 0, &old_val, 1); + if (!old_val) + t->size++; + return old; +} + +/** + * inserts a new value into the art tree (no replace) + * @arg t the tree + * @arg key the key + * @arg key_len the length of the key + * @arg value opaque value. + * @return null if the item was newly inserted, otherwise + * the old value pointer is returned. + */ +void * +art_insert_no_replace(art_tree *t, const unsigned char *key, int key_len, void *value) +{ + int old_val = 0; + void *old = recursive_insert(t->root, &t->root, key, key_len, value, 0, &old_val, 0); + if (!old_val) + t->size++; + return old; +} + +static void +remove_child256(art_node256 *n, art_node **ref, unsigned char c) +{ + n->children[c] = NULL; + n->n.num_children--; + + // Resize to a node48 on underflow, not immediately to prevent + // trashing if we sit on the 48/49 boundary + if (n->n.num_children == 37) { + art_node48 *new_node = (art_node48 *)alloc_node(NODE48); + *ref = (art_node *)new_node; + copy_header((art_node *)new_node, (art_node *)n); + + int pos = 0; + for (int i = 0; i < 256; i++) { + if (n->children[i]) { + new_node->children[pos] = n->children[i]; + new_node->keys[i] = pos + 1; + pos++; + } + } + free(n); + } +} + +static void +remove_child48(art_node48 *n, art_node **ref, unsigned char c) +{ + int pos = n->keys[c]; + n->keys[c] = 0; + n->children[pos - 1] = NULL; + n->n.num_children--; + + if (n->n.num_children == 12) { + art_node16 *new_node = (art_node16 *)alloc_node(NODE16); + *ref = (art_node *)new_node; + copy_header((art_node *)new_node, (art_node *)n); + + int child = 0; + for (int i = 0; i < 256; i++) { + pos = n->keys[i]; + if (pos) { + new_node->keys[child] = i; + new_node->children[child] = n->children[pos - 1]; + child++; + } + } + free(n); + } +} + +static void +remove_child16(art_node16 *n, art_node **ref, art_node **l) +{ + int pos = l - n->children; + memmove(n->keys + pos, n->keys + pos + 1, n->n.num_children - 1 - pos); + memmove(n->children + pos, n->children + pos + 1, (n->n.num_children - 1 - pos) * sizeof(void *)); + n->n.num_children--; + + if (n->n.num_children == 3) { + art_node4 *new_node = (art_node4 *)alloc_node(NODE4); + *ref = (art_node *)new_node; + copy_header((art_node *)new_node, (art_node *)n); + memcpy(new_node->keys, n->keys, 4); + memcpy(new_node->children, n->children, 4 * sizeof(void *)); + free(n); + } +} + +static void +remove_child4(art_node4 *n, art_node **ref, art_node **l) +{ + int pos = l - n->children; + memmove(n->keys + pos, n->keys + pos + 1, n->n.num_children - 1 - pos); + memmove(n->children + pos, n->children + pos + 1, (n->n.num_children - 1 - pos) * sizeof(void *)); + n->n.num_children--; + + // Remove nodes with only a single child + if (n->n.num_children == 1) { + art_node *child = n->children[0]; + if (!IS_LEAF(child)) { + // Concatenate the prefixes + int prefix = n->n.partial_len; + if (prefix < MAX_PREFIX_LEN) { + n->n.partial[prefix] = n->keys[0]; + prefix++; + } + if (prefix < MAX_PREFIX_LEN) { + int sub_prefix = min(child->partial_len, MAX_PREFIX_LEN - prefix); + memcpy(n->n.partial + prefix, child->partial, sub_prefix); + prefix += sub_prefix; + } + + // Store the prefix in the child + memcpy(child->partial, n->n.partial, min(prefix, MAX_PREFIX_LEN)); + child->partial_len += n->n.partial_len + 1; + } + *ref = child; + free(n); + } +} + +static void +remove_child(art_node *n, art_node **ref, unsigned char c, art_node **l) +{ + switch (n->type) { + case NODE4: + return remove_child4((art_node4 *)n, ref, l); + case NODE16: + return remove_child16((art_node16 *)n, ref, l); + case NODE48: + return remove_child48((art_node48 *)n, ref, c); + case NODE256: + return remove_child256((art_node256 *)n, ref, c); + default: + abort(); + } +} + +static art_leaf * +recursive_delete(art_node *n, art_node **ref, const unsigned char *key, int key_len, int depth) +{ + // Search terminated + if (!n) + return NULL; + + // Handle hitting a leaf node + if (IS_LEAF(n)) { + art_leaf *l = LEAF_RAW(n); + if (!leaf_matches(l, key, key_len, depth)) { + *ref = NULL; + return l; + } + return NULL; + } + + // Bail if the prefix does not match + if (n->partial_len) { + int prefix_len = check_prefix(n, key, key_len, depth); + if (prefix_len != min(MAX_PREFIX_LEN, n->partial_len)) { + return NULL; + } + depth = depth + n->partial_len; + } + + // Find child node + art_node **child = find_child(n, key[depth]); + if (!child) + return NULL; + + // If the child is leaf, delete from this node + if (IS_LEAF(*child)) { + art_leaf *l = LEAF_RAW(*child); + if (!leaf_matches(l, key, key_len, depth)) { + remove_child(n, ref, key[depth], child); + return l; + } + return NULL; + + // Recurse + } + else { + return recursive_delete(*child, child, key, key_len, depth + 1); + } +} + +/** + * Deletes a value from the ART tree + * @arg t The tree + * @arg key The key + * @arg key_len The length of the key + * @return NULL if the item was not found, otherwise + * the value pointer is returned. + */ +void * +art_delete(art_tree *t, const unsigned char *key, int key_len) +{ + art_leaf *l = recursive_delete(t->root, &t->root, key, key_len, 0); + if (l) { + t->size--; + void *old = l->value; + free(l); + return old; + } + return NULL; +} + +// Recursively iterates over the tree +static int +recursive_iter(art_node *n, art_callback cb, void *data) +{ + // Handle base cases + if (!n) + return 0; + if (IS_LEAF(n)) { + art_leaf *l = LEAF_RAW(n); + return cb(data, (const unsigned char *)l->key, l->key_len, l->value); + } + + int idx, res; + switch (n->type) { + case NODE4: + for (int i = 0; i < n->num_children; i++) { + res = recursive_iter(((art_node4 *)n)->children[i], cb, data); + if (res) + return res; + } + break; + + case NODE16: + for (int i = 0; i < n->num_children; i++) { + res = recursive_iter(((art_node16 *)n)->children[i], cb, data); + if (res) + return res; + } + break; + + case NODE48: + for (int i = 0; i < 256; i++) { + idx = ((art_node48 *)n)->keys[i]; + if (!idx) + continue; + + res = recursive_iter(((art_node48 *)n)->children[idx - 1], cb, data); + if (res) + return res; + } + break; + + case NODE256: + for (int i = 0; i < 256; i++) { + if (!((art_node256 *)n)->children[i]) + continue; + res = recursive_iter(((art_node256 *)n)->children[i], cb, data); + if (res) + return res; + } + break; + + default: + abort(); + } + return 0; +} + +/** + * Iterates through the entries pairs in the map, + * invoking a callback for each. The call back gets a + * key, value for each and returns an integer stop value. + * If the callback returns non-zero, then the iteration stops. + * @arg t The tree to iterate over + * @arg cb The callback function to invoke + * @arg data Opaque handle passed to the callback + * @return 0 on success, or the return of the callback. + */ +int +art_iter(art_tree *t, art_callback cb, void *data) +{ + return recursive_iter(t->root, cb, data); +} + +/** + * Checks if a leaf prefix matches + * @return 0 on success. + */ +static int +leaf_prefix_matches(const art_leaf *n, const unsigned char *prefix, int prefix_len) +{ + // Fail if the key length is too short + if (n->key_len < (uint32_t)prefix_len) + return 1; + + // Compare the keys + return memcmp(n->key, prefix, prefix_len); +} + +/** + * Iterates through the entries pairs in the map, + * invoking a callback for each that matches a given prefix. + * The call back gets a key, value for each and returns an integer stop value. + * If the callback returns non-zero, then the iteration stops. + * @arg t The tree to iterate over + * @arg prefix The prefix of keys to read + * @arg prefix_len The length of the prefix + * @arg cb The callback function to invoke + * @arg data Opaque handle passed to the callback + * @return 0 on success, or the return of the callback. + */ +int +art_iter_prefix(art_tree *t, const unsigned char *key, int key_len, art_callback cb, void *data) +{ + art_node **child; + art_node * n = t->root; + int prefix_len, depth = 0; + while (n) { + // Might be a leaf + if (IS_LEAF(n)) { + n = (art_node *)LEAF_RAW(n); + // Check if the expanded path matches + if (!leaf_prefix_matches((art_leaf *)n, key, key_len)) { + art_leaf *l = (art_leaf *)n; + return cb(data, (const unsigned char *)l->key, l->key_len, l->value); + } + return 0; + } + + // If the depth matches the prefix, we need to handle this node + if (depth == key_len) { + art_leaf *l = minimum(n); + if (!leaf_prefix_matches(l, key, key_len)) + return recursive_iter(n, cb, data); + return 0; + } + + // Bail if the prefix does not match + if (n->partial_len) { + prefix_len = prefix_mismatch(n, key, key_len, depth); + + // Guard if the mis-match is longer than the MAX_PREFIX_LEN + if ((uint32_t)prefix_len > n->partial_len) { + prefix_len = n->partial_len; + } + + // If there is no match, search is terminated + if (!prefix_len) { + return 0; + + // If we've matched the prefix, iterate on this node + } + else if (depth + prefix_len == key_len) { + return recursive_iter(n, cb, data); + } + + // if there is a full match, go deeper + depth = depth + n->partial_len; + } + + // Recursively search + child = find_child(n, key[depth]); + n = (child) ? *child : NULL; + depth++; + } + return 0; +} \ No newline at end of file diff --git a/src/commons/collections/include/art.h b/src/commons/collections/include/art.h new file mode 100644 index 000000000..0b918a0bf --- /dev/null +++ b/src/commons/collections/include/art.h @@ -0,0 +1,246 @@ +// +// * Copyright (c) 2012, Armon Dadgar +// * All rights reserved. +// * +// * Redistribution and use in source and binary forms, with or without +// * modification, are permitted provided that the following conditions are met: +// * - Redistributions of source code must retain the above copyright +// * notice, this list of conditions and the following disclaimer. +// * - Redistributions in binary form must reproduce the above copyright +// * notice, this list of conditions and the following disclaimer in the +// * documentation and/or other materials provided with the distribution. +// * - Neither the name of the organization nor the +// * names of its contributors may be used to endorse or promote products +// * derived from this software without specific prior written permission. +// * +// * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// * DISCLAIMED. IN NO EVENT SHALL ARMON DADGAR BE LIABLE FOR ANY +// * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include +#include "pdc_malloc.h" + +#ifndef ART_H +#define ART_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define NODE4 1 +#define NODE16 2 +#define NODE48 3 +#define NODE256 4 + +#define MAX_PREFIX_LEN 10 + +#if defined(__GNUC__) && !defined(__clang__) +#if __STDC_VERSION__ >= 199901L && 402 == (__GNUC__ * 100 + __GNUC_MINOR__) +/* + * GCC 4.2.2's C99 inline keyword support is pretty broken; avoid. Introduced in + * GCC 4.2.something, fixed in 4.3.0. So checking for specific major.minor of + * 4.2 is fine. + */ +#define BROKEN_GCC_C99_INLINE +#endif +#endif + +typedef int (*art_callback)(void *data, const unsigned char *key, uint32_t key_len, void *value); + +/** + * This struct is included as part + * of all the various node sizes + */ +typedef struct { + uint32_t partial_len; + uint8_t type; + uint8_t num_children; + unsigned char partial[MAX_PREFIX_LEN]; +} art_node; + +/** + * Small node with only 4 children + */ +typedef struct { + art_node n; + unsigned char keys[4]; + art_node * children[4]; +} art_node4; + +/** + * Node with 16 children + */ +typedef struct { + art_node n; + unsigned char keys[16]; + art_node * children[16]; +} art_node16; + +/** + * Node with 48 children, but + * a full 256 byte field. + */ +typedef struct { + art_node n; + unsigned char keys[256]; + art_node * children[48]; +} art_node48; + +/** + * Full node with 256 children + */ +typedef struct { + art_node n; + art_node *children[256]; +} art_node256; + +/** + * Represents a leaf. These are + * of arbitrary size, as they include the key. + */ +typedef struct { + void * value; + uint32_t key_len; + unsigned char key[]; +} art_leaf; + +/** + * Main struct, points to root. + */ +typedef struct { + art_node *root; + uint64_t size; +} art_tree; + +/** + * Initializes an ART tree + * @return 0 on success. + */ +int art_tree_init(art_tree *t); + +/** + * DEPRECATED + * Initializes an ART tree + * @return 0 on success. + */ +#define init_art_tree(...) art_tree_init(__VA_ARGS__) + +/** + * Destroys an ART tree + * @return 0 on success. + */ +int art_tree_destroy(art_tree *t); + +/** + * DEPRECATED + * Initializes an ART tree + * @return 0 on success. + */ +#define destroy_art_tree(...) art_tree_destroy(__VA_ARGS__) + +/** + * Returns the size of the ART tree. + */ +#ifdef BROKEN_GCC_C99_INLINE +#define art_size(t) ((t)->size) +#else +inline uint64_t +art_size(art_tree *t) +{ + return t->size; +} +#endif + +/** + * inserts a new value into the art tree + * @arg t the tree + * @arg key the key + * @arg key_len the length of the key + * @arg value opaque value. + * @return null if the item was newly inserted, otherwise + * the old value pointer is returned. + */ +void *art_insert(art_tree *t, const unsigned char *key, int key_len, void *value); + +/** + * inserts a new value into the art tree (not replacing) + * @arg t the tree + * @arg key the key + * @arg key_len the length of the key + * @arg value opaque value. + * @return null if the item was newly inserted, otherwise + * the old value pointer is returned. + */ +void *art_insert_no_replace(art_tree *t, const unsigned char *key, int key_len, void *value); + +/** + * Deletes a value from the ART tree + * @arg t The tree + * @arg key The key + * @arg key_len The length of the key + * @return NULL if the item was not found, otherwise + * the value pointer is returned. + */ +void *art_delete(art_tree *t, const unsigned char *key, int key_len); + +/** + * Searches for a value in the ART tree + * @arg t The tree + * @arg key The key + * @arg key_len The length of the key + * @return NULL if the item was not found, otherwise + * the value pointer is returned. + */ +void *art_search(const art_tree *t, const unsigned char *key, int key_len); + +/** + * Returns the minimum valued leaf + * @return The minimum leaf or NULL + */ +art_leaf *art_minimum(art_tree *t); + +/** + * Returns the maximum valued leaf + * @return The maximum leaf or NULL + */ +art_leaf *art_maximum(art_tree *t); + +/** + * Iterates through the entries pairs in the map, + * invoking a callback for each. The call back gets a + * key, value for each and returns an integer stop value. + * If the callback returns non-zero, then the iteration stops. + * @arg t The tree to iterate over + * @arg cb The callback function to invoke + * @arg data Opaque handle passed to the callback + * @return 0 on success, or the return of the callback. + */ +int art_iter(art_tree *t, art_callback cb, void *data); + +/** + * Iterates through the entries pairs in the map, + * invoking a callback for each that matches a given prefix. + * The call back gets a key, value for each and returns an integer stop value. + * If the callback returns non-zero, then the iteration stops. + * @arg t The tree to iterate over + * @arg prefix The prefix of keys to read + * @arg prefix_len The length of the prefix + * @arg cb The callback function to invoke + * @arg data Opaque handle passed to the callback + * @return 0 on success, or the return of the callback. + */ +int art_iter_prefix(art_tree *t, const unsigned char *prefix, int prefix_len, art_callback cb, void *data); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/commons/collections/include/pdc_compare.h b/src/commons/collections/include/pdc_compare.h new file mode 100644 index 000000000..5c037a18d --- /dev/null +++ b/src/commons/collections/include/pdc_compare.h @@ -0,0 +1,146 @@ +/** + * @file pdc_compare.h + * + * Comparison functions for integers, pointers, and strings. + * + * To find the difference between two values pointed at, use + * @ref xxx_compare. + * + * To find if two values pointed at are equal, use @ref xxx_equal. + */ + +#ifndef PDC_COMPARE_H +#define PDC_COMPARE_H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Compare the integer values pointed at by two pointers to determine + * if they are equal. + * + * @param location1 Pointer to the first value to compare. + * @param location2 Pointer to the second value to compare. + * @return Non-zero if the two values are equal, zero if the + * two values are not equal. + */ + +int int_equal(void *location1, void *location2); + +/** + * Compare the integer values pointed at by two pointers. + * + * @param location1 Pointer to the first value to compare. + * @param location2 Pointer to the second value to compare. + * @return A negative value if the first value is less than + * the second value, a positive value if the first + * value is greater than the second value, zero if + * they are equal. + */ + +int int_compare(void *location1, void *location2); + +/** + * Compare the uint64_t integer values pointed at by two pointers to determine + * if they are equal. + * + * @param location1 Pointer to the first value to compare. + * @param location2 Pointer to the second value to compare. + * @return Non-zero if the two values are equal, zero if the + * two values are not equal. + */ + +int ui64_equal(void *location1, void *location2); + +/** + * Compare the uint64_t integer values pointed at by two pointers. + * + * @param location1 Pointer to the first value to compare. + * @param location2 Pointer to the second value to compare. + * @return A negative value if the first value is less than + * the second value, a positive value if the first + * value is greater than the second value, zero if + * they are equal. + */ + +int ui64_compare(void *location1, void *location2); + +/** + * Compare two pointers to determine if they are equal. + * + * @param location1 The first pointer. + * @param location2 The second pointer. + * @return Non-zero if the pointers are equal, zero if they + * are not equal. + */ + +int pointer_equal(void *location1, void *location2); + +/** + * Compare two pointers. + * + * @param location1 The first pointer. + * @param location2 The second pointer. + * @return A negative value if the first pointer is in a lower + * memory address than the second, a positive value if + * the first pointer is in a higher memory address than + * the second, zero if they point to the same location. + */ + +int pointer_compare(void *location1, void *location2); + +/** + * Compare two strings to determine if they are equal. + * + * @param string1 The first string. + * @param string2 The second string. + * @return Non-zero if the strings are equal, zero if they are + * not equal. + */ + +int string_equal(void *string1, void *string2); + +/** + * Compare two strings. + * + * @param string1 The first string. + * @param string2 The second string. + * @return A negative value if the first string should be + * sorted before the second, a positive value if the + * first string should be sorted after the second, + * zero if the two strings are equal. + */ + +int string_compare(void *string1, void *string2); + +/** + * Compare two strings to determine if they are equal, ignoring the + * case of letters. + * + * @param string1 The first string. + * @param string2 The second string. + * @return Non-zero if the strings are equal, zero if they are + * not equal. + */ + +int string_nocase_equal(void *string1, void *string2); + +/** + * Compare two strings, ignoring the case of letters. + * + * @param string1 The first string. + * @param string2 The second string. + * @return A negative value if the first string should be + * sorted before the second, a positive value if the + * first string should be sorted after the second, + * zero if the two strings are equal. + */ + +int string_nocase_compare(void *string1, void *string2); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef PDC_COMPARE_H */ \ No newline at end of file diff --git a/src/commons/collections/include/pdc_deque.h b/src/commons/collections/include/pdc_deque.h new file mode 100644 index 000000000..e8c61cc46 --- /dev/null +++ b/src/commons/collections/include/pdc_deque.h @@ -0,0 +1,33 @@ +#ifndef PDC_DEQUE_H +#define PDC_DEQUE_H + +#include + +// Struct for the deque data structure +typedef struct PDC_deque_t { + void **data; // Pointer to the data array + size_t size; // Current size of the deque + size_t capacity; // Current capacity of the deque + size_t head; // Index of the head of the deque + size_t tail; // Index of the tail of the deque +} PDC_deque_t; + +// Initialize the deque +void PDC_deque_init(PDC_deque_t *deque); + +// Push an element onto the front of the deque +void PDC_deque_push_front(PDC_deque_t *deque, void *value); + +// Push an element onto the back of the deque +void PDC_deque_push_back(PDC_deque_t *deque, void *value); + +// Pop an element from the front of the deque +void *PDC_deque_pop_front(PDC_deque_t *deque); + +// Pop an element from the back of the deque +void *PDC_deque_pop_back(PDC_deque_t *deque); + +// Free memory allocated for the deque +void PDC_deque_free(PDC_deque_t *deque); + +#endif /* PDC_DEQUE_H */ \ No newline at end of file diff --git a/src/commons/collections/include/pdc_hash.h b/src/commons/collections/include/pdc_hash.h new file mode 100644 index 000000000..3eae66926 --- /dev/null +++ b/src/commons/collections/include/pdc_hash.h @@ -0,0 +1,65 @@ +/** + * @file pdc_hash.h + * + * Hash function for integer, pointer, and string. See @ref xxx_hash. + */ + +#ifndef PDC_HASH_H +#define PDC_HASH_H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Generate a hash key for a pointer to an integer. The value pointed + * at is used to generate the key. + * + * @param location The pointer. + * @return A hash key for the value at the location. + */ + +unsigned int int_hash(void *location); + +/** + * Generate a hash key for a pointer to an uint64_t integer. The value pointed + * at is used to generate the key. + * + * @param location The pointer. + * @return A hash key for the value at the location. + */ +unsigned int ui64_hash(void *location); + +/** + * Generate a hash key for a pointer. The value pointed at by the pointer + * is not used, only the pointer itself. + * + * @param location The pointer + * @return A hash key for the pointer. + */ + +unsigned int pointer_hash(void *location); + +/** + * Generate a hash key from a string. + * + * @param string The string. + * @return A hash key for the string. + */ + +unsigned int string_hash(void *string); + +/** + * Generate a hash key from a string, ignoring the case of letters. + * + * @param string The string. + * @return A hash key for the string. + */ + +unsigned int string_nocase_hash(void *string); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef PDC_HASH_H */ \ No newline at end of file diff --git a/src/commons/collections/include/pdc_hash_table.h b/src/commons/collections/include/pdc_hash_table.h new file mode 100644 index 000000000..e5c586fda --- /dev/null +++ b/src/commons/collections/include/pdc_hash_table.h @@ -0,0 +1,280 @@ +/* + +Copyright (c) 2005-2008, Simon Howard + +Permission to use, copy, modify, and/or distribute this software +for any purpose with or without fee is hereby granted, provided +that the above copyright notice and this permission notice appear +in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + */ + +/** + * @file hash-table.h + * + * @brief Hash table. + * + * A hash table stores a set of values which can be addressed by a + * key. Given the key, the corresponding value can be looked up + * quickly. + * + * To create a hash table, use @ref hash_table_new. To destroy a + * hash table, use @ref hash_table_free. + * + * To insert a value into a hash table, use @ref hash_table_insert. + * + * To remove a value from a hash table, use @ref hash_table_remove. + * + * To look up a value by its key, use @ref hash_table_lookup. + * + * To iterate over all values in a hash table, use + * @ref hash_table_iterate to initialise a @ref HashTableIterator + * structure. Each value can then be read in turn using + * @ref hash_table_iter_next and @ref hash_table_iter_has_more. + */ + +#ifndef ALGORITHM_HASH_TABLE_H +#define ALGORITHM_HASH_TABLE_H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * A hash table structure. + */ + +typedef struct _HashTable HashTable; + +/** + * Structure used to iterate over a hash table. + */ + +typedef struct _HashTableIterator HashTableIterator; + +/** + * Internal structure representing an entry in a hash table. + */ + +typedef struct _HashTableEntry HashTableEntry; + +/** + * A key to look up a value in a @ref HashTable. + */ + +typedef void *HashTableKey; + +/** + * A value stored in a @ref HashTable. + */ + +typedef void *HashTableValue; + +/** + * Internal structure representing an entry in hash table + * used as @ref HashTableIterator next result. + */ + +typedef struct _HashTablePair { + HashTableKey key; + HashTableValue value; +} HashTablePair; + +/** + * Definition of a @ref HashTableIterator. + */ + +struct _HashTableIterator { + HashTable * hash_table; + HashTableEntry *next_entry; + unsigned int next_chain; +}; + +/** + * A null @ref HashTableValue. + */ + +#define HASH_TABLE_NULL ((void *)0) + +/** + * Hash function used to generate hash values for keys used in a hash + * table. + * + * @param value The value to generate a hash value for. + * @return The hash value. + */ + +typedef unsigned int (*HashTableHashFunc)(HashTableKey value); + +/** + * Function used to compare two keys for equality. + * + * @return Non-zero if the two keys are equal, zero if the keys are + * not equal. + */ + +typedef int (*HashTableEqualFunc)(HashTableKey value1, HashTableKey value2); + +/** + * Type of function used to free keys when entries are removed from a + * hash table. + */ + +typedef void (*HashTableKeyFreeFunc)(HashTableKey value); + +/** + * Type of function used to free values when entries are removed from a + * hash table. + */ + +typedef void (*HashTableValueFreeFunc)(HashTableValue value); + +/************ Default function pointer **************/ + +extern HashTableHashFunc pdc_default_string_hash_func_ptr; +extern HashTableEqualFunc pdc_default_string_equal_func_ptr; + +extern HashTableHashFunc pdc_default_integer_hash_func_ptr; +extern HashTableEqualFunc pdc_default_integer_equal_func_ptr; + +extern HashTableHashFunc pdc_default_long_hash_func_ptr; +extern HashTableEqualFunc pdc_default_long_equal_func_ptr; + +extern HashTableHashFunc pdc_default_uint64_hash_func_ptr; +extern HashTableEqualFunc pdc_default_uint64_equal_func_ptr; + +extern HashTableHashFunc pdc_default_double_hash; +extern HashTableEqualFunc pdc_default_double_equal; + +/** + * Create a new hash table. + * + * @param hash_func Function used to generate hash keys for the + * keys used in the table. + * @param equal_func Function used to test keys used in the table + * for equality. + * @return A new hash table structure, or NULL if it + * was not possible to allocate the new hash + * table. + */ + +HashTable *hash_table_new(HashTableHashFunc hash_func, HashTableEqualFunc equal_func); + +/** + * Destroy a hash table. + * + * @param hash_table The hash table to destroy. + */ + +void hash_table_free(HashTable *hash_table); + +/** + * Register functions used to free the key and value when an entry is + * removed from a hash table. + * + * @param hash_table The hash table. + * @param key_free_func Function used to free keys. + * @param value_free_func Function used to free values. + */ + +void hash_table_register_free_functions(HashTable *hash_table, HashTableKeyFreeFunc key_free_func, + HashTableValueFreeFunc value_free_func); + +/** + * Insert a value into a hash table, overwriting any existing entry + * using the same key. + * + * @param hash_table The hash table. + * @param key The key for the new value. + * @param value The value to insert. + * @return Non-zero if the value was added successfully, + * or zero if it was not possible to allocate + * memory for the new entry. + */ + +int hash_table_insert(HashTable *hash_table, HashTableKey key, HashTableValue value); + +/** + * Look up a value in a hash table by key. + * + * @param hash_table The hash table. + * @param key The key of the value to look up. + * @return The value, or @ref HASH_TABLE_NULL if there + * is no value with that key in the hash table. + */ + +HashTableValue hash_table_lookup(HashTable *hash_table, HashTableKey key); + +/** + * Remove a value from a hash table. + * + * @param hash_table The hash table. + * @param key The key of the value to remove. + * @return Non-zero if a key was removed, or zero if the + * specified key was not found in the hash table. + */ + +int hash_table_remove(HashTable *hash_table, HashTableKey key); + +/** + * Retrieve the number of entries in a hash table. + * + * @param hash_table The hash table. + * @return The number of entries in the hash table. + */ + +unsigned int hash_table_num_entries(HashTable *hash_table); + +/** + * Initialise a @ref HashTableIterator to iterate over a hash table. + * + * @param hash_table The hash table. + * @param iter Pointer to an iterator structure to + * initialise. + */ + +void hash_table_iterate(HashTable *hash_table, HashTableIterator *iter); + +/** + * Determine if there are more keys in the hash table to iterate + * over. + * + * @param iterator The hash table iterator. + * @return Zero if there are no more values to iterate + * over, non-zero if there are more values to + * iterate over. + */ + +int hash_table_iter_has_more(HashTableIterator *iterator); + +/** + * Using a hash table iterator, retrieve the next @ref HashTablePair. + * + * Note: To avoid @ref HashTableEntry internal @ref HashTablePair + * from being tampered with, and potentially messing with + * internal table structure, the function returns a copy + * of @ref HashTablePair stored internally. + * + * @param iterator The hash table iterator. + * @return The next @ref HashTablePair from the hash + * table, or @ref HASH_TABLE_NULL of Key and + * Value if there are no more keys to iterate + * over. + */ + +HashTablePair hash_table_iter_next(HashTableIterator *iterator); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef ALGORITHM_HASH_TABLE_H */ diff --git a/src/commons/collections/include/pdc_set.h b/src/commons/collections/include/pdc_set.h new file mode 100644 index 000000000..c6f77f20d --- /dev/null +++ b/src/commons/collections/include/pdc_set.h @@ -0,0 +1,263 @@ +/* + +Copyright (c) 2005-2008, Simon Howard + +Permission to use, copy, modify, and/or distribute this software +for any purpose with or without fee is hereby granted, provided +that the above copyright notice and this permission notice appear +in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + */ + +/** + * @file pdc_set.h + * + * @brief Set of values. + * + * A set stores a collection of values. Each value can only exist once in + * the set. + * + * To create a new set, use @ref set_new. To destroy a set, use + * @ref set_free. + * + * To add a value to a set, use @ref set_insert. To remove a value + * from a set, use @ref set_remove. + * + * To find the number of entries in a set, use @ref set_num_entries. + * + * To query if a particular value is in a set, use @ref set_query. + * + * To iterate over all values in a set, use @ref set_iterate to initialise + * a @ref SetIterator structure, with @ref set_iter_next and + * @ref set_iter_has_more to read each value in turn. + * + * Two sets can be combined (union) using @ref set_union, while the + * intersection of two sets can be generated using @ref set_intersection. + */ + +#ifndef PDC_SET_H +#define PDC_SET_H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Represents a set of values. Created using the @ref set_new function and + * destroyed using the @ref set_free function. + */ + +typedef struct _Set Set; + +/** + * An object used to iterate over a set. + * + * @see set_iterate + */ + +typedef struct _SetIterator SetIterator; + +/** + * Internal structure representing an entry in the set. + */ + +typedef struct _SetEntry SetEntry; + +/** + * A value stored in a @ref Set. + */ + +typedef void *SetValue; + +/** + * Definition of a @ref SetIterator. + */ + +struct _SetIterator { + Set * set; + SetEntry * next_entry; + unsigned int next_chain; +}; + +/** + * A null @ref SetValue. + */ + +#define SET_NULL ((void *)0) + +/** + * Hash function. Generates a hash key for values to be stored in a set. + */ + +typedef unsigned int (*SetHashFunc)(SetValue value); + +/** + * Equality function. Compares two values to determine if they are + * equivalent. + */ + +typedef int (*SetEqualFunc)(SetValue value1, SetValue value2); + +/** + * Function used to free values stored in a set. See + * @ref set_register_free_function. + */ + +typedef void (*SetFreeFunc)(SetValue value); + +/** + * Create a new set. + * + * @param hash_func Hash function used on values in the set. + * @param equal_func Compares two values in the set to determine + * if they are equal. + * @return A new set, or NULL if it was not possible to + * allocate the memory for the set. + */ + +Set *set_new(SetHashFunc hash_func, SetEqualFunc equal_func); + +/** + * Destroy a set. + * + * @param set The set to destroy. + */ + +void set_free(Set *set); + +/** + * Register a function to be called when values are removed from + * the set. + * + * @param set The set. + * @param free_func Function to call when values are removed from the + * set. + */ + +void set_register_free_function(Set *set, SetFreeFunc free_func); + +/** + * Add a value to a set. + * + * @param set The set. + * @param data The value to add to the set. + * @return Non-zero (true) if the value was added to the set, + * zero (false) if it already exists in the set, or + * if it was not possible to allocate memory for the + * new entry. + */ + +int set_insert(Set *set, SetValue data); + +/** + * Remove a value from a set. + * + * @param set The set. + * @param data The value to remove from the set. + * @return Non-zero (true) if the value was found and removed + * from the set, zero (false) if the value was not + * found in the set. + */ + +int set_remove(Set *set, SetValue data); + +/** + * Query if a particular value is in a set. + * + * @param set The set. + * @param data The value to query for. + * @return Zero if the value is not in the set, non-zero if the + * value is in the set. + */ + +int set_query(Set *set, SetValue data); + +/** + * Retrieve the number of entries in a set + * + * @param set The set. + * @return A count of the number of entries in the set. + */ + +unsigned int set_num_entries(Set *set); + +/** + * Create an array containing all entries in a set. + * + * @param set The set. + * @return An array containing all entries in the set, + * or NULL if it was not possible to allocate + * memory for the array. + */ + +SetValue *set_to_array(Set *set); + +/** + * Perform a union of two sets. + * + * @param set1 The first set. + * @param set2 The second set. + * @return A new set containing all values which are in the + * first or second sets, or NULL if it was not + * possible to allocate memory for the new set. + */ + +Set *set_union(Set *set1, Set *set2); + +/** + * Perform an intersection of two sets. + * + * @param set1 The first set. + * @param set2 The second set. + * @return A new set containing all values which are in both + * set, or NULL if it was not possible to allocate + * memory for the new set. + */ + +Set *set_intersection(Set *set1, Set *set2); + +/** + * Initialise a @ref SetIterator structure to iterate over the values + * in a set. + * + * @param set The set to iterate over. + * @param iter Pointer to an iterator structure to initialise. + */ + +void set_iterate(Set *set, SetIterator *iter); + +/** + * Determine if there are more values in the set to iterate over. + * + * @param iterator The set iterator object. + * @return Zero if there are no more values in the set + * to iterate over, non-zero if there are more + * values to be read. + */ + +int set_iter_has_more(SetIterator *iterator); + +/** + * Using a set iterator, retrieve the next value from the set. + * + * @param iterator The set iterator. + * @return The next value from the set, or @ref SET_NULL if no + * more values are available. + */ + +SetValue set_iter_next(SetIterator *iterator); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef PDC_SET_H */ diff --git a/src/commons/collections/include/pdc_stack.h b/src/commons/collections/include/pdc_stack.h new file mode 100644 index 000000000..6e899d8fa --- /dev/null +++ b/src/commons/collections/include/pdc_stack.h @@ -0,0 +1,25 @@ +#ifndef PDC_STACK_H +#define PDC_STACK_H + +#include + +// Struct for the stack data structure +typedef struct PDC_stack_t { + void **data; // Pointer to the data array + size_t size; // Current size of the stack + size_t capacity; // Current capacity of the stack +} PDC_stack_t; + +// Initialize the stack +void PDC_stack_init(PDC_stack_t *stack); + +// Push an element onto the stack +void PDC_stack_push(PDC_stack_t *stack, void *value); + +// Pop an element from the stack +void *PDC_stack_pop(PDC_stack_t *stack); + +// Free memory allocated for the stack +void PDC_stack_free(PDC_stack_t *stack); + +#endif /* PDC_STACK_H */ \ No newline at end of file diff --git a/src/commons/collections/include/pdc_vector.h b/src/commons/collections/include/pdc_vector.h new file mode 100644 index 000000000..32fd7ba07 --- /dev/null +++ b/src/commons/collections/include/pdc_vector.h @@ -0,0 +1,111 @@ +#ifndef PDC_VECTOR_H +#define PDC_VECTOR_H + +#include + +/** + * A generic vector data structure that stores a variable number of items of any type. + * A vector is similar to an array, except that it can grow and shrink dynamically. + */ +typedef struct { + void **items; // Pointer to the array of items. + size_t item_count; // Number of items in the vector. + size_t capacity; // Capacity of the array of items. + double expansion_factor; // Factor by which the capacity is expanded. +} PDC_VECTOR; + +/** + * A generic iterator for iterating over the items in a PDC_VECTOR. + */ +typedef struct { + PDC_VECTOR *vector; // The vector being iterated over. + size_t index; // The index of the next item to be returned. +} PDC_VECTOR_ITERATOR; + +/** + * Creates a new PDC_VECTOR with default initial capacity 100 and default expansion factor 2.0. + * @return A pointer to the new PDC_VECTOR. + */ +PDC_VECTOR *pdc_vector_new(); + +/** + * Creates a new PDC_VECTOR with the given initial capacity and expansion factor. + * @param initial_capacity The initial capacity of the vector. + * @param expansion_factor The factor by which the capacity is expanded when the vector is full. + * + * @return A pointer to the new PDC_VECTOR. + */ +PDC_VECTOR *pdc_vector_create(size_t initial_capacity, double expansion_factor); + +/** + * Destroys the given PDC_VECTOR and frees all allocated memory. + * @param vector The PDC_VECTOR to destroy. + */ +void pdc_vector_destroy(PDC_VECTOR *vector); + +/** + * Adds the given item to the end of the given PDC_VECTOR. + * @param vector The PDC_VECTOR to add the item to. + * @param item The item to add to the PDC_VECTOR. + * + */ +void pdc_vector_add(PDC_VECTOR *vector, void *item); + +/** + * Gets the item at the given index in the given PDC_VECTOR. + * @param vector The PDC_VECTOR to get the item from. + * @param index The index of the item to get. + * + * @return A pointer to the item at the given index. + */ +void *pdc_vector_get(PDC_VECTOR *vector, size_t index); + +/** + * Sets the item at the given index in the given PDC_VECTOR. + * @param vector The PDC_VECTOR to set the item in. + * + * @return The number of items in the vector. + */ +size_t pdc_vector_size(PDC_VECTOR *vector); + +/** + * Sets the expansion factor for the given PDC_VECTOR. + * @param vector The PDC_VECTOR to set the expansion factor for. + * @param expansion_factor The factor by which the capacity is expanded when the vector is full. + */ +void pdc_vector_set_expansion_factor(PDC_VECTOR *vector, double expansion_factor); + +/** + * Gets the expansion factor for the given PDC_VECTOR. + * @param vector The PDC_VECTOR to get the expansion factor for. + */ +double pdc_vector_get_expansion_factor(PDC_VECTOR *vector); + +/** + * Creates a new PDC_VECTOR_ITERATOR for the given PDC_VECTOR. + * @param vector The PDC_VECTOR to create the iterator for. + * @return A pointer to the new PDC_VECTOR_ITERATOR. + */ +PDC_VECTOR_ITERATOR *pdc_vector_iterator_new(PDC_VECTOR *vector); + +/** + * Destroys the given PDC_VECTOR_ITERATOR and frees all allocated memory. + * @param iterator The PDC_VECTOR_ITERATOR to destroy. + */ +void pdc_vector_iterator_destroy(PDC_VECTOR_ITERATOR *iterator); + +/** + * Returns the next item in the PDC_VECTOR_ITERATOR. + * @param iterator The PDC_VECTOR_ITERATOR to get the next item from. + * @return A pointer to the next item in the PDC_VECTOR_ITERATOR. + */ +void *pdc_vector_iterator_next(PDC_VECTOR_ITERATOR *iterator); + +/** + * Returns true if the PDC_VECTOR_ITERATOR has more items. + * @param iterator The PDC_VECTOR_ITERATOR to check. + * @return True if the PDC_VECTOR_ITERATOR has more items. + */ +int pdc_vector_iterator_has_next(PDC_VECTOR_ITERATOR *iterator); + +#endif // PDC_VECTOR_H \ No newline at end of file diff --git a/src/commons/collections/pdc_compare.c b/src/commons/collections/pdc_compare.c new file mode 100644 index 000000000..f2176741e --- /dev/null +++ b/src/commons/collections/pdc_compare.c @@ -0,0 +1,177 @@ +#include "pdc_compare.h" + +#include +#include +#include +#include + +/* Comparison functions for a pointer to an integer */ + +int +int_equal(void *vlocation1, void *vlocation2) +{ + int *location1; + int *location2; + + location1 = (int *)vlocation1; + location2 = (int *)vlocation2; + + return *location1 == *location2; +} + +int +int_compare(void *vlocation1, void *vlocation2) +{ + int *location1; + int *location2; + + location1 = (int *)vlocation1; + location2 = (int *)vlocation2; + + if (*location1 < *location2) { + return -1; + } + else if (*location1 > *location2) { + return 1; + } + else { + return 0; + } +} + +int +ui64_equal(void *vlocation1, void *vlocation2) +{ + uint64_t *location1; + uint64_t *location2; + + location1 = (uint64_t *)vlocation1; + location2 = (uint64_t *)vlocation2; + + return *location1 == *location2; +} + +int +ui64_compare(void *vlocation1, void *vlocation2) +{ + uint64_t *location1; + uint64_t *location2; + + location1 = (uint64_t *)vlocation1; + location2 = (uint64_t *)vlocation2; + + if (*location1 < *location2) { + return -1; + } + else if (*location1 > *location2) { + return 1; + } + else { + return 0; + } +} + +/* Comparison functions for a generic void pointer */ + +int +pointer_equal(void *location1, void *location2) +{ + return location1 == location2; +} + +int +pointer_compare(void *location1, void *location2) +{ + if (location1 < location2) { + return -1; + } + else if (location1 > location2) { + return 1; + } + else { + return 0; + } +} + +/* Comparison functions for strings */ + +int +string_equal(void *string1, void *string2) +{ + return strcmp((char *)string1, (char *)string2) == 0; +} + +int +string_compare(void *string1, void *string2) +{ + int result; + + result = strcmp((char *)string1, (char *)string2); + + if (result < 0) { + return -1; + } + else if (result > 0) { + return 1; + } + else { + return 0; + } +} + +/* Comparison functions for strings, which ignore the case of letters. */ + +int +string_nocase_equal(void *string1, void *string2) +{ + return string_nocase_compare((char *)string1, (char *)string2) == 0; +} + +/* On many systems, strcasecmp or stricmp will give the same functionality + * as this function. However, it is non-standard and cannot be relied + * on to be present. */ + +int +string_nocase_compare(void *string1, void *string2) +{ + char *p1; + char *p2; + int c1, c2; + + /* Iterate over each character in the strings */ + + p1 = (char *)string1; + p2 = (char *)string2; + + for (;;) { + + c1 = tolower(*p1); + c2 = tolower(*p2); + + if (c1 != c2) { + + /* Strings are different */ + + if (c1 < c2) { + return -1; + } + else { + return 1; + } + } + + /* End of string */ + + if (c1 == '\0') + break; + + /* Advance to the next character */ + + ++p1; + ++p2; + } + + /* Reached the end of string and no difference found */ + + return 0; +} \ No newline at end of file diff --git a/src/commons/collections/pdc_deque.c b/src/commons/collections/pdc_deque.c new file mode 100644 index 000000000..5d2280c28 --- /dev/null +++ b/src/commons/collections/pdc_deque.c @@ -0,0 +1,94 @@ +#include "pdc_deque.h" +#include + +#define DEFAULT_CAPACITY 16 + +void +PDC_deque_init(PDC_deque_t *deque) +{ + deque->data = malloc(sizeof(void *) * DEFAULT_CAPACITY); + deque->size = 0; + deque->capacity = DEFAULT_CAPACITY; + deque->head = 0; + deque->tail = 0; +} + +static void +resize_deque(PDC_deque_t *deque, size_t new_capacity) +{ + void **new_data = malloc(sizeof(void *) * new_capacity); + if (new_data == NULL) { + printf("Failed to allocate memory for deque!\n"); + exit(1); + } + + size_t i = 0; + while (deque->head != deque->tail) { + new_data[i++] = deque->data[deque->head]; + deque->head = (deque->head + 1) % deque->capacity; + } + + free(deque->data); + deque->data = new_data; + deque->capacity = new_capacity; + deque->head = 0; + deque->tail = i; +} + +void +PDC_deque_push_front(PDC_deque_t *deque, void *value) +{ + if (deque->size == deque->capacity) { + resize_deque(deque, deque->capacity * 2); + } + deque->head = (deque->head - 1 + deque->capacity) % deque->capacity; + deque->data[deque->head] = value; + deque->size++; +} + +void +PDC_deque_push_back(PDC_deque_t *deque, void *value) +{ + if (deque->size == deque->capacity) { + resize_deque(deque, deque->capacity * 2); + } + deque->data[deque->tail] = value; + deque->tail = (deque->tail + 1) % deque->capacity; + deque->size++; +} + +void * +PDC_deque_pop_front(PDC_deque_t *deque) +{ + if (deque->size == 0) { + return NULL; + } + void *value = deque->data[deque->head]; + deque->head = (deque->head + 1) % deque->capacity; + deque->size--; + if (deque->size <= deque->capacity / 4) { + resize_deque(deque, deque->capacity / 2); + } + return value; +} + +void * +PDC_deque_pop_back(PDC_deque_t *deque) +{ + if (deque->size == 0) { + return NULL; + } + deque->tail = (deque->tail - 1 + deque->capacity) % deque->capacity; + void *value = deque->data[deque->tail]; + deque->size--; + if (deque->size <= deque->capacity / 4) { + resize_deque(deque, deque->capacity / 2); + } + return value; +} + +void +PDC_deque_free(PDC_deque_t *deque) +{ + free(deque->data); +} \ No newline at end of file diff --git a/src/commons/collections/pdc_hash.c b/src/commons/collections/pdc_hash.c new file mode 100644 index 000000000..92d394daa --- /dev/null +++ b/src/commons/collections/pdc_hash.c @@ -0,0 +1,75 @@ +#include "pdc_hash.h" +#include +#include +#include + +/* Hash function for a pointer to an integer */ + +unsigned int +int_hash(void *vlocation) +{ + int *location; + + location = (int *)vlocation; + + return (unsigned int)*location; +} + +/* Hash function for a pointer to a uint64_t integer */ + +unsigned int +ui64_hash(void *vlocation) +{ + uint64_t *location; + + location = (uint64_t *)vlocation; + + return (unsigned int)*location; +} + +/* Hash function for a generic pointer */ + +unsigned int +pointer_hash(void *location) +{ + return (unsigned int)(unsigned long)location; +} + +/* String hash function */ + +unsigned int +string_hash(void *string) +{ + /* This is the djb2 string hash function */ + + unsigned int result = 5381; + unsigned char *p; + + p = (unsigned char *)string; + + while (*p != '\0') { + result = (result << 5) + result + *p; + ++p; + } + + return result; +} + +/* The same function, with a tolower on every character so that + * case is ignored. This code is duplicated for performance. */ + +unsigned int +string_nocase_hash(void *string) +{ + unsigned int result = 5381; + unsigned char *p; + + p = (unsigned char *)string; + + while (*p != '\0') { + result = (result << 5) + result + (unsigned int)tolower(*p); + ++p; + } + + return result; +} \ No newline at end of file diff --git a/src/commons/collections/pdc_hash_table.c b/src/commons/collections/pdc_hash_table.c new file mode 100644 index 000000000..c75bac5d9 --- /dev/null +++ b/src/commons/collections/pdc_hash_table.c @@ -0,0 +1,580 @@ +/* + +Copyright (c) 2005-2008, Simon Howard + +Permission to use, copy, modify, and/or distribute this software +for any purpose with or without fee is hereby granted, provided +that the above copyright notice and this permission notice appear +in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + */ + +/* Hash table implementation */ + +#include +#include +#include +#include +#include "pdc_hash_table.h" +#include "pdc_malloc.h" + +#ifdef ALLOC_TESTING +#include "alloc-testing.h" +#endif + +struct _HashTableEntry { + HashTablePair pair; + HashTableEntry *next; +}; + +struct _HashTable { + HashTableEntry ** table; + unsigned int table_size; + HashTableHashFunc hash_func; + HashTableEqualFunc equal_func; + HashTableKeyFreeFunc key_free_func; + HashTableValueFreeFunc value_free_func; + unsigned int entries; + unsigned int prime_index; +}; + +/* This is a set of good hash table prime numbers, from: + * http://planetmath.org/encyclopedia/GoodHashTablePrimes.html + * Each prime is roughly double the previous value, and as far as + * possible from the nearest powers of two. */ + +static const unsigned int hash_table_primes[] = { + 193, 389, 769, 1543, 3079, 6151, 12289, 24593, + 49157, 98317, 196613, 393241, 786433, 1572869, 3145739, 6291469, + 12582917, 25165843, 50331653, 100663319, 201326611, 402653189, 805306457, 1610612741, +}; + +static const unsigned int hash_table_num_primes = sizeof(hash_table_primes) / sizeof(int); + +/************ Default hash/equality function implementation **************/ + +unsigned int +pdc_default_string_hash_func(HashTableKey value) +{ + char * str = (char *)value; + unsigned int hash = 5381; + int c; + + while ((c = *str++)) { + hash = ((hash << 5) + hash) + c; /* hash * 33 + c */ + } + + return hash; +} + +int +pdc_default_string_equal_func(HashTableKey value1, HashTableKey value2) +{ + return strcmp((char *)value1, (char *)value2) == 0; +} + +unsigned int +pdc_default_integer_hash_func(HashTableKey value) +{ + return (unsigned int)*(int *)value; +} + +int +pdc_default_integer_equal_func(HashTableKey value1, HashTableKey value2) +{ + return *(int *)value1 == *(int *)value2; +} + +unsigned int +pdc_default_long_hash_func(HashTableKey value) +{ + return (unsigned int)*(long *)value; +} + +int +pdc_default_long_equal_func(HashTableKey value1, HashTableKey value2) +{ + return *(long *)value1 == *(long *)value2; +} + +unsigned int +pdc_default_uint64_hash_func(HashTableKey value) +{ + return (unsigned int)*(uint64_t *)value; +} + +int +pdc_default_uint64_equal_func(HashTableKey value1, HashTableKey value2) +{ + return *(uint64_t *)value1 == *(uint64_t *)value2; +} + +// For floating-point numbers (double) +unsigned int +pdc_default_double_hash_func(HashTableKey value) +{ + double d = *(const double *)value; + double abs_d = fabs(d); + + if (d == 0) { + return 0; + } + else { + int exp; + double frexp_d = frexp(abs_d, &exp); + return (unsigned int)((frexp_d * (1 << 31)) + exp); + } +} + +int +pdc_default_double_equal_func(HashTableKey value1, HashTableKey value2) +{ + double d1 = *(const double *)value1; + double d2 = *(const double *)value2; + + return fabs(d1 - d2) < 1e-9; // use a small epsilon for comparing floats +} + +/************ Default function pointer **************/ + +HashTableHashFunc pdc_default_string_hash_func_ptr = pdc_default_string_hash_func; +HashTableEqualFunc pdc_default_string_equal_func_ptr = pdc_default_string_equal_func; + +HashTableHashFunc pdc_default_integer_hash_func_ptr = pdc_default_integer_hash_func; +HashTableEqualFunc pdc_default_integer_equal_func_ptr = pdc_default_integer_equal_func; + +HashTableHashFunc pdc_default_long_hash_func_ptr = pdc_default_long_hash_func; +HashTableEqualFunc pdc_default_long_equal_func_ptr = pdc_default_long_equal_func; + +HashTableHashFunc pdc_default_uint64_hash_func_ptr = pdc_default_uint64_hash_func; +HashTableEqualFunc pdc_default_uint64_equal_func_ptr = pdc_default_uint64_equal_func; + +HashTableHashFunc pdc_default_double_hash_func_ptr = pdc_default_double_hash_func; +HashTableEqualFunc pdc_default_double_equal_func_ptr = pdc_default_double_equal_func; + +/* Internal function used to allocate the table on hash table creation + * and when enlarging the table */ + +static int +hash_table_allocate_table(HashTable *hash_table) +{ + unsigned int new_table_size; + + /* Determine the table size based on the current prime index. + * An attempt is made here to ensure sensible behavior if the + * maximum prime is exceeded, but in practice other things are + * likely to break long before that happens. */ + + if (hash_table->prime_index < hash_table_num_primes) { + new_table_size = hash_table_primes[hash_table->prime_index]; + } + else { + new_table_size = hash_table->entries * 10; + } + + hash_table->table_size = new_table_size; + + /* Allocate the table and initialise to NULL for all entries */ + hash_table->table = calloc(hash_table->table_size, sizeof(HashTableEntry *)); + + return hash_table->table != NULL; +} + +/* Free an entry, calling the free functions if there are any registered */ +static void +hash_table_free_entry(HashTable *hash_table, HashTableEntry *entry) +{ + HashTablePair *pair; + + pair = &(entry->pair); + + /* If there is a function registered for freeing keys, use it to free + * the key */ + if (hash_table->key_free_func != NULL) { + hash_table->key_free_func(pair->key); + } + + /* Likewise with the value */ + if (hash_table->value_free_func != NULL) { + hash_table->value_free_func(pair->value); + } + + /* Free the data structure */ + free(entry); +} + +HashTable * +hash_table_new(HashTableHashFunc hash_func, HashTableEqualFunc equal_func) +{ + HashTable *hash_table; + + /* Allocate a new hash table structure */ + hash_table = (HashTable *)malloc(sizeof(HashTable)); + + if (hash_table == NULL) { + return NULL; + } + + hash_table->hash_func = hash_func; + hash_table->equal_func = equal_func; + hash_table->key_free_func = NULL; + hash_table->value_free_func = NULL; + hash_table->entries = 0; + hash_table->prime_index = 0; + + /* Allocate the table */ + if (!hash_table_allocate_table(hash_table)) { + free(hash_table); + + return NULL; + } + + return hash_table; +} + +void +hash_table_free(HashTable *hash_table) +{ + HashTableEntry *rover; + HashTableEntry *next; + unsigned int i; + + /* Free all entries in all chains */ + for (i = 0; i < hash_table->table_size; ++i) { + rover = hash_table->table[i]; + while (rover != NULL) { + next = rover->next; + hash_table_free_entry(hash_table, rover); + rover = next; + } + } + + /* Free the table */ + free(hash_table->table); + + /* Free the hash table structure */ + free(hash_table); +} + +void +hash_table_register_free_functions(HashTable *hash_table, HashTableKeyFreeFunc key_free_func, + HashTableValueFreeFunc value_free_func) +{ + hash_table->key_free_func = key_free_func; + hash_table->value_free_func = value_free_func; +} + +static int +hash_table_enlarge(HashTable *hash_table) +{ + HashTableEntry **old_table; + unsigned int old_table_size; + unsigned int old_prime_index; + HashTableEntry * rover; + HashTablePair * pair; + HashTableEntry * next; + unsigned int index; + unsigned int i; + + /* Store a copy of the old table */ + old_table = hash_table->table; + old_table_size = hash_table->table_size; + old_prime_index = hash_table->prime_index; + + /* Allocate a new, larger table */ + + ++hash_table->prime_index; + + if (!hash_table_allocate_table(hash_table)) { + + /* Failed to allocate the new table */ + hash_table->table = old_table; + hash_table->table_size = old_table_size; + hash_table->prime_index = old_prime_index; + + return 0; + } + + /* Link all entries from all chains into the new table */ + for (i = 0; i < old_table_size; ++i) { + rover = old_table[i]; + + while (rover != NULL) { + next = rover->next; + + /* Fetch rover HashTablePair */ + pair = &(rover->pair); + + /* Find the index into the new table */ + index = hash_table->hash_func(pair->key) % hash_table->table_size; + + /* Link this entry into the chain */ + rover->next = hash_table->table[index]; + hash_table->table[index] = rover; + + /* Advance to next in the chain */ + rover = next; + } + } + + /* Free the old table */ + free(old_table); + + return 1; +} + +int +hash_table_insert(HashTable *hash_table, HashTableKey key, HashTableValue value) +{ + HashTableEntry *rover; + HashTablePair * pair; + HashTableEntry *newentry; + unsigned int index; + + /* If there are too many items in the table with respect to the table + * size, the number of hash collisions increases and performance + * decreases. Enlarge the table size to prevent this happening */ + if ((hash_table->entries * 3) / hash_table->table_size > 0) { + + /* Table is more than 1/3 full */ + if (!hash_table_enlarge(hash_table)) { + /* Failed to enlarge the table */ + return 0; + } + } + + /* Generate the hash of the key and hence the index into the table */ + index = hash_table->hash_func(key) % hash_table->table_size; + + /* Traverse the chain at this location and look for an existing + * entry with the same key */ + rover = hash_table->table[index]; + + while (rover != NULL) { + /* Fetch rover's HashTablePair entry */ + pair = &(rover->pair); + + if (hash_table->equal_func(pair->key, key) != 0) { + + /* Same key: overwrite this entry with new data */ + + /* If there is a value free function, free the old data + * before adding in the new data */ + if (hash_table->value_free_func != NULL) { + hash_table->value_free_func(pair->value); + } + + /* Same with the key: use the new key value and free + * the old one */ + if (hash_table->key_free_func != NULL) { + hash_table->key_free_func(pair->key); + } + + pair->key = key; + pair->value = value; + + /* Finished */ + return 1; + } + + rover = rover->next; + } + + /* Not in the hash table yet. Create a new entry */ + newentry = (HashTableEntry *)PDC_malloc(sizeof(HashTableEntry)); + + if (newentry == NULL) { + return 0; + } + newentry->pair.key = key; + newentry->pair.value = value; + + /* Link into the list */ + newentry->next = hash_table->table[index]; +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_lock(&hash_table_new_mutex_g); +#endif + hash_table->table[index] = newentry; + + /* Maintain the count of the number of entries */ + ++hash_table->entries; + + /* Added successfully */ +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_unlock(&hash_table_new_mutex_g); +#endif + + return 1; +} + +HashTableValue +hash_table_lookup(HashTable *hash_table, HashTableKey key) +{ + HashTableEntry *rover; + HashTablePair * pair; + unsigned int index; + + /* Generate the hash of the key and hence the index into the table */ + index = hash_table->hash_func(key) % hash_table->table_size; + + /* Walk the chain at this index until the corresponding entry is + * found */ + + rover = hash_table->table[index]; + + while (rover != NULL) { + pair = &(rover->pair); + if (hash_table->equal_func(key, pair->key) != 0) { + + /* Found the entry. Return the data. */ + return pair->value; + } + + rover = rover->next; + } + + /* Not found */ + return HASH_TABLE_NULL; +} + +int +hash_table_remove(HashTable *hash_table, HashTableKey key) +{ + HashTableEntry **rover; + HashTableEntry * entry; + HashTablePair * pair; + unsigned int index; + int result; + + /* Generate the hash of the key and hence the index into the table */ + + index = hash_table->hash_func(key) % hash_table->table_size; + + /* Rover points at the pointer which points at the current entry + * in the chain being inspected. ie. the entry in the table, or + * the "next" pointer of the previous entry in the chain. This + * allows us to unlink the entry when we find it. */ + + result = 0; + rover = &hash_table->table[index]; + + while (*rover != NULL) { + + pair = &((*rover)->pair); + + if (hash_table->equal_func(key, pair->key) != 0) { + + /* This is the entry to remove */ + entry = *rover; + + /* Unlink from the list */ + *rover = entry->next; + + /* Destroy the entry structure */ + hash_table_free_entry(hash_table, entry); + + /* Track count of entries */ + --hash_table->entries; + + result = 1; + + break; + } + + /* Advance to the next entry */ + rover = &((*rover)->next); + } + + return result; +} + +unsigned int +hash_table_num_entries(HashTable *hash_table) +{ + return hash_table->entries; +} + +void +hash_table_iterate(HashTable *hash_table, HashTableIterator *iterator) +{ + unsigned int chain; + + iterator->hash_table = hash_table; + + /* Default value of next if no entries are found. */ + iterator->next_entry = NULL; + + /* Find the first entry */ + for (chain = 0; chain < hash_table->table_size; ++chain) { + + if (hash_table->table[chain] != NULL) { + iterator->next_entry = hash_table->table[chain]; + iterator->next_chain = chain; + break; + } + } +} + +int +hash_table_iter_has_more(HashTableIterator *iterator) +{ + return iterator->next_entry != NULL; +} + +HashTablePair +hash_table_iter_next(HashTableIterator *iterator) +{ + HashTableEntry *current_entry; + HashTable * hash_table; + HashTablePair pair = {NULL, NULL}; + unsigned int chain; + + hash_table = iterator->hash_table; + + if (iterator->next_entry == NULL) { + return pair; + } + + /* Result is immediately available */ + current_entry = iterator->next_entry; + pair = current_entry->pair; + + /* Find the next entry */ + if (current_entry->next != NULL) { + + /* Next entry in current chain */ + iterator->next_entry = current_entry->next; + } + else { + /* None left in this chain, so advance to the next chain */ + chain = iterator->next_chain + 1; + + /* Default value if no next chain found */ + iterator->next_entry = NULL; + + while (chain < hash_table->table_size) { + + /* Is there anything in this chain? */ + if (hash_table->table[chain] != NULL) { + iterator->next_entry = hash_table->table[chain]; + break; + } + + /* Try the next chain */ + ++chain; + } + + iterator->next_chain = chain; + } + + return pair; +} diff --git a/src/commons/collections/pdc_set.c b/src/commons/collections/pdc_set.c new file mode 100644 index 000000000..8ec6524ae --- /dev/null +++ b/src/commons/collections/pdc_set.c @@ -0,0 +1,619 @@ +/* + +Copyright (c) 2005-2008, Simon Howard + +Permission to use, copy, modify, and/or distribute this software +for any purpose with or without fee is hereby granted, provided +that the above copyright notice and this permission notice appear +in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + */ + +#include +#include +#include "pdc_set.h" +#include "pdc_malloc.h" + +/* malloc() / free() testing */ + +#ifdef ALLOC_TESTING +#include "alloc-testing.h" +#endif + +/* A set */ + +struct _SetEntry { + SetValue data; + SetEntry *next; +}; + +struct _Set { + SetEntry ** table; + unsigned int entries; + unsigned int table_size; + unsigned int prime_index; + SetHashFunc hash_func; + SetEqualFunc equal_func; + SetFreeFunc free_func; +}; + +/* This is a set of good hash table prime numbers, from: + * http://planetmath.org/encyclopedia/GoodHashTablePrimes.html + * Each prime is roughly double the previous value, and as far as + * possible from the nearest powers of two. */ + +static const unsigned int set_primes[] = { + 193, 389, 769, 1543, 3079, 6151, 12289, 24593, + 49157, 98317, 196613, 393241, 786433, 1572869, 3145739, 6291469, + 12582917, 25165843, 50331653, 100663319, 201326611, 402653189, 805306457, 1610612741, +}; + +static const unsigned int set_num_primes = sizeof(set_primes) / sizeof(int); + +static int +set_allocate_table(Set *set) +{ + /* Determine the table size based on the current prime index. + * An attempt is made here to ensure sensible behavior if the + * maximum prime is exceeded, but in practice other things are + * likely to break long before that happens. */ + + if (set->prime_index < set_num_primes) { + set->table_size = set_primes[set->prime_index]; + } + else { + set->table_size = set->entries * 10; + } + + /* Allocate the table and initialise to NULL */ + + set->table = calloc(set->table_size, sizeof(SetEntry *)); + + return set->table != NULL; +} + +static void +set_free_entry(Set *set, SetEntry *entry) +{ + /* If there is a free function registered, call it to free the + * data for this entry first */ + + if (set->free_func != NULL) { + set->free_func(entry->data); + } + + /* Free the entry structure */ + + free(entry); +} + +Set * +set_new(SetHashFunc hash_func, SetEqualFunc equal_func) +{ + Set *new_set; + + /* Allocate a new set and fill in the fields */ + + new_set = (Set *)PDC_malloc(sizeof(Set)); + + if (new_set == NULL) { + return NULL; + } + + new_set->hash_func = hash_func; + new_set->equal_func = equal_func; + new_set->entries = 0; + new_set->prime_index = 0; + new_set->free_func = NULL; + + /* Allocate the table */ + + if (!set_allocate_table(new_set)) { + free(new_set); + return NULL; + } + + return new_set; +} + +void +set_free(Set *set) +{ + SetEntry * rover; + SetEntry * next; + unsigned int i; + + /* Free all entries in all chains */ + + for (i = 0; i < set->table_size; ++i) { + rover = set->table[i]; + + while (rover != NULL) { + next = rover->next; + + /* Free this entry */ + + set_free_entry(set, rover); + + /* Advance to the next entry in the chain */ + + rover = next; + } + } + + /* Free the table */ + + free(set->table); + + /* Free the set structure */ + + free(set); +} + +void +set_register_free_function(Set *set, SetFreeFunc free_func) +{ + set->free_func = free_func; +} + +static int +set_enlarge(Set *set) +{ + SetEntry * rover; + SetEntry * next; + SetEntry ** old_table; + unsigned int old_table_size; + unsigned int old_prime_index; + unsigned int index; + unsigned int i; + + /* Store the old table */ + + old_table = set->table; + old_table_size = set->table_size; + old_prime_index = set->prime_index; + + /* Use the next table size from the prime number array */ + + ++set->prime_index; + + /* Allocate the new table */ + + if (!set_allocate_table(set)) { + set->table = old_table; + set->table_size = old_table_size; + set->prime_index = old_prime_index; + + return 0; + } + + /* Iterate through all entries in the old table and add them + * to the new one */ + + for (i = 0; i < old_table_size; ++i) { + + /* Walk along this chain */ + + rover = old_table[i]; + + while (rover != NULL) { + + next = rover->next; + + /* Hook this entry into the new table */ + + index = set->hash_func(rover->data) % set->table_size; + rover->next = set->table[index]; + set->table[index] = rover; + + /* Advance to the next entry in the chain */ + + rover = next; + } + } + + /* Free back the old table */ + + free(old_table); + + /* Resized successfully */ + + return 1; +} + +int +set_insert(Set *set, SetValue data) +{ + SetEntry * newentry; + SetEntry * rover; + unsigned int index; + + /* The hash table becomes less efficient as the number of entries + * increases. Check if the percentage used becomes large. */ + + if ((set->entries * 3) / set->table_size > 0) { + + /* The table is more than 1/3 full and must be increased + * in size */ + + if (!set_enlarge(set)) { + return 0; + } + } + + /* Use the hash of the data to determine an index to insert into the + * table at. */ + + index = set->hash_func(data) % set->table_size; + + /* Walk along this chain and attempt to determine if this data has + * already been added to the table */ + + rover = set->table[index]; + + while (rover != NULL) { + + if (set->equal_func(data, rover->data) != 0) { + + /* This data is already in the set */ + + return 0; + } + + rover = rover->next; + } + + /* Not in the set. We must add a new entry. */ + + /* Make a new entry for this data */ + + newentry = (SetEntry *)PDC_malloc(sizeof(SetEntry)); + + if (newentry == NULL) { + return 0; + } + + newentry->data = data; + + /* Link into chain */ + + newentry->next = set->table[index]; + set->table[index] = newentry; + + /* Keep track of the number of entries in the set */ + + ++set->entries; + + /* Added successfully */ + + return 1; +} + +int +set_remove(Set *set, SetValue data) +{ + SetEntry ** rover; + SetEntry * entry; + unsigned int index; + + /* Look up the data by its hash key */ + + index = set->hash_func(data) % set->table_size; + + /* Search this chain, until the corresponding entry is found */ + + rover = &set->table[index]; + + while (*rover != NULL) { + if (set->equal_func(data, (*rover)->data) != 0) { + + /* Found the entry */ + + entry = *rover; + + /* Unlink from the linked list */ + + *rover = entry->next; + + /* Update counter */ + + --set->entries; + + /* Free the entry and return */ + + set_free_entry(set, entry); + + return 1; + } + + /* Advance to the next entry */ + + rover = &((*rover)->next); + } + + /* Not found in set */ + + return 0; +} + +int +set_query(Set *set, SetValue data) +{ + SetEntry * rover; + unsigned int index; + + /* Look up the data by its hash key */ + + index = set->hash_func(data) % set->table_size; + + /* Search this chain, until the corresponding entry is found */ + + rover = set->table[index]; + + while (rover != NULL) { + if (set->equal_func(data, rover->data) != 0) { + + /* Found the entry */ + + return 1; + } + + /* Advance to the next entry in the chain */ + + rover = rover->next; + } + + /* Not found */ + + return 0; +} + +unsigned int +set_num_entries(Set *set) +{ + return set->entries; +} + +SetValue * +set_to_array(Set *set) +{ + SetValue * array; + int array_counter; + unsigned int i; + SetEntry * rover; + + /* Create an array to hold the set entries */ + + array = PDC_malloc(sizeof(SetValue) * set->entries); + + if (array == NULL) { + return NULL; + } + + array_counter = 0; + + /* Iterate over all entries in all chains */ + + for (i = 0; i < set->table_size; ++i) { + + rover = set->table[i]; + + while (rover != NULL) { + + /* Add this value to the array */ + + array[array_counter] = rover->data; + ++array_counter; + + /* Advance to the next entry */ + + rover = rover->next; + } + } + + return array; +} + +Set * +set_union(Set *set1, Set *set2) +{ + SetIterator iterator; + Set * new_set; + SetValue value; + + new_set = set_new(set1->hash_func, set1->equal_func); + + if (new_set == NULL) { + return NULL; + } + + /* Add all values from the first set */ + + set_iterate(set1, &iterator); + + while (set_iter_has_more(&iterator)) { + + /* Read the next value */ + + value = set_iter_next(&iterator); + + /* Copy the value into the new set */ + + if (!set_insert(new_set, value)) { + + /* Failed to insert */ + + set_free(new_set); + return NULL; + } + } + + /* Add all values from the second set */ + + set_iterate(set2, &iterator); + + while (set_iter_has_more(&iterator)) { + + /* Read the next value */ + + value = set_iter_next(&iterator); + + /* Has this value been put into the new set already? + * If so, do not insert this again */ + + if (set_query(new_set, value) == 0) { + if (!set_insert(new_set, value)) { + + /* Failed to insert */ + + set_free(new_set); + return NULL; + } + } + } + + return new_set; +} + +Set * +set_intersection(Set *set1, Set *set2) +{ + Set * new_set; + SetIterator iterator; + SetValue value; + + new_set = set_new(set1->hash_func, set2->equal_func); + + if (new_set == NULL) { + return NULL; + } + + /* Iterate over all values in set 1. */ + + set_iterate(set1, &iterator); + + while (set_iter_has_more(&iterator)) { + + /* Get the next value */ + + value = set_iter_next(&iterator); + + /* Is this value in set 2 as well? If so, it should be + * in the new set. */ + + if (set_query(set2, value) != 0) { + + /* Copy the value first before inserting, + * if necessary */ + + if (!set_insert(new_set, value)) { + set_free(new_set); + + return NULL; + } + } + } + + return new_set; +} + +void +set_iterate(Set *set, SetIterator *iter) +{ + unsigned int chain; + + iter->set = set; + iter->next_entry = NULL; + + /* Find the first entry */ + + for (chain = 0; chain < set->table_size; ++chain) { + + /* There is a value at the start of this chain */ + + if (set->table[chain] != NULL) { + iter->next_entry = set->table[chain]; + break; + } + } + + iter->next_chain = chain; +} + +SetValue +set_iter_next(SetIterator *iterator) +{ + Set * set; + SetValue result; + SetEntry * current_entry; + unsigned int chain; + + set = iterator->set; + + /* No more entries? */ + + if (iterator->next_entry == NULL) { + return SET_NULL; + } + + /* We have the result immediately */ + + current_entry = iterator->next_entry; + result = current_entry->data; + + /* Advance next_entry to the next SetEntry in the Set. */ + + if (current_entry->next != NULL) { + + /* Use the next value in this chain */ + + iterator->next_entry = current_entry->next; + } + else { + + /* Default value if no valid chain is found */ + + iterator->next_entry = NULL; + + /* No more entries in this chain. Search the next chain */ + + chain = iterator->next_chain + 1; + + while (chain < set->table_size) { + + /* Is there a chain at this table entry? */ + + if (set->table[chain] != NULL) { + + /* Valid chain found! */ + + iterator->next_entry = set->table[chain]; + + break; + } + + /* Keep searching until we find an empty chain */ + + ++chain; + } + + iterator->next_chain = chain; + } + + return result; +} + +int +set_iter_has_more(SetIterator *iterator) +{ + return iterator->next_entry != NULL; +} diff --git a/src/commons/collections/pdc_stack.c b/src/commons/collections/pdc_stack.c new file mode 100644 index 000000000..3b332752a --- /dev/null +++ b/src/commons/collections/pdc_stack.c @@ -0,0 +1,46 @@ +#include +#include +#include +#include "pdc_stack.h" + +#define DEFAULT_CAPACITY 16 + +void +stack_init(PDC_stack_t *stack) +{ + stack->data = malloc(sizeof(void *) * DEFAULT_CAPACITY); + stack->size = 0; + stack->capacity = DEFAULT_CAPACITY; +} + +void +stack_push(PDC_stack_t *stack, void *value) +{ + if (stack->size == stack->capacity) { + size_t new_capacity = stack->capacity * 2; + void **new_data = realloc(stack->data, sizeof(void *) * new_capacity); + if (new_data == NULL) { + printf("Failed to reallocate memory for stack!\n"); + exit(1); + } + stack->data = new_data; + stack->capacity = new_capacity; + } + stack->data[stack->size++] = value; +} + +void * +stack_pop(PDC_stack_t *stack) +{ + if (stack->size == 0) { + printf("Stack underflow!\n"); + exit(1); + } + return stack->data[--stack->size]; +} + +void +stack_free(PDC_stack_t *stack) +{ + free(stack->data); +} \ No newline at end of file diff --git a/src/commons/collections/pdc_vector.c b/src/commons/collections/pdc_vector.c new file mode 100644 index 000000000..ac28c646d --- /dev/null +++ b/src/commons/collections/pdc_vector.c @@ -0,0 +1,165 @@ +#include "pdc_vector.h" + +PDC_VECTOR * +pdc_vector_new() +{ + return pdc_vector_create(100, 2.0); +} + +PDC_VECTOR * +pdc_vector_create(size_t initial_capacity, double expansion_factor) +{ + // Allocate memory for the vector struct. + PDC_VECTOR *vector = (PDC_VECTOR *)malloc(sizeof(PDC_VECTOR)); + if (vector == NULL) { + return NULL; + } + + // Allocate memory for the array of items. + vector->items = (void **)malloc(initial_capacity * sizeof(void *)); + if (vector->items == NULL) { + free(vector); + return NULL; + } + + // Initialize the vector fields. + vector->item_count = 0; + vector->capacity = initial_capacity; + vector->expansion_factor = expansion_factor; + + return vector; +} + +void +pdc_vector_destroy(PDC_VECTOR *vector) +{ + if (vector == NULL) { + return; + } + + // Free all allocated memory for each item. + for (size_t i = 0; i < vector->item_count; i++) { + free(vector->items[i]); + } + + // Free the array of items and the vector struct. + free(vector->items); + free(vector); +} + +void +pdc_vector_add(PDC_VECTOR *vector, void *item) +{ + if (vector == NULL || item == NULL) { + return; + } + + // Expand the array of items if necessary. + if (vector->item_count >= vector->capacity) { + vector->capacity *= vector->expansion_factor; + vector->items = (void **)realloc(vector->items, vector->capacity * sizeof(void *)); + if (vector->items == NULL) { + return; + } + } + + // Add the new item to the end of the array. + vector->items[vector->item_count++] = item; +} + +void * +pdc_vector_get(PDC_VECTOR *vector, size_t index) +{ + if (vector == NULL || index >= vector->item_count) { + return NULL; + } + + // Return a pointer to the item at the given index. + return vector->items[index]; +} + +size_t +pdc_vector_size(PDC_VECTOR *vector) +{ + if (vector == NULL) { + return 0; + } + + // Return the number of items in the vector. + return vector->item_count; +} + +void +pdc_vector_set_expansion_factor(PDC_VECTOR *vector, double expansion_factor) +{ + if (vector == NULL) { + return; + } + + // Set the new expansion factor for the vector. + vector->expansion_factor = expansion_factor; +} + +double +pdc_vector_get_expansion_factor(PDC_VECTOR *vector) +{ + if (vector == NULL) { + return 0; + } + + // Return the current expansion factor for the vector. + return vector->expansion_factor; +} + +PDC_VECTOR_ITERATOR * +pdc_vector_iterator_new(PDC_VECTOR *vector) +{ + if (vector == NULL) { + return NULL; + } + + // Allocate memory for the iterator struct. + PDC_VECTOR_ITERATOR *iterator = (PDC_VECTOR_ITERATOR *)malloc(sizeof(PDC_VECTOR_ITERATOR)); + if (iterator == NULL) { + return NULL; + } + + // Initialize the iterator fields. + iterator->vector = vector; + iterator->index = 0; + + return iterator; +} + +void +pdc_vector_iterator_destroy(PDC_VECTOR_ITERATOR *iterator) +{ + if (iterator == NULL) { + return; + } + + // Free the iterator struct. + free(iterator); +} + +void * +pdc_vector_iterator_next(PDC_VECTOR_ITERATOR *iterator) +{ + if (iterator == NULL) { + return NULL; + } + + // Return the next item in the vector. + return pdc_vector_get(iterator->vector, iterator->index++); +} + +int +pdc_vector_iterator_has_next(PDC_VECTOR_ITERATOR *iterator) +{ + if (iterator == NULL) { + return 0; + } + + // Return true if there are more items in the vector. + return iterator->index < pdc_vector_size(iterator->vector); +} \ No newline at end of file diff --git a/src/commons/file/Readme.md b/src/commons/file/Readme.md new file mode 100644 index 000000000..e69de29bb diff --git a/src/commons/file/common_io.c b/src/commons/file/common_io.c new file mode 100644 index 000000000..ae7af7f27 --- /dev/null +++ b/src/commons/file/common_io.c @@ -0,0 +1,188 @@ +#include "common_io.h" + +#include +#include +#include +#include + +FILE * +open_file(char *filename, char *mode) +{ + FILE *fp = fopen(filename, mode); + if (fp == NULL) { + fprintf(stderr, "Error opening file %s: %s\n", filename, strerror(errno)); + exit(1); + } + return fp; +} + +int +close_file(FILE *fp) +{ + if (fclose(fp) != 0) { + fprintf(stderr, "Error closing file\n"); + return 1; + } + return 0; +} + +int +read_file(FILE *fp, io_buffer_t *buffer) +{ + // Determine the file size + fseek(fp, 0L, SEEK_END); + buffer->size = ftell(fp); + rewind(fp); + + // Allocate memory for the buffer + buffer->buffer = (char *)malloc(buffer->size + 1); + if (buffer->buffer == NULL) { + fprintf(stderr, "Error allocating memory for file buffer\n"); + return 1; + } + + // Read the file into the buffer + if (fread(buffer->buffer, 1, buffer->size, fp) != buffer->size) { + fprintf(stderr, "Error reading file\n"); + return 1; + } + buffer->buffer[buffer->size] = '\0'; + + return 0; +} + +int +write_file(FILE *fp, io_buffer_t *buffer) +{ + if (fwrite(buffer->buffer, 1, buffer->size, fp) != buffer->size) { + fprintf(stderr, "Error writing file\n"); + return 1; + } + return 0; +} + +void +print_string(char *string) +{ + printf("%s", string); +} + +int +read_line(FILE *fp, char *buffer, size_t size) +{ + if (fgets(buffer, size, fp) == NULL) { + fprintf(stderr, "Error reading line\n"); + return 1; + } + // Remove the newline character if present + if (strchr(buffer, '\n') != NULL) { + buffer[strcspn(buffer, "\n")] = '\0'; + } + return 0; +} + +int +get_input(char *buffer, size_t size) +{ + if (fgets(buffer, size, stdin) == NULL) { + fprintf(stderr, "Error getting input\n"); + return 1; + } + // Remove the newline character if present + if (strchr(buffer, '\n') != NULL) { + buffer[strcspn(buffer, "\n")] = '\0'; + } + return 0; +} + +void +print_error(char *message) +{ + fprintf(stderr, "Error: %s\n", message); +} + +int +read_text_file(char *filename, void (*callback)(char *line)) +{ + FILE * fp = open_file(filename, IO_MODE_READ); + char * line = NULL; + size_t len = 0; + ssize_t read; + while ((read = getline(&line, &len, fp)) != -1) { + if (line[read - 1] == '\n') { + line[read - 1] = '\0'; + } + callback(line); + } + if (ferror(fp)) { + fprintf(stderr, "Error reading file\n"); + return 1; + } + free(line); + close_file(fp); + return 0; +} + +int +write_text_file(char *filename, char **lines, size_t num_lines) +{ + FILE *fp = open_file(filename, IO_MODE_WRITE); + for (size_t i = 0; i < num_lines; i++) { + if (fprintf(fp, "%s\n", lines[i]) < 0) { + fprintf(stderr, "Error writing to file\n"); + close_file(fp); + return 1; + } + } + close_file(fp); + return 0; +} + +int +read_binary_file(char *filename, void *buffer, size_t size) +{ + FILE *fp = open_file(filename, IO_MODE_BINARY IO_MODE_READ); + if (fread(buffer, 1, size, fp) != size) { + fprintf(stderr, "Error reading file\n"); + close_file(fp); + return 1; + } + close_file(fp); + return 0; +} + +int +write_binary_file(char *filename, void *buffer, size_t size) +{ + FILE *fp = open_file(filename, IO_MODE_BINARY IO_MODE_WRITE); + if (fwrite(buffer, 1, size, fp) != size) { + fprintf(stderr, "Error writing file\n"); + close_file(fp); + return 1; + } + close_file(fp); + return 0; +} + +int +update_binary_file(char *filename, void *buffer, size_t size, unsigned long start_pos, size_t length) +{ + FILE *fp = open_file(filename, IO_MODE_BINARY IO_MODE_READ IO_MODE_WRITE); + if (fseek(fp, start_pos, SEEK_SET) != 0) { + fprintf(stderr, "Error seeking to starting position\n"); + close_file(fp); + return 1; + } + if (fwrite(buffer, 1, size, fp) != size) { + fprintf(stderr, "Error writing to file\n"); + close_file(fp); + return 1; + } + if (length != size && ftruncate(fileno(fp), start_pos + size) != 0) { + fprintf(stderr, "Error truncating file\n"); + close_file(fp); + return 1; + } + close_file(fp); + return 0; +} \ No newline at end of file diff --git a/src/commons/file/include/common_io.h b/src/commons/file/include/common_io.h new file mode 100644 index 000000000..f6b248171 --- /dev/null +++ b/src/commons/file/include/common_io.h @@ -0,0 +1,159 @@ +#ifndef COMMON_IO_H +#define COMMON_IO_H + +#include + +/** + * \def IO_MODE_READ + * \brief Constant for read mode. + */ +#define IO_MODE_READ "r" + +/** + * \def IO_MODE_WRITE + * \brief Constant for write mode. + */ +#define IO_MODE_WRITE "w" + +/** + * \def IO_MODE_APPEND + * \brief Constant for append mode. + */ +#define IO_MODE_APPEND "a" + +/** + * \def IO_MODE_BINARY + * \brief Constant for binary mode. + */ +#define IO_MODE_BINARY "b" + +/** + * \struct io_buffer_t + * \brief Data structure for input/output operations. + */ +typedef struct { + char * buffer; /**< Pointer to the buffer data */ + size_t size; /**< Size of the buffer */ +} io_buffer_t; + +/** + * \fn FILE *open_file(char *filename, char *mode) + * \brief Opens a file. + * \param filename Name of the file to open. + * \param mode Mode in which to open the file. + * \return Pointer to the opened file, or NULL if the file couldn't be opened. + */ +FILE *open_file(char *filename, char *mode); + +/** + * \fn int close_file(FILE *fp) + * \brief Closes a file. + * \param fp Pointer to the file to close. + * \return 0 if the file was closed successfully, EOF otherwise. + */ +int close_file(FILE *fp); + +/** + * \fn int read_file(FILE *fp, io_buffer_t *buffer) + * \brief Reads data from a file into a buffer. + * \param fp Pointer to the file to read from. + * \param buffer Pointer to the buffer where the read data will be stored. + * \return 0 if the data was read successfully, non-zero value otherwise. + */ +int read_file(FILE *fp, io_buffer_t *buffer); + +/** + * \fn int write_file(FILE *fp, io_buffer_t *buffer) + * \brief Writes data from a buffer to a file. + * \param fp Pointer to the file to write to. + * \param buffer Pointer to the buffer from which the data will be written. + * \return 0 if the data was written successfully, non-zero value otherwise. + */ +int write_file(FILE *fp, io_buffer_t *buffer); + +/** + * \fn void print_string(char *string) + * \brief Prints a string to the standard output. + * \param string Pointer to the string to be printed. + */ +void print_string(char *string); + +/** + * \fn int read_line(FILE *fp, char *buffer, size_t size) + * \brief Reads a line from a file into a buffer. + * \param fp Pointer to the file to read from. + * \param buffer Pointer to the buffer where the read line will be stored. + * \param size The maximum number of characters to be read (including the null character). + * \return 0 if the line was read successfully, non-zero value otherwise. + */ +int read_line(FILE *fp, char *buffer, size_t size); + +/** + * \fn int get_input(char *buffer, size_t size) + * \brief Gets user input from the standard input into a buffer. + * \param buffer Pointer to the buffer where the input will be stored. + * \param size The maximum number of characters to be read (including the null character). + * \return 0 if the input was read successfully, non-zero value otherwise. + */ +int get_input(char *buffer, size_t size); + +/** + * \fn void print_error(char *message) + * \brief Prints an error message to the standard error. + * \param message Pointer to the error message to be printed. + */ +void print_error(char *message); + +/** + * \fn int read_text_file(char *filename, void (*callback)(char *line)) + * \brief Reads a text file line by line and applies a callback function to each line. + * \param filename The name of the file to be read. + * \param callback The function to be applied to each line of the file. + * \return 0 if the file was read successfully, non-zero value otherwise. + */ +int read_text_file(char *filename, void (*callback)(char *line)); + +/** + * \fn int write_text_file(char *filename, char **lines, size_t num_lines) + * \brief Writes an array of lines to a text file. + * \param filename The name of the file to be written to. + * \param lines The array of lines to be written to the file. + * \param num_lines The number of lines in the array. + * \return 0 if the file was written successfully, non-zero value otherwise. + */ +int write_text_file(char *filename, char **lines, size_t num_lines); + +/** + * \fn int read_binary_file(char *filename, void *buffer, size_t size) + * \brief Reads a binary file into a buffer. + * \param filename The name of the file to be read. + * \param buffer The buffer where the file content will be stored. + * \param size The maximum number of bytes to be read. + * \return 0 if the file was read successfully, non-zero value otherwise. + */ +int read_binary_file(char *filename, void *buffer, size_t size); + +/** + * \fn int write_binary_file(char *filename, void *buffer, size_t size) + * \brief Writes a buffer to a binary file. + * \param filename The name of the file to be written to. + * \param buffer The buffer whose content will be written to the file. + * \param size The number of bytes to be written from the buffer. + * \return 0 if the file was written successfully, non-zero value otherwise. + */ +int write_binary_file(char *filename, void *buffer, size_t size); + +/** + * \fn int update_binary_file(char *filename, void *buffer, size_t size, unsigned long start_pos, size_t + * length) + * \brief Updates a portion of a binary file with data from a buffer. + * \param filename The name of the file to be updated. + * \param buffer The buffer containing the data to be written to the file. + * \param size The total size of the buffer. + * \param start_pos The starting position in the file where the update will begin. + * \param length The number of bytes to be written from the buffer. + * \return 0 if the file was updated successfully, non-zero value otherwise. + */ +int update_binary_file(char *filename, void *buffer, size_t size, unsigned long start_pos, size_t length); + +#endif /* COMMON_IO_H */ \ No newline at end of file diff --git a/src/commons/generic/include/pdc_generic.h b/src/commons/generic/include/pdc_generic.h new file mode 100644 index 000000000..fc9925ea9 --- /dev/null +++ b/src/commons/generic/include/pdc_generic.h @@ -0,0 +1,215 @@ +#ifndef PDC_GENERIC_H +#define PDC_GENERIC_H + +#include +#include +#include +#include +#include + +#ifndef __cplusplus +#if __STDC_VERSION__ >= 199901L +/* C99 or later */ +#include +#else +/* Pre-C99 */ +typedef enum { false = 0, true = 1 } bool; +#endif +#endif + +typedef enum pdc_c_var_type_t { + PDC_UNKNOWN = -1, /* error */ + PDC_INT = 0, /* integer types (identical to int32_t) */ + PDC_FLOAT = 1, /* floating-point types */ + PDC_DOUBLE = 2, /* double types */ + PDC_CHAR = 3, /* character types */ + PDC_STRING = 4, /* string types */ + PDC_BOOLEAN = 5, /* boolean types */ + PDC_SHORT = 6, /* short types */ + PDC_UINT = 7, /* unsigned integer types (identical to uint32_t) */ + PDC_INT64 = 8, /* 64-bit integer types */ + PDC_UINT64 = 9, /* 64-bit unsigned integer types */ + PDC_INT16 = 10, /* 16-bit integer types */ + PDC_INT8 = 11, /* 8-bit integer types */ + PDC_UINT8 = 12, /* 8-bit unsigned integer types */ + PDC_UINT16 = 13, /* 16-bit unsigned integer types */ + PDC_INT32 = 14, /* 32-bit integer types */ + PDC_UINT32 = 15, /* 32-bit unsigned integer types */ + PDC_LONG = 16, /* long types */ + PDC_VOID_PTR = 17, /* void pointer type */ + PDC_SIZE_T = 18, /* size_t type */ + PDC_TYPE_COUNT = 19 /* this is the number of var types and has to be the last */ +} pdc_c_var_type_t; + +// typedef pdc_c_var_type_t PDC_CType; + +typedef enum pdc_c_var_class_t { + PDC_CLS_SCALAR, + PDC_CLS_ARRAY, + PDC_CLS_ENUM, // not implemented, users can use PDC_CT_INT + PDC_CLS_STRUCT, // not implemented, users can use embedded key value pairs for the members in a struct + PDC_CLS_UNION, // not implemented, users can use embedded key value pairs for the only one member value + // in a union. + PDC_CLS_POINTER, // not implemented, users can use PDC_CT_INT64_T to store the pointer address, but + // won't work for distributed memory. + PDC_CLS_FUNCTION, // not implemented, users can use PDC_CT_INT64_T to store the function address, but + // won't work for distributed memory. + PDC_CLS_COUNT // just the count of the enum. +} pdc_c_var_class_t; + +// typedef pdc_c_var_class_t PDC_CType_Class; + +// clang-format off +static size_t DataTypeSizes[PDC_TYPE_COUNT] = { + sizeof(int), + sizeof(float), + sizeof(double), + sizeof(char), + sizeof(char *), + sizeof(bool), + sizeof(short), + sizeof(unsigned int), + sizeof(int64_t), + sizeof(uint64_t), + sizeof(int16_t), + sizeof(int8_t), + sizeof(uint8_t), + sizeof(uint16_t), + sizeof(int32_t), + sizeof(uint32_t), + sizeof(long), + sizeof(void *), + sizeof(size_t) +}; + +static char *DataTypeNames[PDC_TYPE_COUNT] = { + "int", + "float", + "double", + "char", + "char*", + "bool", + "short", + "unsigned int", + "int64_t", + "uint64_t", + "int16_t", + "int8_t", + "uint8_t", + "uint16_t", + "int32_t", + "uint32_t", + "long", + "void*", + "size_t" +}; + +static char *DataTypeEnumNames[PDC_TYPE_COUNT] = { + "PDC_INT", + "PDC_FLOAT", + "PDC_DOUBLE", + "PDC_CHAR", + "PDC_STRING", + "PDC_BOOLEAN", + "PDC_SHORT", + "PDC_UINT", + "PDC_INT64", + "PDC_UINT64", + "PDC_INT16", + "PDC_INT8", + "PDC_UINT8", + "PDC_UINT16", + "PDC_INT32", + "PDC_UINT32", + "PDC_LONG", + "PDC_VOID_PTR", + "PDC_SIZE_T" +}; + +__attribute__((unused)) +static char *DataTypeFormat[PDC_TYPE_COUNT] = { + "%d", // int + "%f", // float + "%lf", // double + "%c", // char + "%s", // char* + "%d", // bool (represented as an integer) + "%hd", // short + "%u", // unsigned int + "%lld", // int64_t + "%llu", // uint64_t + "%hd", // int16_t + "%hhd", // int8_t + "%hhu", // uint8_t + "%hu", // uint16_t + "%d", // int32_t + "%u", // uint32_t + "%ld", // long + "%p", // void* (pointer) + "%zu" // size_t +}; + +// clang-format on +__attribute__((unused)) static char * +get_enum_name_by_dtype(pdc_c_var_type_t type) +{ + if (type < 0 || type >= PDC_TYPE_COUNT) { + return NULL; + } + return DataTypeEnumNames[type]; +} +__attribute__((unused)) static size_t +get_size_by_dtype(pdc_c_var_type_t type) +{ + if (type < 0 || type >= PDC_TYPE_COUNT) { + return 0; + } + return DataTypeSizes[type]; +} +__attribute__((unused)) static size_t +get_size_by_class_n_type(void *data, size_t item_count, pdc_c_var_class_t pdc_class, + pdc_c_var_type_t pdc_type) +{ + size_t size = 0; + if (pdc_class == PDC_CLS_SCALAR) { + if (pdc_type == PDC_STRING) { + size = (strlen((char *)data) + 1) * sizeof(char); + } + else { + size = get_size_by_dtype(pdc_type); + } + } + else if (pdc_class == PDC_CLS_ARRAY) { + if (pdc_type == PDC_STRING) { + char **str_arr = (char **)data; + size_t i = 0; + for (i = 0; i < item_count; i++) { + size = size + (strlen(str_arr[i]) + 1) * sizeof(char); + } + } + else { + size = item_count * get_size_by_dtype(pdc_type); + } + } + return size; +} +__attribute__((unused)) static char * +get_name_by_dtype(pdc_c_var_type_t type) +{ + if (type < 0 || type >= PDC_TYPE_COUNT) { + return NULL; + } + return DataTypeNames[type]; +} +__attribute__((unused)) static pdc_c_var_type_t +get_dtype_by_enum_name(const char *enumName) +{ + for (int i = 0; i < PDC_TYPE_COUNT; i++) { + if (strcmp(DataTypeEnumNames[i], enumName) == 0) { + return (pdc_c_var_type_t)i; + } + } + return PDC_UNKNOWN; // assuming PDC_UNKNOWN is the enum value for "unknown" +} + +#endif /* PDC_GENERIC_H */ diff --git a/src/api/include/pdc_public.h b/src/commons/include/pdc_public.h similarity index 66% rename from src/api/include/pdc_public.h rename to src/commons/include/pdc_public.h index dc286e5e9..aa628f7fb 100644 --- a/src/api/include/pdc_public.h +++ b/src/commons/include/pdc_public.h @@ -28,6 +28,7 @@ #include #include #include +#include "pdc_generic.h" /*******************/ /* Public Typedefs */ @@ -41,28 +42,21 @@ typedef int PDC_int_t; typedef float PDC_float_t; typedef double PDC_double_t; -typedef enum { - PDC_UNKNOWN = -1, /* error */ - PDC_INT = 0, /* integer types */ - PDC_FLOAT = 1, /* floating-point types */ - PDC_DOUBLE = 2, /* double types */ - PDC_CHAR = 3, /* character types */ - PDC_COMPOUND = 4, /* compound types */ - PDC_ENUM = 5, /* enumeration types */ - PDC_ARRAY = 6, /* Array types */ - PDC_UINT = 7, /* unsigned integer types */ - PDC_INT64 = 8, /* 64-bit integer types */ - PDC_UINT64 = 9, /* 64-bit unsigned integer types */ - PDC_INT16 = 10, - PDC_INT8 = 11, - NCLASSES = 12 /* this must be last */ -} pdc_var_type_t; +typedef pdc_c_var_type_t pdc_var_type_t; + +// FIXME: common data structure should be defined in a group of common header files. +typedef struct pdc_kvtag_t { + char * name; + uint32_t size; + int8_t type; + void * value; +} pdc_kvtag_t; typedef enum { PDC_PERSIST, PDC_TRANSIENT } pdc_lifetime_t; typedef enum { PDC_SERVER_DEFAULT = 0, PDC_SERVER_PER_CLIENT = 1 } pdc_server_selection_t; -typedef struct pdc_histogram_t { //????????? +typedef struct pdc_histogram_t { pdc_var_type_t dtype; int nbin; double incr; @@ -73,4 +67,6 @@ typedef struct pdc_histogram_t { //????????? #define SUCCEED 0 #define FAIL (-1) +#define PDC_SIZE_UNLIMITED UINT64_MAX + #endif /* PDC_PUBLIC_H */ diff --git a/src/commons/index/Readme.md b/src/commons/index/Readme.md new file mode 100644 index 000000000..e69de29bb diff --git a/src/commons/index/dart/dart_algo.c b/src/commons/index/dart/dart_algo.c new file mode 100644 index 000000000..6546d0c39 --- /dev/null +++ b/src/commons/index/dart/dart_algo.c @@ -0,0 +1,212 @@ +#include "dart_algo.h" +#include +#include +#include +#include +#include +#include + +// leftrotate function definition +#define LEFTROTATE(x, c) (((x) << (c)) | ((x) >> (32 - (c)))); + +uint32_t +murmur3_32(const uint8_t *key, size_t len, uint32_t seed) +{ + uint32_t h = seed; + if (len > 3) { + const uint32_t *key_x4 = (const uint32_t *)key; + size_t i = len >> 2; + do { + uint32_t k = *key_x4++; + k *= 0xcc9e2d51; + k = (k << 15) | (k >> 17); + k *= 0x1b873593; + h ^= k; + h = (h << 13) | (h >> 19); + h = (h * 5) + 0xe6546b64; + } while (--i); + key = (const uint8_t *)key_x4; + } + if (len & 3) { + size_t i = len & 3; + uint32_t k = 0; + key = &key[i - 1]; + do { + k <<= 8; + k |= *key--; + } while (--i); + k *= 0xcc9e2d51; + k = (k << 15) | (k >> 17); + k *= 0x1b873593; + h ^= k; + } + h ^= len; + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; +} + +void +to_bytes(uint32_t val, uint8_t *bytes) +{ + bytes[0] = (uint8_t)val; + bytes[1] = (uint8_t)(val >> 8); + bytes[2] = (uint8_t)(val >> 16); + bytes[3] = (uint8_t)(val >> 24); +} + +uint32_t +to_int32(const uint8_t *bytes) +{ + return (uint32_t)bytes[0] | ((uint32_t)bytes[1] << 8) | ((uint32_t)bytes[2] << 16) | + ((uint32_t)bytes[3] << 24); +} + +void +md5(const uint8_t *initial_msg, size_t initial_len, uint8_t *digest) +{ + + // Constants are the integer part of the sines of integers (in radians) * 2^32. + const uint32_t k[64] = { + 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501, + 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821, + 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8, + 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed, 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a, + 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c, 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, + 0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05, 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665, + 0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039, 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1, + 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1, 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391}; + + // r specifies the per-round shift amounts + const uint32_t r[] = {7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, + 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, + 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, + 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21}; + + // These vars will contain the hash + uint32_t h0, h1, h2, h3; + + // Message (to prepare) + uint8_t *msg = NULL; + + size_t new_len, offset; + uint32_t w[16]; + uint32_t a, b, c, d, i, f, g, temp; + + // Initialize variables - simple count in nibbles: + h0 = 0x67452301; + h1 = 0xefcdab89; + h2 = 0x98badcfe; + h3 = 0x10325476; + + // Pre-processing: + // append "1" bit to message + // append "0" bits until message length in bits ≡ 448 (mod 512) + // append length mod (2^64) to message + + for (new_len = initial_len + 1; new_len % (512 / 8) != 448 / 8; new_len++) + ; + + msg = (uint8_t *)malloc(new_len + 8); + memcpy(msg, initial_msg, initial_len); + msg[initial_len] = 0x80; // append the "1" bit; most significant bit is "first" + for (offset = initial_len + 1; offset < new_len; offset++) + msg[offset] = 0; // append "0" bits + + // append the len in bits at the end of the buffer. + to_bytes(initial_len * 8, msg + new_len); + // initial_len>>29 == initial_len*8>>32, but avoids overflow. + to_bytes(initial_len >> 29, msg + new_len + 4); + + // Process the message in successive 512-bit chunks: + // for each 512-bit chunk of message: + for (offset = 0; offset < new_len; offset += (512 / 8)) { + + // break chunk into sixteen 32-bit words w[j], 0 ≤ j ≤ 15 + for (i = 0; i < 16; i++) + w[i] = to_int32(msg + offset + i * 4); + + // Initialize hash value for this chunk: + a = h0; + b = h1; + c = h2; + d = h3; + + // Main loop: + for (i = 0; i < 64; i++) { + + if (i < 16) { + f = (b & c) | ((~b) & d); + g = i; + } + else if (i < 32) { + f = (d & b) | ((~d) & c); + g = (5 * i + 1) % 16; + } + else if (i < 48) { + f = b ^ c ^ d; + g = (3 * i + 5) % 16; + } + else { + f = c ^ (b | (~d)); + g = (7 * i) % 16; + } + + temp = d; + d = c; + c = b; + b = b + LEFTROTATE((a + f + k[i] + w[g]), r[i]); + a = temp; + } + + // Add this chunk's hash to result so far: + h0 += a; + h1 += b; + h2 += c; + h3 += d; + } + + // cleanup + free(msg); + + // var char digest[16] := h0 append h1 append h2 append h3 //(Output is in little-endian) + to_bytes(h0, digest); + to_bytes(h1, digest + 4); + to_bytes(h2, digest + 8); + to_bytes(h3, digest + 12); +} + +uint32_t +djb2_hash(char *str, int len) +{ + uint32_t hash = 5381; + int i = 0; + int c; + + for (i = 0; i < len; i++) { + if (str[i] == '\0') { + c = 0; + } + else { + c = (int)str[i]; + } + hash = ((hash << 5) + hash) + c; /* hash * 33 + c */ + } + + return hash; +} + +uint32_t +md5_hash(int prefix_len, char *word) +{ + + char *prefix = (char *)calloc(prefix_len + 1, sizeof(char)); + strncpy(prefix, word, prefix_len); + uint8_t digest; + md5((uint8_t *)prefix, strlen(prefix), &digest); + uint32_t inthash = to_int32(&digest); + return inthash; +} diff --git a/src/commons/index/dart/dart_core.c b/src/commons/index/dart/dart_core.c new file mode 100644 index 000000000..43410f4b5 --- /dev/null +++ b/src/commons/index/dart/dart_core.c @@ -0,0 +1,690 @@ +#include "dart_utils.h" +#include "dart_algo.h" +#include "dart_math.h" +#include "dart_core.h" + +threadpool dart_thpool_g; + +threadpool +get_dart_temp_thpool(int count) +{ + return thpool_init(count); +} + +threadpool +get_dart_thpool_g() +{ + return dart_thpool_g; +} + +int +is_index_write_op(dart_op_type_t op_type) +{ + return (op_type == OP_INSERT || op_type == OP_DELETE); +} + +void +dart_space_init(DART *dart, int num_client, int num_server, int alphabet_size, int extra_tree_height, + int replication_factor) +{ + if (dart == NULL) { + dart = (DART *)calloc(1, sizeof(DART)); + } + dart->alphabet_size = alphabet_size; + // initialize clients; + dart->num_client = num_client; + // initialize servers; + dart->num_server = num_server; + + dart->dart_tree_height = (int)ceil(log_with_base((double)dart->alphabet_size, (double)dart->num_server)) + + 1 + extra_tree_height; + // calculate number of all leaf nodes + dart->num_vnode = (uint64_t)pow(dart->alphabet_size, dart->dart_tree_height); + dart->replication_factor = replication_factor; + dart_thpool_g = thpool_init(num_server); + dart->suffix_tree_mode = 1; +} +/** + * A utility function for dummies. + * Get server id by given virtual node id. + */ +uint64_t +get_server_id_by_vnode_id(DART *dart, uint64_t vnode_id) +{ + // printf("vnode_id = %d, dart->num_server = %d\n", vnode_id, dart->num_server); + int num_vnode_per_server = dart->num_vnode / dart->num_server; + return (vnode_id / num_vnode_per_server) % dart->num_server; +} + +/** + * This function make the client request counter increment by 1. + * + */ +int +dart_client_request_count_incr(DART *dart_g) +{ + if (dart_g->client_request_count < 0) { + dart_g->client_request_count = 0; + } + dart_g->client_request_count = dart_g->client_request_count + 1; + return dart_g->client_request_count; +} + +/** + * This function is for getting the base virtual node ID by a given string. + * + */ +uint64_t +get_base_virtual_node_id_by_string(DART *dart, char *str) +{ + int n = 0; + uint64_t c; + uint64_t rst = 0; + uint64_t i_t_n; + int met_end = 0; + for (n = 1; n <= dart->dart_tree_height; n++) { + if (str[n - 1] == '\0') { // test if the string ends. + met_end = 1; + } + if (str[n - 1] != '\0' && met_end == 0) { // if not, we calculate the character index. + i_t_n = ((uint64_t)str[n - 1]) % dart->alphabet_size; + } + c = (i_t_n) * ((uint64_t)uint32_pow(dart->alphabet_size, dart->dart_tree_height - n)); + rst += c; + } + return (rst % (uint64_t)dart->num_vnode); +} + +/** + * This function is for getting the alternative virtual node ID. + * + */ +uint64_t +get_reconciled_vnode_id_with_power_of_two_choice_rehashing(DART *dart, uint64_t base_vnode_idx, char *word, + get_server_info_callback get_server_cb) +{ + + int ir_idx = (int)ceil((double)(dart->alphabet_size / 2)); + + // base virtual node address always in the first element of the array. + uint64_t rst = base_vnode_idx; + + // determine the tree height. + int tree_height = dart->dart_tree_height; + + // get serverID for base virtual node + uint64_t serverId = get_server_id_by_vnode_id(dart, base_vnode_idx); + + // we first let reconciled virtual node to be the base virtual node. + uint64_t reconciled_vnode_idx = base_vnode_idx; + + if (dart->dart_tree_height <= 1) { + return reconciled_vnode_idx; + } + + // The procedure of picking alternative virtual node is important. + // We first need to know what is the character lying on the leaves of DART partition tree. + int last_c_index = tree_height - 1; + + int post_leaf_index = 0; + int pre_leaf_index = 0; + + // The pre_leaf_index is the index of character right before the leaf character. + // The post_leaf_index is the index of character right after the leaf character. + if (strlen(word) <= tree_height) { + // if the word is not longer than the tree height, then there is no post-leaf character, therefore + // post_leaf_index should be 0 + post_leaf_index = 0; + // then the last_c_index should be the index of the last character in the alphabet. + last_c_index = strlen(word) - 1; + if (strlen(word) <= 1) { + // if the word contains 0-1 character, there is no pre-leaf character and therefore pre_leaf_index + // should be 0. + pre_leaf_index = 0; + } + else { + // otherwise, pre_leaf_index should be the index of proceeding character of the + // leaf-level character in the alphabet. + pre_leaf_index = (int)word[last_c_index - 1] % dart->alphabet_size; + } + } + else { + // if the length of the word exceeds the height of the DART partition tree, + // definitely, post-leaf character exists. + post_leaf_index = (int)word[last_c_index + 1] % dart->alphabet_size; + // but, there is a case where DART partition tree is of height 1. + // in this case, there will be no pre-leaf character. + if (tree_height <= 1) { + pre_leaf_index = 0; + } + else { + // otherwise, there will be a pre-leaf character. + pre_leaf_index = (int)word[last_c_index - 1] % dart->alphabet_size; + } + } + int leaf_index = (int)word[last_c_index] % dart->alphabet_size; + + int leaf_post_sum = leaf_index + pre_leaf_index + post_leaf_index; + int leaf_post_diff = abs(post_leaf_index - leaf_index - pre_leaf_index); + + // int leaf_post_sum = leaf_index + pre_leaf_index + 0; + // int leaf_post_diff = abs(leaf_index-pre_leaf_index); + + // We calculate the region size: + int region_size = dart->num_vnode / dart->alphabet_size; // d=1, rs = 1; d = 2, rs = k; d = 3, rs =k^2; + // We calculate the sub-region size: + int sub_region_size = region_size / dart->alphabet_size; // d=1, srs = 0; d = 2, srs = 1; d = 3, srs = k; + + // We calculate the major offset which possibly pick a virtual node in another sub-region. + int major_offset = (leaf_post_sum % dart->alphabet_size) * (sub_region_size); + // We calcuate the minor offset which will possibly pick a different virtual node within the same + // sub-region. + int minor_offset = leaf_post_diff; + // Finally the region offset will be some certain virtual node in one region. + // uint64_t region_offset = (reconciled_vnode_idx + (uint64_t)major_offset - (uint64_t)minor_offset) + // % (uint64_t)region_size; + uint64_t region_offset = + (reconciled_vnode_idx + (uint64_t)major_offset - (uint64_t)minor_offset) % (uint64_t)region_size; + // Invert region Index: ceil(alphabet_size / 2); + + int n = 0; + uint64_t c; + // uint64_t rst = 0; + uint64_t i_t_n; + int met_end = 0; + for (n = 1; n <= dart->dart_tree_height; n++) { + if (word[n - 1] == '\0') { + met_end = 1; + } + if (word[n - 1] != '\0' && met_end == 0) { + if (n == 1) { + i_t_n = ((int)word[n - 1] + ir_idx) % dart->alphabet_size; + } + else if (n == (dart->dart_tree_height - 1)) { + i_t_n = ((int)word[n - 1] + leaf_post_sum) % dart->alphabet_size; + } + else if (n == dart->dart_tree_height) { + i_t_n = abs((int)word[n - 1] - leaf_post_diff) % dart->alphabet_size; + } + } + c = (i_t_n) * ((uint64_t)uint32_pow(dart->alphabet_size, dart->dart_tree_height - n)); + rst += c; + } + + int alterV = (rst % (uint64_t)dart->num_vnode); + + // // We also calculate the region start position. + // uint64_t region_start = ((((int)word[0]+ir_idx) % dart->alphabet_size)) * region_size;// + // ((reconciled_vnode_idx)/region_size) * (region_size); + // // Finally, the reconciled vnode index is calculated. + // // reconciled_vnode_idx = (0 + region_start + region_offset) % dart->num_vnode; + // reconciled_vnode_idx = (reconciled_vnode_idx + region_start + region_offset) % dart->num_vnode; + + // Only when inserting a word, we do such load detection. + // get alternative virtual node and therefore the alternative server ID. + int reconcile_serverId = get_server_id_by_vnode_id(dart, alterV); + if (get_server_cb != NULL) { + // Check both physical server to see which one has smaller number of indexed keywords on it. + dart_server origin_server = get_server_cb(serverId); + dart_server reconciled_server = get_server_cb(reconcile_serverId); + // printf("For keyword %s, choosing between %d and %d\n", word, serverId, reconcile_serverId); + + if (origin_server.indexed_word_count > reconciled_server.indexed_word_count) { + // printf("Reconcile happened. from %d to %d\n", vnode_idx , reconciled_vnode_idx); + rst = alterV; + } + } + else { + rst = alterV; + } + return rst; +} + +/** + * This function is for getting the alternative virtual node ID. + * + */ +uint64_t +get_reconciled_vnode_id_with_power_of_two_choice_rehashing_2(DART *dart, uint64_t base_vnode_idx, char *word, + get_server_info_callback get_server_cb) +{ + + // base virtual node address always in the first element of the array. + uint64_t rst = base_vnode_idx; + + // determine the tree height. + int tree_height = dart->dart_tree_height; + + // get serverID for base virtual node + uint64_t serverId = get_server_id_by_vnode_id(dart, base_vnode_idx); + + // we first let reconciled virtual node to be the base virtual node. + uint64_t reconciled_vnode_idx = base_vnode_idx; + + if (dart->dart_tree_height <= 1) { + return reconciled_vnode_idx; + } + + // The procedure of picking alternative virtual node is important. + // We first need to know what is the character lying on the leaves of DART partition tree. + int last_c_index = tree_height - 1; + + int post_leaf_index = 0; + int pre_leaf_index = 0; + + // The pre_leaf_index is the index of character right before the leaf character. + // The post_leaf_index is the index of character right after the leaf character. + if (strlen(word) <= tree_height) { + // if the word is not longer than the tree height, then there is no post-leaf character, therefore + // post_leaf_index should be 0 + post_leaf_index = 0; + // then the last_c_index should be the index of the last character in the alphabet. + last_c_index = strlen(word) - 1; + if (strlen(word) <= 1) { + // if the word contains 0-1 character, there is no pre-leaf character and therefore pre_leaf_index + // should be 0. + pre_leaf_index = 0; + } + else { + // otherwise, pre_leaf_index should be the index of proceeding character of the + // leaf-level character in the alphabet. + pre_leaf_index = (int)word[last_c_index - 1] % dart->alphabet_size; + } + } + else { + // if the length of the word exceeds the height of the DART partition tree, + // definitely, post-leaf character exists. + post_leaf_index = (int)word[last_c_index + 1] % dart->alphabet_size; + // but, there is a case where DART partition tree is of height 1. + // in this case, there will be no pre-leaf character. + if (tree_height <= 1) { + pre_leaf_index = 0; + } + else { + // otherwise, there will be a pre-leaf character. + pre_leaf_index = (int)word[last_c_index - 1] % dart->alphabet_size; + } + } + int leaf_index = (int)word[last_c_index] % dart->alphabet_size; + + int leaf_post_sum = leaf_index + pre_leaf_index + post_leaf_index; + int leaf_post_diff = abs(post_leaf_index - leaf_index - pre_leaf_index); + + // int leaf_post_sum = leaf_index + pre_leaf_index + 0; + // int leaf_post_diff = abs(leaf_index-pre_leaf_index); + + // We calculate the region size: + int region_size = dart->num_vnode / dart->alphabet_size; // d=1, rs = 1; d = 2, rs = k; d = 3, rs =k^2; + // We calculate the sub-region size: + int sub_region_size = region_size / dart->alphabet_size; // d=1, srs = 0; d = 2, srs = 1; d = 3, srs = k; + + // We calculate the major offset which possibly pick a virtual node in another sub-region. + int major_offset = (leaf_post_sum % dart->alphabet_size) * (sub_region_size); + // We calcuate the minor offset which will possibly pick a different virtual node within the same + // sub-region. + int minor_offset = leaf_post_diff; + // Finally the region offset will be some certain virtual node in one region. + uint64_t region_offset = + (reconciled_vnode_idx + (uint64_t)major_offset - (uint64_t)minor_offset) % (uint64_t)region_size; + // Invert region Index: ceil(alphabet_size / 2); + + int ir_idx = (int)ceil((double)(dart->alphabet_size / 2)); + + // We also calculate the region start position. + uint64_t region_start = ((((int)word[0] + ir_idx) % dart->alphabet_size)) * + region_size; // ((reconciled_vnode_idx)/region_size) * (region_size); + // Finally, the reconciled vnode index is calculated. + // reconciled_vnode_idx = (0 + region_start + region_offset) % dart->num_vnode; + reconciled_vnode_idx = (reconciled_vnode_idx + region_start + region_offset) % dart->num_vnode; + + // Only when inserting a word, we do such load detection. + // get alternative virtual node and therefore the alternative server ID. + int reconcile_serverId = get_server_id_by_vnode_id(dart, reconciled_vnode_idx); + if (get_server_cb != NULL) { + // Check both physical server to see which one has smaller number of indexed keywords on it. + dart_server origin_server = get_server_cb(serverId); + dart_server reconciled_server = get_server_cb(reconcile_serverId); + // printf("For keyword %s, choosing between %d and %d\n", word, serverId, reconcile_serverId); + + if (origin_server.indexed_word_count > reconciled_server.indexed_word_count) { + // printf("Reconcile happened. from %d to %d\n", vnode_idx , reconciled_vnode_idx); + rst = reconciled_vnode_idx; + } + } + else { + rst = reconciled_vnode_idx; + } + return rst; +} + +/** + * Get IDs of all virtual nodes of replicas by given string and overall tree-height setting. + * When is_physical = 1, it returns the IDs of all physical nodes of replicas. + * + * return The length of the ID array. + */ +int +get_replica_node_ids(DART *dart, uint64_t master_node_id, int is_physical, uint64_t **out) +{ + if (out == NULL) { + return 0; + } + out[0] = (uint64_t *)calloc(dart->replication_factor, sizeof(uint64_t)); + + out[0][0] = master_node_id; + + uint64_t vnode_distance = dart->num_vnode / dart->alphabet_size; + + int r = 1; + for (r = 1; r < dart->replication_factor; r++) { + out[0][r] = (out[0][0] + (uint64_t)r * vnode_distance) % dart->num_vnode; + } + + if (is_physical == 1) { + for (r = 0; r < dart->replication_factor; r++) { + out[0][r] = get_server_id_by_vnode_id(dart, out[0][r]); + } + } + + return dart->replication_factor; +} + +/** + * This function takes a keyword, creates an index record in DART and returns all server ids + * including replica servers. The caller of this function should perform index create PRC call + * to all the servers specified by the ID list. + * + * \param dart_g [IN] Pointer of Global state of DART + * \param keyword [IN] The keyword which should be indexed. + * \param get_server_cb [IN] A callback function which determines the lighter-load server between + * two servers. \param out [OUT] The pointer of the server ID array. \return The length of + * the server ID. + */ +int +get_server_ids_for_insert(DART *dart_g, char *keyword, get_server_info_callback get_server_cb, uint64_t **out) +{ + if (out == NULL) { + return 0; + } + int rst = 0; // not success + uint64_t base_virtual_node_id = get_base_virtual_node_id_by_string(dart_g, keyword); + // For insert operations, we only return the reconciled virtual node. + // The reconciled virtual node can be either the base virtual node or the alternative virtual node. + uint64_t alter_virtual_node_id = get_reconciled_vnode_id_with_power_of_two_choice_rehashing_2( + dart_g, base_virtual_node_id, keyword, get_server_cb); + // We call the following function to calculate all the server IDs. + int is_physical = 1; + int rst_len = get_replica_node_ids(dart_g, alter_virtual_node_id, is_physical, out); + return rst_len; +} + +/** + * This function gives all the server ID according to the given query token. + * The query token should not contain any wildcard sign. + * + * Once the server IDs are retrieved, the query should be broadcasted to all these servers. + * + * Also, this function can be used for index updating and index deletion operations. + * + * Notice that, for index updating and index deletion operations, the given token must represent + * an exact keyword, thus, the result will be an ID list of two servers which are the possible + * locations of the specified keyword. + * + * Two different type of updating operations should be considered. + * + * 1. Add/Remove one ID from the set of IDs related to the keywords. + * 2. Updating the attribute value of an object according to object ID. + * + * *** If bulk transfer may overwhelm the system, + * *** We need to define a protocol for client and server to communicate without bulk transfer. + * + * + * \param dart_g [IN] Pointer of Global state of DART + * \param token [IN] The token which will searched for. It should not contain any wildcard + * sign. \param op_type [IN] Give the operation type. \param out [IN/OUT] The + * pointer of the server ID array. To accelerate, we require caller function to pass an ID array of length M + * into this function, where M is the number of total physical servers. Then, this function will fill up this + * array. \return [OUT] The valid length of the server ID array. + */ +int +get_server_ids_for_query(DART *dart_g, char *token, dart_op_type_t op_type, uint64_t **out) +{ + if (out == NULL) { + return 0; + } + if (op_type == OP_INSERT) { + return 0; + } // For INSERT operation ,we return nothing here. + + // We first eliminate possibility of INFIX query. + // Note: if suffix tree mode is ON, we don't have to search all servers. +#ifndef PDC_DART_SFX_TREE + if (op_type == OP_INFIX_QUERY) { + out[0] = (uint64_t *)calloc(dart_g->num_server, sizeof(uint64_t)); + int i = 0; + for (i = 0; i < dart_g->num_server; i++) { + out[0][i] = i; + } + return dart_g->num_server; + } +#endif + + // for prefix/suffix query, we seek for proper parent region and only perform query broadcast for that + // region. when suffix tree mode is ON: suffix query == exact query, infix query == prefix query. + if (strlen(token) < dart_g->dart_tree_height && (op_type == OP_PREFIX_QUERY +#ifdef PDC_DART_SFX_TREE + || op_type == OP_INFIX_QUERY +#else + || op_type == OP_SUFFIX_QUERY +#endif + )) { + + // TODO: currently broadcast request to all virtual nodes in one region. + // TODO: we have to consider if we can reduce the number of nodes we need to + // access as the prefix length grows. + int region_size = dart_g->num_vnode / dart_g->alphabet_size; + int region_start = ((((int)token[0] % dart_g->alphabet_size) + 0) * region_size); + + uint64_t server_start = get_server_id_by_vnode_id(dart_g, region_start); + uint64_t server_end = get_server_id_by_vnode_id(dart_g, region_start + region_size - 1); + + int num_srvs = server_end - server_start + 1; + out[0] = (uint64_t *)calloc(num_srvs, sizeof(uint64_t)); + int num_srv_ids = 0; + + uint64_t srvId = server_start; + for (; srvId <= server_end; srvId++) { + out[0][num_srv_ids++] = srvId; + } + return num_srvs; + } + else { + // this branch handles the following cases: + // 1. exact search (suffix tree mode can be either ON or OFF): + // the logic is pretty much the same as insert operation, + // the only difference is to reserve two servers for query. + // 2. suffix search with insufficient token length (in suffix tree mode), + // this is treated as an exact query. + // 2. prefix/infix (suffix tree mode can be ON or OFF) with sufficiently long token, + // this is pretty much the same with exact queries. + // 3. delete operation: we need to delete all replicas, + // including base replicas and alternative replicas. + // Explanation: For exact search or prefix/suffix search, when the given token length is large enough, + // we consider a different query procedure. in this case, there is no difference between a long + // prefix/suffix/infix and an exact query against a long keyword. + + uint64_t base_virtual_node_id = get_base_virtual_node_id_by_string(dart_g, token); + // For insert operations, we only return the reconciled virtual node. + // The reconciled virtual node can be either the base virtual node or the + // alternative virtual node. + uint64_t reconciled_vnode_id = get_reconciled_vnode_id_with_power_of_two_choice_rehashing_2( + dart_g, base_virtual_node_id, token, NULL); + + uint64_t *base_replicas; + uint64_t *alter_replicas; + + int num_base_reps = get_replica_node_ids(dart_g, base_virtual_node_id, 1, &base_replicas); + int num_alter_reps = get_replica_node_ids(dart_g, reconciled_vnode_id, 1, &alter_replicas); + if (op_type == OP_DELETE) { + // for delete operations, we need to perform delete on all replicas + out[0] = (uint64_t *)calloc(num_base_reps + num_alter_reps, sizeof(uint64_t)); + int i = 0; + for (i = 0; i < num_base_reps; i++) { + out[0][i] = base_replicas[i]; + } + for (i = 0; i < num_alter_reps; i++) { + out[0][num_base_reps + i] = alter_replicas[i]; + } + return num_base_reps + num_alter_reps; + } + else { + // for other query operations, we only provide two servers for them to access + // and the server has to be any two of the possible replicas. + int req_count = dart_client_request_count_incr(dart_g); + int rep_index = req_count % num_base_reps; + out[0] = (uint64_t *)calloc(2, sizeof(uint64_t)); + int rst_size = 1; + out[0][0] = base_replicas[rep_index]; + if (base_replicas[rep_index] != alter_replicas[rep_index]) { + rst_size = 2; + out[0][1] = alter_replicas[rep_index]; + } + return rst_size; + } + } +} + +/** + * This is a routing function for different possible operations in DART. + * It will only generate the array of IDs of servers that client should touch. + * + * The return value is the length of valid elements in the array. + * A for loop can use the return value as the upper bound to iterate all elements in the array. + * + * @param dart_g [IN] Pointer of Global state of DART + * @param key [IN] The given key. For queries, it is already a bare keyword without '*'. + * @param op_type [IN] Give the operation type. + * @param get_server_cb [IN] The callback function for getting the statistical information of a server. + * Signature: dart_server (*get_server_info_callback)(uint32_t server_id) + * @param out [IN/OUT] The pointer of the server ID array. To accelerate, we require caller + * function to pass an ID array of length M into this function, where M is + * the number of total physical servers. Then, this function will fill up + * this array. + * @return [OUT] The valid length of the server ID array. + * + */ +int +DART_hash(DART *dart_g, char *key, dart_op_type_t op_type, get_server_info_callback get_server_cb, + index_hash_result_t **out) +{ + int ret_value = 0; + + if (out == NULL) { + return ret_value; + } + + uint64_t *temp_out = NULL; + int tmp_out_len = 0; + char * tok = NULL; + *out = NULL; + + // regardless of suffix tree mode, we only need to get the DART hash result for one time. + int loop_count = 1; + if (is_index_write_op(op_type)) { +#ifdef PDC_DART_SFX_TREE + // suffix tree mode is ON, we can iterate all suffixes for insert/delete operations. + // if there are all N suffixes, we need to get the DART hash results for N times, + // and each time we get the result for a single suffix. + loop_count = strlen(key); +#else + // suffix tree mode is OFF, we iterate the original string and its reverse. (2 iterations) + loop_count = 2; +#endif + } + int iter = 0; + for (iter = 0; iter < loop_count; iter++) { +#ifdef PDC_DART_SFX_TREE + tok = substring(key, iter, strlen(key)); +#else + tok = iter == 0 ? strdup(key) : reverse_str(key); +#endif + /* ************ [START] CORE DART HASH FOR EVERY SINGLE TOKEN ************** */ + if (op_type == OP_INSERT) { + // An INSERT operation happens + tmp_out_len = get_server_ids_for_insert(dart_g, tok, get_server_cb, &temp_out); + } + else { + // A query operation happens + tmp_out_len = get_server_ids_for_query(dart_g, tok, op_type, &temp_out); + } + /* ************ [END] CORE DART HASH FOR EVERY SINGLE TOKEN ************** */ + // now we have the result of DART hash, we need to merge it with the previous results. + ret_value += tmp_out_len; + if (*out == NULL) { + *out = (index_hash_result_t *)calloc(ret_value, sizeof(index_hash_result_t)); + } + else { + *out = (index_hash_result_t *)realloc(*out, ret_value * sizeof(index_hash_result_t)); + } + for (int j = 0; j < tmp_out_len; j++) { + (*out)[ret_value - tmp_out_len + j].server_id = temp_out[j]; + (*out)[ret_value - tmp_out_len + j].key = tok; + } + if (temp_out != NULL) + free(temp_out); + } + return ret_value; +} + +/** + * This function performs DHT hashing, solely for comparison purpose. + * + */ +int +DHT_hash(DART *dart_g, size_t len, char *key, dart_op_type_t op_type, index_hash_result_t **out) +{ + if (out == NULL) { + return 0; + } + uint64_t hashVal = djb2_hash(key, (int)len); + uint64_t server_id = hashVal % (dart_g->num_server); + int i = 0; + *out = NULL; + + int ret_value = 0; + int is_full_scan = 0; + if (is_index_write_op(op_type)) { + is_full_scan = 0; + } + else if (op_type == OP_EXACT_QUERY) { + is_full_scan = 0; + } + else if (op_type == OP_PREFIX_QUERY || op_type == OP_SUFFIX_QUERY) { + if (len > 1) { + is_full_scan = 1; + } + else { + is_full_scan = 0; + } + } + else if (op_type == OP_INFIX_QUERY) { + is_full_scan = 1; + } + + // according to different scan type, we return different results. + if (is_full_scan) { + ret_value = dart_g->num_server; + *out = (index_hash_result_t *)calloc(ret_value, sizeof(index_hash_result_t)); + for (i = 0; i < dart_g->num_server; i++) { + (*out)[i].server_id = i; + (*out)[i].key = key; + } + } + else { + ret_value = 1; + *out = (index_hash_result_t *)calloc(ret_value, sizeof(index_hash_result_t)); + (*out)[0].server_id = server_id; + (*out)[0].key = key; + } + return ret_value; +} \ No newline at end of file diff --git a/src/commons/index/dart/dart_core_test.c b/src/commons/index/dart/dart_core_test.c new file mode 100644 index 000000000..30f17c7d6 --- /dev/null +++ b/src/commons/index/dart/dart_core_test.c @@ -0,0 +1,92 @@ +#include "dart_core.h" + +int32_t request_count_g; + +dart_server *all_servers; + +dart_server +virtual_dart_retrieve_server_info_cb(uint32_t server_id) +{ + return all_servers[server_id]; +} + +int +main(int argc, char *argv[]) +{ + + if (argc != 4) { + printf("Usage: %s \n", argv[0]); + return 1; + } + + DART dart; + char *query_str = argv[1]; + int num_server = atoi(argv[2]); + int num_client = num_server * 120; + int alphabet_size = 27; + int extra_tree_height = 0; + int replication_factor = atoi(argv[3]); + replication_factor = replication_factor == 0 ? num_server / 10 : replication_factor; + + // Init all servers + all_servers = (dart_server *)malloc(num_server * sizeof(dart_server)); + for (int i = 0; i < num_server; i++) { + all_servers[i].id = i; + all_servers[i].indexed_word_count = 0; + all_servers[i].request_count = 0; + } + + dart_space_init(&dart, num_client, num_server, alphabet_size, extra_tree_height, replication_factor); + + println( + "num_server: %d, num_client: %d, alphabet_size: %d, extra_tree_height: %d, replication_factor: %d", + num_server, num_client, alphabet_size, extra_tree_height, replication_factor); + println("DART: num_vnode: %lu", dart.num_vnode); + + index_hash_result_t *out; + int array_len = DART_hash(&dart, query_str, OP_INSERT, NULL, &out); + // print out in the same line + printf("server ids for insert (%d): \n{", array_len); + for (int i = 0; i < array_len; i++) { + printf(" %d : %s,\n", out[i].server_id, out[i].key); + } + printf("\n}\n"); + + free(out); + + array_len = DART_hash(&dart, query_str, OP_EXACT_QUERY, NULL, &out); + printf("server ids for exact search (%d): \n{", array_len); + for (int i = 0; i < array_len; i++) { + printf(" %d : %s,\n", out[i].server_id, out[i].key); + } + printf("\n}\n"); + + free(out); + + array_len = DART_hash(&dart, substring(query_str, 0, strlen(query_str) - 3), OP_PREFIX_QUERY, NULL, &out); + printf("server ids for prefix search (%d): \n{", array_len); + for (int i = 0; i < array_len; i++) { + printf(" %d : %s,\n", out[i].server_id, out[i].key); + } + printf("\n}\n"); + + free(out); + + array_len = DART_hash(&dart, substring(query_str, 3, strlen(query_str)), OP_SUFFIX_QUERY, NULL, &out); + printf("server ids for suffix search (%d): \n{", array_len); + for (int i = 0; i < array_len; i++) { + printf(" %d : %s,\n", out[i].server_id, out[i].key); + } + printf("\n}\n"); + + free(out); + + array_len = DART_hash(&dart, substring(query_str, 2, strlen(query_str) - 2), OP_INFIX_QUERY, NULL, &out); + printf("server ids for infix search (%d): \n{", array_len); + for (int i = 0; i < array_len; i++) { + printf(" %d : %s,\n", out[i].server_id, out[i].key); + } + printf("\n}\n"); + + return 0; +} \ No newline at end of file diff --git a/src/commons/index/dart/dart_math.c b/src/commons/index/dart/dart_math.c new file mode 100644 index 000000000..4818be1b0 --- /dev/null +++ b/src/commons/index/dart/dart_math.c @@ -0,0 +1,43 @@ +#include "dart_math.h" + +#include +#include +#include +#include + +double +log_with_base(double base, double x) +{ + return log(x) / log(base); +} + +uint32_t +uint32_pow(uint32_t base, uint32_t pow) +{ + uint32_t p = 0; + uint32_t rst = 1; + for (p = 0; p < pow; p++) { + rst = base * rst; + } + return rst; +} + +uint64_t +uint64_pow(uint64_t base, uint64_t pow) +{ + uint64_t p = 0; + uint64_t rst = 1; + for (p = 0; p < pow; p++) { + rst = base * rst; + } + return rst; +} + +int +int_abs(int a) +{ + if (a < 0) { + return 0 - a; + } + return a; +} \ No newline at end of file diff --git a/src/commons/index/dart/dart_utils.c b/src/commons/index/dart/dart_utils.c new file mode 100644 index 000000000..2024b9bbc --- /dev/null +++ b/src/commons/index/dart/dart_utils.c @@ -0,0 +1,97 @@ +#include +#include +#include +#include +#include +#include "dart_utils.h" +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +void +random_seed(long int seed) +{ + if (seed > 0) + srand(seed); + else if (seed == 0) + srand(time(NULL)); +} + +double +random_range(double min, double max) +{ + // return numbers between a given range + double range = (max - min); + double div = RAND_MAX / range; + return min + (rand() / div); +} + +double +random_uniform() +{ + // return numbers between 0 and 1 + return (double)rand() / (RAND_MAX + 1.0); +} + +double +random_normal(double mean, double dev) +{ + // returns numbers based on normal distribution given mean and std dev + + // normal distribution centered on 0 with std dev of 1 + // Box-Muller transform + double randomNum_normal = + sqrt(-2 * log((rand() + 1.0) / (RAND_MAX + 1.0))) * cos(2 * M_PI * (rand() + 1.0) / (RAND_MAX + 1.0)); + double y = mean + dev * randomNum_normal; + + return y; +} + +double +random_exponential(double mean) +{ + // returns numbers based on exp distr given mean + double x = 1.0 / mean; // inverse lambda + + double u; // this will be my uniform random variable + double exp_value; + + // Pull a uniform random number (0 < z < 1) + do { + u = random_uniform(); + } while ((u == 0) || (u == 1)); + exp_value = -x * log(u); + + return (exp_value); +} + +int64_t +get_timestamp_ns() +{ + struct timespec current; + int64_t rst = -1; + + if (clock_gettime(CLOCK_REALTIME, ¤t) == -1) { + return rst; + } + + return current.tv_sec * (int64_t)1e9 + current.tv_nsec; +} + +int64_t +get_timestamp_us() +{ + return get_timestamp_ns() / 1000; +} + +int64_t +get_timestamp_ms() +{ + return get_timestamp_us() / 1000; +} + +int64_t +get_timestamp_s() +{ + return get_timestamp_ms() / 1000; +} \ No newline at end of file diff --git a/src/commons/index/dart/include/dart_algo.h b/src/commons/index/dart/include/dart_algo.h new file mode 100644 index 000000000..e5cd99636 --- /dev/null +++ b/src/commons/index/dart/include/dart_algo.h @@ -0,0 +1,24 @@ +#ifndef DART_ALGO_H +#define DART_ALGO_H + +#include +#include +#include +#include +#include +#include +#include "dart_math.h" + +void to_bytes(uint32_t val, uint8_t *bytes); + +uint32_t to_int32(const uint8_t *bytes); + +void md5(const uint8_t *initial_msg, size_t initial_len, uint8_t *digest); + +uint32_t murmur3_32(const uint8_t *key, size_t len, uint32_t seed); + +uint32_t md5_hash(int prefix_len, char *word); + +uint32_t djb2_hash(char *str, int len); + +#endif // DART_ALGO_H \ No newline at end of file diff --git a/src/commons/index/dart/include/dart_core.h b/src/commons/index/dart/include/dart_core.h new file mode 100644 index 000000000..fedfbcaa1 --- /dev/null +++ b/src/commons/index/dart/include/dart_core.h @@ -0,0 +1,198 @@ +/** + * The purpose of this file is to provide all the function declarations in DART library. + * DART should be designed as an library/framework handling the distributed inverted index. + * The current version should focus on managing distributed inverted index in HPC environment. + * + * Alternative version will be provided using JVM compliant language. + */ + +#ifndef DART_H +#define DART_H + +#include +#include "dart_utils.h" +#include "dart_algo.h" +#include "dart_math.h" +#include "thpool.h" +#include "string_utils.h" +#include "pdc_config.h" + +typedef enum { NUMERIC = 1, TIME = 2, CHAR = 3, BINARY = 4 } dart_indexed_value_type_t; + +typedef enum { DHT_FULL_HASH = 1, DHT_INITIAL_HASH = 2, DART_HASH = 3 } dart_hash_algo_t; + +typedef enum { + OP_INSERT = 1, + OP_EXACT_QUERY = 2, + OP_PREFIX_QUERY = 3, + OP_SUFFIX_QUERY = 4, + OP_INFIX_QUERY = 5, + OP_DELETE = 6 +} dart_op_type_t; + +typedef enum { REF_PRIMARY_ID = 1, REF_SECONDARY_ID = 2, REF_SERVER_ID = 3 } dart_object_ref_type_t; + +typedef struct { + uint32_t id; + uint32_t num_dst_server; + uint32_t *dst_server_ids; +} dart_vnode; + +typedef struct { + int alphabet_size; + int dart_tree_height; + int replication_factor; + int client_request_count; + uint32_t num_client; + uint32_t num_server; + uint64_t num_vnode; + dart_vnode *vnodes; + int suffix_tree_mode; +} DART; + +typedef struct { + uint32_t id; + DART * dart; +} dart_client; + +typedef struct { + uint32_t id; + int64_t indexed_word_count; + int64_t request_count; +} dart_server; + +typedef struct { + uint32_t server_id; + char * key; +} index_hash_result_t; + +// Defining a function pointer by which the server load information can be retrieved. +// The returning data type should be dart_server, which is a struct. +// The parameter should be a uint32_t. +// The function name can be anything. +typedef dart_server (*get_server_info_callback)(uint32_t server_id); + +/** + * Initialize the DART space. + * + * + */ +void dart_space_init(DART *dart, int num_client, int num_server, int alphabet_size, int extra_tree_height, + int replication_factor); + +/** + * This function make the client request counter increment by 1. + * ** We should consider to make this as an atomic operation. + */ +int dart_client_request_count_incr(DART *dart_g); + +/** + * A utility function for dummies. + * Get server id by given virtual node id. + */ +uint64_t get_server_id_by_vnode_id(DART *dart, uint64_t vnode_id); + +/** + * This function is for getting the base virtual node ID by a given string. + * + */ +uint64_t get_base_virtual_node_id_by_string(DART *dart, char *str); + +/** + * This function is for getting the alternative virtual node ID. + * + */ +uint64_t get_reconciled_vnode_id_with_power_of_two_choice_rehashing(DART *dart, uint64_t base_vnode_idx, + char * word, + get_server_info_callback get_server_cb); + +/** + * Get IDs of all virtual nodes of replicas by given string and overall tree-height setting. + * + * + * return The length of the ID array. + */ +int get_replica_node_ids(DART *dart, uint64_t master_node_id, int is_physical, uint64_t **out); + +/** + * This function takes a keyword, creates an index record in DART and returns all server ids + * including replica servers. The caller of this function should perform index create PRC call + * to all the servers specified by the ID list. + * + * \param dart_g [IN] Pointer of Global state of DART + * \param keyword [IN] The keyword which should be indexed. + * \param get_server_cb [IN] A callback function which determines the lighter-load server between + * two servers. \param out [OUT] The pointer of the server ID array. \return The length of + * the server ID. + */ +int get_server_ids_for_insert(DART *dart_g, char *keyword, get_server_info_callback get_server_cb, + uint64_t **out); + +/** + * This function gives all the server ID according to the given query token. + * The query token should not contain any wildcard sign. + * + * Once the server IDs are retrieved, the query should be broadcasted to all these servers. + * + * Also, this function can be used for index updating and index deletion operations. + * + * Notice that, for index updating and index deletion operations, the given token must represent + * an exact keyword, thus, the result will be an ID list of two servers which are the possible + * locations of the specified keyword. + * + * Two different type of updating operations should be considered. + * + * 1. Add/Remove one ID from the set of IDs related to the keywords. + * 2. Updating the attribute value of an object according to object ID. + * + * *** If bulk transfer may overwhelm the system, + * *** We need to define a protocol for client and server to communicate without bulk transfer. + * + * + * \param dart_g [IN] Pointer of Global state of DART + * \param token [IN] The token which will searched for. It should not contain any wildcard + * sign. \param op_type [IN] Give the operation type. \param out [IN/OUT] The + * pointer of the server ID array. To accelerate, we require caller function to pass an ID array of length M + * into this function, where M is the number of total physical servers. Then, this function will fill up this + * array. \return [OUT] The valid length of the server ID array. + */ +int get_server_ids_for_query(DART *dart_g, char *token, dart_op_type_t is_prefix, uint64_t **out); + +/** + * This is a routing function for different possible operations in DART. + * It will only generate the array of IDs of servers that client should touch. + * + * The return value is the length of valid elements in the array. + * A for loop can use the return value as the upper bound to iterate all elements in the array. + * + * \param dart_g [IN] Pointer of Global state of DART + * \param key [IN] The given key. For queries, it is already a bare keyword without '*'. + * \param op_type [IN] Give the operation type. + * \param get_server_cb [IN] The callback function for getting the statistical information of a server. + * Signature: dart_server (*get_server_info_callback)(uint32_t server_id) \param out [IN/OUT] The + * pointer of the server ID array. To accelerate, we require caller function to pass an ID array of length M + * into this function, where M is the number of total physical servers. Then, this function will fill up this + * array. \return [OUT] The valid length of the server ID array. + * + */ +int DART_hash(DART *dart_g, char *key, dart_op_type_t op_type, get_server_info_callback get_server_cb, + index_hash_result_t **out); + +/** + * This function performs DHT hashing, solely for comparison purpose. + * + * out will take an array of the IDs of all servers which should be accessed + * return value is the length of the array. + */ +int DHT_hash(DART *dart_g, size_t len, char *key, dart_op_type_t op_type, index_hash_result_t **out); + +/** + * This is a quick function to test if the operation type any type of operation that will write to the index. + * These operations currently include: insert, delete. + * + * @param op_type [IN] The operation type. + * @return 1 if the operation type is for index write operations, 0 otherwise. + */ +int is_index_write_op(dart_op_type_t op_type); + +#endif // DART_H \ No newline at end of file diff --git a/src/commons/index/dart/include/dart_math.h b/src/commons/index/dart/include/dart_math.h new file mode 100644 index 000000000..d4a85546b --- /dev/null +++ b/src/commons/index/dart/include/dart_math.h @@ -0,0 +1,16 @@ +#ifndef DART_MATH_H +#define DART_MATH_H + +#include +#include +#include +#include +#include +#include "dart_utils.h" + +double log_with_base(double base, double x); +uint32_t uint32_pow(uint32_t base, uint32_t pow); +uint64_t uint64_pow(uint64_t base, uint64_t pow); +int int_abs(int a); + +#endif // END DART_MATH_H \ No newline at end of file diff --git a/src/commons/index/dart/include/dart_utils.h b/src/commons/index/dart/include/dart_utils.h new file mode 100644 index 000000000..a8caa419e --- /dev/null +++ b/src/commons/index/dart/include/dart_utils.h @@ -0,0 +1,25 @@ +#ifndef DART_UTILS_H +#define DART_UTILS_H + +#include +#include +#include +#include +#include +#include + +void random_seed(long int seed); +double random_uniform(); +double random_range(double min, double max); +double random_normal(double mean, double dev); +double random_exponential(double mean); + +int64_t get_timestamp_ns(); + +int64_t get_timestamp_us(); + +int64_t get_timestamp_ms(); + +int64_t get_timestamp_s(); + +#endif // DART_UTILS_H \ No newline at end of file diff --git a/src/api/profiling/include/pdc_hashtab.h b/src/commons/profiling/include/pdc_hashtab.h similarity index 100% rename from src/api/profiling/include/pdc_hashtab.h rename to src/commons/profiling/include/pdc_hashtab.h diff --git a/src/api/profiling/include/pdc_stack_ops.h b/src/commons/profiling/include/pdc_stack_ops.h similarity index 100% rename from src/api/profiling/include/pdc_stack_ops.h rename to src/commons/profiling/include/pdc_stack_ops.h diff --git a/src/api/profiling/pdc_hashtab.c b/src/commons/profiling/pdc_hashtab.c similarity index 100% rename from src/api/profiling/pdc_hashtab.c rename to src/commons/profiling/pdc_hashtab.c diff --git a/src/api/profiling/pdc_stack_ops.c b/src/commons/profiling/pdc_stack_ops.c similarity index 100% rename from src/api/profiling/pdc_stack_ops.c rename to src/commons/profiling/pdc_stack_ops.c diff --git a/src/commons/serde/bulki.c b/src/commons/serde/bulki.c new file mode 100644 index 000000000..a60441f31 --- /dev/null +++ b/src/commons/serde/bulki.c @@ -0,0 +1,115 @@ +#include "bulki.h" + +BULKI * +pdc_serde_init(int initial_field_count) +{ + BULKI *data = malloc(sizeof(BULKI)); + data->numKeys = initial_field_count; + data->header = malloc(sizeof(BULKI_Header)); + data->header->keys = malloc(sizeof(BULKI_Key) * initial_field_count); + data->header->totalSize = 0; + data->data = malloc(sizeof(BULKI_Data)); + data->data->values = malloc(sizeof(BULKI_Value) * initial_field_count); + data->data->totalSize = 0; + return data; +} + +void +pdc_serde_append_key_value(BULKI *data, BULKI_Key *key, BULKI_Value *value) +{ + data->header->keys[data->numKeys] = *key; + // append bytes for type, size, and key + data->header->totalSize += (sizeof(uint8_t) + sizeof(uint64_t) + key->size); + + data->data->values[data->numValues] = *value; + // append bytes for class, type, size, and data + data->data->totalSize += (sizeof(uint8_t) + sizeof(uint8_t) + sizeof(uint64_t) + value->size); + + data->numKeys++; + data->totalSize = data->header->totalSize + data->data->totalSize + sizeof(uint64_t) * 6; +} + +BULKI_Key * +BULKI_KEY(void *key, pdc_c_var_type_t pdc_type, uint64_t size) +{ + BULKI_Key *pdc_key = (BULKI_Key *)malloc(sizeof(BULKI_Key)); + uint64_t key_size = (uint64_t)get_size_by_class_n_type(key, size, PDC_CLS_SCALAR, pdc_type); + pdc_key->key = malloc(key_size); + memcpy(pdc_key->key, key, key_size); + pdc_key->pdc_type = pdc_type; + pdc_key->size = key_size; + return pdc_key; +} + +BULKI_Value * +BULKI_VALUE(void *data, pdc_c_var_type_t pdc_type, pdc_c_var_class_t pdc_class, uint64_t size) +{ + BULKI_Value *pdc_value = (BULKI_Value *)malloc(sizeof(BULKI_Value)); + size_t value_size = size; + if (pdc_class == PDC_CLS_STRUCT) { + // we are postponing the serialization of the embedded SERDE_SerializedData, so no need to check the + // type here. + BULKI *struct_data = (BULKI *)data; + value_size = struct_data->totalSize; + pdc_value->data = data; + } + else if (pdc_class <= PDC_CLS_ARRAY) { + value_size = (size_t)get_size_by_class_n_type(data, size, pdc_class, pdc_type); + pdc_value->data = malloc(value_size); + memcpy(pdc_value->data, data, value_size); + } + else { + printf("Error: unsupported class %d\n", pdc_class); + return NULL; + } + pdc_value->pdc_class = pdc_class; + pdc_value->pdc_type = pdc_type; + pdc_value->size = value_size; + return pdc_value; +} + +void +pdc_serde_free(BULKI *data) +{ + for (size_t i = 0; i < data->header->numKeys; i++) { + free(data->header->keys[i].key); + } + free(data->header->keys); + for (size_t i = 0; i < data->data->numValues; i++) { + free(data->data->values[i].data); + } + free(data->data->values); + free(data->header); + free(data->data); + free(data); +} + +void +pdc_serde_print(BULKI *data) +{ + printf("Header:\n"); + printf(" numKeys: %zu\n", data->header->numKeys); + printf(" totalSize: %zu\n", data->header->totalSize); + for (size_t i = 0; i < data->header->numKeys; i++) { + printf(" key %ld:\n", i); + printf(" type: %d\n", data->header->keys[i].pdc_type); + printf(" size: %zu\n", data->header->keys[i].size); + printf(" key: %s\n", (char *)data->header->keys[i].key); + } + printf("Data:\n"); + printf(" numValues: %zu\n", data->data->numValues); + printf(" totalSize: %zu\n", data->data->totalSize); + for (size_t i = 0; i < data->data->numValues; i++) { + printf(" value %ld:\n", i); + printf(" class: %d\n", data->data->values[i].pdc_class); + printf(" type: %d\n", data->data->values[i].pdc_type); + printf(" size: %zu\n", data->data->values[i].size); + printf(" data: "); + if (data->data->values[i].pdc_type == PDC_STRING) { + printf("%s\n", (char *)data->data->values[i].data); + } + else { + printf("\n"); + } + } +} \ No newline at end of file diff --git a/src/commons/serde/bulki_serde.c b/src/commons/serde/bulki_serde.c new file mode 100644 index 000000000..3d6a0caf1 --- /dev/null +++ b/src/commons/serde/bulki_serde.c @@ -0,0 +1,229 @@ +#include "bulki_serde.h" + +uint64_t +get_total_size_for_serialized_data(BULKI *data) +{ + + if (data->totalSize <= 0) { + size_t total_size = data->header->totalSize + data->data->totalSize + sizeof(uint64_t) * 6; + data->totalSize = total_size; + } + return data->totalSize; +} + +// clang-format off +/** + * This function serializes the entire BULKI structure. + * + * The overview of the serialized binary data layout is: + * +---------------------+---------------------+---------------------+----------------------+----------------------+----------------------+ + * | Size of the Header | Size of the Data | Header Region | Data Offset | Data Region | Data Offset | + * | (uint64_t) | (uint64_t) | | (uint64_t) | | (uint64_t) | + * +---------------------+---------------------+---------------------+----------------------+----------------------+----------------------+ + * + * The first 3 field is called meta-header, which provides metadata about size of the header region and the size of the data region. + * Note that the size of the header region doesn't include the 'Number of Keys' field. + * Also, the size of the data region doesn't include the 'Data Offset' field. + * + * Then the following is the header region with two keys: + * +----------------------+-------------------------+-----------------------------+---------------------------+--------------------------+-----------------------------+---------------------------+ + * | Number of K-Vs | Key 1 Type | Key 1 Size | Key 1 Data | Key 2 Type | Key 2 Size | Key 2 Data | + * | (uint64_t) | (uint8_t) | (uint64_t) | (Variable size depending | (uint8_t) | (uint64_t) | (Variable size depending | + * | | | | on Key 1 Size) | | | on Key 2 Size) | + * +----------------------+-------------------------+-----------------------------+---------------------------+--------------------------+-----------------------------+---------------------------+ + * + * Then, the following is the layout of the data region with the final offset validation point. + * + * |----------------------------------------------------------------------------------------------------------------| + * | Number of K-V Pairs (uint64_t) | Value 1 Class (uint8_t) | Value 1 Type (uint8_t) | Value 1 Size (uint64_t)| + * |----------------------------------------------------------------------------------------------------------------| + * | Value 1 Data (Variable size depending on Value 1 Size) | Value 2 Class (uint8_t) | Value 2 Type (uint8_t) | + * |----------------------------------------------------------------------------------------------------------------| + * | Value 2 Size (uint64_t) | Value 2 Data (Variable size depending on Value 2 Size) | + * |----------------------------------------------------------------------------------------------------------------| + * | ...repeated for the number of value entries in the data... | + * |----------------------------------------------------------------------------------------------------------------| + * | Final Data Offset (uint64_t) | + * |----------------------------------------------------------------------------------------------------------------| + * + * Please refer to `get_size_by_class_n_type` function in pdc_generic.h for size calculation on scalar values and array values. + * + */ +// clang-format on +void * +BULKI_serde_serialize(BULKI *data) +{ + // The buffer contains: + // the size of the header (size_t) + + // the size of the data (size_t) + + // the number of keys (size_t) + + // the header region + + // the data offset (size_t) + + // the number of value entries (size_t) + + // the data region + void *buffer = malloc(get_total_size_for_serialized_data(data)); + // serialize the meta header, which contains only the size of the header and the size of the data region. + memcpy(buffer, &data->header->totalSize, sizeof(size_t)); + memcpy(buffer + sizeof(size_t), &data->data->totalSize, sizeof(size_t)); + + // serialize the header + // start with the number of keys + memcpy(buffer + sizeof(size_t) * 2, &data->header->numKeys, sizeof(size_t)); + // then the keys + size_t offset = sizeof(size_t) * 3; + for (size_t i = 0; i < data->header->numKeys; i++) { + int8_t pdc_type = (int8_t)(data->header->keys[i].pdc_type); + memcpy(buffer + offset, &pdc_type, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(buffer + offset, &data->header->keys[i].size, sizeof(size_t)); + offset += sizeof(size_t); + memcpy(buffer + offset, data->header->keys[i].key, data->header->keys[i].size); + offset += data->header->keys[i].size; + } + + // serialize the data offset, this is for validation purpose to see if header region is corrupted. + memcpy(buffer + offset, &offset, sizeof(size_t)); + offset += sizeof(size_t); + + // serialize the data + // start with the number of value entries + memcpy(buffer + offset, &data->data->numValues, sizeof(size_t)); + offset += sizeof(size_t); + // then the values + for (size_t i = 0; i < data->data->numValues; i++) { + int8_t pdc_class = (int8_t)data->data->values[i].pdc_class; + int8_t pdc_type = (int8_t)data->data->values[i].pdc_type; + memcpy(buffer + offset, &pdc_class, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(buffer + offset, &pdc_type, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(buffer + offset, &data->data->values[i].size, sizeof(size_t)); + offset += sizeof(size_t); + + if (data->data->values[i].pdc_class == PDC_CLS_STRUCT) { + void *sdata = BULKI_serde_serialize((BULKI *)(data->data->values[i].data)); + memcpy(buffer + offset, sdata, data->data->values[i].size); + } + else if (data->data->values[i].pdc_class <= PDC_CLS_ARRAY) { + memcpy(buffer + offset, data->data->values[i].data, data->data->values[i].size); + } + else { + printf("Error: unsupported class type %d\n", data->data->values[i].pdc_class); + return NULL; + } + + offset += data->data->values[i].size; + memcpy(buffer + offset, data->data->values[i].data, data->data->values[i].size); + offset += data->data->values[i].size; + } + // serialize the data offset again, this is for validation purpose to see if data region is corrupted. + memcpy(buffer + offset, &offset, sizeof(size_t)); + offset += sizeof(size_t); + return buffer; +} + +BULKI * +BULKI_serde_deserialize(void *buffer) +{ + size_t offset = 0; + // read the meta header + size_t headerSize; + size_t dataSize; + memcpy(&headerSize, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + memcpy(&dataSize, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + + printf("headerSize: %zu\n", headerSize); + printf("dataSize: %zu\n", dataSize); + + // read the header + size_t numKeys; + memcpy(&numKeys, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + + printf("numKeys: %zu\n", numKeys); + + BULKI_Header *header = malloc(sizeof(BULKI_Header)); + header->keys = malloc(sizeof(BULKI_Key) * numKeys); + header->numKeys = numKeys; + header->totalSize = headerSize; + + printf("iterating %zu keys in the header\n", numKeys); + + for (size_t i = 0; i < numKeys; i++) { + int8_t pdc_type; + size_t size; + memcpy(&pdc_type, buffer + offset, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(&size, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + void *key = malloc(size); + memcpy(key, buffer + offset, size); + offset += size; + header->keys[i].key = key; + header->keys[i].pdc_type = (pdc_c_var_type_t)pdc_type; + header->keys[i].size = size; + + printf("key %zu: %s, size: %zu, type: %s\n", i, (char *)key, size, get_name_by_dtype(pdc_type)); + } + + // read the data offset + size_t dataOffset; + memcpy(&dataOffset, buffer + offset, sizeof(size_t)); + // check the data offset + if (dataOffset != offset) { + printf("Error: data offset does not match the expected offset.\n"); + return NULL; + } + offset += sizeof(size_t); + + // read the data + size_t numValues; + memcpy(&numValues, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + BULKI_Data *data = malloc(sizeof(BULKI_Data)); + data->values = malloc(sizeof(BULKI_Value) * numValues); + data->numValues = numValues; + data->totalSize = dataSize; + for (size_t i = 0; i < numValues; i++) { + int8_t pdc_class; + int8_t pdc_type; + size_t size; + memcpy(&pdc_class, buffer + offset, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(&pdc_type, buffer + offset, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(&size, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + void *value = malloc(size); + memcpy(value, buffer + offset, size); + offset += size; + + // TODO: postponed deserialization of struct data, need to be finished here. + data->values[i].pdc_class = (pdc_c_var_class_t)pdc_class; + data->values[i].pdc_type = (pdc_c_var_type_t)pdc_type; + data->values[i].size = size; + data->values[i].data = value; + printf("value %zu: size: %zu, type: %s\n", i, size, get_name_by_dtype(pdc_type)); + } + // check the total size + memcpy(&dataOffset, buffer + offset, sizeof(size_t)); + // check the data offset + if (dataOffset != offset) { + printf("Error: data offset does not match the expected offset.\n"); + return NULL; + } + offset += sizeof(size_t); + if (offset != headerSize + sizeof(size_t) * 6 + dataSize) { + printf("Error: total size does not match the expected size.\n"); + return NULL; + } + // create the serialized data + BULKI *serializedData = malloc(sizeof(BULKI)); + serializedData->header = header; + serializedData->data = data; + serializedData->totalSize = headerSize + dataSize + sizeof(size_t) * 6; + + return serializedData; +} diff --git a/src/commons/serde/bulki_serde_test.c b/src/commons/serde/bulki_serde_test.c new file mode 100644 index 000000000..59ebbae9d --- /dev/null +++ b/src/commons/serde/bulki_serde_test.c @@ -0,0 +1,84 @@ +#include "bulki_serde.h" + +int +test_serde_framework() +{ + // Initialize a serialized data structure + BULKI *data = BULKI_serde_init(5); + + // Create and append key-value pairs for different data types + char * intKey_str = "int"; + int intVal = 42; + BULKI_Key * intKey = BULKI_KEY(intKey_str, PDC_STRING, sizeof(intKey_str)); + BULKI_Value *intValue = BULKI_VALUE(&intVal, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); + BULKI_serde_append_key_value(data, intKey, intValue); + + char * doubleKey_str = "double"; + double doubleVal = 3.14159; + BULKI_Key * doubleKey = BULKI_KEY(doubleKey_str, PDC_STRING, sizeof(doubleKey_str)); + BULKI_Value *doubleValue = BULKI_VALUE(&doubleVal, PDC_DOUBLE, PDC_CLS_SCALAR, sizeof(double)); + BULKI_serde_append_key_value(data, doubleKey, doubleValue); + + char * strKey_str = "string"; + char * strVal = "Hello, World!"; + BULKI_Key * strKey = BULKI_KEY(strKey_str, PDC_STRING, (strlen(strKey_str) + 1) * sizeof(char)); + BULKI_Value *strValue = + BULKI_VALUE(strVal, PDC_STRING, PDC_CLS_SCALAR, (strlen(strVal) + 1) * sizeof(char)); + BULKI_serde_append_key_value(data, strKey, strValue); + + char * arrayKey_str = "array"; + int intArray[3] = {1, 2, 3}; + BULKI_Key * arrayKey = BULKI_KEY(arrayKey_str, PDC_STRING, sizeof(arrayKey_str)); + BULKI_Value *arrayValue = BULKI_VALUE(intArray, PDC_INT, PDC_CLS_ARRAY, 3); + BULKI_serde_append_key_value(data, arrayKey, arrayValue); + + typedef struct { + int x; + int y; + } Point; + + Point pointVal = {10, 20}; + + // prepare the data of a struct + BULKI * point_data = BULKI_serde_init(2); + BULKI_Key * x_name = BULKI_KEY("x", PDC_STRING, sizeof(char *)); + BULKI_Value *x_value = BULKI_VALUE(&pointVal.x, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); + BULKI_Key * y_name = BULKI_KEY("y", PDC_STRING, sizeof(char *)); + BULKI_Value *y_value = BULKI_VALUE(&pointVal.y, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); + + BULKI_serde_append_key_value(point_data, x_name, x_value); + BULKI_serde_append_key_value(point_data, y_name, y_value); + + // append the struct data as a key value pair, along with a key. + char * pointKey = "point"; + BULKI_Key * structKey = BULKI_KEY(pointKey, PDC_STRING, sizeof(pointKey)); + BULKI_Value *structValue = BULKI_VALUE(point_data, PDC_VOID_PTR, PDC_CLS_STRUCT, sizeof(Point)); + BULKI_serde_append_key_value(data, structKey, structValue); + + // Serialize the data + void *buffer = BULKI_serde_serialize(data); + + printf("Serialized data:\n"); + BULKI_serde_print(data); + + // Deserialize the buffer + BULKI *deserializedData = BULKI_serde_deserialize(buffer); + + printf("Deserialized data:\n"); + + // Print the deserialized data + BULKI_serde_print(deserializedData); + + // Free the memory + BULKI_serde_free(data); + BULKI_serde_free(deserializedData); + free(buffer); + + return 0; +} + +int +main(int argc, char *argv[]) +{ + return test_serde_framework(); +} \ No newline at end of file diff --git a/src/commons/serde/include/bulki.h b/src/commons/serde/include/bulki.h new file mode 100644 index 000000000..168af576e --- /dev/null +++ b/src/commons/serde/include/bulki.h @@ -0,0 +1,101 @@ +#ifndef BULKI_H +#define BULKI_H + +#include +#include +#include +#include +#include +#include "pdc_generic.h" + +typedef struct { + pdc_c_var_type_t pdc_type; /**< Data type of the key */ + uint64_t size; /**< Size of the key */ + void * key; /**< Pointer to the key data */ +} BULKI_Key; + +typedef struct { + pdc_c_var_class_t pdc_class; /**< Class of the value */ + pdc_c_var_type_t pdc_type; /**< Data type of the value */ + uint64_t size; // size of the data. If a string, it is strlen(data) + 1; + // if an array, it is the number of elements; + // if a struct, it is the totalSize of the data chunk of the struct, etc. + void *data; /**< Pointer to the value data */ +} BULKI_Value; + +typedef struct { + BULKI_Key *keys; /**< Array of keys */ + size_t totalSize; /**< Total size of the header */ +} BULKI_Header; + +typedef struct { + BULKI_Value *values; /**< Array of values */ + uint64_t totalSize; /**< Total size of the data */ +} BULKI_Data; + +typedef struct { + BULKI_Header *header; /**< Pointer to the header */ + BULKI_Data * data; /**< Pointer to the data */ + uint64_t totalSize; /**< Total size of the serialized data */ + uint64_t numKeys; /**< Number of keys */ +} BULKI; + +/** + * @brief Initialize a serialized data structure + * + * @param initial_field_count Number of initial fields to allocate space for + * + * @return Pointer to the initialized BULKI structure + */ +BULKI *BULKI_serde_init(int initial_field_count); + +/** + * @brief Append a key-value pair to the serialized data structure + * + * @param data Pointer to the BULKI structure + * @param key Pointer to the BULKI_Key structure representing the key + * @param value Pointer to the BULKI_Value structure representing the value + */ +void BULKI_serde_append_key_value(BULKI *data, BULKI_Key *key, BULKI_Value *value); + +/** + * @brief Free the memory allocated for the serialized data structure + * + * @param data Pointer to the BULKI structure to be freed + */ +void BULKI_serde_free(BULKI *data); + +/** + * @brief Print the contents of the serialized data structure + * + * @param data Pointer to the BULKI structure to be printed + */ +void BULKI_serde_print(BULKI *data); + +/** + * @brief Create a BULKI_Key structure + * + * @param key Pointer to the key data + * @param pdc_type Data type of the key. For BULKI_Key, we only support PDC_CLS_SCALAR class. + * @param size Size of the key data + * + * @return Pointer to the created BULKI_Key structure + */ +BULKI_Key *BULKI_KEY(void *key, pdc_c_var_type_t pdc_type, uint64_t size); + +/** + * @brief Create a BULKI_Value structure + * + * @param data Pointer to the value data + * @param pdc_type Data type of the value + * @param pdc_class Class of the value + * @param size Size of the value data. + * For scalar value, it is the result of sizeof(type) function; + * for array, it is the number of elements; + * for struct, it is the totalSize of the data chunk of the struct, etc. + * + * @return Pointer to the created BULKI_Value structure + */ +BULKI_Value *BULKI_VALUE(void *data, pdc_c_var_type_t pdc_type, pdc_c_var_class_t pdc_class, uint64_t size); + +#endif /* BULKI_H */ \ No newline at end of file diff --git a/src/commons/serde/include/bulki_serde.h b/src/commons/serde/include/bulki_serde.h new file mode 100644 index 000000000..bd139881a --- /dev/null +++ b/src/commons/serde/include/bulki_serde.h @@ -0,0 +1,42 @@ +#ifndef BULKI_SERDE_H +#define BULKI_SERDE_H + +#include +#include +#include +#include +#include +#include "pdc_generic.h" +#include "bulki.h" + +#define MAX_KEYS 10 +#define MAX_BUFFER_SIZE 1000 + +/** + * @brief get the total size of BULKI structure instance + * + * @param data Pointer to the BULKI structure instance + * + * @return total size of the BULKI structure instance + */ +uint64_t get_total_size_for_serialized_data(BULKI *data); + +/** + * @brief Serialize the data in the serialized data structure and return the buffer + * + * @param data Pointer to the BULKI structure + * + * @return Pointer to the buffer containing the serialized data + */ +void *BULKI_serde_serialize(BULKI *data); + +/** + * @brief Deserialize the buffer and return the deserialized data structure + * + * @param buffer Pointer to the buffer containing the serialized data + * + * @return Pointer to the deserialized BULKI structure + */ +BULKI *BULKI_serde_deserialize(void *buffer); + +#endif /* BULKI_SERDE_H */ \ No newline at end of file diff --git a/src/commons/utils/Readme.md b/src/commons/utils/Readme.md new file mode 100644 index 000000000..e69de29bb diff --git a/src/utils/include/pdc_id_pkg.h b/src/commons/utils/include/pdc_id_pkg.h similarity index 96% rename from src/utils/include/pdc_id_pkg.h rename to src/commons/utils/include/pdc_id_pkg.h index 9623661fd..ba858c82d 100644 --- a/src/utils/include/pdc_id_pkg.h +++ b/src/commons/utils/include/pdc_id_pkg.h @@ -25,7 +25,7 @@ #ifndef PDC_ID_PKG_H #define PDC_ID_PKG_H -#include "pdc_private.h" +#include "pdc_public.h" #include "pdc_linkedlist.h" #include "mercury_atomic.h" /* @@ -47,6 +47,11 @@ /* Map an atom to an ID type number */ #define PDC_TYPE(a) ((PDC_type_t)(((pdcid_t)(a) >> ID_BITS) & TYPE_MASK)) +struct _pdc_class { + char * name; + pdcid_t local_id; +}; + struct _pdc_id_info { pdcid_t id; /* ID for this info */ hg_atomic_int32_t count; /* ref. count for this atom */ diff --git a/src/utils/include/pdc_linkedlist.h b/src/commons/utils/include/pdc_linkedlist.h similarity index 99% rename from src/utils/include/pdc_linkedlist.h rename to src/commons/utils/include/pdc_linkedlist.h index 6fa7ce08a..a9de691b0 100644 --- a/src/utils/include/pdc_linkedlist.h +++ b/src/commons/utils/include/pdc_linkedlist.h @@ -43,8 +43,8 @@ #ifndef PDC_LINKEDLIST_H #define PDC_LINKEDLIST_H -#include "pdc_cont_pkg.h" -#include "pdc_cont.h" +// #include "pdc_cont_pkg.h" +// #include "pdc_cont.h" #include "mercury_thread_mutex.h" #include diff --git a/src/utils/include/pdc_malloc.h b/src/commons/utils/include/pdc_malloc.h similarity index 66% rename from src/utils/include/pdc_malloc.h rename to src/commons/utils/include/pdc_malloc.h index e8ea1941e..8c3a36335 100644 --- a/src/utils/include/pdc_malloc.h +++ b/src/commons/utils/include/pdc_malloc.h @@ -31,28 +31,53 @@ /* Library-private Function Prototypes */ /***************************************/ /** - * Create an object + * allocate memory * * \param size [IN] Size of the struct to be malloced */ void *PDC_malloc(size_t size); /** - * Create an object + * allocate memory and add size to specified memory size pointer + */ +void *PDC_malloc_addsize(size_t size, size_t *mem_usage_ptr); + +/** + * allocate memory and set to zero * * \param size [IN] Size of the struct to be calloced */ -void *PDC_calloc(size_t size); +void *PDC_calloc(size_t count, size_t size); + +/** + * allocate zero-filled memory and add size to specified memory size pointer + */ +void *PDC_calloc_addsize(size_t count, size_t size, size_t *mem_usage_ptr); /** - * Create an object + * adjust the size of the memory block pointed to by ptr + */ +void *PDC_realloc(void *ptr, size_t size); + +/** + * realloc memory and add size to specified memory size pointer + */ +void *PDC_realloc_addsize(void *ptr, size_t size, size_t *mem_usage_ptr); + +/** + * free allocated memory * * \param mem [IN] Starting address of memory */ void *PDC_free(void *mem); -#define PDC_MALLOC(t) (t *)PDC_malloc(sizeof(t)) -#define PDC_CALLOC(t) (t *)PDC_calloc(sizeof(t)) +/** + * Get total memory usage from the global variable + */ +size_t PDC_get_global_mem_usage(); + +#define PDC_MALLOC(t) (t *)PDC_malloc(sizeof(t)) +#define PDC_CALLOC(c, t) (t *)PDC_calloc(c, sizeof(t)) #define PDC_FREE(t, obj) (t *)(intptr_t) PDC_free(obj) diff --git a/src/utils/include/pdc_private.h b/src/commons/utils/include/pdc_private.h similarity index 63% rename from src/utils/include/pdc_private.h rename to src/commons/utils/include/pdc_private.h index b6ca3bcfd..8ae8886a8 100644 --- a/src/utils/include/pdc_private.h +++ b/src/commons/utils/include/pdc_private.h @@ -28,6 +28,22 @@ #include "pdc_config.h" #include "pdc_public.h" #include +#include + +#include "mercury.h" +#include "mercury_macros.h" +#include "mercury_proc_string.h" +#include "mercury_list.h" +#include "mercury_config.h" +#include "mercury_thread_pool.h" +#include "mercury_atomic.h" + +#ifdef ENABLE_MULTITHREAD +#include "mercury_thread_pool.h" +#include "mercury_thread_condition.h" +#include "mercury_thread_mutex.h" +#endif + // #include /* gettimeofday() */ /****************************/ @@ -60,18 +76,9 @@ typedef enum { PDC_Q_MATCH_GREATER_THAN /* greater than */ } _pdc_query_op_t; -typedef enum { ROW_major, COL_major } _pdc_major_type_t; - -typedef enum { C_lang = 0, FORTRAN_lang, PYTHON_lang, JULIA_lang, N_LANGUAGES } _pdc_analysis_language_t; - /***************************/ /* Library Private Structs */ /***************************/ -struct _pdc_class { - char * name; - pdcid_t local_id; -}; - #ifdef __cplusplus #define ATTRIBUTE(a) #else /* __cplusplus */ @@ -146,7 +153,7 @@ extern pbool_t err_occurred; /* Include a basic profiling interface */ #ifdef ENABLE_PROFILING -#include "stack_ops.h" +#include "pdc_stack_ops.h" #define FUNC_ENTER(X) \ do { \ @@ -199,4 +206,60 @@ extern pbool_t err_occurred; } while (0) #endif +#if defined(IS_PDC_SERVER) && defined(ENABLE_MULTITHREAD) + +// Macros for multi-thread callback, grabbed from Mercury/Testing/mercury_rpc_cb.c +#define HG_TEST_RPC_CB(func_name, handle) static hg_return_t func_name##_thread_cb(hg_handle_t handle) + +/* Assuming func_name_cb is defined, calling HG_TEST_THREAD_CB(func_name) + * will define func_name_thread and func_name_thread_cb that can be used + * to execute RPC callback from a thread + */ +#define HG_TEST_THREAD_CB(func_name) \ + static HG_INLINE HG_THREAD_RETURN_TYPE func_name##_thread(void *arg) \ + { \ + hg_handle_t handle = (hg_handle_t)arg; \ + hg_thread_ret_t thread_ret = (hg_thread_ret_t)0; \ + \ + func_name##_thread_cb(handle); \ + \ + return thread_ret; \ + } \ + hg_return_t func_name##_cb(hg_handle_t handle) \ + { \ + struct hg_thread_work *work = HG_Get_data(handle); \ + hg_return_t ret = HG_SUCCESS; \ + \ + work->func = func_name##_thread; \ + work->args = handle; \ + hg_thread_pool_post(hg_test_thread_pool_g, work); \ + \ + return ret; \ + } +#else +#define HG_TEST_RPC_CB(func_name, handle) hg_return_t func_name##_cb(hg_handle_t handle) +#define HG_TEST_THREAD_CB(func_name) + +#endif // End of ENABLE_MULTITHREAD + +#define PDC_FUNC_DECLARE_REGISTER(x) \ + hg_id_t PDC_##x##_register(hg_class_t *hg_class) \ + { \ + hg_id_t ret_value; \ + FUNC_ENTER(NULL); \ + ret_value = MERCURY_REGISTER(hg_class, #x, x##_in_t, x##_out_t, x##_cb); \ + FUNC_LEAVE(ret_value); \ + return ret_value; \ + } + +#define PDC_FUNC_DECLARE_REGISTER_IN_OUT(x, y, z) \ + hg_id_t PDC_##x##_register(hg_class_t *hg_class) \ + { \ + hg_id_t ret_value; \ + FUNC_ENTER(NULL); \ + ret_value = MERCURY_REGISTER(hg_class, #x, y, z, x##_cb); \ + FUNC_LEAVE(ret_value); \ + return ret_value; \ + } + #endif /* PDC_PRIVATE_H */ diff --git a/src/utils/include/pdc_timing.h b/src/commons/utils/include/pdc_timing.h similarity index 99% rename from src/utils/include/pdc_timing.h rename to src/commons/utils/include/pdc_timing.h index 1ea2f475b..22b39148f 100644 --- a/src/utils/include/pdc_timing.h +++ b/src/commons/utils/include/pdc_timing.h @@ -10,7 +10,9 @@ #endif /* HOST_NAME_MAX */ #include "pdc_config.h" +#ifdef ENABLE_MPI #include +#endif #include #include #include diff --git a/src/commons/utils/include/query_utils.h b/src/commons/utils/include/query_utils.h new file mode 100644 index 000000000..513fa0916 --- /dev/null +++ b/src/commons/utils/include/query_utils.h @@ -0,0 +1,120 @@ +// +// Created by Wei Zhang on 7/10/17. +// +#ifndef PDC_QUERY_UTILS_H +#define PDC_QUERY_UTILS_H + +#define TAG_DELIMITER "," +#include "string_utils.h" +#include "pdc_public.h" + +typedef struct query_gen_input { + pdc_kvtag_t *base_tag; + int key_query_type; + int value_query_type; + int affix_len; + int range_lo; + int range_hi; +} query_gen_input_t; + +typedef struct query_gen_output { + char * key_query; + size_t key_query_len; + char * value_query; + size_t value_query_len; +} query_gen_output_t; + +/** + * Generate query strings for key and value according to the given input. + * The query strings will be stored in the output. + * @param input + * @param output + */ +void gen_query_key_value(query_gen_input_t *input, query_gen_output_t *output); + +/** + * Generate concatenated query strings for key and value according to the given input. + * @param query_gen_output + * @return concatenated query string + */ +char *gen_query_str(query_gen_output_t *query_gen_output); + +/** + * Free the memory allocated for the output of query generation + * This function does not free the memory allocated for the query_gen_output_t struct itself. + * @param output + */ +void free_query_output(query_gen_output_t *output); + +/** + * Test tag generation in a for loop + */ +void gen_tags_in_loop(); +/** + * Generate tags according to a seed + * @return + */ +char *gen_tags(int); +/** + * Get key string from a key value pair + * @param kv_pair + * @param delim + * @return + */ +char *get_key(const char *kv_pair, char delim); +/** + * Get value string from a key value pair + * @param kv_pair + * @param delim + * @return + */ +char *get_value(const char *kv_pair, char delim); +/** + * To test if a key value pair matches the given conditions + * @param tagslist + * @param key_pattern + * @param value_pattern + * @return + */ +char *k_v_matches_p(const char *tagslist, const char *key_pattern, const char *value_pattern); +/** + * To test if there is the specified tag within a list of tags. + * @param tagslist + * @param tagname + * @return + */ +int has_tag(const char *tagslist, const char *tagname); +/** + * To test if there is a tag matching the given pattern in the list of tags + * @param tagslist + * @param pattern + * @return + */ +int has_tag_p(const char *tagslist, const char *pattern); +/** + * To test if there is a tag in the list of tags with its value equals to the given value + * @param tagslist + * @param tagname + * @param val + * @return + */ +int is_value_match(const char *tagslist, const char *tagname, const char *val); +/** + * To test if there is a tag with its value matching the given pattern in the list of tags + * @param tagslist + * @param tagname + * @param pattern + * @return + */ +int is_value_match_p(const char *tagslist, const char *tagname, const char *pattern); +/** + * To test if there is a tag in the tag list with its value ranges from *from* to *to* + * @param tagslist + * @param tagname + * @param from + * @param to + * @return + */ +int is_value_in_range(const char *tagslist, const char *tagname, int from, int to); + +#endif // PDC_QUERY_UTILS_H diff --git a/src/commons/utils/include/string_utils.h b/src/commons/utils/include/string_utils.h new file mode 100644 index 000000000..675944dda --- /dev/null +++ b/src/commons/utils/include/string_utils.h @@ -0,0 +1,196 @@ +// +// Created by Wei Zhang on 7/12/17. +// + +#ifndef PDC_STRING_UTILS_H +#define PDC_STRING_UTILS_H + +#include +#include +#include +#include + +// #define PATTERN_EXACT 0 +// #define PATTERN_SUFFIX 1 +// #define PATTERN_PREFIX 2 +// #define PATTERN_MIDDLE 3 + +#define ANSI_COLOR_RED "\x1b[31m" +#define ANSI_COLOR_GREEN "\x1b[32m" +#define ANSI_COLOR_YELLOW "\x1b[33m" +#define ANSI_COLOR_BLUE "\x1b[34m" +#define ANSI_COLOR_MAGENTA "\x1b[35m" +#define ANSI_COLOR_CYAN "\x1b[36m" +#define ANSI_COLOR_RESET "\x1b[0m" + +typedef enum { PATTERN_EXACT = 2, PATTERN_PREFIX = 3, PATTERN_SUFFIX = 4, PATTERN_MIDDLE = 5 } pattern_type_t; + +typedef struct { + char * start; + size_t length; +} string; + +/** + * take the part starting from the start position + * you need to free str after use. + * @param str + * @param start + * @return + */ +char *substr(const char *str, int start); +/** + * take the part before end position + * you need to free str after use. + * @param str + * @param end + * @return + */ +char *subrstr(const char *str, int end); +/** + * take the substring starting from start, ending at end-1 + * you need to free str after use. + * @param str + * @param start + * @param end + * @return + */ +char *substring(const char *str, int start, int end); +/** + * determine the pattern type. + * Currently, only support for different types: + * + * PREFIX : ab* + * SUFFIX : *ab + * MIDDLE : *ab* + * EXACT : ab + * + * @param pattern + * @return + */ +pattern_type_t determine_pattern_type(const char *pattern); +/** + * return the index of token tok in given string str. + * if not found, return -1 + * + * @param str + * @param tok + * @return + */ +int indexOfStr(const char *str, char *tok); +/** + * return the index of character c in given string str. + * if not found, return -1 + * + * @param str + * @param c + * @return + */ +int indexOf(const char *str, char c); + +/** + * to determine if a string str starts with prefix pre + * @param pre + * @param str + * @return + */ +int startsWith(const char *str, const char *pre); +/** + * to determine if a string str ends with suffix suf + * @param suf + * @param str + * @return + */ +int endsWith(const char *str, const char *suf); +/** + * to determine if a string str contains token tok + * @param tok + * @param str + * @return + */ +int contains(const char *str, const char *tok); +/** + * to determine if a string str exactly matches a token tok. + * @param tok + * @param str + * @return + */ +int equals(const char *str, const char *tok); + +/** + * dynamically generate string for you according to the format you pass. + * Since usually, it is hard to predict how much memory should be allocated + * before generating an arbitrary string. + * + * remember to free the generated string after use. + * + * @param format + * @param ... + * @return + */ +char *dsprintf(const char *format, ...); + +/** + * Print anything on stdout. + * Always put a line feed after the string you want to print. + * + * @param format + * @param ... + */ +void println(const char *format, ...); + +/** + * Print anything on stderr. + * Always put a line feed after the string you want to print. + * + * @param format + * @param ... + */ +void stderr_println(const char *format, ...); +/** + * Only support expressions like: + * + * *ab + * ab* + * *ab* + * ab + * + * @param str + * @param pattern + * @return 1 if matches, 0 if not. + */ +int simple_matches(const char *str, const char *pattern); + +/** + * get the reverse of a given string. + * @param str + * @return a reversed string + * @note remember to free the reversed string after use. + */ +char *reverse_str(char *str); + +/** + * split a string into several parts according to the delimiter. + * @param str the string to be split + * @param delim the delimiter, which is a regular expression. + * @param result the result array + * @param result_len the length of the result array + * @return the number of parts, if -1, means error. + */ +int split_string(const char *str, const char *delim, char ***result, int *result_len); + +/** + * Generate given number of random strings with given length. + * You may need to seed the random number generator before calling this function. + * The minimum size of alphabet is 26, accordingly, that alphabet will be [a-z]. + * If you would like to have [a-z0-9] , alphabet_size should be 36. + * If you would like to have [a-z0-9!@#$%^&*()-_=+[]{}|;:'",.<>?/`~], alphabet_size should be 67. + * If you would like to have [a-z0-9!@#$%^&*()-_=+[]{}|;:'",.<>?/`~A-Z], alphabet_size should be 93. + * @param count the number of strings to be generated + * @param minlen the minimum length of the string + * @param maxlen the maximum length of the string + * @param alphabet_size the size of the alphabet + * @return an array of strings + */ +char **gen_random_strings(int count, int minlen, int maxlen, int alphabet_size); + +#endif // PDC_STRING_UTILS_H diff --git a/src/commons/utils/include/thpool.h b/src/commons/utils/include/thpool.h new file mode 100644 index 000000000..268792dd1 --- /dev/null +++ b/src/commons/utils/include/thpool.h @@ -0,0 +1,178 @@ +/********************************** + * @author Johan Hanssen Seferidis + * License: MIT + * + **********************************/ + +#ifndef _THPOOL_ +#define _THPOOL_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* =================================== API ======================================= */ + +typedef struct thpool_ *threadpool; + +/** + * @brief Initialize threadpool + * + * Initializes a threadpool. This function will not return untill all + * threads have initialized successfully. + * + * @example + * + * .. + * threadpool thpool; //First we declare a threadpool + * thpool = thpool_init(4); //then we initialize it to 4 threads + * .. + * + * @param num_threads number of threads to be created in the threadpool + * @return threadpool created threadpool on success, + * NULL on error + */ +threadpool thpool_init(int num_threads); + +/** + * @brief Add work to the job queue + * + * Takes an action and its argument and adds it to the threadpool's job queue. + * If you want to add to work a function with more than one arguments then + * a way to implement this is by passing a pointer to a structure. + * + * NOTICE: You have to cast both the function and argument to not get warnings. + * + * @example + * + * void print_num(int num){ + * printf("%d\n", num); + * } + * + * int main() { + * .. + * int a = 10; + * thpool_add_work(thpool, (void*)print_num, (void*)a); + * .. + * } + * + * @param threadpool threadpool to which the work will be added + * @param function_p pointer to function to add as work + * @param arg_p pointer to an argument + * @return 0 on successs, -1 otherwise. + */ +int thpool_add_work(threadpool, void (*function_p)(void *), void *arg_p); + +/** + * @brief Wait for all queued jobs to finish + * + * Will wait for all jobs - both queued and currently running to finish. + * Once the queue is empty and all work has completed, the calling thread + * (probably the main program) will continue. + * + * Smart polling is used in wait. The polling is initially 0 - meaning that + * there is virtually no polling at all. If after 1 seconds the threads + * haven't finished, the polling interval starts growing exponentially + * untill it reaches max_secs seconds. Then it jumps down to a maximum polling + * interval assuming that heavy processing is being used in the threadpool. + * + * @example + * + * .. + * threadpool thpool = thpool_init(4); + * .. + * // Add a bunch of work + * .. + * thpool_wait(thpool); + * puts("All added work has finished"); + * .. + * + * @param threadpool the threadpool to wait for + * @return nothing + */ +void thpool_wait(threadpool); + +/** + * @brief Pauses all threads immediately + * + * The threads will be paused no matter if they are idle or working. + * The threads return to their previous states once thpool_resume + * is called. + * + * While the thread is being paused, new work can be added. + * + * @example + * + * threadpool thpool = thpool_init(4); + * thpool_pause(thpool); + * .. + * // Add a bunch of work + * .. + * thpool_resume(thpool); // Let the threads start their magic + * + * @param threadpool the threadpool where the threads should be paused + * @return nothing + */ +void thpool_pause(threadpool); + +/** + * @brief Unpauses all threads if they are paused + * + * @example + * .. + * thpool_pause(thpool); + * sleep(10); // Delay execution 10 seconds + * thpool_resume(thpool); + * .. + * + * @param threadpool the threadpool where the threads should be unpaused + * @return nothing + */ +void thpool_resume(threadpool); + +/** + * @brief Destroy the threadpool + * + * This will wait for the currently active threads to finish and then 'kill' + * the whole threadpool to free up memory. + * + * @example + * int main() { + * threadpool thpool1 = thpool_init(2); + * threadpool thpool2 = thpool_init(2); + * .. + * thpool_destroy(thpool1); + * .. + * return 0; + * } + * + * @param threadpool the threadpool to destroy + * @return nothing + */ +void thpool_destroy(threadpool); + +/** + * @brief Show currently working threads + * + * Working threads are the threads that are performing work (not idle). + * + * @example + * int main() { + * threadpool thpool1 = thpool_init(2); + * threadpool thpool2 = thpool_init(2); + * .. + * printf("Working threads: %d\n", thpool_num_threads_working(thpool1)); + * .. + * return 0; + * } + * + * @param threadpool the threadpool of interest + * @return integer number of threads working + */ +int thpool_num_threads_working(threadpool); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/commons/utils/include/timer_utils.h b/src/commons/utils/include/timer_utils.h new file mode 100644 index 000000000..1078fefa7 --- /dev/null +++ b/src/commons/utils/include/timer_utils.h @@ -0,0 +1,77 @@ +/** timer.h + * Microsecond timer in C, using `gettimeofday()`. + * + * Note: `gettimeofday()` is Unix, GNU/Linux and + * Mac OS X system-specific. + * + * The only portable function is time.h's `clock()`, + * but it isn't very precise. + * + * See: http://www.songho.ca/misc/timer/timer.html + * + * Copyright (c) 2011,2013 Alexandre Dantas + */ + +#ifndef TIMER_UTILS_H_DEFINED +#define TIMER_UTILS_H_DEFINED + +#include +#include /* gettimeofday() */ +#include /* bool */ +#include /* int64_t */ + +typedef struct stopwatch_t { + int64_t start_mark; /* Timer start point */ + int64_t pause_mark; /* In case we pause the timer */ + bool running; /* Is it running? */ + bool paused; /* Is it paused? */ + +} stopwatch_t; + +int64_t timer_us_timestamp(); + +int64_t timer_ms_timestamp(); + +int64_t timer_s_timestamp(); + +/** Starts the timer. + * + * @note If called multiple times, restarts the timer. + */ +void timer_start(stopwatch_t *t); + +/** Pauses the timer. + * + * @note If called multiple times, it does nothing. + */ +void timer_pause(stopwatch_t *t); + +/** Unpauses the timer. + * + * @note If the timer's not paused or this is called + * multiple times, it does nothing. + */ +void timer_unpause(stopwatch_t *t); + +/** Returns the time difference in microseconds + * + * @note (1/1000000 seconds) + */ +double timer_delta_us(stopwatch_t *t); + +/** Returns the time difference in miliseconds. + * + * @note (1/1000 seconds) + */ +double timer_delta_ms(stopwatch_t *t); + +/** Returns the time difference in seconds. */ +double timer_delta_s(stopwatch_t *t); + +/** Returns the time difference in minutes (60 seconds). */ +double timer_delta_m(stopwatch_t *t); + +/** Returns the time difference in hours (3600 seconds). */ +double timer_delta_h(stopwatch_t *t); + +#endif /* TIMER_UTILS_H_DEFINED */ diff --git a/src/commons/utils/pdc_malloc.c b/src/commons/utils/pdc_malloc.c new file mode 100644 index 000000000..c7ca6aa5f --- /dev/null +++ b/src/commons/utils/pdc_malloc.c @@ -0,0 +1,188 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#include +#include +#include "pdc_malloc.h" +#include "pdc_private.h" + +#ifdef HAVE_MALLOC_USABLE_SIZE +#include +#else +// alternative header files for malloc_usable_size or similar functions. +#endif + +size_t PDC_mem_usage_g; + +void * +PDC_malloc(size_t size) +{ + void *ret_value; + + FUNC_ENTER(NULL); + + assert(size); + + if (size) + ret_value = malloc(size); + else + ret_value = NULL; + + if (ret_value) + PDC_mem_usage_g += size; + + FUNC_LEAVE(ret_value); +} + +void * +PDC_malloc_addsize(size_t size, size_t *mem_usage_ptr) +{ + void *ret_value = PDC_malloc(size); + if (ret_value && mem_usage_ptr) + *mem_usage_ptr += size; + return ret_value; +} + +void * +PDC_calloc(size_t count, size_t size) +{ + void *ret_value; + + FUNC_ENTER(NULL); + + assert(count); + assert(size); + + if (count && size) + ret_value = calloc(count, size); + else + ret_value = NULL; + + if (ret_value) + PDC_mem_usage_g += size; + + FUNC_LEAVE(ret_value); +} + +void * +PDC_calloc_addsize(size_t count, size_t size, size_t *mem_usage_ptr) +{ + void *ret_value = PDC_calloc(count, size); + if (ret_value && mem_usage_ptr) + *mem_usage_ptr += size; + return ret_value; +} + +void * +PDC_realloc_knowing_oldsize(void *ptr, size_t size, size_t old_size) +{ + void *ret_value; + + FUNC_ENTER(NULL); + + assert(size); + size_t _old_size = old_size; + if (size) + ret_value = realloc(ptr, size); + else + ret_value = NULL; + + if (ret_value) { + PDC_mem_usage_g += size; + if (_old_size) { + PDC_mem_usage_g -= _old_size; + } + } + + FUNC_LEAVE(ret_value); +} + +void * +PDC_realloc(void *ptr, size_t size) +{ + size_t _old_size = 0; +#ifdef HAVE_MALLOC_USABLE_SIZE + _old_size = malloc_usable_size(ptr); +#endif + return PDC_realloc_knowing_oldsize(ptr, size, _old_size); +} + +void * +PDC_realloc_addsize_knowing_oldsize(void *ptr, size_t size, size_t old_size, size_t *mem_usage_ptr) +{ + size_t _old_size = old_size; + void * ret_value = PDC_realloc_knowing_oldsize(ptr, size, _old_size); + if (ret_value && mem_usage_ptr) { + *mem_usage_ptr += size; + if (_old_size) { + *mem_usage_ptr -= _old_size; + } + } + return ret_value; +} + +void * +PDC_realloc_addsize(void *ptr, size_t size, size_t *mem_usage_ptr) +{ + size_t _old_size = 0; +#ifdef HAVE_MALLOC_USABLE_SIZE + _old_size = malloc_usable_size(ptr); +#endif + return PDC_realloc_addsize_knowing_oldsize(ptr, size, _old_size, mem_usage_ptr); +} + +void * +PDC_free_knowing_old_size(void *mem, size_t old_size) +{ + size_t _old_size = old_size; + + void *ret_value = NULL; + + FUNC_ENTER(NULL); + + if (mem) { + free(mem); + if (_old_size) { + PDC_mem_usage_g -= _old_size; + } + } + + FUNC_LEAVE(ret_value); +} + +void * +PDC_free(void *mem) +{ + size_t _old_size = 0; +#ifdef HAVE_MALLOC_USABLE_SIZE + _old_size = malloc_usable_size(mem); +#endif + return PDC_free_knowing_old_size(mem, _old_size); +} + +size_t +PDC_get_global_mem_usage() +{ + return PDC_mem_usage_g; +} \ No newline at end of file diff --git a/src/utils/pdc_timing.c b/src/commons/utils/pdc_timing.c similarity index 100% rename from src/utils/pdc_timing.c rename to src/commons/utils/pdc_timing.c diff --git a/src/commons/utils/query_utils.c b/src/commons/utils/query_utils.c new file mode 100644 index 000000000..c0ea64032 --- /dev/null +++ b/src/commons/utils/query_utils.c @@ -0,0 +1,346 @@ +// +// Created by Wei Zhang on 7/10/17. +// +#include "query_utils.h" + +int +_gen_affix_for_token(char *token_str, int affix_type, size_t affix_len, char **out_str) +{ + + size_t token_len = strlen(token_str); + affix_len = affix_len < token_len ? affix_len : token_len; + size_t copy_len = affix_type == 0 ? token_len : affix_len; + char * source = affix_type <= 1 ? token_str : &(token_str[token_len - affix_len]); + *out_str = (char *)calloc(copy_len + 3, sizeof(char)); + strncpy(*out_str, source, copy_len + 1); + + if (affix_type == 0) { // exact + // nothing to do here. + } + else if (affix_type == 1) { // prefix + // "hello" -> "hell*" or "hell" -> "hell*" + (*out_str)[affix_len] = '*'; + (*out_str)[affix_len + 1] = '\0'; + } + else if (affix_type == 2) { // suffix + // "hello" -> '*ello' or 'hell' -> '*hell' + for (int k = affix_len; k > 0; k--) { + (*out_str)[k] = (*out_str)[k - 1]; + } + (*out_str)[0] = '*'; + (*out_str)[affix_len + 1] = '\0'; + } + else if (affix_type == 3) { // infix + // "hello" -> '*ello*' or 'hell' -> '*hell*' + for (int k = affix_len; k > 0; k--) { + (*out_str)[k] = (*out_str)[k - 1]; + } + (*out_str)[0] = '*'; + (*out_str)[affix_len + 1] = '*'; + (*out_str)[affix_len + 2] = '\0'; + } + else { + printf("Invalid affix type!\n"); + return 0; + } + return strlen(*out_str); +} + +void +gen_query_key_value(query_gen_input_t *input, query_gen_output_t *output) +{ + char * key_ptr = NULL; + size_t key_ptr_len = 0; + char * value_ptr = NULL; + size_t value_ptr_len = 0; + // check base_tag->name length + if (strlen(input->base_tag->name) < 3) { + char *new_tag_name = (char *)calloc(4, sizeof(char)); + memset(new_tag_name, input->base_tag->name[0], 3); + input->base_tag->name = new_tag_name; + input->affix_len = 1; + } + // check base_tag->value length if it is a string + if (input->base_tag->type == PDC_STRING && strlen((char *)input->base_tag->value) < 3) { + char *new_tag_value = (char *)calloc(4, sizeof(char)); + memset(new_tag_value, ((char *)input->base_tag->value)[0], 3); + input->base_tag->value = (void *)new_tag_value; + input->affix_len = 1; + } + size_t affix_len = input->affix_len; + + // "hello" + key_ptr_len = _gen_affix_for_token(input->base_tag->name, input->key_query_type, affix_len, &key_ptr); + if (key_ptr_len == 0) { + printf("Failed to generate key query!\n"); + return; + } + + // process value in base_tag + if (input->base_tag->type == PDC_STRING) { + value_ptr_len = _gen_affix_for_token((char *)input->base_tag->value, input->value_query_type, + affix_len, &value_ptr); + if (value_ptr_len == 0) { + printf("Failed to generate value query!\n"); + return; + } + } + else if (input->base_tag->type == PDC_INT) { + if (input->value_query_type == 4) { + value_ptr_len = snprintf(NULL, 0, "%d", ((int *)input->base_tag->value)[0]); + value_ptr = (char *)calloc(value_ptr_len + 1, sizeof(char)); + snprintf(value_ptr, value_ptr_len + 1, "%d", ((int *)input->base_tag->value)[0]); + } + else if (input->value_query_type == 5) { + size_t lo_len = snprintf(NULL, 0, "%d", input->range_lo); + size_t hi_len = snprintf(NULL, 0, "%d", input->range_hi); + value_ptr_len = lo_len + hi_len + 1; + value_ptr = (char *)calloc(value_ptr_len + 1, sizeof(char)); + snprintf(value_ptr, value_ptr_len + 1, "%d~%d", input->range_lo, input->range_hi); + } + else { + printf("Invalid value query type for integer!\n"); + return; + } + } + else { + printf("Invalid tag type!\n"); + return; + } + + output->key_query = key_ptr; + output->key_query_len = key_ptr_len; + output->value_query = value_ptr; + output->value_query_len = value_ptr_len; +} + +char * +gen_query_str(query_gen_output_t *query_gen_output) +{ + char *final_query_str = + (char *)calloc(query_gen_output->key_query_len + query_gen_output->value_query_len + 2, sizeof(char)); + strcat(final_query_str, query_gen_output->key_query); + strcat(final_query_str, "="); + strcat(final_query_str, query_gen_output->value_query); + return final_query_str; +} + +void +free_query_output(query_gen_output_t *output) +{ + if (output->key_query != NULL) { + free(output->key_query); + } + if (output->value_query != NULL) { + free(output->value_query); + } +} + +/** + * + * return the key from a kv_pair string connected by delim character. + * + * return NULL if key cannot be retrieved, either because the kv_pair is invalid, or + * because no delimiter is found in kv_pair string. + * + * @param kv_pair + * @param delim + * @return + */ +char * +get_key(const char *kv_pair, char delim) +{ + + char *ret = NULL; + int idx = indexOf(kv_pair, delim); + + if (idx < 0) { + return ret; + } + return subrstr(kv_pair, idx); +} + +/** + * return the key from a kv_pair string connected by delim character. + * + * return NULL if key cannot be retrieved, either because the kv_pair is invalid, or + * because no delimiter is found in kv_pair string. + * + * @param kv_pair + * @param delim + * @return + */ +char * +get_value(const char *kv_pair, char delim) +{ + + char *ret = NULL; + int idx = indexOf(kv_pair, delim); + + if (idx < 0) { + return ret; + } + + return substr(kv_pair, idx + 1); +} + +/** + * Generate tags based on obj_id. + * + * It is just for simulation, pretty random. + * + * @param obj_id + * @return + */ +char * +gen_tags(int obj_id) +{ + int j; + int tag_num = obj_id % 20; + char *ret = ""; + for (j = 0; j <= tag_num; j++) { + char *fspace = ret; + ret = dsprintf("%stag%d=%d%d,", ret, j, obj_id, j); + if (strlen(fspace) > 0) { + free(fspace); + } + } + ret[strlen(ret) - 1] = '\0'; + return ret; +} + +/** + * generate multiple tagslists for many objects. + * + * This is just a test. + */ +void +gen_tags_in_loop() +{ + + int my_count = 1000; + int i; + for (i = 0; i < my_count; i++) { + int tag_num = i % 20; + char *ret = gen_tags(tag_num); + println("helloworld, %s", ret); + if (ret != NULL) { + free(ret); + } + } +} +/** + * returns 1 if the tag is found, otherwise, returns 0. + * @param tagslist + * @param tagname + * @return + */ +int +has_tag(const char *tagslist, const char *tagname) +{ + /* + char *pattern = strdup(tagname); + if (startsWith("*", pattern)) { + pattern = &pattern[1]; + } + if (endsWith("*", pattern)) { + pattern[strlen(pattern)]='\0'; + } + */ + return has_tag_p(tagslist, tagname); +} +/** + * Check if there is any tag in the tags list that matches the given pattern. + * + * @param tagslist + * @param pattern + * @return + */ +int +has_tag_p(const char *tagslist, const char *pattern) +{ + return (k_v_matches_p(tagslist, pattern, NULL) != NULL); +} + +char * +k_v_matches_p(const char *tagslist, const char *key_pattern, const char *value_pattern) +{ + char *rst_kv = NULL; + char *_tags_list = NULL; + + if (tagslist == NULL || key_pattern == NULL) { + return rst_kv; + } + _tags_list = strdup(tagslist); + // GO THROUGH EACH KV PAIR + char *tag_kv = strtok(_tags_list, TAG_DELIMITER); + while (tag_kv != NULL) { + /** + * Check to see if the current key-value pair is valid + */ + if (strchr(tag_kv, '=') != NULL) { + // get key and value + char *key = NULL; + key = get_key(tag_kv, '='); + char *value = NULL; + value = get_value(tag_kv, '='); + + /** + * if no value pattern is specified, we only match key pattern. + * otherwise, we match both. + */ + int is_key_matched = simple_matches(key, key_pattern); + + int is_value_matched = (value_pattern == NULL ? 0 : simple_matches(value, value_pattern)); + + int pattern_matches = + (value_pattern == NULL ? is_key_matched : (is_key_matched && is_value_matched)); + + if (key != NULL) { + free(key); + } + if (value != NULL) { + free(value); + } + + if (pattern_matches) { + rst_kv = tag_kv; + break; + } + } + tag_kv = strtok(NULL, TAG_DELIMITER); + } + + if (_tags_list != NULL) { + // free(_tags_list); + } + return rst_kv; +} + +int +is_value_match(const char *tagslist, const char *tagname, const char *val) +{ + /* + char *pattern = strdup(val); + if (startsWith("*", pattern)) { + pattern = &pattern[1]; + } + if (endsWith("*", pattern)) { + pattern[strlen(pattern)]='\0'; + } + */ + return is_value_match_p(tagslist, tagname, val); +} +int +is_value_match_p(const char *tagslist, const char *tagname, const char *pattern) +{ + return (k_v_matches_p(tagslist, tagname, pattern) != NULL); +} +int +is_value_in_range(const char *tagslist, const char *tagname, int from, int to) +{ + const char *matched_kv = k_v_matches_p(tagslist, tagname, NULL); + char * value = get_value(matched_kv, '='); + int v = atoi(value); + return (v >= from && v <= to); +} diff --git a/src/commons/utils/query_utils_test.c b/src/commons/utils/query_utils_test.c new file mode 100644 index 000000000..73ebd940a --- /dev/null +++ b/src/commons/utils/query_utils_test.c @@ -0,0 +1,64 @@ +#include "query_utils.h" + +void +print_query_output(query_gen_output_t *output) +{ + println("key query: %s, len: %lu", output->key_query, output->key_query_len); + println("value query: %s, len: %lu", output->value_query, output->value_query_len); + char *final_query_str = gen_query_str(output); + println("final query: %s, len: %lu", final_query_str, strlen(final_query_str)); + free(final_query_str); +} + +int +main(int argc, char *argv[]) +{ + int affix_length = atoi(argv[1]); + pdc_kvtag_t *base_string_tag; + base_string_tag = (pdc_kvtag_t *)calloc(1, sizeof(pdc_kvtag_t)); + base_string_tag->name = "testname"; + base_string_tag->type = PDC_STRING; + base_string_tag->value = "testvalue"; + + for (int i = 0; i < 4; i++) { + int key_query_type = i; + int value_query_type = i; + query_gen_input_t input; + query_gen_output_t output; + input.base_tag = base_string_tag; + input.key_query_type = key_query_type; + input.value_query_type = value_query_type; + input.affix_len = affix_length; + gen_query_key_value(&input, &output); + print_query_output(&output); + free_query_output(&output); + } + + pdc_kvtag_t *base_int_tag; + int int_value = 234; + base_int_tag = (pdc_kvtag_t *)calloc(1, sizeof(pdc_kvtag_t)); + base_int_tag->name = "testname"; + base_int_tag->type = PDC_INT; + base_int_tag->value = (void *)calloc(1, sizeof(int)); + ((int *)base_int_tag->value)[0] = int_value; + + for (int i = 4; i < 6; i++) { + int key_query_type = 0; + int value_query_type = i; + int range_offset = 10; + int range_lo = int_value - range_offset; + int range_hi = int_value + range_offset; + query_gen_input_t input; + query_gen_output_t output; + input.base_tag = base_int_tag; + input.key_query_type = key_query_type; + input.value_query_type = value_query_type; + input.range_lo = range_lo; + input.range_hi = range_hi; + gen_query_key_value(&input, &output); + print_query_output(&output); + free_query_output(&output); + } + + return 0; +} \ No newline at end of file diff --git a/src/commons/utils/string_utils.c b/src/commons/utils/string_utils.c new file mode 100644 index 000000000..546d5679d --- /dev/null +++ b/src/commons/utils/string_utils.c @@ -0,0 +1,330 @@ +// +// Created by Wei Zhang on 7/12/17. +// + +#include "string_utils.h" +#include + +const char *VISIBLE_ALPHABET = + "abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()-_=+[]{}|;:'\",.<>?/`~ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + +int +startsWith(const char *str, const char *pre) +{ + if (str == NULL || pre == NULL) + return 0; + return strncmp(str, pre, strlen(pre)) == 0; +} + +int +endsWith(const char *str, const char *suf) +{ + if (str == NULL || suf == NULL) + return 0; + size_t lensuf = strlen(suf), lenstr = strlen(str); + return lenstr < lensuf ? 0 : (strncmp(str + lenstr - lensuf, suf, lensuf) == 0); +} + +int +contains(const char *str, const char *tok) +{ + return strstr(str, tok) != NULL; +} + +int +equals(const char *str, const char *tok) +{ + return strcmp(tok, str) == 0; +} + +int +simple_matches(const char *str, const char *pattern) +{ + int result = 0; + // Ensure both str and pattern cannot be empty. + if (str == NULL || pattern == NULL) { + return result; + } + int pattern_type = determine_pattern_type(pattern); + + char *tok = NULL; + switch (pattern_type) { + case PATTERN_EXACT: + result = equals(str, pattern); + break; + case PATTERN_PREFIX: + tok = subrstr(pattern, strlen(pattern) - 1); + result = (tok == NULL ? 0 : startsWith(str, tok)); + break; + case PATTERN_SUFFIX: + tok = substr(pattern, 1); + result = (tok == NULL ? 0 : endsWith(str, tok)); + break; + case PATTERN_MIDDLE: + tok = substring(pattern, 1, strlen(pattern) - 1); + result = (tok == NULL ? 0 : contains(str, tok)); + break; + default: + break; + } + if (tok != NULL) { + free(tok); + } + return result; +} + +pattern_type_t +determine_pattern_type(const char *pattern) +{ + + if (startsWith(pattern, "*")) { + if (endsWith(pattern, "*")) { + return PATTERN_MIDDLE; + } + else { + return PATTERN_SUFFIX; + } + } + else { + if (endsWith(pattern, "*")) { + return PATTERN_PREFIX; + } + else { + return PATTERN_EXACT; + } + } +} +char * +substr(const char *str, int start) +{ + return substring(str, start, strlen(str) + 1); +} +char * +subrstr(const char *str, int end) +{ + return substring(str, 0, end); +} + +char * +substring(const char *str, int start, int end) +{ + // Check for invalid parameters + if (str == NULL || end < start || start < 0 || end < 0) { + return NULL; + } + + // Length of the original string + int str_len = strlen(str); + + // Adjust end if it is beyond the length of the string + if (end > str_len) { + end = str_len; + } + + // Calculate the length of the substring + int substr_len = end - start; + + // Allocate memory for the new string (including null-terminator) + char *substr = (char *)malloc((substr_len + 1) * sizeof(char)); + if (substr == NULL) { // Check if malloc succeeded + return NULL; + } + + // Copy the substring into the new string + memcpy(substr, &str[start], substr_len); + + // Null-terminate the new string + substr[substr_len] = '\0'; + + return substr; +} + +int +indexOfStr(const char *str, char *tok) +{ + const char *p = strstr(str, tok); + if (p) { + return p - str; + } + return -1; +} +int +indexOf(const char *str, char c) +{ + const char *p = strchr(str, c); + if (p) { + return p - str; + } + return -1; +} + +char * +dsprintf(const char *format, ...) +{ + char *ret; + // 1. declare argument list + va_list args; + // 2. starting argument list + va_start(args, format); + // 3. get arguments value + int numbytes = vsnprintf((char *)NULL, 0, format, args); + ret = (char *)calloc((numbytes + 1), sizeof(char)); + + va_start(args, format); + vsprintf(ret, format, args); + // 4. ending argument list + va_end(args); + return ret; +} + +void +println(const char *format, ...) +{ + // 1. declare argument list + va_list args; + // 2. starting argument list + va_start(args, format); + // 3. get arguments value + vfprintf(stdout, format, args); + // 4. ending argument list + va_end(args); + fputc('\n', stdout); + fflush(stdout); +} + +void +stderr_println(const char *format, ...) +{ + // 1. declare argument list + va_list args; + // 2. starting argument list + va_start(args, format); + // 3. get arguments value + vfprintf(stderr, format, args); + // 4. ending argument list + va_end(args); + fputc('\n', stderr); + fflush(stderr); +} + +char * +reverse_str(char *str) +{ + if (str == NULL) { + return NULL; + } + + int length = strlen(str); + char *reversed = (char *)malloc(length + 1); // +1 for the null-terminator + + if (reversed == NULL) { + return NULL; // Return NULL if memory allocation fails + } + + for (int i = 0; i < length; i++) { + reversed[i] = str[length - 1 - i]; + } + + reversed[length] = '\0'; // Null-terminate the new string + + return reversed; +} + +int +split_string(const char *str, const char *delim, char ***result, int *result_len) +{ + if (str == NULL || delim == NULL || result == NULL || result_len == NULL) { + return -1; + } + + regex_t regex; + int reti; + + // Compile regular expression + reti = regcomp(®ex, delim, 0); + if (reti) { + fprintf(stderr, "Could not compile regex\n"); + return -1; + } + + const char *tmp = str; + int count = 0; + regmatch_t pmatch[1]; + + // Count matches + while (regexec(®ex, tmp, 1, pmatch, 0) != REG_NOMATCH) { + count++; + tmp += pmatch[0].rm_eo; + } + + *result_len = count + 1; + *result = (char **)malloc((*result_len) * sizeof(char *)); + if (!*result) { + return -1; // Memory allocation failed + } + + tmp = str; // Reset tmp + const char *start = str; + int i = 0; + + while (i < count && regexec(®ex, tmp, 1, pmatch, 0) != REG_NOMATCH) { + int len = pmatch[0].rm_so; + + (*result)[i] = (char *)malloc((len + 1) * sizeof(char)); + if (!(*result)[i]) { + for (int j = 0; j < i; j++) { + free((*result)[j]); + } + free(*result); + *result = NULL; + regfree(®ex); + return -1; + } + + memcpy((*result)[i], start, len); + (*result)[i][len] = '\0'; + + tmp += pmatch[0].rm_eo; + start = tmp; + i++; + } + + (*result)[i] = strdup(start); + if (!(*result)[i]) { + for (int j = 0; j < i; j++) { + free((*result)[j]); + } + free(*result); + *result = NULL; + regfree(®ex); + return -1; + } + + regfree(®ex); + return *result_len; +} + +char ** +gen_random_strings(int count, int minlen, int maxlen, int alphabet_size) +{ + int c = 0; + int i = 0; + char **result = (char **)calloc(count, sizeof(char *)); + int abc_size = alphabet_size > strlen(VISIBLE_ALPHABET) ? strlen(VISIBLE_ALPHABET) : alphabet_size; + abc_size = abc_size < 1 ? 26 : abc_size; // the minimum alphabet size is 26 + for (c = 0; c < count; c++) { + int len = (rand() % maxlen) + 1; + len = len < minlen ? minlen : len; // Ensure at least minlen character + char *str = (char *)calloc(len + 1, sizeof(char)); + for (i = 0; i < len; i++) { + int randnum = rand(); + if (randnum < 0) + randnum *= -1; + char chr = VISIBLE_ALPHABET[randnum % abc_size]; + str[i] = chr; + } + str[len] = '\0'; // Null-terminate the string + result[c] = str; + } + return result; +} \ No newline at end of file diff --git a/src/commons/utils/thpool.c b/src/commons/utils/thpool.c new file mode 100644 index 000000000..2e3fec47b --- /dev/null +++ b/src/commons/utils/thpool.c @@ -0,0 +1,542 @@ +/* ******************************** + * Author: Johan Hanssen Seferidis + * License: MIT + * Description: Library providing a threading pool where you can add + * work. For usage, check the thpool.h file or README.md + * + */ +/** @file thpool.h */ /* + * + ********************************/ + +#if defined(__APPLE__) +#include +#else +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200809L +#endif +#endif +#include +#include +#include +#include +#include +#include +#include +#if defined(__linux__) +#include +#endif + +#include "thpool.h" + +#ifdef THPOOL_DEBUG +#define THPOOL_DEBUG 1 +#else +#define THPOOL_DEBUG 0 +#endif + +#if !defined(DISABLE_PRINT) || defined(THPOOL_DEBUG) +#define err(str) fprintf(stderr, str) +#else +#define err(str) +#endif + +static volatile int threads_keepalive; +static volatile int threads_on_hold; + +/* ========================== STRUCTURES ============================ */ + +/* Binary semaphore */ +typedef struct bsem { + pthread_mutex_t mutex; + pthread_cond_t cond; + int v; +} bsem; + +/* Job */ +typedef struct job { + struct job *prev; /* pointer to previous job */ + void (*function)(void *arg); /* function pointer */ + void *arg; /* function's argument */ +} job; + +/* Job queue */ +typedef struct jobqueue { + pthread_mutex_t rwmutex; /* used for queue r/w access */ + job * front; /* pointer to front of queue */ + job * rear; /* pointer to rear of queue */ + bsem * has_jobs; /* flag as binary semaphore */ + int len; /* number of jobs in queue */ +} jobqueue; + +/* Thread */ +typedef struct thread { + int id; /* friendly id */ + pthread_t pthread; /* pointer to actual thread */ + struct thpool_ *thpool_p; /* access to thpool */ +} thread; + +/* Threadpool */ +typedef struct thpool_ { + thread ** threads; /* pointer to threads */ + volatile int num_threads_alive; /* threads currently alive */ + volatile int num_threads_working; /* threads currently working */ + pthread_mutex_t thcount_lock; /* used for thread count etc */ + pthread_cond_t threads_all_idle; /* signal to thpool_wait */ + jobqueue jobqueue; /* job queue */ +} thpool_; + +/* ========================== PROTOTYPES ============================ */ + +static int thread_init(thpool_ *thpool_p, struct thread **thread_p, int id); +static void *thread_do(struct thread *thread_p); +static void thread_hold(int sig_id); +static void thread_destroy(struct thread *thread_p); + +static int jobqueue_init(jobqueue *jobqueue_p); +static void jobqueue_clear(jobqueue *jobqueue_p); +static void jobqueue_push(jobqueue *jobqueue_p, struct job *newjob_p); +static struct job *jobqueue_pull(jobqueue *jobqueue_p); +static void jobqueue_destroy(jobqueue *jobqueue_p); + +static void bsem_init(struct bsem *bsem_p, int value); +static void bsem_reset(struct bsem *bsem_p); +static void bsem_post(struct bsem *bsem_p); +static void bsem_post_all(struct bsem *bsem_p); +static void bsem_wait(struct bsem *bsem_p); + +/* ========================== THREADPOOL ============================ */ + +/* Initialise thread pool */ +struct thpool_ * +thpool_init(int num_threads) +{ + + threads_on_hold = 0; + threads_keepalive = 1; + + if (num_threads < 0) { + num_threads = 0; + } + + /* Make new thread pool */ + thpool_ *thpool_p; + thpool_p = (struct thpool_ *)malloc(sizeof(struct thpool_)); + if (thpool_p == NULL) { + err("thpool_init(): Could not allocate memory for thread pool\n"); + return NULL; + } + thpool_p->num_threads_alive = 0; + thpool_p->num_threads_working = 0; + + /* Initialise the job queue */ + if (jobqueue_init(&thpool_p->jobqueue) == -1) { + err("thpool_init(): Could not allocate memory for job queue\n"); + free(thpool_p); + return NULL; + } + + /* Make threads in pool */ + thpool_p->threads = (struct thread **)malloc(num_threads * sizeof(struct thread *)); + if (thpool_p->threads == NULL) { + err("thpool_init(): Could not allocate memory for threads\n"); + jobqueue_destroy(&thpool_p->jobqueue); + free(thpool_p); + return NULL; + } + + pthread_mutex_init(&(thpool_p->thcount_lock), NULL); + pthread_cond_init(&thpool_p->threads_all_idle, NULL); + + /* Thread init */ + int n; + for (n = 0; n < num_threads; n++) { + thread_init(thpool_p, &thpool_p->threads[n], n); +#if THPOOL_DEBUG + printf("THPOOL_DEBUG: Created thread %d in pool \n", n); +#endif + } + + /* Wait for threads to initialize */ + while (thpool_p->num_threads_alive != num_threads) { + } + + return thpool_p; +} + +/* Add work to the thread pool */ +int +thpool_add_work(thpool_ *thpool_p, void (*function_p)(void *), void *arg_p) +{ + job *newjob; + + newjob = (struct job *)malloc(sizeof(struct job)); + if (newjob == NULL) { + err("thpool_add_work(): Could not allocate memory for new job\n"); + return -1; + } + + /* add function and argument */ + newjob->function = function_p; + newjob->arg = arg_p; + + /* add job to queue */ + jobqueue_push(&thpool_p->jobqueue, newjob); + + return 0; +} + +/* Wait until all jobs have finished */ +void +thpool_wait(thpool_ *thpool_p) +{ + pthread_mutex_lock(&thpool_p->thcount_lock); + while (thpool_p->jobqueue.len || thpool_p->num_threads_working) { + pthread_cond_wait(&thpool_p->threads_all_idle, &thpool_p->thcount_lock); + } + pthread_mutex_unlock(&thpool_p->thcount_lock); +} + +/* Destroy the threadpool */ +void +thpool_destroy(thpool_ *thpool_p) +{ + /* No need to destory if it's NULL */ + if (thpool_p == NULL) + return; + + volatile int threads_total = thpool_p->num_threads_alive; + + /* End each thread 's infinite loop */ + threads_keepalive = 0; + + /* Give one second to kill idle threads */ + double TIMEOUT = 1.0; + time_t start, end; + double tpassed = 0.0; + time(&start); + while (tpassed < TIMEOUT && thpool_p->num_threads_alive) { + bsem_post_all(thpool_p->jobqueue.has_jobs); + time(&end); + tpassed = difftime(end, start); + } + + /* Poll remaining threads */ + while (thpool_p->num_threads_alive) { + bsem_post_all(thpool_p->jobqueue.has_jobs); + sleep(1); + } + + /* Job queue cleanup */ + jobqueue_destroy(&thpool_p->jobqueue); + /* Deallocs */ + int n; + for (n = 0; n < threads_total; n++) { + thread_destroy(thpool_p->threads[n]); + } + free(thpool_p->threads); + free(thpool_p); +} + +/* Pause all threads in threadpool */ +void +thpool_pause(thpool_ *thpool_p) +{ + int n; + for (n = 0; n < thpool_p->num_threads_alive; n++) { + pthread_kill(thpool_p->threads[n]->pthread, SIGUSR1); + } +} + +/* Resume all threads in threadpool */ +void +thpool_resume(thpool_ *thpool_p) +{ + // resuming a single threadpool hasn't been + // implemented yet, meanwhile this supresses + // the warnings + (void)thpool_p; + + threads_on_hold = 0; +} + +int +thpool_num_threads_working(thpool_ *thpool_p) +{ + return thpool_p->num_threads_working; +} + +/* ============================ THREAD ============================== */ + +/* Initialize a thread in the thread pool + * + * @param thread address to the pointer of the thread to be created + * @param id id to be given to the thread + * @return 0 on success, -1 otherwise. + */ +static int +thread_init(thpool_ *thpool_p, struct thread **thread_p, int id) +{ + + *thread_p = (struct thread *)malloc(sizeof(struct thread)); + if (thread_p == NULL) { + err("thread_init(): Could not allocate memory for thread\n"); + return -1; + } + + (*thread_p)->thpool_p = thpool_p; + (*thread_p)->id = id; + + pthread_create(&(*thread_p)->pthread, NULL, (void *)thread_do, (*thread_p)); + pthread_detach((*thread_p)->pthread); + return 0; +} + +/* Sets the calling thread on hold */ +static void +thread_hold(int sig_id) +{ + (void)sig_id; + threads_on_hold = 1; + while (threads_on_hold) { + sleep(1); + } +} + +/* What each thread is doing + * + * In principle this is an endless loop. The only time this loop gets interuppted is once + * thpool_destroy() is invoked or the program exits. + * + * @param thread thread that will run this function + * @return nothing + */ +static void * +thread_do(struct thread *thread_p) +{ + + /* Set thread name for profiling and debuging */ + char thread_name[128] = {0}; + sprintf(thread_name, "thread-pool-%d", thread_p->id); + +#if defined(__linux__) + /* Use prctl instead to prevent using _GNU_SOURCE flag and implicit declaration */ + prctl(PR_SET_NAME, thread_name); +#elif defined(__APPLE__) && defined(__MACH__) + pthread_setname_np(thread_name); +#else + err("thread_do(): pthread_setname_np is not supported on this system"); +#endif + + /* Assure all threads have been created before starting serving */ + thpool_ *thpool_p = thread_p->thpool_p; + + /* Register signal handler */ + struct sigaction act; + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = thread_hold; + if (sigaction(SIGUSR1, &act, NULL) == -1) { + err("thread_do(): cannot handle SIGUSR1"); + } + + /* Mark thread as alive (initialized) */ + pthread_mutex_lock(&thpool_p->thcount_lock); + thpool_p->num_threads_alive += 1; + pthread_mutex_unlock(&thpool_p->thcount_lock); + + while (threads_keepalive) { + + bsem_wait(thpool_p->jobqueue.has_jobs); + + if (threads_keepalive) { + + pthread_mutex_lock(&thpool_p->thcount_lock); + thpool_p->num_threads_working++; + pthread_mutex_unlock(&thpool_p->thcount_lock); + + /* Read job from queue and execute it */ + void (*func_buff)(void *); + void *arg_buff; + job * job_p = jobqueue_pull(&thpool_p->jobqueue); + if (job_p) { + func_buff = job_p->function; + arg_buff = job_p->arg; + func_buff(arg_buff); + free(job_p); + } + + pthread_mutex_lock(&thpool_p->thcount_lock); + thpool_p->num_threads_working--; + if (!thpool_p->num_threads_working) { + pthread_cond_signal(&thpool_p->threads_all_idle); + } + pthread_mutex_unlock(&thpool_p->thcount_lock); + } + } + pthread_mutex_lock(&thpool_p->thcount_lock); + thpool_p->num_threads_alive--; + pthread_mutex_unlock(&thpool_p->thcount_lock); + + return NULL; +} + +/* Frees a thread */ +static void +thread_destroy(thread *thread_p) +{ + free(thread_p); +} + +/* ============================ JOB QUEUE =========================== */ + +/* Initialize queue */ +static int +jobqueue_init(jobqueue *jobqueue_p) +{ + jobqueue_p->len = 0; + jobqueue_p->front = NULL; + jobqueue_p->rear = NULL; + + jobqueue_p->has_jobs = (struct bsem *)malloc(sizeof(struct bsem)); + if (jobqueue_p->has_jobs == NULL) { + return -1; + } + + pthread_mutex_init(&(jobqueue_p->rwmutex), NULL); + bsem_init(jobqueue_p->has_jobs, 0); + + return 0; +} + +/* Clear the queue */ +static void +jobqueue_clear(jobqueue *jobqueue_p) +{ + + while (jobqueue_p->len) { + free(jobqueue_pull(jobqueue_p)); + } + + jobqueue_p->front = NULL; + jobqueue_p->rear = NULL; + bsem_reset(jobqueue_p->has_jobs); + jobqueue_p->len = 0; +} + +/* Add (allocated) job to queue + */ +static void +jobqueue_push(jobqueue *jobqueue_p, struct job *newjob) +{ + + pthread_mutex_lock(&jobqueue_p->rwmutex); + newjob->prev = NULL; + + switch (jobqueue_p->len) { + + case 0: /* if no jobs in queue */ + jobqueue_p->front = newjob; + jobqueue_p->rear = newjob; + break; + + default: /* if jobs in queue */ + jobqueue_p->rear->prev = newjob; + jobqueue_p->rear = newjob; + } + jobqueue_p->len++; + + bsem_post(jobqueue_p->has_jobs); + pthread_mutex_unlock(&jobqueue_p->rwmutex); +} + +static struct job * +jobqueue_pull(jobqueue *jobqueue_p) +{ + + pthread_mutex_lock(&jobqueue_p->rwmutex); + job *job_p = jobqueue_p->front; + + switch (jobqueue_p->len) { + + case 0: /* if no jobs in queue */ + break; + + case 1: /* if one job in queue */ + jobqueue_p->front = NULL; + jobqueue_p->rear = NULL; + jobqueue_p->len = 0; + break; + + default: /* if >1 jobs in queue */ + jobqueue_p->front = job_p->prev; + jobqueue_p->len--; + /* more than one job in queue -> post it */ + bsem_post(jobqueue_p->has_jobs); + } + + pthread_mutex_unlock(&jobqueue_p->rwmutex); + return job_p; +} + +/* Free all queue resources back to the system */ +static void +jobqueue_destroy(jobqueue *jobqueue_p) +{ + jobqueue_clear(jobqueue_p); + free(jobqueue_p->has_jobs); +} + +/* ======================== SYNCHRONISATION ========================= */ + +/* Init semaphore to 1 or 0 */ +static void +bsem_init(bsem *bsem_p, int value) +{ + if (value < 0 || value > 1) { + err("bsem_init(): Binary semaphore can take only values 1 or 0"); + exit(1); + } + pthread_mutex_init(&(bsem_p->mutex), NULL); + pthread_cond_init(&(bsem_p->cond), NULL); + bsem_p->v = value; +} + +/* Reset semaphore to 0 */ +static void +bsem_reset(bsem *bsem_p) +{ + bsem_init(bsem_p, 0); +} + +/* Post to at least one thread */ +static void +bsem_post(bsem *bsem_p) +{ + pthread_mutex_lock(&bsem_p->mutex); + bsem_p->v = 1; + pthread_cond_signal(&bsem_p->cond); + pthread_mutex_unlock(&bsem_p->mutex); +} + +/* Post to all threads */ +static void +bsem_post_all(bsem *bsem_p) +{ + pthread_mutex_lock(&bsem_p->mutex); + bsem_p->v = 1; + pthread_cond_broadcast(&bsem_p->cond); + pthread_mutex_unlock(&bsem_p->mutex); +} + +/* Wait on semaphore until semaphore has value 0 */ +static void +bsem_wait(bsem *bsem_p) +{ + pthread_mutex_lock(&bsem_p->mutex); + while (bsem_p->v != 1) { + pthread_cond_wait(&bsem_p->cond, &bsem_p->mutex); + } + bsem_p->v = 0; + pthread_mutex_unlock(&bsem_p->mutex); +} \ No newline at end of file diff --git a/src/commons/utils/timer_utils.c b/src/commons/utils/timer_utils.c new file mode 100644 index 000000000..a768ae19a --- /dev/null +++ b/src/commons/utils/timer_utils.c @@ -0,0 +1,103 @@ +#include "timer_utils.h" + +#define MICROSECONDS_IN_SECONDS 1000000 + +/** + * Function that returns the current timestamp in microseconds + * since the Epoch (1970-01-01 00:00:00:000:000 UTC). + */ +int64_t +timer_us_timestamp() +{ + struct timeval tmp; + gettimeofday(&(tmp), NULL); + return ((int64_t)tmp.tv_usec) + (int64_t)(tmp.tv_sec * MICROSECONDS_IN_SECONDS); +} + +/** + * Function that returns the current timestamp in miliseconds + * since the Epoch (1970-01-01 00:00:00:000 UTC). + */ +int64_t +timer_ms_timestamp() +{ + return (timer_us_timestamp() / 1000); +} + +/** + * Function that returns the current timestamp in seconds + * since the Epoch (1970-01-01 00:00:00 UTC). + */ +int64_t +timer_s_timestamp() +{ + return (timer_ms_timestamp() / 1000); +} + +void +timer_start(stopwatch_t *t) +{ + t->start_mark = timer_us_timestamp(); + t->pause_mark = 0; + t->running = true; + t->paused = false; +} + +void +timer_pause(stopwatch_t *t) +{ + if (!(t->running) || (t->paused)) + return; + + t->pause_mark = timer_us_timestamp() - (t->start_mark); + t->running = false; + t->paused = true; +} + +void +timer_unpause(stopwatch_t *t) +{ + if (t->running || !(t->paused)) + return; + + t->start_mark = timer_us_timestamp() - (t->pause_mark); + t->running = true; + t->paused = false; +} + +double +timer_delta_us(stopwatch_t *t) +{ + if (t->running) + return timer_us_timestamp() - (t->start_mark); + + if (t->paused) + return t->pause_mark; + + // Will never actually get here + return (double)((t->pause_mark) - (t->start_mark)); +} + +double +timer_delta_ms(stopwatch_t *t) +{ + return (timer_delta_us(t) / 1000.0); +} + +double +timer_delta_s(stopwatch_t *t) +{ + return (timer_delta_ms(t) / 1000.0); +} + +double +timer_delta_m(stopwatch_t *t) +{ + return (timer_delta_s(t) / 60.0); +} + +double +timer_delta_h(stopwatch_t *t) +{ + return (timer_delta_m(t) / 60.0); +} diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 449b64f10..aaedcd4ae 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -7,6 +7,7 @@ if(PDC_ENABLE_FASTBIT) endif() include_directories( + ${PDC_COMMON_INCLUDE_DIRS} ${PDC_INCLUDES_BUILD_TIME} ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR} @@ -35,8 +36,10 @@ if(PDC_ENABLE_TIMING) add_definitions(-DPDC_TIMING=1) endif() -add_executable(pdc_server.exe + +add_library(pdc_server_lib pdc_server.c + pdc_server_metadata_index.c pdc_server_metadata.c pdc_client_server_common.c dablooms/pdc_dablooms.c @@ -48,17 +51,28 @@ add_executable(pdc_server.exe ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c ${PDC_SOURCE_DIR}/src/utils/pdc_region_utils.c - ${PDC_SOURCE_DIR}/src/utils/pdc_timing.c ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_analysis_common.c ${PDC_SOURCE_DIR}/src/api/pdc_transform/pdc_transforms_common.c ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_hist_pkg.c ) +if(PDC_ENABLE_FASTBIT) + message(STATUS "Enabled fastbit") + target_link_libraries(pdc_server_lib mercury ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES} ${FASTBIT_LIBRARY}/libfastbit.so) +else() + target_link_libraries(pdc_server_lib mercury ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES}) +endif() + +add_executable(pdc_server.exe + pdc_server_main.c +) +target_link_libraries(pdc_server.exe pdc_server_lib) + + +add_executable(pdc_server_metadata_index_test + pdc_server_metadata_index_test.c +) +target_link_libraries(pdc_server_metadata_index_test pdc_server_lib) -#install( -# TARGETS -# pdc_server.exe -# DESTINATION ${PDC_INSTALL_BIN_DIR} -#) if(NOT ${PDC_INSTALL_BIN_DIR} MATCHES ${PROJECT_BINARY_DIR}/bin) install( @@ -68,11 +82,6 @@ install( ) endif() -if(PDC_ENABLE_FASTBIT) - message(STATUS "Enabled fastbit") - target_link_libraries(pdc_server.exe mercury pdcprof -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES} ${FASTBIT_LIBRARY}/libfastbit.so) -else() - target_link_libraries(pdc_server.exe mercury pdcprof -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES}) -endif() + diff --git a/src/server/dablooms/pdc_dablooms.c b/src/server/dablooms/pdc_dablooms.c index 321777000..2be56c884 100644 --- a/src/server/dablooms/pdc_dablooms.c +++ b/src/server/dablooms/pdc_dablooms.c @@ -13,6 +13,7 @@ #include #include #include +#include "pdc_malloc.h" #include "pdc_murmur.h" #include "pdc_dablooms.h" @@ -37,7 +38,7 @@ free_bitmap(bitmap_t *bitmap) bitmap_t * bitmap_resize(bitmap_t *bitmap, size_t new_size) { - bitmap->array = malloc(new_size); + bitmap->array = PDC_malloc(new_size); bitmap->bytes = new_size; return bitmap; } @@ -50,7 +51,7 @@ new_bitmap(size_t bytes) { bitmap_t *bitmap; - if ((bitmap = (bitmap_t *)malloc(sizeof(bitmap_t))) == NULL) { + if ((bitmap = (bitmap_t *)PDC_malloc(sizeof(bitmap_t))) == NULL) { return NULL; } @@ -181,7 +182,7 @@ counting_bloom_init(unsigned int capacity, double error_rate, long offset) { counting_bloom_t *bloom; - if ((bloom = malloc(sizeof(counting_bloom_t))) == NULL) { + if ((bloom = PDC_malloc(sizeof(counting_bloom_t))) == NULL) { fprintf(stderr, "Error, could not realloc a new bloom filter\n"); return NULL; } @@ -194,7 +195,7 @@ counting_bloom_init(unsigned int capacity, double error_rate, long offset) bloom->size = bloom->nfuncs * bloom->counts_per_func; /* rounding-up integer divide by 2 of bloom->size */ bloom->num_bytes = ((bloom->size + 1) / 2) + sizeof(counting_bloom_header_t); - bloom->hashes = calloc(bloom->nfuncs, sizeof(uint32_t)); + bloom->hashes = PDC_calloc(bloom->nfuncs, sizeof(uint32_t)); return bloom; } diff --git a/src/server/include/pdc_client_server_common.h b/src/server/include/pdc_client_server_common.h index 7be76a1b3..98edcacdc 100644 --- a/src/server/include/pdc_client_server_common.h +++ b/src/server/include/pdc_client_server_common.h @@ -26,26 +26,16 @@ #define PDC_CLIENT_SERVER_COMMON_H #include "pdc_config.h" +#include "pdc_private.h" #include "pdc_linkedlist.h" #include "pdc_prop_pkg.h" #include "pdc_analysis_and_transforms_common.h" #include "pdc_query.h" -#include "mercury_macros.h" -#include "mercury_proc_string.h" -#include "mercury_list.h" -#include "mercury_config.h" -#include "mercury_thread_pool.h" #include "pdc_timing.h" #include "pdc_server_region_transfer_metadata_query.h" #include "pdc_server_region_transfer.h" -#ifdef ENABLE_MULTITHREAD -#include "mercury_thread_pool.h" -#include "mercury_thread_condition.h" -#include "mercury_thread_mutex.h" -#endif - #include #include #include @@ -57,18 +47,14 @@ hg_thread_mutex_t meta_buf_map_mutex_g; hg_thread_mutex_t meta_obj_map_mutex_g; #endif -#ifndef HOST_NAME_MAX -#if defined(__APPLE__) -#define HOST_NAME_MAX 255 -#else -#define HOST_NAME_MAX 64 -#endif /* __APPLE__ */ -#endif /* HOST_NAME_MAX */ - #define PAGE_SIZE 4096 -#define ADDR_MAX 512 +#define ADDR_MAX 1024 +#define NA_STRING_INFO_LEN ADDR_MAX / 2 +#define HOSTNAME_LEN ADDR_MAX / 8 +#define TMP_DIR_STRING_LEN ADDR_MAX / 2 #define DIM_MAX 4 #define TAG_LEN_MAX 2048 +#define OBJ_NAME_MAX TAG_LEN_MAX / 2 #define PDC_SERVER_ID_INTERVEL 1000000000ull #define PDC_SERVER_MAX_PROC_PER_NODE 32 #define PDC_SERIALIZE_MAX_SIZE 256 @@ -78,8 +64,20 @@ hg_thread_mutex_t meta_obj_map_mutex_g; #define PDC_SEQ_ID_INIT_VALUE 1000 #define PDC_UPDATE_CACHE 111 #define PDC_UPDATE_STORAGE 101 +#define DART_ALPHABET_SIZE 27 + +#ifndef HOST_NAME_MAX +#if defined(__APPLE__) +#define HOST_NAME_MAX ADDR_MAX / 4 +#define HOSTNAME_LEN HOST_NAME_MAX +#else +#define HOST_NAME_MAX ADDR_MAX / 8 +#define HOSTNAME_LEN HOST_NAME_MAX +#endif /* __APPLE__ */ +#endif /* HOST_NAME_MAX */ #define pdc_server_cfg_name_g "server.cfg" +#define pdc_app_lock_name_g "pdc_app.lck" #define ADD_OBJ 1 #define DEL_OBJ 2 @@ -98,42 +96,6 @@ hg_handle_t close_all_server_handle_g; #define PDC_MAX(x, y) (((x) > (y)) ? (x) : (y)) #define PDC_MIN(x, y) (((x) < (y)) ? (x) : (y)) -#if defined(IS_PDC_SERVER) && defined(ENABLE_MULTITHREAD) - -// Macros for multi-thread callback, grabbed from Mercury/Testing/mercury_rpc_cb.c -#define HG_TEST_RPC_CB(func_name, handle) static hg_return_t func_name##_thread_cb(hg_handle_t handle) - -/* Assuming func_name_cb is defined, calling HG_TEST_THREAD_CB(func_name) - * will define func_name_thread and func_name_thread_cb that can be used - * to execute RPC callback from a thread - */ -#define HG_TEST_THREAD_CB(func_name) \ - static HG_INLINE HG_THREAD_RETURN_TYPE func_name##_thread(void *arg) \ - { \ - hg_handle_t handle = (hg_handle_t)arg; \ - hg_thread_ret_t thread_ret = (hg_thread_ret_t)0; \ - \ - func_name##_thread_cb(handle); \ - \ - return thread_ret; \ - } \ - hg_return_t func_name##_cb(hg_handle_t handle) \ - { \ - struct hg_thread_work *work = HG_Get_data(handle); \ - hg_return_t ret = HG_SUCCESS; \ - \ - work->func = func_name##_thread; \ - work->args = handle; \ - hg_thread_pool_post(hg_test_thread_pool_g, work); \ - \ - return ret; \ - } -#else -#define HG_TEST_RPC_CB(func_name, handle) hg_return_t func_name##_cb(hg_handle_t handle) -#define HG_TEST_THREAD_CB(func_name) - -#endif // End of ENABLE_MULTITHREAD - struct _pdc_iterator_info *PDC_Block_iterator_cache; /****************************/ @@ -282,6 +244,7 @@ typedef struct { typedef struct { int32_t client_id; int32_t nclient; + int32_t is_init; char client_addr[ADDR_MAX]; } client_test_connect_args; @@ -390,8 +353,8 @@ typedef struct data_server_region_unmap_t { // For storing metadata typedef struct pdc_metadata_t { int user_id; // Both server and client gets it and do security check - char app_name[ADDR_MAX]; - char obj_name[ADDR_MAX]; + char app_name[OBJ_NAME_MAX]; + char obj_name[OBJ_NAME_MAX]; int time_step; // Above four are the unique identifier for objects @@ -474,6 +437,9 @@ typedef struct metadata_query_transfer_in_t { size_t ndim; const char *tags; + const char *k_query; + const char *vfrom_query; + const char *vto_query; } metadata_query_transfer_in_t; /* Define metadata_query_in_t */ @@ -647,6 +613,8 @@ typedef struct { /* Define metadata_query_transfer_out_t */ typedef struct { int32_t ret; + int64_t server_time_elapsed; + int64_t server_memory_consumption; hg_bulk_t bulk_handle; } metadata_query_transfer_out_t; @@ -688,6 +656,7 @@ typedef struct { typedef struct { uint32_t client_id; int32_t nclient; + int is_init; hg_string_t client_addr; } client_test_connect_in_t; @@ -1171,6 +1140,47 @@ typedef struct query_storage_region_transfer_t { pdc_histogram_t hist; } query_storage_region_transfer_t; +/* Define dart_get_server_info_in_t */ +typedef struct { + uint32_t serverId; +} dart_get_server_info_in_t; + +/* Define dart_get_server_info_out_t */ +typedef struct { + int64_t indexed_word_count; + int64_t request_count; +} dart_get_server_info_out_t; + +/* Define dart_perform_one_server_in_t */ +typedef struct { + int8_t op_type; + int8_t hash_algo; + hg_const_string_t attr_key; + hg_const_string_t attr_val; + int8_t obj_ref_type; + uint64_t obj_primary_ref; + uint64_t obj_secondary_ref; + uint64_t obj_server_ref; + int64_t timestamp; +} dart_perform_one_server_in_t; + +/* Define dart_perform_one_server_out_t */ +typedef struct { + int8_t op_type; + int32_t ret; + hg_bulk_t bulk_handle; + int64_t indexed_word_count; + int64_t request_count; + int8_t has_bulk; + int64_t n_items; + int64_t timestamp; + int64_t server_time_elapsed; + int64_t server_memory_consumption; +} dart_perform_one_server_out_t; + +/*****************************************/ +/* Serialization and Deserialization */ +/*****************************************/ /* Define hg_proc_pdc_kvtag_t */ static hg_return_t hg_proc_pdc_kvtag_t(hg_proc_t proc, void *data) @@ -1188,6 +1198,11 @@ hg_proc_pdc_kvtag_t(hg_proc_t proc, void *data) // HG_LOG_ERROR("Proc error"); return ret; } + ret = hg_proc_int8_t(proc, &struct_data->type); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } if (struct_data->size) { switch (hg_proc_get_op(proc)) { case HG_DECODE: @@ -1548,6 +1563,21 @@ hg_proc_metadata_query_transfer_in_t(hg_proc_t proc, void *data) // HG_LOG_ERROR("Proc error"); return ret; } + ret = hg_proc_hg_string_t(proc, &struct_data->k_query); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_hg_string_t(proc, &struct_data->vfrom_query); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_hg_string_t(proc, &struct_data->vto_query); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } return ret; } @@ -1563,6 +1593,16 @@ hg_proc_metadata_query_transfer_out_t(hg_proc_t proc, void *data) // HG_LOG_ERROR("Proc error"); return ret; } + ret = hg_proc_int64_t(proc, &struct_data->server_time_elapsed); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_int64_t(proc, &struct_data->server_memory_consumption); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } ret = hg_proc_hg_bulk_t(proc, &struct_data->bulk_handle); if (ret != HG_SUCCESS) { // HG_LOG_ERROR("Proc error"); @@ -2140,6 +2180,11 @@ hg_proc_client_test_connect_in_t(hg_proc_t proc, void *data) // HG_LOG_ERROR("Proc error"); return ret; } + ret = hg_proc_int32_t(proc, &struct_data->is_init); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } ret = hg_proc_hg_string_t(proc, &struct_data->client_addr); if (ret != HG_SUCCESS) { // HG_LOG_ERROR("Proc error"); @@ -3756,23 +3801,176 @@ hg_proc_query_storage_region_transfer_t(hg_proc_t proc, void *data) return ret; } +static HG_INLINE hg_return_t +hg_proc_dart_get_server_info_in_t(hg_proc_t proc, void *data) +{ + hg_return_t ret; + + dart_get_server_info_in_t *struct_data = (dart_get_server_info_in_t *)data; + + ret = hg_proc_uint32_t(proc, &struct_data->serverId); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + return ret; +} + +static HG_INLINE hg_return_t +hg_proc_dart_get_server_info_out_t(hg_proc_t proc, void *data) +{ + hg_return_t ret; + dart_get_server_info_out_t *struct_data = (dart_get_server_info_out_t *)data; + + ret = hg_proc_int64_t(proc, &struct_data->indexed_word_count); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + + ret = hg_proc_int64_t(proc, &struct_data->request_count); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + return ret; +} + +static HG_INLINE hg_return_t +hg_proc_dart_perform_one_server_in_t(hg_proc_t proc, void *data) +{ + hg_return_t ret; + dart_perform_one_server_in_t *struct_data = (dart_perform_one_server_in_t *)data; + ret = hg_proc_int8_t(proc, &struct_data->op_type); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_int8_t(proc, &struct_data->hash_algo); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_hg_const_string_t(proc, &struct_data->attr_key); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_hg_const_string_t(proc, &struct_data->attr_val); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_int8_t(proc, &struct_data->obj_ref_type); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_uint64_t(proc, &struct_data->obj_primary_ref); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_uint64_t(proc, &struct_data->obj_secondary_ref); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_uint64_t(proc, &struct_data->obj_server_ref); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + + ret = hg_proc_int64_t(proc, &struct_data->timestamp); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + return ret; +} + +static HG_INLINE hg_return_t +hg_proc_dart_perform_one_server_out_t(hg_proc_t proc, void *data) +{ + hg_return_t ret; + dart_perform_one_server_out_t *struct_data = (dart_perform_one_server_out_t *)data; + ret = hg_proc_int8_t(proc, &struct_data->op_type); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_int32_t(proc, &struct_data->ret); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_hg_bulk_t(proc, &struct_data->bulk_handle); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_int64_t(proc, &struct_data->indexed_word_count); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_int64_t(proc, &struct_data->request_count); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_int8_t(proc, &struct_data->has_bulk); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_int64_t(proc, &struct_data->n_items); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_int64_t(proc, &struct_data->timestamp); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_int64_t(proc, &struct_data->server_time_elapsed); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + ret = hg_proc_int64_t(proc, &struct_data->server_memory_consumption); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } + return ret; +} + /***************************/ /* Library Private Structs */ /***************************/ struct bulk_args_t { - int cnt; - int op; - uint64_t cont_id; - hg_handle_t handle; - hg_bulk_t bulk_handle; - size_t nbytes; - int origin; - size_t ret; - pdc_metadata_t **meta_arr; - uint32_t n_meta; - uint64_t obj_id; - uint64_t * obj_ids; - int client_seq_id; + int cnt; + int op; + uint64_t cont_id; + hg_handle_t handle; + hg_bulk_t bulk_handle; + size_t nbytes; + int origin; + size_t ret; + pdc_metadata_t ** meta_arr; + uint32_t n_meta; + uint64_t obj_id; + uint64_t * obj_ids; + int client_seq_id; + int is_id; // if is_id == true, then use uint64_t; otherwise, pdc_metadata_t + int8_t op_type; + hg_atomic_int32_t bulk_done_flag; + int server_id; + int64_t server_time_elapsed; + int64_t server_memory_consumption; int query_id; @@ -4087,6 +4285,10 @@ hg_id_t PDC_get_sel_data_rpc_register(hg_class_t *hg_class); hg_id_t PDC_send_data_query_rpc_register(hg_class_t *hg_class); hg_id_t PDC_send_data_query_region_register(hg_class_t *hg_class); +// DART index +hg_id_t PDC_dart_get_server_info_register(hg_class_t *hg_class); +hg_id_t PDC_dart_perform_one_server_register(hg_class_t *hg_class); + /** * Calculate time from start to end * diff --git a/src/server/include/pdc_server.h b/src/server/include/pdc_server.h index a4b79f6d5..09f527013 100644 --- a/src/server/include/pdc_server.h +++ b/src/server/include/pdc_server.h @@ -191,4 +191,13 @@ hg_return_t PDC_Server_checkpoint_cb(); */ hg_return_t PDC_Server_recv_shm_cb(const struct hg_cb_info *callback_info); +/** + * This is for main function in the executable to call. + * + * \param argc[IN] Number of command line arguments + * \param argv[IN] Command line arguments + * \return Non-negative on success/Negative on failure + */ +int server_run(int argc, char *argv[]); + #endif /* PDC_SERVER_H */ diff --git a/src/server/include/pdc_server_metadata.h b/src/server/include/pdc_server_metadata.h index f81225a7d..d7832b6f9 100644 --- a/src/server/include/pdc_server_metadata.h +++ b/src/server/include/pdc_server_metadata.h @@ -36,6 +36,9 @@ #include "pdc_server_common.h" #include "pdc_client_server_common.h" +#include "pdc_server_metadata_index.h" + +#include "pdc_malloc.h" #define CREATE_BLOOM_THRESHOLD 64 @@ -44,7 +47,7 @@ /*****************************/ extern int pdc_server_rank_g; extern int pdc_server_size_g; -extern char pdc_server_tmp_dir_g[ADDR_MAX]; +extern char pdc_server_tmp_dir_g[TMP_DIR_STRING_LEN]; extern uint32_t n_metadata_g; extern HashTable * metadata_hash_table_g; extern HashTable * container_hash_table_g; diff --git a/src/server/include/pdc_server_metadata_index.h b/src/server/include/pdc_server_metadata_index.h new file mode 100644 index 000000000..d5360539c --- /dev/null +++ b/src/server/include/pdc_server_metadata_index.h @@ -0,0 +1,84 @@ +#ifndef PDC_SERVER_METADATA_INDEX_H +#define PDC_SERVER_METADATA_INDEX_H + +#include "pdc_client_server_common.h" + +// #include "hashtab.h" +#include "query_utils.h" +#include "timer_utils.h" +#include "art.h" +#include "pdc_set.h" +#include "pdc_hash.h" +#include "pdc_compare.h" +#include "dart_core.h" +#include "pdc_hash-table.h" + +typedef struct { + // On the leaf of ART, we maintain a hash table of IDs of all objects containing that key. + HashTable *server_id_obj_id_table; + + dart_indexed_value_type_t data_type; + // Also, for key lookup ART, we also maintain the pointer to the value tree + void *extra_prefix_index; + void *extra_suffix_index; + void *extra_range_index; + void *extra_infix_index; +} key_index_leaf_content; + +typedef struct pdc_art_iterator_param { + char * query_str; + char * level_one_infix; + char * level_two_infix; + uint32_t total_count; + Set * out; +} pdc_art_iterator_param_t; + +/** + * @brief Initialize the ART index + */ +void PDC_Server_dart_init(); + +// /** +// * @brief Create the metadata index +// * @param in [IN] Input parameters for the create operation +// * @param out [OUT] Output parameters for the create operation +// * @return perr_t SUCCESS on success, FAIL on failure +// */ +// perr_t PDC_Server_metadata_index_create(metadata_index_create_in_t *in, metadata_index_create_out_t *out); + +// /** +// * @brief Delete the metadata index +// * @param in [IN] Input parameters for the delete operation +// * @param out [OUT] Output parameters for the delete operation +// * @return perr_t SUCCESS on success, FAIL on failure +// */ +// perr_t PDC_Server_metadata_index_delete(metadata_index_delete_in_t *in, metadata_index_delete_out_t *out); + +// /** +// * @brief Search the metadata index +// * @param in [IN] Input parameters for the search operation +// * @param out [OUT] Output parameters for the search operation +// * @return perr_t SUCCESS on success, FAIL on failure +// */ +// perr_t PDC_Server_metadata_index_search(metadata_index_search_in_t *in, metadata_index_search_out_t *out, +// uint64_t *n_obj_ids_ptr, uint64_t **buf_ptrs); + +/** + * @brief Get the server information for the metadata index + * @param in [IN] Input parameters for the server info + * @param out [OUT] Output parameters for the server info + * @return perr_t SUCCESS on success, FAIL on failure + */ +perr_t PDC_Server_dart_get_server_info(dart_get_server_info_in_t *in, dart_get_server_info_out_t *out); + +/** + * @brief Perform various of DART operations on one single server. + * @param in [IN] Input parameters for the DART operation + * @param out [OUT] Output parameters for the DART operation + * @return perr_t SUCCESS on success, FAIL on failure + */ +perr_t PDC_Server_dart_perform_one_server(dart_perform_one_server_in_t * in, + dart_perform_one_server_out_t *out, uint64_t *n_obj_ids_ptr, + uint64_t **buf_ptrs); + +#endif /* PDC_SERVER_METADATA_INDEX_H */ \ No newline at end of file diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index 289c20527..d90f83bff 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -265,7 +265,14 @@ PDC_get_var_type_size(pdc_var_type_t dtype) break; case PDC_INT8: ret_value = sizeof(int8_t); - + goto done; + break; + case PDC_UINT8: + ret_value = sizeof(uint8_t); + goto done; + break; + case PDC_UINT16: + ret_value = sizeof(uint16_t); goto done; break; case PDC_INT64: @@ -394,7 +401,7 @@ PDC_metadata_cmp(pdc_metadata_t *a, pdc_metadata_t *b) void PDC_mkdir(const char *dir) { - char tmp[ADDR_MAX]; + char tmp[TMP_DIR_STRING_LEN]; char *p = NULL; FUNC_ENTER(NULL); @@ -469,8 +476,8 @@ PDC_metadata_init(pdc_metadata_t *a) a->ndim = 0; a->data_server_id = 0; - memset(a->app_name, 0, sizeof(char) * ADDR_MAX); - memset(a->obj_name, 0, sizeof(char) * ADDR_MAX); + memset(a->app_name, 0, sizeof(char) * NAME_MAX); + memset(a->obj_name, 0, sizeof(char) * NAME_MAX); memset(a->tags, 0, sizeof(char) * TAG_LEN_MAX); memset(a->data_location, 0, sizeof(char) * ADDR_MAX); memset(a->dims, 0, sizeof(uint64_t) * DIM_MAX); @@ -1478,6 +1485,21 @@ PDC_Server_recv_get_sel_data(const struct hg_cb_info *callback_info ATTRIBUTE(un { return HG_SUCCESS; } +perr_t +PDC_Server_dart_get_server_info(dart_get_server_info_in_t *in ATTRIBUTE(unused), + dart_get_server_info_out_t *out ATTRIBUTE(unused)) +{ + return SUCCEED; +} +perr_t +PDC_Server_dart_perform_one_server(dart_perform_one_server_in_t *in ATTRIBUTE(unused), + dart_perform_one_server_out_t *out ATTRIBUTE(unused), + uint64_t *n_obj_ids_ptr ATTRIBUTE(unused), + uint64_t **buf_ptrs ATTRIBUTE(unused)) +{ + return SUCCEED; +} + #else hg_return_t PDC_Client_work_done_cb(const struct hg_cb_info *callback_info ATTRIBUTE(unused)) @@ -1649,6 +1671,7 @@ HG_TEST_RPC_CB(client_test_connect, handle) #endif args->client_id = in.client_id; args->nclient = in.nclient; + args->is_init = in.is_init; sprintf(args->client_addr, "%s", in.client_addr); #ifdef ENABLE_MULTITHREAD hg_thread_mutex_unlock(&pdc_client_info_mutex_g); @@ -1677,7 +1700,10 @@ HG_TEST_RPC_CB(container_query, handle) HG_Get_input(handle, &in); PDC_Server_find_container_by_name(in.cont_name, &cont_entry); - out.cont_id = cont_entry->cont_id; + if (cont_entry) + out.cont_id = cont_entry->cont_id; + else + out.cont_id = 0; HG_Respond(handle, NULL, NULL, &out); @@ -3088,7 +3114,7 @@ HG_TEST_RPC_CB(region_release, handle) size2 = HG_Bulk_get_size(remote_bulk_handle); if (size != size2) { error = 1; - printf("==PDC_SERVER: local size %lu, remote %lu\n", size, size2); + printf("==PDC_SERVER: local size %llu, remote %llu\n", size, size2); PGOTO_ERROR(HG_OTHER_ERROR, "===PDC SERVER: HG_TEST_RPC_CB(region_release, " "handle) local and remote bulk size does not match"); } @@ -3261,7 +3287,7 @@ HG_TEST_RPC_CB(region_release, handle) size2 = HG_Bulk_get_size(remote_bulk_handle); if (size != size2) { error = 1; - printf("==PDC_SERVER: local size %lu, remote %lu\n", size, size2); + printf("==PDC_SERVER: local size %llu, remote %llu\n", size, size2); /* PGOTO_ERROR(HG_OTHER_ERROR, "===PDC SERVER: HG_TEST_RPC_CB(region_release, * handle) local and remote bulk size does not match"); */ } @@ -4520,8 +4546,15 @@ HG_TEST_RPC_CB(query_partial, handle) n_meta_ptr = (uint32_t *)malloc(sizeof(uint32_t)); + stopwatch_t server_timer; + timer_start(&server_timer); + PDC_Server_get_partial_query_result(&in, n_meta_ptr, &buf_ptrs); + timer_pause(&server_timer); + out.server_time_elapsed = (int64_t)timer_delta_us(&server_timer); + out.server_memory_consumption = (int64_t)PDC_get_global_mem_usage(); + // No result found if (*n_meta_ptr == 0) { out.bulk_handle = HG_BULK_NULL; @@ -4604,7 +4637,15 @@ HG_TEST_RPC_CB(query_kvtag, handle) // Decode input HG_Get_input(handle, &in); + stopwatch_t server_timer; + timer_start(&server_timer); + ret_value = PDC_Server_get_kvtag_query_result(&in, &nmeta, &buf_ptr); + + timer_pause(&server_timer); + out.server_time_elapsed = (int64_t)timer_delta_us(&server_timer); + out.server_memory_consumption = (int64_t)PDC_get_global_mem_usage(); + if (ret_value != SUCCEED || nmeta == 0) { out.bulk_handle = HG_BULK_NULL; out.ret = 0; @@ -6358,6 +6399,111 @@ HG_TEST_RPC_CB(send_bulk_rpc, handle) FUNC_LEAVE(ret_value); } +/* static hg_return_t */ +/* dart_get_server_info_cb(hg_handle_t handle) */ +HG_TEST_RPC_CB(dart_get_server_info, handle) +{ + hg_return_t ret = HG_SUCCESS; + dart_get_server_info_in_t in; + dart_get_server_info_out_t out; + + FUNC_ENTER(NULL); + // Extract input from handle + HG_Get_input(handle, &in); + // retrieve server info from desigated server + PDC_Server_dart_get_server_info(&in, &out); + + // Send response to client + HG_Respond(handle, NULL, NULL, &out); + /* printf("==PDC_SERVER: dart_get_server_info_cb(): returned %llu\n", out.indexed_word_count); */ + // Free input + HG_Free_input(handle, &in); + // Free handle + HG_Destroy(handle); + + return ret; +} + +/* static hg_return_t */ +// dart_perform_one_server_cb(hg_handle_t handle) +HG_TEST_RPC_CB(dart_perform_one_server, handle) +{ + hg_return_t ret = HG_SUCCESS; + hg_return_t hg_ret = HG_SUCCESS; + dart_perform_one_server_in_t in; + dart_perform_one_server_out_t out; + + hg_bulk_t bulk_handle = HG_BULK_NULL; + uint64_t * n_obj_ids_ptr; + uint64_t n_buf; + uint64_t **buf_ptrs; + size_t * buf_sizes; + uint32_t i; + + FUNC_ENTER(NULL); + // Extract input from handle + HG_Get_input(handle, &in); + + n_obj_ids_ptr = (uint64_t *)calloc(1, sizeof(uint64_t)); + buf_ptrs = (uint64_t **)calloc(1, sizeof(uint64_t *)); + + stopwatch_t server_timer; + timer_start(&server_timer); + + PDC_Server_dart_perform_one_server(&in, &out, n_obj_ids_ptr, buf_ptrs); + + timer_pause(&server_timer); + out.server_time_elapsed = (int64_t)timer_delta_us(&server_timer); + out.server_memory_consumption = (int64_t)PDC_get_global_mem_usage(); + + // printf("perform_server_cb. n_obj_ids_ptr on op_type = %d = %d\n", in.op_type ,*n_obj_ids_ptr); + out.op_type = in.op_type; + // printf("out.n_items= %d\n", out.n_items); + // No result found + if (*n_obj_ids_ptr == 0) { + out.bulk_handle = HG_BULK_NULL; + out.ret = 0; + // printf("No object ids returned for the query\n"); + ret = HG_Respond(handle, NULL, NULL, &out); + goto done; + } + + n_buf = 1; + buf_sizes = (size_t *)calloc(n_buf, sizeof(size_t)); + buf_sizes[0] = sizeof(uint64_t) * (*n_obj_ids_ptr); + + // Create bulk handle + hg_ret = HG_Bulk_create(hg_class_g, n_buf, (void **)buf_ptrs, (const hg_size_t *)buf_sizes, + HG_BULK_READ_ONLY, &bulk_handle); + if (hg_ret != HG_SUCCESS) { + fprintf(stderr, "Could not create bulk data handle\n"); + return EXIT_FAILURE; + } + + // Fill bulk handle and return number of metadata that satisfy the query + out.bulk_handle = bulk_handle; + out.ret = *n_obj_ids_ptr; + // printf("out.ret = %d\n", out.ret); + + // FIXME: Memory leak? buf_ptrs is not freed + // TODO: To confirm how we can know the bulk data has been sent to client completely + + // Send bulk handle to client + /* printf("query_partial_cb(): Sending bulk handle to client\n"); */ + /* fflush(stdout); */ + /* HG_Respond(handle, PDC_server_bulk_respond_cb, NULL, &out); */ + ret = HG_Respond(handle, NULL, NULL, &out); + +done: + /* printf("==PDC_SERVER: metadata_index_search_cb(): returned %llu\n", out.ret); */ + // Free input + HG_Free_input(handle, &in); + // Free handle + HG_Destroy(handle); + + FUNC_LEAVE(ret); +} + HG_TEST_THREAD_CB(server_lookup_client) HG_TEST_THREAD_CB(gen_obj_id) HG_TEST_THREAD_CB(gen_cont_id) @@ -6423,26 +6569,6 @@ HG_TEST_THREAD_CB(send_bulk_rpc) HG_TEST_THREAD_CB(get_sel_data_rpc) HG_TEST_THREAD_CB(send_read_sel_obj_id_rpc) -#define PDC_FUNC_DECLARE_REGISTER(x) \ - hg_id_t PDC_##x##_register(hg_class_t *hg_class) \ - { \ - hg_id_t ret_value; \ - FUNC_ENTER(NULL); \ - ret_value = MERCURY_REGISTER(hg_class, #x, x##_in_t, x##_out_t, x##_cb); \ - FUNC_LEAVE(ret_value); \ - return ret_value; \ - } - -#define PDC_FUNC_DECLARE_REGISTER_IN_OUT(x, y, z) \ - hg_id_t PDC_##x##_register(hg_class_t *hg_class) \ - { \ - hg_id_t ret_value; \ - FUNC_ENTER(NULL); \ - ret_value = MERCURY_REGISTER(hg_class, #x, y, z, x##_cb); \ - FUNC_LEAVE(ret_value); \ - return ret_value; \ - } - PDC_FUNC_DECLARE_REGISTER(gen_obj_id) PDC_FUNC_DECLARE_REGISTER(gen_cont_id) PDC_FUNC_DECLARE_REGISTER(server_lookup_client) @@ -6510,6 +6636,9 @@ PDC_FUNC_DECLARE_REGISTER_IN_OUT(send_nhits, send_nhits_t, pdc_int_ret_t) PDC_FUNC_DECLARE_REGISTER_IN_OUT(send_bulk_rpc, bulk_rpc_in_t, pdc_int_ret_t) PDC_FUNC_DECLARE_REGISTER_IN_OUT(get_sel_data_rpc, get_sel_data_rpc_in_t, pdc_int_ret_t) PDC_FUNC_DECLARE_REGISTER_IN_OUT(send_read_sel_obj_id_rpc, get_sel_data_rpc_in_t, pdc_int_ret_t) +// DART Index +PDC_FUNC_DECLARE_REGISTER(dart_get_server_info) +PDC_FUNC_DECLARE_REGISTER(dart_perform_one_server) /* * Check if two 1D segments overlaps @@ -6838,11 +6967,11 @@ PDC_kvtag_dup(pdc_kvtag_t *from, pdc_kvtag_t **to) if (from == NULL || to == NULL) PGOTO_DONE(FAIL); - (*to) = (pdc_kvtag_t *)calloc(1, sizeof(pdc_kvtag_t)); - (*to)->name = (char *)malloc(strlen(from->name) + 1); - (*to)->size = from->size; - - (*to)->value = (void *)malloc(from->size); + (*to) = (pdc_kvtag_t *)PDC_calloc(1, sizeof(pdc_kvtag_t)); + (*to)->name = (char *)PDC_malloc(strlen(from->name) + 1); + (*to)->size = from->size; + (*to)->type = from->type; + (*to)->value = (void *)PDC_malloc(from->size); memcpy((void *)(*to)->name, (void *)from->name, strlen(from->name) + 1); memcpy((void *)(*to)->value, (void *)from->value, from->size); diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 1ffddb283..a07e5596b 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -78,14 +78,13 @@ pdc_task_list_t *pdc_server_agg_task_head_g = NULL; pdc_task_list_t *pdc_server_s2s_task_head_g = NULL; int pdc_server_task_id_g = PDC_SERVER_TASK_INIT_VALUE; -pdc_client_info_t * pdc_client_info_g = NULL; -pdc_remote_server_info_t *pdc_remote_server_info_g = NULL; -char * all_addr_strings_1d_g = NULL; -char ** all_addr_strings_g = NULL; -int is_all_client_connected_g = 0; -int is_hash_table_init_g = 0; -int lustre_stripe_size_mb_g = 16; -int lustre_total_ost_g = 0; +pdc_client_info_t * pdc_client_info_g = NULL; +pdc_remote_server_info_t *pdc_remote_server_info_g = NULL; +char * all_addr_strings_1d_g = NULL; +char ** all_addr_strings_g = NULL; +int is_hash_table_init_g = 0; +int lustre_stripe_size_mb_g = 16; +int lustre_total_ost_g = 0; hg_id_t get_remote_metadata_register_id_g; hg_id_t buf_map_server_register_id_g; @@ -112,7 +111,7 @@ extern hg_thread_pool_t *hg_test_thread_pool_g; extern hg_thread_pool_t *hg_test_thread_pool_fs_g; hg_atomic_int32_t close_server_g; -char pdc_server_tmp_dir_g[ADDR_MAX]; +char pdc_server_tmp_dir_g[TMP_DIR_STRING_LEN]; int is_restart_g = 0; int pdc_server_rank_g = 0; int pdc_server_size_g = 1; @@ -265,13 +264,15 @@ PDC_Server_get_client_addr(const struct hg_cb_info *callback_info) hg_thread_mutex_lock(&pdc_client_addr_mutex_g); #endif - if (is_all_client_connected_g == 1) { - printf("==PDC_SERVER[%d]: new application run detected, create new client info\n", pdc_server_rank_g); - fflush(stdout); + if (pdc_client_info_g && in->is_init == 1) { + if (is_debug_g && pdc_server_rank_g == 0) { + printf("==PDC_SERVER[%d]: new application run detected, create new client info\n", + pdc_server_rank_g); + fflush(stdout); + } PDC_Server_destroy_client_info(pdc_client_info_g); - pdc_client_info_g = NULL; - is_all_client_connected_g = 0; + pdc_client_info_g = NULL; } #ifdef ENABLE_MULTITHREAD @@ -692,8 +693,8 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) perr_t ret_value = SUCCEED; int i = 0; char self_addr_string[ADDR_MAX]; - char na_info_string[ADDR_MAX]; - char hostname[1024]; + char na_info_string[NA_STRING_INFO_LEN]; + char hostname[HOSTNAME_LEN]; struct hg_init_info init_info = {0}; /* Set the default mercury transport @@ -727,9 +728,9 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) if ((hg_transport = getenv("HG_TRANSPORT")) == NULL) { hg_transport = default_hg_transport; } - memset(hostname, 0, 1024); - gethostname(hostname, 1023); - snprintf(na_info_string, ADDR_MAX, "%s://%s:%d", hg_transport, hostname, port); + memset(hostname, 0, HOSTNAME_LEN); + gethostname(hostname, HOSTNAME_LEN - 1); + snprintf(na_info_string, NA_STRING_INFO_LEN, "%s://%s:%d", hg_transport, hostname, port); if (pdc_server_rank_g == 0) printf("==PDC_SERVER[%d]: using %.7s\n", pdc_server_rank_g, na_info_string); @@ -904,6 +905,9 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) n_metadata_g = 0; + // Initialize DART + PDC_Server_dart_init(); + // PDC transfer_request infrastructures PDC_server_transfer_request_init(); #ifdef PDC_SERVER_CACHE @@ -1223,6 +1227,7 @@ PDC_Server_checkpoint() fwrite(&key_len, sizeof(int), 1, file); fwrite(kvlist_elt->kvtag->name, key_len, 1, file); fwrite(&kvlist_elt->kvtag->size, sizeof(uint32_t), 1, file); + fwrite(&kvlist_elt->kvtag->type, sizeof(int8_t), 1, file); fwrite(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, 1, file); } @@ -1401,9 +1406,6 @@ PDC_Server_restart(char *filename) printf("Error getting slurm job id from SLURM_JOB_ID!\n"); } - // init hash table - PDC_Server_init_hash_table(); - if (fread(&n_cont, sizeof(int), 1, file) != 1) { printf("Read failed for n_count\n"); } @@ -1491,6 +1493,9 @@ PDC_Server_restart(char *filename) if (fread(&kvtag_list->kvtag->size, sizeof(uint32_t), 1, file) != 1) { printf("Read failed for kvtag_list->kvtag->size\n"); } + if (fread(&kvtag_list->kvtag->type, sizeof(int8_t), 1, file) != 1) { + printf("Read failed for kvtag_list->kvtag->type\n"); + } kvtag_list->kvtag->value = malloc(kvtag_list->kvtag->size); if (fread(kvtag_list->kvtag->value, kvtag_list->kvtag->size, 1, file) != 1) { printf("Read failed for kvtag_list->kvtag->value\n"); @@ -1951,6 +1956,10 @@ PDC_Server_mercury_register() PDC_region_transform_release_register(hg_class_g); PDC_region_analysis_release_register(hg_class_g); + // DART Index + PDC_dart_get_server_info_register(hg_class_g); + PDC_dart_perform_one_server_register(hg_class_g); + // Server to client RPC server_lookup_client_register_id_g = PDC_server_lookup_client_register(hg_class_g); notify_io_complete_register_id_g = PDC_notify_io_complete_register(hg_class_g); @@ -1988,7 +1997,7 @@ PDC_Server_get_env() if (tmp_env_char == NULL) tmp_env_char = "./pdc_tmp"; - snprintf(pdc_server_tmp_dir_g, ADDR_MAX, "%s/", tmp_env_char); + snprintf(pdc_server_tmp_dir_g, TMP_DIR_STRING_LEN, "%s/", tmp_env_char); lustre_total_ost_g = 1; #ifdef ENABLE_LUSTRE @@ -2070,16 +2079,8 @@ PDC_Server_get_env() } } -/* - * Main function of PDC server - * - * \param argc[IN] Number of command line arguments - * \param argv[IN] Command line arguments - * - * \return Non-negative on success/Negative on failure - */ int -main(int argc, char *argv[]) +server_run(int argc, char *argv[]) { int port; perr_t ret; @@ -2182,4 +2183,4 @@ main(int argc, char *argv[]) MPI_Finalize(); #endif return 0; -} +} \ No newline at end of file diff --git a/src/server/pdc_server_main.c b/src/server/pdc_server_main.c new file mode 100644 index 000000000..1cda39457 --- /dev/null +++ b/src/server/pdc_server_main.c @@ -0,0 +1,7 @@ +#include "pdc_server.h" + +int +main(int argc, char *argv[]) +{ + return server_run(argc, argv); +} \ No newline at end of file diff --git a/src/server/pdc_server_metadata.c b/src/server/pdc_server_metadata.c index 2331df3b1..7eb2abd66 100644 --- a/src/server/pdc_server_metadata.c +++ b/src/server/pdc_server_metadata.c @@ -48,6 +48,8 @@ #include "pdc_client_server_common.h" #include "pdc_server_metadata.h" #include "pdc_server.h" +#include "mercury_hash_table.h" +#include "pdc_malloc.h" #define BLOOM_TYPE_T counting_bloom_t #define BLOOM_NEW new_counting_bloom @@ -1216,7 +1218,7 @@ PDC_insert_metadata_to_hash_table(gen_obj_id_in_t *in, gen_obj_id_out_t *out) gettimeofday(&pdc_timer_start, 0); #endif - metadata = (pdc_metadata_t *)malloc(sizeof(pdc_metadata_t)); + metadata = (pdc_metadata_t *)PDC_malloc(sizeof(pdc_metadata_t)); if (metadata == NULL) { printf("Cannot allocate pdc_metadata_t!\n"); goto done; @@ -1251,7 +1253,7 @@ PDC_insert_metadata_to_hash_table(gen_obj_id_in_t *in, gen_obj_id_out_t *out) strcpy(metadata->tags, in->data.tags); strcpy(metadata->data_location, in->data.data_location); - hash_key = (uint32_t *)malloc(sizeof(uint32_t)); + hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); if (hash_key == NULL) { printf("Cannot allocate hash_key!\n"); goto done; @@ -1300,7 +1302,7 @@ PDC_insert_metadata_to_hash_table(gen_obj_id_in_t *in, gen_obj_id_out_t *out) fflush(stdout); pdc_hash_table_entry_head *entry = - (pdc_hash_table_entry_head *)malloc(sizeof(pdc_hash_table_entry_head)); + (pdc_hash_table_entry_head *)PDC_malloc(sizeof(pdc_hash_table_entry_head)); entry->bloom = NULL; entry->metadata = NULL; entry->n_obj = 0; @@ -1526,6 +1528,7 @@ is_metadata_satisfy_constraint(pdc_metadata_t *metadata, metadata_query_transfer } // TODO: Currently only supports searching with one tag if (strcmp(constraints->tags, " ") != 0 && strstr(metadata->tags, constraints->tags) == NULL) { + ret_value = -1; goto done; } @@ -1589,8 +1592,39 @@ PDC_Server_get_partial_query_result(metadata_query_transfer_in_t *in, uint32_t * FUNC_LEAVE(ret_value); } +pbool_t +_is_matching_kvtag(pdc_kvtag_t *in, pdc_kvtag_t *kvtag) +{ + pbool_t ret_value = TRUE; + FUNC_ENTER(NULL); + // match attribute name + if (!simple_matches(kvtag->name, in->name)) { + return FALSE; + } + + // test attribute type + if (in->type != kvtag->type) { + return FALSE; + } + if (in->type == (int8_t)PDC_STRING) { + // FIXME: need to address kvtag->type serialization problem. + char *pattern = (char *)in->value; + if (!simple_matches(kvtag->value, pattern)) { + return FALSE; + } + } + else { // FIXME: for all numeric types, we use memcmp to compare, for exact value query, but we also + // have to support range query. + if (memcmp(in->value, kvtag->value, in->size) != 0) + return FALSE; + } + + FUNC_LEAVE(ret_value); +} + perr_t -PDC_Server_get_kvtag_query_result(pdc_kvtag_t *in, uint32_t *n_meta, uint64_t **obj_ids) +PDC_Server_get_kvtag_query_result(pdc_kvtag_t *in /*FIXME: query input should be string-based*/, + uint32_t *n_meta, uint64_t **obj_ids) { perr_t ret_value = SUCCEED; uint32_t iter = 0; @@ -1605,7 +1639,7 @@ PDC_Server_get_kvtag_query_result(pdc_kvtag_t *in, uint32_t *n_meta, uint64_t ** FUNC_ENTER(NULL); *n_meta = 0; - // TODO: free obj_ids + *obj_ids = (void *)calloc(alloc_size, sizeof(uint64_t)); if (metadata_hash_table_g != NULL) { @@ -1620,33 +1654,27 @@ PDC_Server_get_kvtag_query_result(pdc_kvtag_t *in, uint32_t *n_meta, uint64_t ** { DL_FOREACH(elt->kvtag_list_head, kvtag_list_elt) { - is_name_match = 0; - is_value_match = 0; - if (in->name[0] != ' ') { - if (strcmp(in->name, kvtag_list_elt->kvtag->name) == 0) - is_name_match = 1; - else - continue; - } - else - is_name_match = 1; - - if (((char *)(in->value))[0] != ' ') { - if (memcmp(in->value, kvtag_list_elt->kvtag->value, in->size) == 0) - is_value_match = 1; - else - continue; - } - else - is_value_match = 1; - - if (is_name_match == 1 && is_value_match == 1) { +#ifdef PDC_DEBUG_OUTPUT + printf("==PDC_SERVER: Matching kvtag [\"%s\":\"%s\"] of object %s on condition in->key: " + "%s, in->value: %s ", + (char *)kvtag_list_elt->kvtag->name, (char *)kvtag_list_elt->kvtag->value, + elt->obj_name, in->name, in->value); +#endif + if (_is_matching_kvtag(in, kvtag_list_elt->kvtag) == TRUE) { +#ifdef PDC_DEBUG_OUTPUT + println("[Found]"); +#endif if (iter >= alloc_size) { alloc_size *= 2; *obj_ids = (void *)realloc(*obj_ids, alloc_size * sizeof(uint64_t)); } (*obj_ids)[iter++] = elt->obj_id; - break; + // break; // FIXME: shall we break here? or continue to check other kvtags? + } + else { +#ifdef PDC_DEBUG_OUTPUT + println("[NOT FOUND]"); +#endif } } // End for each kvtag @@ -2198,7 +2226,7 @@ PDC_Server_container_add_objs(int n_obj, uint64_t *obj_ids, uint64_t cont_id) // Check if need to allocate space if (cont_entry->n_allocated == 0) { cont_entry->n_allocated = PDC_ALLOC_BASE_NUM; - cont_entry->obj_ids = (uint64_t *)calloc(sizeof(uint64_t), PDC_ALLOC_BASE_NUM); + cont_entry->obj_ids = (uint64_t *)PDC_calloc(sizeof(uint64_t), PDC_ALLOC_BASE_NUM); total_mem_usage_g += sizeof(uint64_t) * cont_entry->n_allocated; } else if (cont_entry->n_allocated < cont_entry->n_obj + n_obj) { @@ -2212,7 +2240,7 @@ PDC_Server_container_add_objs(int n_obj, uint64_t *obj_ids, uint64_t cont_id) cont_entry->n_allocated, realloc_size / sizeof(uint64_t)); } - cont_entry->obj_ids = (uint64_t *)realloc(cont_entry->obj_ids, realloc_size); + cont_entry->obj_ids = (uint64_t *)PDC_realloc(cont_entry->obj_ids, realloc_size); if (NULL == cont_entry->obj_ids) { printf("==PDC_SERVER[%d]: %s - ERROR with realloc!\n", pdc_server_rank_g, __func__); ret_value = FAIL; @@ -2522,7 +2550,7 @@ PDC_add_kvtag_to_list(pdc_kvtag_list_t **list_head, pdc_kvtag_t *tag) FUNC_ENTER(NULL); PDC_kvtag_dup(tag, &newtag); - new_list_item = (pdc_kvtag_list_t *)calloc(1, sizeof(pdc_kvtag_list_t)); + new_list_item = PDC_CALLOC(1, pdc_kvtag_list_t); new_list_item->kvtag = newtag; DL_APPEND(*list_head, new_list_item); @@ -2576,7 +2604,6 @@ PDC_Server_add_kvtag(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out) ret_value = FAIL; out->ret = -1; } - } // if lookup_value != NULL else { // look for containers cont_lookup_value = hash_table_lookup(container_hash_table_g, &hash_key); @@ -2778,7 +2805,8 @@ PDC_Server_del_kvtag(metadata_get_kvtag_in_t *in, metadata_add_tag_out_t *out) #ifdef ENABLE_MULTITHREAD int unlocked; #endif - pdc_hash_table_entry_head *lookup_value; + pdc_hash_table_entry_head * lookup_value; + pdc_cont_hash_table_entry_t *cont_lookup_value; FUNC_ENTER(NULL); @@ -2794,62 +2822,63 @@ PDC_Server_del_kvtag(metadata_get_kvtag_in_t *in, metadata_add_tag_out_t *out) #ifdef ENABLE_MULTITHREAD // Obtain lock for hash table - unlocked = 0; hg_thread_mutex_lock(&pdc_metadata_hash_table_mutex_g); #endif + // Look obj tags first lookup_value = hash_table_lookup(metadata_hash_table_g, &hash_key); if (lookup_value != NULL) { pdc_metadata_t *target; target = find_metadata_by_id_from_list(lookup_value->metadata, obj_id); if (target != NULL) { - PDC_del_kvtag_value_from_list(&target->kvtag_list_head, in->key); - out->ret = 1; + ret_value = PDC_del_kvtag_value_from_list(&target->kvtag_list_head, in->key); + out->ret = 1; } else { ret_value = FAIL; out->ret = -1; + printf("==PDC_SERVER[%d]: %s - failed to find requested kvtag [%s]\n", pdc_server_rank_g, + __func__, in->key); + goto done; } } else { - ret_value = FAIL; - out->ret = -1; - } - - if (ret_value != SUCCEED) { - printf("==PDC_SERVER[%d]: %s - error \n", pdc_server_rank_g, __func__); - goto done; + cont_lookup_value = hash_table_lookup(container_hash_table_g, &hash_key); + if (cont_lookup_value != NULL) { + PDC_del_kvtag_value_from_list(&cont_lookup_value->kvtag_list_head, in->key); + out->ret = 1; + } + else { + ret_value = FAIL; + out->ret = -1; + printf("==PDC_SERVER[%d]: %s - failed to find requested kvtag [%s]\n", pdc_server_rank_g, + __func__, in->key); + goto done; + } } +done: #ifdef ENABLE_MULTITHREAD - // ^ Release hash table lock hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); - unlocked = 1; #endif #ifdef ENABLE_TIMING // Timing gettimeofday(&pdc_timer_end, 0); ht_total_sec = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); -#endif #ifdef ENABLE_MULTITHREAD hg_thread_mutex_lock(&pdc_time_mutex_g); #endif -#ifdef ENABLE_TIMING server_update_time_g += ht_total_sec; -#endif #ifdef ENABLE_MULTITHREAD hg_thread_mutex_unlock(&pdc_time_mutex_g); #endif -done: -#ifdef ENABLE_MULTITHREAD - if (unlocked == 0) - hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); -#endif +#endif // End ENABLE_TIMING + fflush(stdout); FUNC_LEAVE(ret_value); diff --git a/src/server/pdc_server_metadata_index.c b/src/server/pdc_server_metadata_index.c new file mode 100644 index 000000000..1e91c42f7 --- /dev/null +++ b/src/server/pdc_server_metadata_index.c @@ -0,0 +1,730 @@ +#include "pdc_server_metadata_index.h" + +#define DART_SERVER_DEBUG 0 + +// DART search +int64_t indexed_word_count_g = 0; +int64_t server_request_count_g = 0; +int64_t request_per_unit_time_g = 0; +double unit_time_to_update_request = 5000.0; // ms. +art_tree *art_key_prefix_tree_g = NULL; +art_tree *art_key_suffix_tree_g = NULL; + +// void +// create_hash_table_for_keyword(char *keyword, char *value, size_t len, void *data) +// { +// uint32_t hashVal = djb2_hash(keyword, (int)len); +// printf("%d:", hashVal); +// gen_obj_id_in_t in; +// gen_obj_id_out_t out; + +// in.data.obj_name = keyword; +// in.data.time_step = (int32_t)data; +// in.data.user_id = (uint32_t)data; +// char *taglist = (char *)calloc(256, sizeof(char)); +// printf("%s=%s", keyword, value); +// sprintf(taglist, "%s=%s", keyword, value); +// in.data.tags = taglist; +// in.data.data_location = " "; +// in.data.app_name = " "; +// in.data.ndim = 1; +// in.hash_value = hashVal; + +// PDC_insert_metadata_to_hash_table(&in, &out); +// } + +// int +// brutal_force_partial_search(metadata_query_transfer_in_t *in, uint32_t *n_meta, void ***buf_ptrs, +// char *k_query, char *vfrom_query, char *vto_query, uint32_t *hash_value) +// { +// int result = 0; + +// uint32_t iter = 0; +// HashTableIterator hash_table_iter; +// HashTableValue * head = NULL; +// pdc_metadata_t * elt; +// int n_entry; + +// if (metadata_hash_table_g != NULL) { +// if (hash_value != NULL) { +// head = hash_table_lookup(metadata_hash_table_g, hash_value); +// if (head != NULL) { +// DL_FOREACH(head->metadata, elt) +// { +// // List all objects, no need to check other constraints +// if (in->is_list_all == 1) { +// (*buf_ptrs)[iter++] = elt; +// } +// // check if current metadata matches search constraint +// else if (is_metadata_satisfy_constraint(elt, in) == 1) { +// (*buf_ptrs)[iter++] = elt; +// } +// } +// } +// } +// else { +// n_entry = hash_table_num_entries(metadata_hash_table_g); +// hash_table_iterate(metadata_hash_table_g, &hash_table_iter); + +// while (n_entry != 0 && hash_table_iter_has_more(&hash_table_iter)) { +// head = hash_table_iter_next(&hash_table_iter); +// DL_FOREACH(head->metadata, elt) +// { +// // List all objects, no need to check other constraints +// if (in->is_list_all == 1) { +// (*buf_ptrs)[iter++] = elt; +// } +// // check if current metadata matches search constraint +// else if (is_metadata_satisfy_constraint(elt, in) == 1) { +// (*buf_ptrs)[iter++] = elt; +// } +// } +// } +// } +// *n_meta = iter; + +// printf("==PDC_SERVER: brutal_force_partial_search: Total matching results: %d\n", *n_meta); +// result = 1; +// } // if (metadata_hash_table_g != NULL) +// else { +// printf("==PDC_SERVER: metadata_hash_table_g not initilized!\n"); +// result = 0; +// } + +// return result; +// } + +// void +// search_through_hash_table(char *k_query, uint32_t index_type, pattern_type_t pattern_type, +// pdc_art_iterator_param_t *param) +// { + +// metadata_query_transfer_in_t in; +// in.is_list_all = -1; +// in.user_id = -1; +// in.app_name = " "; +// in.obj_name = " "; +// in.time_step_from = -1; +// in.time_step_to = -1; +// in.ndim = -1; +// in.tags = " "; +// char * qType_string; +// uint32_t n_meta; +// void ** buf_ptrs; +// char * tok; + +// uint32_t *hash_ptr = NULL; +// uint32_t hash_value = -1; + +// switch (pattern_type) { +// case PATTERN_EXACT: +// qType_string = "Exact"; +// tok = k_query; +// if (index_type == 1) { +// hash_value = djb2_hash(tok, (int)strlen(tok)); +// hash_ptr = &hash_value; +// } +// else if (index_type == 2) { +// hash_value = djb2_hash(tok, 1); +// hash_ptr = &hash_value; +// } +// break; +// case PATTERN_PREFIX: +// qType_string = "Prefix"; +// tok = subrstr(k_query, strlen(k_query) - 1); +// if (index_type == 2) { +// hash_value = djb2_hash(tok, 1); +// hash_ptr = &hash_value; +// } +// else { +// hash_ptr = NULL; +// } +// break; +// case PATTERN_SUFFIX: +// qType_string = "Suffix"; +// tok = substr(k_query, 1); +// tok = reverse_str(tok); +// if (index_type == 2) { +// hash_value = djb2_hash(tok, 1); +// hash_ptr = &hash_value; +// } +// else { +// hash_ptr = NULL; +// } +// break; +// case PATTERN_MIDDLE: +// qType_string = "Infix"; +// tok = substring(k_query, 1, strlen(k_query) - 1); +// break; +// default: +// break; +// } + +// int search_rst = brutal_force_partial_search(&in, &n_meta, &buf_ptrs, k_query, NULL, NULL, hash_ptr); +// int i = 0; +// for (i = 0; i < n_meta; i++) { +// pdc_metadata_t *metadata = (pdc_metadata_t *)buf_ptrs[i]; +// hashset_add(param->out, (metadata->user_id)); +// param->total_count = param->total_count + 1; +// } +// } + +// void +// delete_hash_table_for_keyword(char *keyword, size_t len, void *data) +// { +// uint32_t hashVal = djb2_hash(keyword, (int)len); + +// metadata_delete_in_t in; +// metadata_delete_out_t out; + +// in.obj_name = keyword; +// in.time_step = (int32_t)data; +// in.hash_value = hashVal; + +// PDC_delete_metadata_from_hash_table(&in, &out); +// } + +/****************************/ +/* Initialize DART */ +/****************************/ +void +PDC_Server_dart_init() +{ + + indexed_word_count_g = 0; + server_request_count_g = 0; + art_key_prefix_tree_g = (art_tree *)calloc(1, sizeof(art_tree)); + art_key_suffix_tree_g = (art_tree *)calloc(1, sizeof(art_tree)); + + art_tree_init(art_key_prefix_tree_g); + art_tree_init(art_key_suffix_tree_g); +} + +/****************************/ +/* Create index item for KV in DART */ +/****************************/ + +// #define PDC_DART_SFX_TREE + +perr_t +create_prefix_index_for_attr_value(void **index, unsigned char *attr_value, void *data) +{ + perr_t ret = SUCCEED; + if (*index == NULL) { + *index = (art_tree *)PDC_calloc(1, sizeof(art_tree)); + art_tree_init(*index); + } + + art_tree *art_value_prefix_tree = (art_tree *)*index; + + int len = strlen((const char *)attr_value); + Set *obj_id_set = (Set *)art_search(art_value_prefix_tree, attr_value, len); + if (obj_id_set == NULL) { + obj_id_set = set_new(ui64_hash, ui64_equal); + set_register_free_function(obj_id_set, free); + art_insert(art_value_prefix_tree, attr_value, len, (void *)obj_id_set); + } + + int indexed = set_insert(obj_id_set, data); + + if (indexed == -1) { + return FAIL; + } + + return ret; +} + +art_tree * +create_index_for_attr_name(char *attr_name, char *attr_value, void *data) +{ + + int len = strlen(attr_name); + key_index_leaf_content *leaf_content = NULL; + art_tree * nm_trie = NULL; + unsigned char * nm_key = NULL; + +#ifndef PDC_DART_SFX_TREE + int rr = 0; + for (rr = 0; rr < 2; rr++) { + nm_key = (rr == 1) ? (unsigned char *)reverse_str(attr_name) : (unsigned char *)attr_name; + nm_trie = (rr == 1) ? art_key_suffix_tree_g : art_key_prefix_tree_g; +#else + int sub_loop_count = len; // should be 'len', but we already iterate all suffixes at client side + nm_trie = art_key_prefix_tree_g; + for (int j = 0; j < sub_loop_count; j++) { + nm_key = (unsigned char *)substring(attr_name, j, len); +#endif + key_index_leaf_content *leafcnt = + (key_index_leaf_content *)art_search(nm_trie, nm_key, strlen((const char *)nm_key)); + if (leafcnt == NULL) { + leafcnt = (key_index_leaf_content *)PDC_calloc(1, sizeof(key_index_leaf_content)); + leafcnt->extra_prefix_index = (art_tree *)PDC_calloc(1, sizeof(art_tree)); + art_tree_init((art_tree *)leafcnt->extra_prefix_index); +#ifndef PDC_DART_SFX_TREE + // we only enable suffix index when suffix tree mode is off. + leafcnt->extra_suffix_index = (art_tree *)PDC_calloc(1, sizeof(art_tree)); + art_tree_init((art_tree *)leafcnt->extra_suffix_index); +#endif + // TODO: build local index for range query. + leafcnt->extra_range_index = (art_tree *)PDC_calloc(1, sizeof(art_tree)); + art_tree_init((art_tree *)leafcnt->extra_range_index); + + art_insert(nm_trie, nm_key, strlen((const char *)nm_key), leafcnt); + } + + art_tree *secondary_trie = NULL; + +#ifndef PDC_DART_SFX_TREE + int r = 0; + for (r = 0; r < 2; r++) { + unsigned char *val_key = + (r == 1 ? (unsigned char *)reverse_str(attr_value) : (unsigned char *)attr_value); + secondary_trie = (r == 1 ? (art_tree *)(leafcnt->extra_suffix_index) + : (art_tree *)(leafcnt->extra_prefix_index)); + +#else + secondary_trie = (art_tree *)(leafcnt->extra_prefix_index); + int val_len = strlen(attr_value); + for (int jj = 0; jj < val_len; jj++) { + unsigned char *val_key = (unsigned char *)substring(attr_value, jj, val_len); +#endif + create_prefix_index_for_attr_value((void **)&secondary_trie, val_key, data); + } // this matches with the 'r' loop or 'jj' loop + } // this matches with the 'rr' loop or 'j' loop + return nm_trie; +} + +perr_t +metadata_index_create(char *attr_key, char *attr_value, uint64_t obj_locator, int8_t index_type) +{ + perr_t ret_value = FAIL; + stopwatch_t timer; + timer_start(&timer); + uint64_t *data = (uint64_t *)calloc(1, sizeof(uint64_t)); + *data = obj_locator; + + // if (index_type == DHT_FULL_HASH) { + // FIXME: remember to check obj_locator type inside of this function below + // create_hash_table_for_keyword(attr_key, attr_value, strlen(attr_key), (void *)data); + // } + // else if (index_type == DHT_INITIAL_HASH) { + // FIXME: remember to check obj_locator type inside of this function below + // create_hash_table_for_keyword(attr_key, attr_value, 1, (void *)data); + // } + // else if (index_type == DART_HASH) { + create_index_for_attr_name(attr_key, attr_value, (void *)data); + // } + timer_pause(&timer); + // if (DART_SERVER_DEBUG) { + // printf("[Server_Side_Insert_%d] Timer to insert a keyword %s : %s into index = %.4f + // microseconds\n", + // pdc_server_rank_g, attr_key, attr_value, timer_delta_us(&timer)); + // } + indexed_word_count_g++; + ret_value = SUCCEED; + return ret_value; +} + +/****************************/ +/* Delete index item for KV in DART */ +/****************************/ + +perr_t +delete_prefix_index_for_attr_value(void **index, unsigned char *attr_value, void *data) +{ + perr_t ret = SUCCEED; + if (*index == NULL) { + // println("The value prefix tree is NULL, there is nothing to delete."); + return ret; + } + + art_tree *art_value_prefix_tree = (art_tree *)*index; + + int len = strlen((const char *)attr_value); + Set *obj_id_set = (Set *)art_search(art_value_prefix_tree, attr_value, len); + if (obj_id_set == NULL) { + // println("The obj_id_set is NULL, there nothing more to delete."); + if (art_size(art_value_prefix_tree) == 0) { + art_tree_destroy(*index); + } + return ret; + } + + if (set_query(obj_id_set, data) != 0) { + set_remove(obj_id_set, data); + } + + if (set_num_entries(obj_id_set) == 0) { + art_delete(art_value_prefix_tree, attr_value, len); + set_free(obj_id_set); + } + return ret; +} + +void +delete_index_for_attr_name(char *attr_name, char *attr_value, void *data) +{ + int len = strlen(attr_name); + key_index_leaf_content *leaf_content = NULL; + art_tree * nm_trie = NULL; + unsigned char * nm_key = NULL; + +#ifndef PDC_DART_SFX_TREE + int rr = 0; + for (rr = 0; rr < 2; rr++) { + nm_key = rr == 1 ? (unsigned char *)reverse_str(attr_name) : (unsigned char *)attr_name; + nm_trie = rr == 1 ? art_key_suffix_tree_g : art_key_prefix_tree_g; +#else + int sub_loop_count = 1; // should be 'len', but we already iterate all suffixes at client side; + nm_trie = art_key_prefix_tree_g; + for (int j = 0; j < sub_loop_count; j++) { + nm_key = (unsigned char *)substring(attr_name, j, len); +#endif + key_index_leaf_content *leafcnt = + (key_index_leaf_content *)art_search(nm_trie, nm_key, strlen((const char *)nm_key)); + if (leafcnt == NULL) { + art_delete(nm_trie, nm_key, strlen((const char *)nm_key)); + } + else { + art_tree *secondary_trie = NULL; +#ifndef PDC_DART_SFX_TREE + int r = 0; + for (r = 0; r < 2; r++) { + secondary_trie = (r == 1 ? (art_tree *)(leafcnt->extra_suffix_index) + : (art_tree *)(leafcnt->extra_prefix_index)); + unsigned char *val_key = + (r == 1 ? (unsigned char *)reverse_str(attr_value) : (unsigned char *)attr_value); +#else + secondary_trie = (art_tree *)(leafcnt->extra_prefix_index); + for (int jj = 0; jj < strlen(attr_value); jj++) { + unsigned char *val_key = (unsigned char *)substring(attr_value, jj, strlen(attr_value)); +#endif + delete_prefix_index_for_attr_value((void **)&secondary_trie, val_key, data); + } + if (leafcnt->extra_suffix_index == NULL && leafcnt->extra_prefix_index == NULL) { + art_delete(nm_trie, nm_key, len); + leafcnt = NULL; + } + // TODO: deal with index for range query. + } // this matches with the 'r' loop or 'jj' loop + } // this matches with the 'rr' loop or 'j' loop +} + +perr_t +metadata_index_delete(char *attr_key, char *attr_value, uint64_t obj_locator, int8_t index_type) +{ + perr_t ret_value = FAIL; + stopwatch_t timer; + timer_start(&timer); + uint64_t *data = (uint64_t *)calloc(1, sizeof(uint64_t)); + *data = obj_locator; + + // if (index_type == DHT_FULL_HASH) { + // delete_hash_table_for_keyword(attr_key, strlen(attr_key), (void *)obj_locator); + // } + // else if (index_type == DHT_INITIAL_HASH) { + // delete_hash_table_for_keyword(attr_key, 1, (void *)obj_locator); + // } + // else if (index_type == DART_HASH) { + delete_index_for_attr_name(attr_key, attr_value, (void *)data); + // } + + timer_pause(&timer); + // if (DART_SERVER_DEBUG) { + // printf("[Server_Side_Delete_%d] Timer to delete a keyword %s : %s from index = %.4f + // microseconds\n", + // pdc_server_rank_g, attr_key, attr_value, timer_delta_us(&timer)); + // } + indexed_word_count_g--; + ret_value = SUCCEED; + return ret_value; +} + +/****************************/ +/* DART Get Server Info */ +/****************************/ + +perr_t +PDC_Server_dart_get_server_info(dart_get_server_info_in_t *in, dart_get_server_info_out_t *out) +{ + perr_t ret_value = SUCCEED; + FUNC_ENTER(NULL); + uint32_t serverId = in->serverId; + out->indexed_word_count = indexed_word_count_g; + out->request_count = server_request_count_g; + FUNC_LEAVE(ret_value); +} + +/** + * The callback function performs on each prefix on secondary art_tree + * + * + */ +int +level_two_art_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + pdc_art_iterator_param_t *param = (pdc_art_iterator_param_t *)(data); + // println("Level two start"); + if (param->level_two_infix != NULL) { + if (contains((const char *)key, (const char *)param->level_two_infix) == 0) { + return 0; + } + } + if (value != NULL) { + Set * obj_id_set = (Set *)value; + SetIterator value_set_iter; + set_iterate(obj_id_set, &value_set_iter); + + while (set_iter_has_more(&value_set_iter)) { + uint64_t *item = (uint64_t *)set_iter_next(&value_set_iter); + uint64_t *itemValue = (uint64_t *)calloc(1, sizeof(uint64_t)); + *itemValue = *item; + set_insert(param->out, itemValue); + } + } + // println("Level two finish"); + return 0; +} + +/** + * The callback function performs on each prefix on a art_tree. + * + */ +int +level_one_art_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + key_index_leaf_content * leafcnt = (key_index_leaf_content *)value; + pdc_art_iterator_param_t *param = (pdc_art_iterator_param_t *)(data); + + if (param->level_one_infix != NULL) { + if (contains((char *)key, param->level_one_infix) == 0) { + return 0; + } + } + + char *secondary_query = param->query_str; + // param->total_count = 0; + // param->out = NULL; + if (strchr(secondary_query, '~')) { + // TODO: DO RANGE QUERY HERE. currently no solution for range query. + // + } + else { + // DO TEXT QUERY HERE. + pattern_type_t level_two_ptn_type = determine_pattern_type(secondary_query); + char * tok = NULL; + switch (level_two_ptn_type) { + case PATTERN_EXACT: + tok = secondary_query; + if (leafcnt->extra_prefix_index != NULL) { + Set *obj_id_set = + (Set *)art_search(leafcnt->extra_prefix_index, (unsigned char *)tok, strlen(tok)); + if (obj_id_set != NULL) { + level_two_art_callback((void *)param, (unsigned char *)tok, strlen(tok), + (void *)obj_id_set); + } + } + break; + case PATTERN_PREFIX: + tok = subrstr(secondary_query, strlen(secondary_query) - 1); + if (leafcnt->extra_prefix_index != NULL) { + art_iter_prefix((art_tree *)leafcnt->extra_prefix_index, (unsigned char *)tok, + strlen(tok), level_two_art_callback, param); + } + break; + case PATTERN_SUFFIX: + tok = substr(secondary_query, 1); + art_tree *secondary_trie = NULL; +#ifndef PDC_DART_SFX_TREE + tok = reverse_str(tok); + secondary_trie = (art_tree *)leafcnt->extra_suffix_index; +#else + secondary_trie = (art_tree *)leafcnt->extra_prefix_index; +#endif + if (secondary_trie != NULL) { +#ifndef PDC_DART_SFX_TREE + art_iter_prefix(secondary_trie, (unsigned char *)tok, strlen(tok), level_two_art_callback, + param); +#else + Set *obj_id_set = (Set *)art_search(secondary_trie, (unsigned char *)tok, strlen(tok)); + if (obj_id_set != NULL) { + level_two_art_callback((void *)param, (unsigned char *)tok, strlen(tok), + (void *)obj_id_set); + } +#endif + } + break; + case PATTERN_MIDDLE: + tok = substring(secondary_query, 1, strlen(secondary_query) - 1); + secondary_trie = (art_tree *)leafcnt->extra_prefix_index; + if (secondary_trie != NULL) { +#ifndef PDC_DART_SFX_TREE + param->level_two_infix = tok; + art_iter(secondary_trie, level_two_art_callback, param); +#else + art_iter_prefix(secondary_trie, (unsigned char *)tok, strlen(tok), level_two_art_callback, + param); +#endif + } + break; + default: + break; + } + } + return 0; +} + +perr_t +metadata_index_search(char *query, int index_type, uint64_t *n_obj_ids_ptr, uint64_t **buf_ptrs) +{ + + perr_t result = SUCCEED; + stopwatch_t index_timer; + + char *kdelim_ptr = strchr(query, (int)'='); + + char *k_query = get_key(query, '='); + char *v_query = get_value(query, '='); + + if (DART_SERVER_DEBUG) { + println("[Server_Side_Query_%d] k_query = '%s' | v_query = '%s' ", pdc_server_rank_g, k_query, + v_query); + } + + pdc_art_iterator_param_t *param = (pdc_art_iterator_param_t *)calloc(1, sizeof(pdc_art_iterator_param_t)); + param->level_one_infix = NULL; + param->level_two_infix = NULL; + param->query_str = v_query; + param->out = set_new(ui64_hash, ui64_equal); + set_register_free_function(param->out, free); + + timer_start(&index_timer); + + char *qType_string = "Exact"; + + if (NULL == kdelim_ptr) { + if (DART_SERVER_DEBUG) { + println("[Server_Side_Query_%d]query string '%s' is not valid.", pdc_server_rank_g, query); + } + *n_obj_ids_ptr = 0; + return result; + } + else { + char *tok; + // println("k_query %s, v_query %s", k_query, v_query); + pattern_type_t level_one_ptn_type = determine_pattern_type(k_query); + key_index_leaf_content *leafcnt = NULL; + // if (index_type == DHT_FULL_HASH || index_type == DHT_INITIAL_HASH) { + // search_through_hash_table(k_query, index_type, level_one_ptn_type, param); + // } + // else { + switch (level_one_ptn_type) { + case PATTERN_EXACT: + qType_string = "Exact"; + tok = k_query; + leafcnt = (key_index_leaf_content *)art_search(art_key_prefix_tree_g, (unsigned char *)tok, + strlen(tok)); + if (leafcnt != NULL) { + level_one_art_callback((void *)param, (unsigned char *)tok, strlen(tok), (void *)leafcnt); + } + break; + case PATTERN_PREFIX: + qType_string = "Prefix"; + tok = subrstr(k_query, strlen(k_query) - 1); + art_iter_prefix((art_tree *)art_key_prefix_tree_g, (unsigned char *)tok, strlen(tok), + level_one_art_callback, param); + break; + case PATTERN_SUFFIX: + qType_string = "Suffix"; + tok = substr(k_query, 1); +#ifndef PDC_DART_SFX_TREE + tok = reverse_str(tok); + art_iter_prefix((art_tree *)art_key_suffix_tree_g, (unsigned char *)tok, strlen(tok), + level_one_art_callback, param); +#else + leafcnt = (key_index_leaf_content *)art_search(art_key_prefix_tree_g, (unsigned char *)tok, + strlen(tok)); + if (leafcnt != NULL) { + level_one_art_callback((void *)param, (unsigned char *)tok, strlen(tok), (void *)leafcnt); + } +#endif + break; + case PATTERN_MIDDLE: + qType_string = "Infix"; + tok = substring(k_query, 1, strlen(k_query) - 1); +#ifndef PDC_DART_SFX_TREE + param->level_one_infix = tok; + art_iter(art_key_prefix_tree_g, level_one_art_callback, param); +#else + art_iter_prefix(art_key_prefix_tree_g, (unsigned char *)tok, strlen(tok), + level_one_art_callback, param); +#endif + break; + default: + break; + } + // } + } + + uint32_t i = 0; + + *n_obj_ids_ptr = set_num_entries(param->out); + *buf_ptrs = (uint64_t *)calloc(*n_obj_ids_ptr, sizeof(uint64_t)); + + SetIterator iter; + set_iterate(param->out, &iter); + while (set_iter_has_more(&iter)) { + uint64_t *item = (uint64_t *)set_iter_next(&iter); + (*buf_ptrs)[i] = *item; + i++; + } + set_free(param->out); + + timer_pause(&index_timer); + if (DART_SERVER_DEBUG) { + printf("[Server_Side_%s_%d] Time to address query '%s' and get %d results = %.4f microseconds\n", + qType_string, pdc_server_rank_g, query, *n_obj_ids_ptr, timer_delta_us(&index_timer)); + } + server_request_count_g++; + return result; +} + +perr_t +PDC_Server_dart_perform_one_server(dart_perform_one_server_in_t *in, dart_perform_one_server_out_t *out, + uint64_t *n_obj_ids_ptr, uint64_t **buf_ptrs) +{ + perr_t result = SUCCEED; + dart_op_type_t op_type = in->op_type; + dart_hash_algo_t hash_algo = in->hash_algo; + char * attr_key = (char *)in->attr_key; + char * attr_val = (char *)in->attr_val; + dart_object_ref_type_t ref_type = in->obj_ref_type; + + uint64_t obj_locator = in->obj_primary_ref; + if (ref_type == REF_PRIMARY_ID) { + obj_locator = in->obj_primary_ref; + } + else if (ref_type == REF_SECONDARY_ID) { + obj_locator = in->obj_secondary_ref; + } + else if (ref_type == REF_SERVER_ID) { + obj_locator = in->obj_server_ref; + } + out->has_bulk = 0; + // printf("Respond to: in->op_type=%d\n", in->op_type ); + if (op_type == OP_INSERT) { + metadata_index_create(attr_key, attr_val, obj_locator, hash_algo); + } + else if (op_type == OP_DELETE) { + metadata_index_delete(attr_key, attr_val, obj_locator, hash_algo); + } + else { + char *query = (char *)in->attr_key; + result = metadata_index_search(query, hash_algo, n_obj_ids_ptr, buf_ptrs); + out->n_items = (*n_obj_ids_ptr); + if ((*n_obj_ids_ptr) > 0) { + out->has_bulk = 1; + } + } + return result; +} \ No newline at end of file diff --git a/src/server/pdc_server_metadata_index_test.c b/src/server/pdc_server_metadata_index_test.c new file mode 100644 index 000000000..d385c717f --- /dev/null +++ b/src/server/pdc_server_metadata_index_test.c @@ -0,0 +1,74 @@ +#include +#include +#include +#include "pdc_server_metadata_index.h" + +void +insert_kv_to_index(char *key, char *value, uint64_t obj_id) +{ + dart_perform_one_server_in_t input; + dart_perform_one_server_out_t output; + + input.obj_ref_type = REF_PRIMARY_ID; + input.hash_algo = DART_HASH; + // Test Insert Index + input.op_type = OP_INSERT; + input.attr_val = value; + input.obj_primary_ref = obj_id; + + for (int i = 0; i < strlen(key); i++) { + input.attr_key = substring(key, i, strlen(key)); + assert(PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL) == SUCCEED); + // printf("Index Insertion Successful!\n"); + } +} + +void +query_result_from_kvtag(char *key_value_query, int8_t op_type) +{ + dart_perform_one_server_in_t input; + dart_perform_one_server_out_t output; + uint64_t n_obj_ids = 0; + uint64_t * buf_ptr = NULL; + input.op_type = op_type; + input.attr_key = key_value_query; + assert(PDC_Server_dart_perform_one_server(&input, &output, &n_obj_ids, &buf_ptr) == SUCCEED); + printf("Query Successful! %d Results: ", n_obj_ids); + for (int i = 0; i < n_obj_ids; i++) { + printf("%llu, ", buf_ptr[i]); + } + printf("\n\n"); +} + +void +test_PDC_Server_dart_perform_one_server() +{ + + PDC_Server_dart_init(); + + char *key = (char *)calloc(100, sizeof(char)); + char *val = (char *)calloc(100, sizeof(char)); + + for (int i = 0; i < 100; i++) { + sprintf(key, "key%03dkey", i); + sprintf(val, "val%03dval", i); + printf("Inserting %s, %s\n", key, val); + insert_kv_to_index(key, val, 10000 + i); + } + + insert_kv_to_index("0key", "0val", 20000); + insert_kv_to_index("key000key", "val000val", 30000); + + query_result_from_kvtag("key000key=val000val", OP_EXACT_QUERY); + query_result_from_kvtag("0key=0val", OP_EXACT_QUERY); + query_result_from_kvtag("key01*=val01*", OP_PREFIX_QUERY); + query_result_from_kvtag("*3key=*3val", OP_SUFFIX_QUERY); + query_result_from_kvtag("*9*=*9*", OP_INFIX_QUERY); +} + +int +main() +{ + test_PDC_Server_dart_perform_one_server(); + return 0; +} \ No newline at end of file diff --git a/src/server/pdc_server_region/include/pdc_server_data.h b/src/server/pdc_server_region/include/pdc_server_data.h index 4bcdac9dd..a204aa437 100644 --- a/src/server/pdc_server_region/include/pdc_server_data.h +++ b/src/server/pdc_server_region/include/pdc_server_data.h @@ -244,7 +244,7 @@ typedef struct cache_storage_region_t { /*****************************/ extern int pdc_server_rank_g; extern int pdc_server_size_g; -extern char pdc_server_tmp_dir_g[ADDR_MAX]; +extern char pdc_server_tmp_dir_g[TMP_DIR_STRING_LEN]; extern double server_write_time_g; extern double server_read_time_g; extern double server_get_storage_info_time_g; diff --git a/src/server/pdc_server_region/pdc_server_data.c b/src/server/pdc_server_region/pdc_server_data.c index 8388957b0..9b3ae5b6f 100644 --- a/src/server/pdc_server_region/pdc_server_data.c +++ b/src/server/pdc_server_region/pdc_server_data.c @@ -130,7 +130,7 @@ PDC_Server_set_lustre_stripe(const char *path, int stripe_count, int stripe_size perr_t ret_value = SUCCEED; size_t len; int i, index; - char tmp[ADDR_MAX]; + char tmp[TMP_DIR_STRING_LEN]; char cmd[TAG_LEN_MAX]; FUNC_ENTER(NULL); @@ -4695,7 +4695,8 @@ PDC_Server_posix_write(int fd, void *buf, uint64_t write_size) while (write_size > max_write_size) { ret = write(fd, buf, max_write_size); if (ret < 0 || ret != (ssize_t)max_write_size) { - printf("==PDC_SERVER[%d]: write %d failed\n", pdc_server_rank_g, fd); + printf("==PDC_SERVER[%d]: in-loop: write %d failed, ret = %ld, max_write_size = %llu\n", + pdc_server_rank_g, fd, ret, max_write_size); ret_value = FAIL; goto done; } @@ -4706,7 +4707,7 @@ PDC_Server_posix_write(int fd, void *buf, uint64_t write_size) ret = write(fd, buf, write_size); if (ret < 0 || ret != (ssize_t)write_size) { - printf("==PDC_SERVER[%d]: write %d failed, not all data written %lu/%lu\n", pdc_server_rank_g, fd, + printf("==PDC_SERVER[%d]: write %d failed, not all data written %llu/%llu\n", pdc_server_rank_g, fd, write_bytes, write_size); ret_value = FAIL; goto done; @@ -4736,7 +4737,7 @@ PDC_Server_data_write_out(uint64_t obj_id, struct pdc_region_info *region_info, double start = MPI_Wtime(), start_posix; #endif - uint64_t write_size; + uint64_t write_size = 0; if (region_info->ndim >= 1) write_size = unit * region_info->size[0]; if (region_info->ndim >= 2) @@ -4789,7 +4790,7 @@ PDC_Server_data_write_out(uint64_t obj_id, struct pdc_region_info *region_info, // 1D can overwrite data in region directly pos = (overlap_offset[0] - overlap_region->start[0]) * unit; if (pos > write_size) { - printf("==PDC_SERVER[%d]: Error with buf pos calculation %lu / %ld! @ line %d\n", + printf("==PDC_SERVER[%d]: Error with buf pos calculation %llu / %llu! @ line %d\n", pdc_server_rank_g, pos, write_size, __LINE__); ret_value = -1; goto done; @@ -5101,19 +5102,18 @@ PDC_Server_data_read_from(uint64_t obj_id, struct pdc_region_info *region_info, if (overlap_offset) { if (region_info->ndim == 1) { - // 1D can overwrite data in region directly pos = (overlap_offset[0] - overlap_region->start[0]) * unit; - if ((ssize_t)pos > request_bytes) { - printf("==PDC_SERVER[%d]: Error with buf pos calculation %lu / %ld! @ line %d\n", - pdc_server_rank_g, pos, request_bytes, __LINE__); - ret_value = -1; - goto done; - } + /* if ((ssize_t)pos > request_bytes) { */ + /* printf("==PDC_SERVER[%d]: Error with buf pos calculation %lu / %ld! @ line %d\n", */ + /* pdc_server_rank_g, pos, request_bytes, __LINE__); */ + /* ret_value = -1; */ + /* goto done; */ + /* } */ #ifdef PDC_TIMING start_posix = MPI_Wtime(); #endif - printf("POSIX read from file offset %lu, region start = %lu, region size = %lu\n", - overlap_region->offset, overlap_region->start[0], overlap_region->count[0]); + /* printf("POSIX read from file offset %lu, region start = %lu, region size = %lu\n", */ + /* overlap_region->offset, overlap_region->start[0], overlap_region->count[0]); */ if (pread(region->fd, buf + (overlap_offset[0] - region_info->offset[0]) * unit, overlap_size[0] * unit, overlap_region->offset + pos) != (ssize_t)(overlap_size[0] * unit)) { @@ -7252,15 +7252,18 @@ PDC_Server_query_evaluate_merge_opt(pdc_query_t *query, query_task_t *task, pdc_ pdc_selection_t *sel = query->sel; uint64_t nelem; size_t i, j, unit_size; - pdc_query_op_t op = PDC_QUERY_OR, lop = PDC_QUERY_OR, rop = PDC_QUERY_OR; - float flo = .0, fhi = .0; - double dlo = .0, dhi = .0; - int ilo = 0, ihi = 0, ndim, count = 0; - uint32_t ulo = 0, uhi = 0; - int64_t i64lo = 0, i64hi = 0; - uint64_t ui64lo = 0, ui64hi = 0; - void * value = NULL, *buf = NULL; - int n_eval_region = 0, can_skip, region_iter = 0; + // FIXME: need to check the types of these 'op's. I think they should be of the following (or don't even + // need to be initilized): + pdc_query_op_t op = PDC_EQ, lop = PDC_EQ, rop = PDC_EQ; + // pdc_query_op_t op = PDC_QUERY_OR, lop = PDC_QUERY_OR, rop = PDC_QUERY_OR; + float flo = .0, fhi = .0; + double dlo = .0, dhi = .0; + int ilo = 0, ihi = 0, ndim, count = 0; + uint32_t ulo = 0, uhi = 0; + int64_t i64lo = 0, i64hi = 0; + uint64_t ui64lo = 0, ui64hi = 0; + void * value = NULL, *buf = NULL; + int n_eval_region = 0, can_skip, region_iter = 0; printf("==PDC_SERVER[%d]: %s - start query evaluation!\n", pdc_server_rank_g, __func__); fflush(stdout); @@ -9483,7 +9486,7 @@ PDC_Server_recv_get_sel_data(const struct hg_cb_info *callback_info) get_sel_data_rpc_in_t *in = (get_sel_data_rpc_in_t *)callback_info->arg; query_task_t * task_elt, *task = NULL; pdc_metadata_t * meta; - struct hg_cb_info fake_callback_info; + struct hg_cb_info fake_callback_info = {0}; DL_FOREACH(query_task_list_head_g, task_elt) { diff --git a/src/server/pdc_server_region/pdc_server_region_cache.c b/src/server/pdc_server_region/pdc_server_region_cache.c index 6bcf5217b..40564e784 100644 --- a/src/server/pdc_server_region/pdc_server_region_cache.c +++ b/src/server/pdc_server_region/pdc_server_region_cache.c @@ -3,7 +3,17 @@ #ifdef PDC_SERVER_CACHE -#define MAX_CACHE_SIZE 1610612736 +#ifdef PDC_SERVER_CACHE_MAX_SIZE +#define MAX_CACHE_SIZE PDC_SERVER_CACHE_MAX_GB * 1024 * 1024 * 1024 +#else +#define MAX_CACHE_SIZE 34359738368 +#endif + +#ifdef PDC_SERVER_CACHE_FLUSH_TIME +#define PDC_CACHE_FLUSH_TIME_INT PDC_SERVER_CACHE_FLUSH_TIME +#else +#define PDC_CACHE_FLUSH_TIME_INT 30 +#endif typedef struct pdc_region_cache { struct pdc_region_info * region_cache_info; @@ -392,8 +402,11 @@ PDC_region_cache_register(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dim struct pdc_region_info *region_cache_info; if (obj_ndim != ndim && obj_ndim > 0) { printf("PDC_region_cache_register reports obj_ndim != ndim, %d != %d\n", obj_ndim, ndim); + return FAIL; } + pthread_mutex_lock(&pdc_obj_cache_list_mutex); + obj_cache_iter = obj_cache_list; while (obj_cache_iter != NULL) { if (obj_cache_iter->obj_id == obj_id) { @@ -459,7 +472,15 @@ PDC_region_cache_register(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dim memcpy(region_cache_info->buf, buf, sizeof(char) * buf_size); total_cache_size += buf_size; + pthread_mutex_unlock(&pdc_obj_cache_list_mutex); + if (total_cache_size > maximum_cache_size) { + int server_rank = 0; +#ifdef ENABLE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &server_rank); +#endif + printf("==PDC_SERVER[%d]: server cache full %.1f / %.1f MB, will flush to storage\n", server_rank, + total_cache_size / 1048576.0, maximum_cache_size / 1048576.0); PDC_region_cache_flush_all(); } @@ -570,11 +591,11 @@ PDC_transfer_request_data_write_out(uint64_t obj_id, int obj_ndim, const uint64_ region_cache_iter = region_cache_iter->next; } } + pthread_mutex_unlock(&pdc_obj_cache_list_mutex); if (!flag) { PDC_region_cache_register(obj_id, obj_ndim, obj_dims, buf, write_size, region_info->offset, region_info->size, region_info->ndim, unit); } - pthread_mutex_unlock(&pdc_obj_cache_list_mutex); // PDC_Server_data_write_out2(obj_id, region_info, buf, unit); #ifdef PDC_TIMING @@ -667,6 +688,7 @@ PDC_region_cache_flush_by_pointer(uint64_t obj_id, pdc_obj_cache *obj_cache) char ** buf, **new_buf, *buf_ptr = NULL; uint64_t * start, *end, *new_start, *new_end; int merged_request_size = 0; + int server_rank = 0; uint64_t unit; struct pdc_region_info **obj_regions; #ifdef PDC_TIMING @@ -739,6 +761,9 @@ PDC_region_cache_flush_by_pointer(uint64_t obj_id, pdc_obj_cache *obj_cache) nflush += merged_request_size; } +#ifdef ENABLE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &server_rank); +#endif // Iterate through all cache regions and use POSIX I/O to write them back to file system. region_cache_iter = obj_cache->region_cache; while (region_cache_iter != NULL) { @@ -752,6 +777,9 @@ PDC_region_cache_flush_by_pointer(uint64_t obj_id, pdc_obj_cache *obj_cache) if (obj_cache->ndim >= 3) write_size *= region_cache_info->size[2]; + printf("==PDC_SERVER[%d]: server flushed %.1f / %.1f MB to storage\n", server_rank, + write_size / 1048576.0, total_cache_size / 1048576.0); + total_cache_size -= write_size; free(region_cache_info->offset); if (obj_cache->ndim > 1) { @@ -825,7 +853,7 @@ PDC_region_cache_clock_cycle(void *ptr) struct timeval current_time; struct timeval finish_time; int nflush = 0; - double flush_frequency_s = 2.0, elapsed_time; + double flush_frequency_s = PDC_CACHE_FLUSH_TIME_INT, elapsed_time; int server_rank = 0; char *p = getenv("PDC_SERVER_CACHE_FLUSH_FREQUENCY_S"); @@ -861,10 +889,9 @@ PDC_region_cache_clock_cycle(void *ptr) gettimeofday(&finish_time, NULL); elapsed_time = finish_time.tv_sec - current_time.tv_sec + (finish_time.tv_usec - current_time.tv_usec) / 1000000.0; - fprintf( - stderr, - "==PDC_SERVER[%d]: flushed %d regions from cache to storage (every %.1fs), took %.4fs\n", - server_rank, nflush, flush_frequency_s, elapsed_time); + fprintf(stderr, + "==PDC_SERVER[%d]: flushed %d regions to storage (full/every %.0fs), took %.4fs\n", + server_rank, nflush, flush_frequency_s, elapsed_time); } pthread_mutex_unlock(&pdc_obj_cache_list_mutex); } @@ -956,4 +983,4 @@ PDC_region_fetch(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, struct } return 0; } -#endif +#endif \ No newline at end of file diff --git a/src/server/pdc_server_region/pdc_server_region_request_handler.h b/src/server/pdc_server_region/pdc_server_region_request_handler.h index f3dc64cde..04ab48481 100644 --- a/src/server/pdc_server_region/pdc_server_region_request_handler.h +++ b/src/server/pdc_server_region/pdc_server_region_request_handler.h @@ -811,7 +811,7 @@ HG_TEST_RPC_CB(transfer_request, handle) ret_value = HG_Respond(handle, NULL, NULL, &out); if (in.access_type == PDC_WRITE) { ret_value = HG_Bulk_create(info->hg_class, 1, &(local_bulk_args->data_buf), - &(local_bulk_args->total_mem_size), HG_BULK_READWRITE, + (const hg_size_t *)&(local_bulk_args->total_mem_size), HG_BULK_READWRITE, &(local_bulk_args->bulk_handle)); if (ret_value != HG_SUCCESS) { printf("Error at HG_TEST_RPC_CB(transfer_request, handle): @ line %d \n", __LINE__); @@ -881,7 +881,7 @@ HG_TEST_RPC_CB(transfer_request, handle) *((int *)(local_bulk_args->data_buf + sizeof(int)))); */ ret_value = HG_Bulk_create(info->hg_class, 1, &(local_bulk_args->data_buf), - &(local_bulk_args->total_mem_size), HG_BULK_READWRITE, + (const hg_size_t *)&(local_bulk_args->total_mem_size), HG_BULK_READWRITE, &(local_bulk_args->bulk_handle)); if (ret_value != HG_SUCCESS) { printf("Error at HG_TEST_RPC_CB(transfer_request, handle): @ line %d \n", __LINE__); diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 499421880..a780c409c 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -1,6 +1,7 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/helper/include ${PDC_INCLUDES_BUILD_TIME} ${PROJECT_BINARY_DIR} ${PDC_SOURCE_DIR} @@ -9,6 +10,21 @@ include_directories( $ENV{HOME}/include ) +# ************************************************* +# Julia Support +# ************************************************* +if (PDC_ENABLE_JULIA) + include_directories(${JULIA_INCLUDE_DIRS}) +endif(PDC_ENABLE_JULIA) + +# ************************************************* +# * Find UUID library +# ************************************************* +find_package(UUID REQUIRED) +if(UUID_FOUND) + include_directories(${UUID_INCLUDE_DIRS}) +endif(UUID_FOUND) + set(PROGRAMS pdc_init # create_prop @@ -63,8 +79,6 @@ set(PROGRAMS # data_server_meta_test kvtag_add_get # kvtag_get - kvtag_add_get_benchmark - kvtag_add_get_scale # kvtag_query kvtag_query_scale # obj_transformation @@ -105,11 +119,69 @@ set(PROGRAMS query_data ) +set(MPI_PROGRAMS + kvtag_query_scale_col + kvtag_query_mpi + kvtag_add_get_benchmark + kvtag_add_get_scale + ) + +set(TEST_EXT_LIB "") +set(TEST_EXT_INCLUDE_DIRS "") +set(EXTRA_SRC_FILE "") + foreach(program ${PROGRAMS}) add_executable(${program} ${program}.c) - target_link_libraries(${program} pdc) + target_link_libraries(${program} pdc pdc_commons ${TEST_EXT_LIB}) + target_include_directories(${program} PRIVATE ${TEST_EXT_INCLUDE_DIRS}) endforeach(program) +if(BUILD_MPI_TESTING) + foreach(program ${MPI_PROGRAMS}) + add_executable(${program} ${program}.c) + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${program} PRIVATE ${SUPPRESSED_LIST}) + endif() + target_link_libraries(${program} pdc pdc_commons ${TEST_EXT_LIB}) + target_include_directories(${program} PRIVATE ${TEST_EXT_INCLUDE_DIRS}) + endforeach(program) +endif() + + +if(UUID_FOUND) + message(STATUS "UUID library found: ${UUID_LIBRARIES}") + message(STATUS "UUID include directory: ${UUID_INCLUDE_DIRS}") + set(ENHANCED_PROGRAMS + dart_algo_sim + dart_attr_dist_test + dart_func_test + dart_test + ) + + set(TEST_EXT_LIB ${TEST_EXT_LIB} ${UUID_LIBRARIES}) + set(TEST_EXT_INCLUDE_DIRS ${TEST_EXT_INCLUDE_DIRS} ${UUID_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/helper/include) + + set(EXTRA_SRC_FILE "") + if (PDC_ENABLE_JULIA) + list(APPEND EXTRA_SRC_FILE + ${CMAKE_CURRENT_SOURCE_DIR}/helper/julia_helper_loader.c + ) + list(APPEND TEST_EXT_LIB + ${JULIA_LIBRARIES} + ) + list(APPEND TEST_EXT_INCLUDE_DIRS + ${JULIA_INCLUDE_DIRS} + ) + endif(PDC_ENABLE_JULIA) + + foreach(program ${ENHANCED_PROGRAMS}) + add_executable(${program} ${program}.c ${EXTRA_SRC_FILE}) + target_link_libraries(${program} pdc pdc_commons ${TEST_EXT_LIB}) + target_include_directories(${program} PRIVATE ${TEST_EXT_INCLUDE_DIRS}) + endforeach(program) +endif(UUID_FOUND) + + set(SCRIPTS run_test.sh mpi_test.sh @@ -358,7 +430,7 @@ if(BUILD_MPI_TESTING) add_test(NAME region_transfer_all3_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./region_transfer_all ${MPI_RUN_CMD} 4 6 0 0) add_test(NAME region_transfer_all3_2D_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./region_transfer_all_2D ${MPI_RUN_CMD} 4 6 0 0) add_test(NAME region_transfer_all3_3D_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./region_transfer_all_3D ${MPI_RUN_CMD} 4 6 0 0) - add_test(NAME region_transfer_all_append3_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./region_transfer_all_append ${MPI_RUN_CMD} 4 60 0) + add_test(NAME region_transfer_all_append3_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./region_transfer_all_append ${MPI_RUN_CMD} 4 6 0 0) add_test(NAME region_transfer_all_append3_2D_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./region_transfer_all_append_2D ${MPI_RUN_CMD} 4 6 0 0) add_test(NAME region_transfer_all_append3_3D_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./region_transfer_all_append_3D ${MPI_RUN_CMD} 4 6 0 0) add_test(NAME region_transfer_all4_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./region_transfer_all ${MPI_RUN_CMD} 4 6 1 0) diff --git a/src/tests/bdcats_v2.c b/src/tests/bdcats_v2.c index aeea45134..88a78b094 100644 --- a/src/tests/bdcats_v2.c +++ b/src/tests/bdcats_v2.c @@ -226,7 +226,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to map with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to map with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -275,7 +275,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to lock with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to lock with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -324,7 +324,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to relese lock with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to relese lock with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -372,7 +372,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to read data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to read data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/client_server.c b/src/tests/client_server.c index cd1d8810f..f63e659de 100644 --- a/src/tests/client_server.c +++ b/src/tests/client_server.c @@ -36,8 +36,8 @@ #define MAX_SERVER_NUM 1024 -static hg_id_t gen_obj_id_client_id_g; -static int work_todo_g = 0; +static hg_id_t gen_obj_id_client_id_g; +static hg_atomic_int32_t atomic_work_todo_g; int PDC_Client_read_server_addr_from_file(char target_addr_string[MAX_SERVER_NUM][PATH_MAX]) @@ -79,7 +79,7 @@ client_rpc_cb(const struct hg_cb_info *callback_info) /* printf("Return value=%llu\n", output.ret); */ client_lookup_args->obj_id = output.ret; - work_todo_g--; + hg_atomic_decr32(&atomic_work_todo_g); return HG_SUCCESS; } @@ -153,7 +153,7 @@ PDC_Client_check_response(hg_context_t **hg_context) /* printf("actual_count=%d\n",actual_count); */ /* Do not try to make progress anymore if we're done */ - if (work_todo_g <= 0) + if (hg_atomic_get32(&atomic_work_todo_g) <= 0) break; hg_ret = HG_Progress(*hg_context, HG_MAX_IDLE_TIME); @@ -209,13 +209,13 @@ main(int argc, char **argv) printf("Error with Mercury Init, exiting...\n"); exit(0); } + hg_atomic_init32(&atomic_work_todo_g, 0); // Obj name and ID char **test_obj_names = (char **)malloc(sizeof(char *) * n_server); for (i = 0; i < n_server; i++) test_obj_names[i] = (char *)malloc(sizeof(char) * 128); - work_todo_g = 0; struct client_lookup_args *client_lookup_args = (struct client_lookup_args *)malloc(sizeof(struct client_lookup_args) * n_server); @@ -227,7 +227,7 @@ main(int argc, char **argv) client_lookup_args[i].obj_name = test_obj_names[i]; client_lookup_args[i].obj_id = -1; - work_todo_g++; + hg_atomic_incr32(&atomic_work_todo_g); printf("Target addr:%s\n", target_addr_string[i]); send_name_recv_id(&client_lookup_args[i], target_addr_string[i]); } @@ -237,7 +237,7 @@ main(int argc, char **argv) hg_time_get_current(&end_time); elapsed_time = hg_time_subtract(end_time, start_time); elapsed_time_double = hg_time_to_double(elapsed_time); - printf("Total elapsed time for PDC server connection: %.6fs\n", elapsed_time_double); + printf("Total elapsed time for PDC server connection: %.5e s\n", elapsed_time_double); for (i = 0; i < n_server; i++) { printf("\"%s\" obj_id = %d\n", client_lookup_args[i].obj_name, client_lookup_args[i].obj_id); diff --git a/src/tests/cont_add_del.c b/src/tests/cont_add_del.c index 993a41f16..de55ff6e7 100644 --- a/src/tests/cont_add_del.c +++ b/src/tests/cont_add_del.c @@ -198,7 +198,7 @@ main(int argc, char **argv) ht_total_end.tv_usec - ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("%10d created ... %.4f s\n", i * size, ht_total_sec); + printf("%10d created ... %.5e s\n", i * size, ht_total_sec); fflush(stdout); } #ifdef ENABLE_MPI @@ -215,7 +215,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to create %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to create %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/cont_del.c b/src/tests/cont_del.c index ab708685f..5ecdf2e6b 100644 --- a/src/tests/cont_del.c +++ b/src/tests/cont_del.c @@ -79,6 +79,11 @@ main(int argc, char **argv) printf("successfully close container c1\n"); } + printf("trying to open a deleted container, should fail\n"); + cont = PDCcont_open("VPIC_cont", pdc); + if (cont > 0) + printf("Error: opened a container that was just deleted @ line %d!\n", __LINE__); + // close a container property if (PDCprop_close(create_prop) < 0) { printf("Fail to close property @ line %d\n", __LINE__); diff --git a/src/tests/cont_tags.c b/src/tests/cont_tags.c index 04132c332..41a5acd60 100644 --- a/src/tests/cont_tags.c +++ b/src/tests/cont_tags.c @@ -32,12 +32,12 @@ main(int argc, char **argv) { pdcid_t pdc, cont_prop, cont, cont2; perr_t ret; - int ret_value = 0; int rank = 0, size = 1; - char tag_value[128], tag_value2[128], *tag_value_ret; - psize_t value_size; + char tag_value[128], tag_value2[128], *tag_value_ret; + pdc_var_type_t value_type; + psize_t value_size; strcpy(tag_value, "some tag value"); strcpy(tag_value2, "some tag value 2 is longer than tag 1"); @@ -57,7 +57,7 @@ main(int argc, char **argv) } else { printf("Fail to create container property @ line %d!\n", __LINE__); - ret_value = 1; + return -1; } // create a container cont = PDCcont_create("c1", cont_prop); @@ -66,7 +66,7 @@ main(int argc, char **argv) } else { printf("Fail to create container @ line %d!\n", __LINE__); - ret_value = 1; + return -1; } cont2 = PDCcont_create("c2", cont_prop); @@ -75,88 +75,124 @@ main(int argc, char **argv) } else { printf("Fail to create container @ line %d!\n", __LINE__); - ret_value = 1; + return -1; } - ret = PDCcont_put_tag(cont, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCcont_put_tag(cont, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 1\n"); - ret_value = 1; + return -1; } - ret = PDCcont_put_tag(cont, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCcont_put_tag(cont, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 1\n"); - ret_value = 1; + return -1; } - ret = PDCcont_put_tag(cont2, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCcont_put_tag(cont2, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 2\n"); - ret_value = 1; + return -1; } - ret = PDCcont_put_tag(cont2, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCcont_put_tag(cont2, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 2\n"); - ret_value = 1; + return -1; } - ret = PDCcont_get_tag(cont, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 1\n"); - ret_value = 1; + return -1; } if (strcmp(tag_value, tag_value_ret) != 0) { printf("Wrong tag value at container 1, expected = [%s], get [%s]\n", tag_value, tag_value_ret); - ret_value = 1; + return -1; } - ret = PDCcont_get_tag(cont, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 1\n"); - ret_value = 1; + return -1; } if (strcmp(tag_value2, tag_value_ret) != 0) { printf("Wrong tag value at container 1, expected = [%s], get [%s]\n", tag_value2, tag_value_ret); - ret_value = 1; + return -1; } - ret = PDCcont_get_tag(cont2, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont2, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 2\n"); - ret_value = 1; + return -1; } if (strcmp(tag_value, tag_value_ret) != 0) { printf("Wrong tag value at container 2, expected = [%s], get [%s]\n", tag_value, tag_value_ret); - ret_value = 1; + return -1; } - ret = PDCcont_get_tag(cont2, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont2, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 2\n"); - ret_value = 1; + return -1; } if (strcmp(tag_value2, tag_value_ret) != 0) { printf("Wrong tag value at container 2, expected = [%s], get [%s]\n", tag_value2, tag_value_ret); - ret_value = 1; + return -1; + } + + ret = PDCcont_del_tag(cont2, "some tag 2"); + if (ret != SUCCEED) { + printf("Delete tag failed at container 2\n"); + return -1; + } + else { + printf("successfully deleted a tag from container c2\n"); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + ret = PDCcont_get_tag(cont2, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); + if (ret != SUCCEED) { + printf("Get tag failed at container 2\n"); + return -1; + } + + if (tag_value_ret != NULL || value_size != 0) { + printf("Error: got non-empty tag after deletion\n"); + return -1; + } + else { + printf("verified the tag has been deleted successfully\n"); } // close a container if (PDCcont_close(cont) < 0) { printf("fail to close container c1\n"); - ret_value = 1; + return -1; } else { printf("successfully close container c1\n"); } + // close a container + if (PDCcont_close(cont2) < 0) { + printf("fail to close container c1\n"); + return -1; + } + else { + printf("successfully close container c1\n"); + } + // close a container property if (PDCprop_close(cont_prop) < 0) { printf("Fail to close property @ line %d\n", __LINE__); - ret_value = 1; + return -1; } else { printf("successfully close container property\n"); @@ -164,10 +200,10 @@ main(int argc, char **argv) // close pdc if (PDCclose(pdc) < 0) { printf("fail to close PDC\n"); - ret_value = 1; + return -1; } #ifdef ENABLE_MPI MPI_Finalize(); #endif - return ret_value; + return 0; } diff --git a/src/tests/create_obj_scale.c b/src/tests/create_obj_scale.c index 61244a63d..ebd0d373e 100644 --- a/src/tests/create_obj_scale.c +++ b/src/tests/create_obj_scale.c @@ -201,7 +201,7 @@ main(int argc, char **argv) ht_total_end.tv_usec - ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("%10d created ... %.4f s\n", i * size, ht_total_sec); + printf("%10d created ... %.5e s\n", i * size, ht_total_sec); fflush(stdout); } #ifdef ENABLE_MPI @@ -218,7 +218,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to create %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to create %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/dart_algo_sim.c b/src/tests/dart_algo_sim.c new file mode 100644 index 000000000..5b216dd8e --- /dev/null +++ b/src/tests/dart_algo_sim.c @@ -0,0 +1,468 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "string_utils.h" +#include "timer_utils.h" +#include "dart_core.h" + +#define HASH_MD5 0 +#define HASH_MURMUR 1 +#define HASH_DART 2 + +typedef void (*insert_key_cb)(char *key, int prefix_len); +typedef int (*search_key_cb)(char *query_string, int prefix_len); + +const int INPUT_RANDOM_STRING = 0; +const int INPUT_UUID = 1; +const int INPUT_DICTIONARY = 2; +const int INPUT_WIKI_KEYWORD = 3; + +DART dart_g; + +int32_t request_count_g; + +dart_server *all_servers; + +dart_server +virtual_dart_retrieve_server_info_cb(uint32_t server_id) +{ + return all_servers[server_id]; +} + +// void +// md5_keyword_insert(char *key, int prefix_len) +// { +// if (key == NULL) +// return; +// int len = prefix_len == 0 ? strlen(key) : prefix_len; +// uint32_t hashVal = md5_hash(len, key); +// uint32_t server_id = hashVal % dart_g.num_server; +// all_servers[server_id].indexed_word_count = all_servers[server_id].indexed_word_count + 1; +// } + +// int +// md5_keyword_search(char *key, int prefix_len) +// { +// if (key == NULL) +// return 0; +// int len = prefix_len == 0 ? strlen(key) : prefix_len; +// uint32_t hashVal = md5_hash(len, key); +// uint32_t server_id = hashVal % dart_g.num_server; +// all_servers[server_id].request_count = all_servers[server_id].request_count + 1; +// } + +// void +// murmurhash_keyword_insert(char *key, int prefix_len) +// { +// if (key == NULL) +// return; +// int len = prefix_len == 0 ? strlen(key) : prefix_len; +// uint32_t hashVal = murmur3_32(key, len, 1); +// uint32_t server_id = hashVal % dart_g.num_server; +// all_servers[server_id].indexed_word_count = all_servers[server_id].indexed_word_count + 1; +// } + +// int +// murmurhash_keyword_search(char *key, int prefix_len) +// { +// if (key == NULL) +// return 0; +// int len = prefix_len == 0 ? strlen(key) : prefix_len; +// uint32_t hashVal = murmur3_32(key, len, 1); +// uint32_t server_id = hashVal % dart_g.num_server; +// all_servers[server_id].request_count = all_servers[server_id].request_count + 1; +// } + +// int +// djb2_hash_keyword_insert(char *key, int prefix_len) +// { +// if (key == NULL) +// return; +// int len = prefix_len == 0 ? strlen(key) : prefix_len; +// uint32_t hashVal = djb2_hash(key, len); +// uint32_t server_id = hashVal % dart_g.num_server; +// all_servers[server_id].indexed_word_count = all_servers[server_id].indexed_word_count + 1; +// } + +// int +// djb2_hash_keyword_search(char *key, int prefix_len) +// { +// if (key == NULL) +// return 0; +// int len = prefix_len == 0 ? strlen(key) : prefix_len; +// uint32_t hashVal = djb2_hash(key, len); +// uint32_t server_id = hashVal % dart_g.num_server; +// all_servers[server_id].request_count = all_servers[server_id].request_count + 1; +// } + +// int +// djb2_hash_keyword_insert_full(char *key, int prefix_len) +// { +// if (key == NULL) +// return; +// int len = prefix_len == 0 ? strlen(key) : prefix_len; +// uint32_t hashVal = djb2_hash(key, strlen(key)); +// uint32_t server_id = hashVal % dart_g.num_server; +// all_servers[server_id].indexed_word_count = all_servers[server_id].indexed_word_count + 1; +// } + +// int +// djb2_hash_keyword_search_full(char *key, int prefix_len) +// { +// if (key == NULL) +// return 0; +// int len = prefix_len == 0 ? strlen(key) : prefix_len; +// uint32_t hashVal = djb2_hash(key, strlen(key)); +// uint32_t server_id = hashVal % dart_g.num_server; +// all_servers[server_id].request_count = all_servers[server_id].request_count + 1; +// } + +void +DHT_INITIAL_keyword_insert(char *key, int prefix_len) +{ + if (key == NULL) + return; + index_hash_result_t *out; + int arr_len = DHT_hash(&dart_g, 1, key, OP_INSERT, &out); + int replica = 0; + for (replica = 0; replica < arr_len; replica++) { + all_servers[out[replica].server_id].indexed_word_count = + all_servers[out[replica].server_id].indexed_word_count + 1; + } +} + +int +DHT_INITIAL_keyword_search(char *key, int prefix_len) +{ + if (key == NULL) + return 0; + index_hash_result_t *out; + int arr_len = DHT_hash(&dart_g, 1, key, OP_INSERT, &out); + int i = 0; + for (i = 0; i < arr_len; i++) { // Perhaps we can use openmp for this loop? + all_servers[out[i].server_id].request_count = all_servers[out[i].server_id].request_count + 1; + } + return 1; +} + +void +DHT_FULL_keyword_insert(char *key, int prefix_len) +{ + if (key == NULL) + return; + index_hash_result_t *out; + int arr_len = DHT_hash(&dart_g, strlen(key), key, OP_INSERT, &out); + int replica = 0; + for (replica = 0; replica < arr_len; replica++) { + all_servers[out[replica].server_id].indexed_word_count = + all_servers[out[replica].server_id].indexed_word_count + 1; + } +} + +int +DHT_FULL_keyword_search(char *key, int prefix_len) +{ + if (key == NULL) + return 0; + index_hash_result_t *out; + int arr_len = DHT_hash(&dart_g, strlen(key), key, OP_INSERT, &out); + int i = 0; + for (i = 0; i < arr_len; i++) { // Perhaps we can use openmp for this loop? + all_servers[out[i].server_id].request_count = all_servers[out[i].server_id].request_count + 1; + } + return 1; +} + +void +dart_keyword_insert(char *key, int prefix_len) +{ + if (key == NULL) + return; + index_hash_result_t *out; + int arr_len = DART_hash(&dart_g, key, OP_INSERT, virtual_dart_retrieve_server_info_cb, &out); + int replica = 0; + for (replica = 0; replica < arr_len; replica++) { + all_servers[out[replica].server_id].indexed_word_count = + all_servers[out[replica].server_id].indexed_word_count + 1; + } +} + +int +dart_keyword_search(char *key, int prefix_len) +{ + if (key == NULL) + return 0; + index_hash_result_t *out; + int arr_len = DART_hash(&dart_g, key, OP_EXACT_QUERY, virtual_dart_retrieve_server_info_cb, &out); + int i = 0; + for (i = 0; i < arr_len; i++) { // Perhaps we can use openmp for this loop? + all_servers[out[i].server_id].request_count = all_servers[out[i].server_id].request_count + 1; + } + return 1; +} + +void +get_key_distribution(int num_server, char *algo) +{ + int srv_cnt = 0; + double sqrt_sum = 0; + double sum = 0; + double min = 999999999999999.0; + double max = 0.0; + for (srv_cnt = 0; srv_cnt < num_server; srv_cnt++) { + dart_server server_abstract = all_servers[srv_cnt]; + double num_indexed_word = (double)server_abstract.indexed_word_count; + if (min > num_indexed_word) { + min = num_indexed_word; + } + if (max < num_indexed_word) { + max = num_indexed_word; + } + println("[%s Key Distribution] The total number of indexed keys on server %d = %.0f", algo, srv_cnt, + num_indexed_word); + sum += (double)num_indexed_word; + sqrt_sum += (double)((double)num_indexed_word * (double)num_indexed_word); + } + double maxRange = max - min; + double mean = sum / (double)num_server; + double variance = sqrt_sum / num_server - mean * mean; + double stddev = sqrt(variance); + + println("[%s Key Distribution] STDDEV = %.3f and CV = %.3f for %d servers and %.0f keys in total.", algo, + stddev, stddev / mean, num_server, sum); +} + +void +get_request_distribution(int num_server, char *algo) +{ + int srv_cnt = 0; + double sqrt_sum = 0; + double sum = 0; + + for (srv_cnt = 0; srv_cnt < num_server; srv_cnt++) { + dart_server server_abstract = all_servers[srv_cnt]; + double request_count = (double)server_abstract.request_count; + println("[%s Load Balance All] The total number of query requests on server %d = %.0f", algo, srv_cnt, + request_count); + sum += (double)request_count; + sqrt_sum += (double)((double)request_count * (double)request_count); + } + double mean = sum / (double)num_server; + double variance = sqrt_sum / num_server - mean * mean; + double stddev = sqrt(variance); + println("[%s Load Balance All] STDDEV = %.3f and CV = %.3f for %d servers and %.0f request in total.", + algo, stddev, stddev / mean, num_server, sum); +} + +void +reset_request_count(int num_server) +{ + int i = 0; + for (i = 0; i < num_server; i++) { + all_servers[i].request_count = 0; + } +} + +char ** +gen_uuids(int count, int prefix_len, insert_key_cb insert_cb, search_key_cb search_cb) +{ + uuid_t out; + int c = 0; + char **result = (char **)calloc(count, sizeof(char *)); + for (c = 0; c < count; c++) { + uuid_generate_random(out); + result[c] = (char *)calloc(37, sizeof(char)); + uuid_unparse_lower(out, result[c]); + + if (insert_cb != NULL && search_cb != NULL) { + insert_cb(result[c], prefix_len); + + int sch = 0; + for (sch = 0; sch < 1; sch++) { + search_cb(result[c], prefix_len); + } + } + } + return result; +} + +char ** +gen_random_strings_with_cb(int count, int minlen, int maxlen, int alphabet_size, int prefix_len, + insert_key_cb insert_cb, search_key_cb search_cb) +{ + + int c = 0; + int i = 0; + char **result = (char **)calloc(count, sizeof(char *)); + for (c = 0; c < count; c++) { + // int len = maxlen;//rand()%maxlen; + int len = (rand() % maxlen) + 1; + len = len < minlen ? minlen : len; + char *str = (char *)calloc(len + 1, sizeof(len)); + for (i = 0; i < len; i++) { + int randnum = rand(); + if (randnum < 0) + randnum *= -1; + char chr = (char)((randnum % alphabet_size) + 65); + str[i] = chr; + } + // printf("generated %s\n", str); + result[c] = str; + if (insert_cb != NULL && search_cb != NULL) { + insert_cb(result[c], prefix_len); + + int sch = 0; + for (sch = 0; sch < 1; sch++) { + search_cb(result[c], prefix_len); + } + } + } + return result; +} + +char ** +read_words_from_text(const char *fileName, int *word_count, int **req_count, int prefix_len, + insert_key_cb insert_cb, search_key_cb search_cb) +{ + + FILE *file = fopen(fileName, "r"); /* should check the result */ + if (file == NULL) { + println("File not available\n"); + exit(4); + } + int lines_allocated = 128; + int max_line_len = 512; + + int i; + int line_count = 0; + char *line = (char *)malloc(sizeof(char) * (2 * max_line_len)); + char *word = (char *)malloc(sizeof(char) * (max_line_len)); + for (i = 0; 1; i++) { + int j; + if (fgets(line, max_line_len - 1, file) == NULL) { + break; + } + /* Get rid of CR or LF at end of line */ + + for (j = strlen(line) - 1; j >= 0 && (line[j] == '\n' || line[j] == '\r'); j--) + ; + line[j + 1] = '\0'; + int rc; + sscanf(line, "%d %s", &rc, word); + // (*req_count)[line_count]=rc; + + if (insert_cb != NULL && search_cb != NULL) { + insert_cb(word, prefix_len); + + int sch = 0; + for (sch = 0; sch < rc; sch++) { + search_cb(word, prefix_len); + } + } + line_count++; + } + free(line); + free(word); + *word_count = line_count; + //*total_word_count = i; + + fclose(file); + return NULL; +} + +void +print_usage() +{ + println("dart_sim.exe " + " "); +} + +int +main(int argc, char **argv) +{ + + if (argc < 9) { + print_usage(); + exit(1); + } + + int hashalgo = atoi(argv[1]); + int num_server = atoi(argv[2]); + int INPUT_TYPE = atoi(argv[3]); + char *txtFilePath = argv[4]; + + int alphabet_size, replication_factor, word_count, prefix_len; + + alphabet_size = atoi(argv[5]); + replication_factor = atoi(argv[6]); + + word_count = atoi(argv[7]); + prefix_len = atoi(argv[8]); + char **input_word_list = NULL; + int * req_count = NULL; + + int i = 0; + + // Init all servers + all_servers = (dart_server *)malloc(num_server * sizeof(dart_server)); + for (i = 0; i < num_server; i++) { + all_servers[i].id = i; + all_servers[i].indexed_word_count = 0; + all_servers[i].request_count = 0; + } + int extra_tree_height = 0; + char *algo_name = ""; + if (hashalgo == HASH_MD5) { + algo_name = "MD5"; + } + else if (hashalgo == HASH_MURMUR) { + algo_name = "MURMUR"; + } + else if (hashalgo == HASH_DART) { + algo_name = "DART"; + } + + println("HASH = %s", algo_name); + + void (*keyword_insert[])(char *, int) = {DHT_INITIAL_keyword_insert, DHT_FULL_keyword_insert, + dart_keyword_insert}; + int (*keyword_search[])(char *, int) = {DHT_INITIAL_keyword_search, DHT_FULL_keyword_search, + dart_keyword_search}; + + if (INPUT_TYPE == INPUT_DICTIONARY) { + // Init dart space. + alphabet_size = 29; + dart_space_init(&dart_g, num_server, num_server, alphabet_size, extra_tree_height, + replication_factor); + read_words_from_text(txtFilePath, &word_count, &req_count, prefix_len, keyword_insert[hashalgo], + keyword_search[hashalgo]); + } + else if (INPUT_TYPE == INPUT_RANDOM_STRING) { + alphabet_size = 129; + dart_space_init(&dart_g, num_server, num_server, alphabet_size, extra_tree_height, + replication_factor); + gen_random_strings_with_cb(word_count, 6, 16, alphabet_size, prefix_len, keyword_insert[hashalgo], + keyword_search[hashalgo]); + } + else if (INPUT_TYPE == INPUT_UUID) { + alphabet_size = 37; + dart_space_init(&dart_g, num_server, num_server, alphabet_size, extra_tree_height, + replication_factor); + gen_uuids(word_count, prefix_len, keyword_insert[hashalgo], keyword_search[hashalgo]); + } + else if (INPUT_TYPE == INPUT_WIKI_KEYWORD) { + alphabet_size = 129; + dart_space_init(&dart_g, num_server, num_server, alphabet_size, extra_tree_height, + replication_factor); + read_words_from_text(txtFilePath, &word_count, &req_count, prefix_len, keyword_insert[hashalgo], + keyword_search[hashalgo]); + } + + get_key_distribution(num_server, algo_name); + get_request_distribution(num_server, algo_name); +} diff --git a/src/tests/dart_attr_dist_test.c b/src/tests/dart_attr_dist_test.c new file mode 100644 index 000000000..04c6eab42 --- /dev/null +++ b/src/tests/dart_attr_dist_test.c @@ -0,0 +1,377 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "string_utils.h" +#include "timer_utils.h" +#include "dart_core.h" + +// #define ENABLE_MPI 1 + +#ifdef ENABLE_MPI +#include "mpi.h" +#endif + +#include "pdc.h" +#include "pdc_client_connect.h" + +// #define PDC_ENABLE_JULIA 1 + +#ifdef PDC_ENABLE_JULIA +#include "julia_helper_loader.h" +#define JULIA_HELPER_NAME "JuliaHelper" +// only define the following once, in an executable (not in a shared library) if you want fast +// code. +JULIA_DEFINE_FAST_TLS + +void +generate_incremental_associations(int64_t num_attr, int64_t num_obj, int64_t num_groups, int64_t **arr, + size_t *len) +{ + /* run generate_incremental_associations with parameters */ + jl_fn_args_t *args = (jl_fn_args_t *)calloc(1, sizeof(jl_fn_args_t)); + args->nargs = 3; + args->args = (jl_value_t **)calloc(args->nargs, sizeof(jl_value_t *)); + args->args[0] = jl_box_int64(num_attr); + args->args[1] = jl_box_int64(num_obj); + args->args[2] = jl_box_int64(num_groups); + + run_jl_get_int64_array(JULIA_HELPER_NAME, "generate_incremental_associations", args, arr, len); +} + +void +generate_attribute_occurrences(int64_t num_attr, int64_t num_obj, const char *dist, int64_t **arr, + size_t *len) +{ + /* run generate_incremental_associations with parameters */ + jl_fn_args_t *args = (jl_fn_args_t *)calloc(1, sizeof(jl_fn_args_t)); + args->nargs = 3; + args->args = (jl_value_t **)calloc(args->nargs, sizeof(jl_value_t *)); + args->args[0] = jl_box_int64(num_attr); + args->args[1] = jl_box_int64(num_obj); + args->args[2] = jl_cstr_to_string(dist); + + /* run generate_attribute_occurrences with parameters */ + run_jl_get_int64_array(JULIA_HELPER_NAME, "generate_attribute_occurrences", args, arr, len); +} +#endif + +void +generate_attr_obj_association(int64_t num_attr, int64_t num_obj, int64_t **arr, size_t *len) +{ + *arr = (int64_t *)calloc(num_attr, sizeof(int64_t)); + *len = num_attr; + for (int64_t i = 0; i < num_attr; i++) { + (*arr)[i] = (num_obj / num_attr) * (i + 1); + } +} + +int +main(int argc, char *argv[]) +{ + int rank = 0, size = 1; + +#ifdef ENABLE_MPI + fflush(stdout); + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + int64_t *attr_2_obj_array = NULL; + size_t arr_len = 0; + size_t total_num_obj = atoi(argv[1]); + size_t total_num_attr = atoi(argv[2]); + pdcid_t *obj_ids; + int i, j, k, pct, q_repeat_count = 100; + double stime, total_time; + int val; + + char pdc_context_name[40]; + char pdc_container_name[40]; + char pdc_obj_name[128]; + + char key[32]; + char value[32]; + char exact_query[48]; + char prefix_query[48]; + char suffix_query[48]; + char infix_query[48]; + +#ifdef PDC_ENABLE_JULIA + jl_module_list_t modules = {.julia_modules = (char *[]){JULIA_HELPER_NAME}, .num_modules = 1}; + init_julia(&modules); +#endif + + if (rank == 0) { + // calling julia helper to get the array. + +#ifdef PDC_ENABLE_JULIA + generate_incremental_associations(total_num_attr, total_num_obj, total_num_attr, &attr_2_obj_array, + &arr_len); +#else + generate_attr_obj_association(total_num_attr, total_num_obj, &attr_2_obj_array, &arr_len); +#endif + + // generate_attribute_occurrences(total_num_attr, total_num_obj, "uniform", &attr_2_obj_array, + // &arr_len); + + // broadcast the size from rank 0 to all other processes + MPI_Bcast(&arr_len, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD); + } + else { + // receive the size on all other ranks + MPI_Bcast(&arr_len, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD); + + // allocate memory for the array + attr_2_obj_array = (int64_t *)malloc(arr_len * sizeof(int64_t)); + } + // broadcast the array itself + MPI_Bcast(attr_2_obj_array, arr_len, MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); + + // print array. + for (i = 0; i < arr_len; ++i) { + printf("rank %d: %ld\n", rank, attr_2_obj_array[i]); + } + + sprintf(pdc_context_name, "pdc_%d", rank); + pdcid_t pdc = PDCinit(pdc_context_name); + + pdcid_t cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + printf("Fail to create container property @ line %d!\n", __LINE__); + + sprintf(pdc_container_name, "c1_%d", rank); + pdcid_t cont = PDCcont_create(pdc_container_name, cont_prop); + if (cont <= 0) + printf("Fail to create container @ line %d!\n", __LINE__); + + pdcid_t obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); + if (obj_prop <= 0) + printf("Fail to create object property @ line %d!\n", __LINE__); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + + // create enough objects + obj_ids = (pdcid_t *)calloc(total_num_obj, sizeof(pdcid_t)); + for (i = 0; i < total_num_obj; i++) { + if (i % size == rank) { + sprintf(pdc_obj_name, "obj%d", i); + obj_ids[i] = PDCobj_create(cont, pdc_obj_name, obj_prop); + if (obj_ids[i] <= 0) + printf("Fail to create object @ line %d!\n", __LINE__); + } + else { + obj_ids[i] = -1; + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; + stime = MPI_Wtime(); +#endif + if (rank == 0) + printf("[Summary] Create %zu objects with %d ranks, time: %.6f\n", total_num_obj, size, total_time); + // ========== ATTACH TAGS TO OBJECTS ========== + stopwatch_t timer_obj; + stopwatch_t timer_dart; + double duration_obj_ms = 0.0; + double duration_dart_ms = 0.0; + + // attr 1: 10 + // attr 2: 20 + // attr 3: 30 + // + // 10 objs with attr 1, 2, 3 + // 10 objs with attr 2, 3, + // 10 objs with attr 3. + // m attrs, n objs, where m << n + // avg number of obj per attr: n/m + // so, every n/m objs will have different number of attr + // 1st n/m objs: 1 attrs + // 2nd n/m objs: 2 attrs + // 3rd n/m objs: 3 attrs + // ... + // mth n/m objs: m attrs + + for (i = 0; i < arr_len; i++) { // iterate through all attributes + val = i + 23456; + pct = 0; + int num_obj_per_attr = attr_2_obj_array[i]; + sprintf(key, "k%ld", i + 12345); + sprintf(value, "v%ld", val); + println("attaching attribute #%d [%s:%s] to %d objects", i, key, value, num_obj_per_attr); + for (j = 0; j < num_obj_per_attr; j++) { + // each attribute is attached to a specific number of objects, and this is how we make up + // different selectivity. + if (j % size == rank) { // evenly distribute the task to all ranks. + // attach attribute to object + timer_start(&timer_obj); + if (PDCobj_put_tag(obj_ids[j], key, (void *)&val, PDC_INT, sizeof(int)) < 0) + printf("fail to add a kvtag to o%d\n", j); + timer_pause(&timer_obj); + duration_obj_ms += (double)timer_delta_ms(&timer_obj); + } + size_t num_object_per_pct = num_obj_per_attr / 100; + size_t num_object_per_ton_thousand = num_obj_per_attr / 10000; + if (j % num_object_per_pct == 0) + pct += 1; + if (rank == 0 && j % num_object_per_ton_thousand == 0) { + printf("[Client_Side_Insert] %d\%: Insert '%s=%s' for %llu objs within %.4f ms\n", pct, key, + value, j, duration_obj_ms); + } + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; + stime = MPI_Wtime(); +#endif + + if (rank == 0) + printf("[Summary] Inserted %d attributes for %zu objects with %d ranks, obj time: %.6f\n", + total_num_attr, total_num_obj, size, total_time); + // ========== INSERT OBJECT REFERENCE INTO DART ========== + dart_object_ref_type_t ref_type = REF_PRIMARY_ID; + dart_hash_algo_t hash_algo = DART_HASH; + + for (i = 0; i < arr_len; i++) { + sprintf(key, "k%ld", i + 12345); + val = i + 23456; + sprintf(value, "v%ld", val); + pct = 0; + for (j = 0; j < attr_2_obj_array[i]; j++) { + if (j % size == rank) { + // insert object reference into DART + timer_start(&timer_dart); + PDC_Client_insert_obj_ref_into_dart(hash_algo, key, value, ref_type, j); + timer_pause(&timer_dart); + duration_dart_ms += (double)timer_delta_ms(&timer_dart); + } + size_t num_object_per_pct = attr_2_obj_array[i] / 100; + size_t num_object_per_ton_thousand = attr_2_obj_array[i] / 10000; + if (j % num_object_per_pct == 0) + pct += 1; + if (rank == 0 && j % num_object_per_ton_thousand == 0) { + printf("[Client_Side_Insert] %d\%: Insert '%s=%s' for %llu objs, index time " + "%.4f ms\n", + pct, key, value, j, duration_dart_ms); + } + } + } +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; + stime = MPI_Wtime(); +#endif + if (rank == 0) + printf("[Summary] Inserted %d attributes for %zu objects with %d ranks, dart time: " + "%.6f\n", + total_num_attr, total_num_obj, size, total_time); + // ========== EXACT QUERY with Naive Approach ========== + pdc_kvtag_t kvtag; + duration_obj_ms = 0.0; + duration_dart_ms = 0.0; + + for (i = 0; i < arr_len; i++) { + if (i % arr_len == rank) { + uint64_t *out1; + int rest_count1 = 0; + timer_start(&timer_obj); + for (k = 0; k < q_repeat_count; k++) { + sprintf(key, "k%ld", i + 12345); + val = i + 23456; + + kvtag.name = key; + kvtag.value = (void *)&val; + kvtag.size = sizeof(int); + kvtag.type = PDC_INT; + + // naive query methods + if (PDC_Client_query_kvtag(&kvtag, &rest_count1, &out1) < 0) { + printf("fail to query kvtag\n"); + break; + } + } + timer_pause(&timer_obj); + duration_obj_ms += timer_delta_ms(&timer_obj); + println("[Client_Side_Exact] Search '%s' for %d times and get %d results, obj time: %.4f ms\n", + key, q_repeat_count, rest_count1, duration_obj_ms); + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; + stime = MPI_Wtime(); +#endif + + if (rank == 0) + printf("[Summary] Exact query %d attributes for %zu objects with %d ranks, obj time: %.6f\n", + total_num_attr, total_num_obj, size, total_time); + // ========== EXACT QUERY with DART ========== + duration_obj_ms = 0.0; + duration_dart_ms = 0.0; + + for (i = 0; i < arr_len; i++) { + if (i % arr_len == rank) { + uint64_t *out1; + int rest_count1 = 0; + timer_start(&timer_dart); + for (k = 0; k < q_repeat_count; k++) { + sprintf(key, "k%ld", i + 12345); + sprintf(value, "v%ld", i + 23456); + sprintf(exact_query, "%s=%s", key, value); + + // DART query methods + PDC_Client_search_obj_ref_through_dart(hash_algo, exact_query, ref_type, &rest_count1, &out1); + } + timer_pause(&timer_dart); + duration_dart_ms += timer_delta_ms(&timer_dart); + println("[Client_Side_Exact] Search '%s' for %d times and get %d results, dart " + "time: %.4f ms\n", + key, q_repeat_count, rest_count1, duration_dart_ms); + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + + if (rank == 0) + printf("[Summary] Exact query %d attributes for %zu objects with %d ranks, dart " + "time: %.6f ms\n", + total_num_attr, total_num_obj, size, total_time); + + if (PDCcont_close(cont) < 0) + printf("fail to close container %lld\n", cont); + + if (PDCprop_close(cont_prop) < 0) + printf("Fail to close property @ line %d\n", __LINE__); + + if (PDCclose(pdc) < 0) + printf("fail to close PDC\n"); + + // free attr_2_obj_array + free(attr_2_obj_array); + +#ifdef PDC_ENABLE_JULIA + close_julia(); +#endif + +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return 0; +} \ No newline at end of file diff --git a/src/tests/dart_func_test.c b/src/tests/dart_func_test.c new file mode 100644 index 000000000..8d29379b8 --- /dev/null +++ b/src/tests/dart_func_test.c @@ -0,0 +1,110 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "string_utils.h" +#include "timer_utils.h" +#include "dart_core.h" + +// #define ENABLE_MPI 1 + +#ifdef ENABLE_MPI +#include "mpi.h" +#endif + +#include "pdc.h" +#include "pdc_client_connect.h" + +int +main(int argc, char **argv) +{ + + int rank = 0, size = 1; + +#ifdef ENABLE_MPI + // println("MPI enabled!\n"); + fflush(stdout); + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + // if (rank == 0) { + pdcid_t pdc = PDCinit("pdc"); + + pdcid_t cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + printf("Fail to create container property @ line %d!\n", __LINE__); + + pdcid_t cont = PDCcont_create("c1", cont_prop); + if (cont <= 0) + printf("Fail to create container @ line %d!\n", __LINE__); + + pdcid_t obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); + if (obj_prop <= 0) + printf("Fail to create object property @ line %d!\n", __LINE__); + + dart_object_ref_type_t ref_type = REF_PRIMARY_ID; + dart_hash_algo_t hash_algo = DART_HASH; + + char * key = "abcd"; + char * value = "1234"; + uint64_t data = 12341234; + // if (rank == 0) { + PDC_Client_insert_obj_ref_into_dart(hash_algo, key, value, ref_type, data); + println("[Client_Side_Insert] Insert '%s=%s' for ref %llu", key, value, data); + + // This is for testing exact search + char * exact_query = "abcd=1234"; + uint64_t *out1; + int rest_count1 = 0; + PDC_Client_search_obj_ref_through_dart(hash_algo, exact_query, ref_type, &rest_count1, &out1); + + println("[Client_Side_Exact] Search '%s' and get %d results : %llu", exact_query, rest_count1, out1[0]); + + // This function test is for testing the prefix search + char * prefix_query = "ab*=12*"; + uint64_t *out2; + int rest_count2 = 0; + PDC_Client_search_obj_ref_through_dart(hash_algo, prefix_query, ref_type, &rest_count2, &out2); + + println("[Client_Side_Prefix] Search '%s' and get %d results : %llu", prefix_query, rest_count2, out2[0]); + + // This function test is for testing the suffix search. + char * suffix_query = "*cd=*34"; + uint64_t *out3; + int rest_count3 = 0; + PDC_Client_search_obj_ref_through_dart(hash_algo, suffix_query, ref_type, &rest_count3, &out3); + + println("[Client_Side_Suffix] Search '%s' and get %d results : %llu", suffix_query, rest_count3, out3[0]); + + // This is for testing infix search. + char * infix_query = "*bc*=*23*"; + uint64_t *out4; + int rest_count4 = 0; + PDC_Client_search_obj_ref_through_dart(hash_algo, infix_query, ref_type, &rest_count4, &out4); + + println("[Client_Side_Infix] Search '%s' and get %d results : %llu", infix_query, rest_count4, out4[0]); + + // } + + // done: + + if (PDCcont_close(cont) < 0) + printf("fail to close container %lld\n", cont); + + if (PDCprop_close(cont_prop) < 0) + printf("Fail to close property @ line %d\n", __LINE__); + + if (PDCclose(pdc) < 0) + printf("fail to close PDC\n"); + // } +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return 0; +} diff --git a/src/tests/dart_test.c b/src/tests/dart_test.c new file mode 100644 index 000000000..25a5e3da0 --- /dev/null +++ b/src/tests/dart_test.c @@ -0,0 +1,590 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "string_utils.h" +#include "timer_utils.h" + +// #define ENABLE_MPI 1 + +#ifdef ENABLE_MPI +#include "mpi.h" +#endif + +#include "pdc.h" +#include "pdc_client_connect.h" + +const int INPUT_RANDOM_STRING = 0; +const int INPUT_UUID = 1; +const int INPUT_DICTIONARY = 2; +const int INPUT_WIKI_KEYWORD = 3; + +void +print_usage() +{ + println("Usage: srun -n ./dart_test " + " /path/to/txtfile \n" + "alphabet_size: 26 for random string, 37 for uuid, 29 for dictionary, 129 for wiki keyword\n" + "replication_factor: 1-5\n" + "word_count: The number of words you would like to generate. For txt files, it doesn't matter.\n" + "txtfile: /path/to/txtfile, for random string and uuid, just provide random/uuid.\n" + "index_type: 1 for full hashing, 2 for initial hashing, 3 for DART\n" + "Example: srun -n 4 ./dart_test 0 26 2 2048 random 3\n"); +} + +char ** +gen_uuids(int count) +{ + uuid_t out; + int c = 0; + char **result = (char **)calloc(count, sizeof(char *)); + for (c = 0; c < count; c++) { + uuid_generate_random(out); + result[c] = (char *)calloc(37, sizeof(char)); + uuid_unparse_lower(out, result[c]); + } + return result; +} + +char ** +read_words_from_text(const char *fileName, int *word_count, int *total_word_count, int mpi_rank) +{ + + FILE *file = fopen(fileName, "r"); /* should check the result */ + if (file == NULL) { + println("File not available\n"); + exit(4); + } + int lines_allocated = 128; + int max_line_len = 512; + char **words = (char **)malloc(sizeof(char *) * lines_allocated); + if (words == NULL) { + fprintf(stderr, "Out of memory\n"); + exit(1); + } + int i; + int line_count = 0; + for (i = 0; 1; i++) { + int j; + +#ifdef ENABLE_MPI + if (i % (mpi_rank + 1) != 0) { + // char *trash_skip_buf=(char *)calloc(max_line_len, sizeof(char)); + char trash_skip_buf[512]; + if (fgets(trash_skip_buf, max_line_len - 1, file) == NULL) { + // free(trash_skip_buf); + break; + } + // println("skip '%s'", trash_skip_buf); + // free(trash_skip_buf); + // i++; + continue; + } +#endif + if (i >= lines_allocated) { + int new_size; + new_size = lines_allocated * 2; + char **new_wordlist_ptr = (char **)realloc(words, sizeof(char *) * new_size); + if (new_wordlist_ptr == NULL) { + fprintf(stderr, "Out of memory\n"); + exit(3); + } + words = new_wordlist_ptr; + lines_allocated = new_size; + } + words[line_count] = (char *)malloc(sizeof(char) * max_line_len); + if (words[line_count] == NULL) { + fprintf(stderr, "out of memory\n"); + exit(4); + } + if (fgets(words[line_count], max_line_len - 1, file) == NULL) { + break; + } + /* Get rid of CR or LF at end of line */ + for (j = strlen(words[line_count]) - 1; + j >= 0 && (words[line_count][j] == '\n' || words[line_count][j] == '\r'); j--) + ; + words[line_count][j + 1] = '\0'; + line_count++; + } + *word_count = line_count; + *total_word_count = i; + + fclose(file); + return words; +} + +int +main(int argc, char **argv) +{ + + int rank = 0, size = 1; + +#ifdef ENABLE_MPI + // println("MPI enabled!\n"); + fflush(stdout); + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + if (argc < 6) { + print_usage(); + exit(1); + } + + random_seed(0); + + int alphabet_size = atoi(argv[1]); + int replication_factor = atoi(argv[2]); + + int num_client = size; + + char **input_word_list = NULL; + int total_word_count = 2048; + + // total_word_count = atoi(argv[3]); + + int word_count = total_word_count / size; + + const char *dict_filename = argv[4]; + + int index_type = atoi(argv[5]); + dart_hash_algo_t hash_algo = (dart_hash_algo_t)index_type; + dart_object_ref_type_t ref_type = REF_PRIMARY_ID; + + if (dict_filename == NULL) { + print_usage(); + exit(1); + } + + if (strcmp(dict_filename, "random") == 0) { + input_word_list = gen_random_strings(word_count, 6, 16, alphabet_size); + alphabet_size = 129; + } + else if (strcmp(dict_filename, "uuid") == 0) { + input_word_list = gen_uuids(word_count); + alphabet_size = 37; + } + else { + input_word_list = read_words_from_text(dict_filename, &word_count, &total_word_count, rank); + alphabet_size = 29; + if (indexOfStr(dict_filename, "wiki") != -1) { + alphabet_size = 129; + } + } + + if (rank == 0) { + println("word_count = %d", word_count); + } + + pdcid_t pdc = PDCinit("pdc"); + + pdcid_t cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + printf("Fail to create container property @ line %d!\n", __LINE__); + + pdcid_t cont = PDCcont_create("c1", cont_prop); + if (cont <= 0) + printf("Fail to create container @ line %d!\n", __LINE__); + + pdcid_t obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); + if (obj_prop <= 0) + printf("Fail to create object property @ line %d!\n", __LINE__); + + DART *dart_g = get_dart_g(); + dart_g->replication_factor = replication_factor; + + int round = 0; + for (round = 0; round < 2; round++) { + stopwatch_t timer; + stopwatch_t detailed_timer; + double insert_throughput = 0; + double query_throughput = 0; + double delete_throughput = 0; + double update_throughput = 0; + int i = 0; + +// /* =============== Setup Connection By Inserting and Deleting for one time ======================= */ +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + for (i = 0; i < word_count; i++) { + int data = i; + PDC_Client_insert_obj_ref_into_dart(index_type, input_word_list[i], input_word_list[i], ref_type, + (uint64_t)data); + } + + for (i = 0; i < word_count; i++) { + int data = i; + PDC_Client_delete_obj_ref_from_dart(index_type, input_word_list[i], input_word_list[i], ref_type, + (uint64_t)data); + } + +/* =============== Insert testing ======================= */ +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + timer_start(&timer); + + for (i = 0; i < word_count; i++) { + timer_start(&detailed_timer); + int data = i; + PDC_Client_insert_obj_ref_into_dart(index_type, input_word_list[i], input_word_list[i], ref_type, + (uint64_t)data); + timer_pause(&detailed_timer); + if (round == 1) + println("[Client_Side_Insert] Time to insert key %s for both prefix index and suffix index = " + "%d microseconds", + input_word_list[i], timer_delta_us(&detailed_timer)); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + timer_pause(&timer); + + if (rank == 0) { + + insert_throughput = (double)((double)(total_word_count) / (double)timer_delta_ms(&timer)); + if (round == 1) + println("[Client_Side_Insert_Throughput] %.3f ops/ms", insert_throughput); + + // int srv_cnt = 0; + // double sqrt_sum = 0; + // double sum = 0; + + // for (srv_cnt = 0; srv_cnt < dart_g->num_server; srv_cnt++){ + // dart_server server_abstract = dart_retrieve_server_info_cb((uint32_t)srv_cnt); + // int64_t num_indexed_word = server_abstract.indexed_word_count/2; + // println("[DART Key Distribution] Server %d has %d words indexed", srv_cnt, + // num_indexed_word); sum += (double)num_indexed_word; sqrt_sum += + // (double)((double)num_indexed_word * (double)num_indexed_word); + // } + // double mean = sum/(double)dart_g->num_server; + // double variance = sqrt_sum/dart_g->num_server - mean * mean; + // double stddev = sqrt(variance); + // println("[DART Key Distribution] STDDEV = %.3f for %d servers and %d keys in total.", stddev, + // dart_g->num_server, word_count * size); + } + + // /* =============== Exact Query testing ======================= */ + + char **query_input_list = (char **)calloc(word_count, sizeof(char *)); + for (i = 0; i < word_count; i++) { + query_input_list[i] = (char *)calloc((strlen(input_word_list[i]) + 1), sizeof(char)); + strcpy(query_input_list[i], input_word_list[i]); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + timer_start(&timer); + for (i = 0; i < word_count; i++) { + timer_start(&detailed_timer); + int data = i; + char *key = query_input_list[i]; + char query_str[100]; + sprintf(query_str, "%s=%s", key, key); + uint64_t *out; + int rest_count = 0; + PDC_Client_search_obj_ref_through_dart(hash_algo, query_str, ref_type, &rest_count, &out); + timer_pause(&detailed_timer); + if (round == 1) + println("[Client_Side_Exact] Time to search '%s' and get %d results = %d microseconds for " + "rank %d", + query_str, rest_count, timer_delta_us(&detailed_timer), rank); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + timer_pause(&timer); + + if (rank == 0) { + query_throughput = (double)((double)(total_word_count) / (double)timer_delta_ms(&timer)); + if (round == 1) + println("[Client_Side_Exact_Throughput] %.3f ops/ms", query_throughput); + + int srv_cnt = 0; + for (srv_cnt = 0; srv_cnt < dart_g->num_server; srv_cnt++) { + dart_server server_abstract = dart_retrieve_server_info_cb((uint32_t)srv_cnt); + if (round == 1) + println("[DART Load Balance 1] Server %d has query requests = %d", srv_cnt, + server_abstract.request_count); + } + } + + /* =============== Prefix Query testing ======================= */ + + for (i = 0; i < word_count; i++) { + query_input_list[i] = (char *)calloc((strlen(input_word_list[i]) + 1), sizeof(char)); + strcpy(query_input_list[i], input_word_list[i]); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + timer_start(&timer); + for (i = 0; i < word_count; i++) { + timer_start(&detailed_timer); + int data = i; + char *key = query_input_list[i]; + if (strlen(key) > 4) { + key[4] = '\0'; // trim to prefix of 4. + } + char query_str[80]; + sprintf(query_str, "%s*=%s*", key, key); + uint64_t *out; + int rest_count = 0; + PDC_Client_search_obj_ref_through_dart(hash_algo, query_str, ref_type, &rest_count, &out); + timer_pause(&detailed_timer); + if (round == 1) + println("[Client_Side_Prefix] Time to search '%s' and get %d results = %d microseconds for " + "rank %d", + query_str, rest_count, timer_delta_us(&detailed_timer), rank); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + timer_pause(&timer); + + if (rank == 0) { + query_throughput = (double)((double)(total_word_count) / (double)timer_delta_ms(&timer)); + if (round == 1) + println("[Client_Side_Prefix_Throughput] %.3f ops/ms", query_throughput); + + int srv_cnt = 0; + for (srv_cnt = 0; srv_cnt < dart_g->num_server; srv_cnt++) { + dart_server server_abstract = dart_retrieve_server_info_cb((uint32_t)srv_cnt); + if (round == 1) + println("[DART Load Balance 2] Server %d has query requests = %d", srv_cnt, + server_abstract.request_count); + } + } + + /* =============== Suffix Query testing ======================= */ + for (i = 0; i < word_count; i++) { + query_input_list[i] = (char *)calloc((strlen(input_word_list[i]) + 1), sizeof(char)); + strcpy(query_input_list[i], input_word_list[i]); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + timer_start(&timer); + for (i = 0; i < word_count; i++) { + timer_start(&detailed_timer); + int data = i; + char *key = query_input_list[i]; + int key_len = strlen(key); + if (key_len > 4) { + key = &key[key_len - 4]; // trim to suffix of 4. + } + char query_str[80]; + sprintf(query_str, "*%s=*%s", key, key); + uint64_t *out; + int rest_count = 0; + PDC_Client_search_obj_ref_through_dart(hash_algo, query_str, ref_type, &rest_count, &out); + timer_pause(&detailed_timer); + if (round == 1) + println("[Client_Side_Suffix] Time to search '%s' and get %d results = %d microseconds for " + "rank %d", + query_str, rest_count, timer_delta_us(&detailed_timer), rank); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + timer_pause(&timer); + + if (rank == 0) { + query_throughput = (double)((double)(total_word_count) / (double)timer_delta_ms(&timer)); + if (round == 1) + println("[Client_Side_Suffix_Throughput] %.3f ops/ms", query_throughput); + + int srv_cnt = 0; + for (srv_cnt = 0; srv_cnt < dart_g->num_server; srv_cnt++) { + dart_server server_abstract = dart_retrieve_server_info_cb((uint32_t)srv_cnt); + if (round == 1) + println("[DART Load Balance 3] Server %d has query requests = %d", srv_cnt, + server_abstract.request_count); + } + } + + /* =============== Infix Query testing ======================= */ + for (i = 0; i < word_count; i++) { + query_input_list[i] = (char *)calloc((strlen(input_word_list[i]) + 1), sizeof(char)); + strcpy(query_input_list[i], input_word_list[i]); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + timer_start(&timer); + for (i = 0; i < word_count; i++) { + timer_start(&detailed_timer); + int data = i; + char *key = query_input_list[i]; + // if (strlen(key)>4){ + // key[4] = '\0'; //trim to prefix of 4. + // } + if (strlen(key) > 6) { + key = &(key[1]); + key[4] = '\0'; // trim to prefix of 4. + } + char query_str[80]; + sprintf(query_str, "*%s*=*%s*", key, key); + uint64_t *out; + int rest_count = 0; + PDC_Client_search_obj_ref_through_dart(hash_algo, query_str, ref_type, &rest_count, &out); + timer_pause(&detailed_timer); + if (round == 1) + println("[Client_Side_Infix] Time to search '%s' and get %d results = %d microseconds for " + "rank %d", + query_str, rest_count, timer_delta_us(&detailed_timer), rank); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + timer_pause(&timer); + + if (rank == 0) { + query_throughput = (double)((double)(total_word_count) / (double)timer_delta_ms(&timer)); + if (round == 1) + println("[Client_Side_Infix_Throughput] %.3f ops/ms", query_throughput); + + int srv_cnt = 0; + for (srv_cnt = 0; srv_cnt < dart_g->num_server; srv_cnt++) { + dart_server server_abstract = dart_retrieve_server_info_cb((uint32_t)srv_cnt); + if (round == 1) + println("[DART Load Balance 4] Server %d has query requests = %d", srv_cnt, + server_abstract.request_count); + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + if (rank == 0) { + int srv_cnt = 0; + double sqrt_sum = 0; + double sum = 0; + for (srv_cnt = 0; srv_cnt < dart_g->num_server; srv_cnt++) { + dart_server server_abstract = dart_retrieve_server_info_cb((uint32_t)srv_cnt); + int64_t num_request = server_abstract.request_count; + if (round == 1) + println("[DART Load Balance All] The total number of query requests on server %d = %d", + srv_cnt, num_request); + sum += (double)num_request; + sqrt_sum += (double)((double)num_request * (double)num_request); + } + double mean = sum / (double)dart_g->num_server; + double variance = sqrt_sum / dart_g->num_server - mean * mean; + double stddev = sqrt(variance); + + // double normalSum = (sum - (mean*(double)dart_g->num_server))/stddev; + // double normalSqrtSum = sqrt_sum-2*sum*mean+((sum*sum)/(double)dart_g->num_server); + // double normalMean = normalSum / (double)dart_g->num_server; + // double normalVariance = normalSqrtSum/dart_g->num_server - normalMean * normalMean; + // double normalStdDev = sqrt(normalVariance); + if (round == 1) + println( + "[DART DART Load Balance All] STDDEV = %.3f for %d servers and %.1f request in total.", + stddev, dart_g->num_server, sum); + + srv_cnt = 0; + sqrt_sum = 0; + sum = 0; + + for (srv_cnt = 0; srv_cnt < dart_g->num_server; srv_cnt++) { + dart_server server_abstract = dart_retrieve_server_info_cb((uint32_t)srv_cnt); + int64_t num_indexed_word = server_abstract.indexed_word_count / 2; + if (round == 1) + println("[DART Key Distribution] Server %d has %d words indexed", srv_cnt, + num_indexed_word); + sum += (double)num_indexed_word; + sqrt_sum += (double)((double)num_indexed_word * (double)num_indexed_word); + } + mean = sum / (double)dart_g->num_server; + variance = sqrt_sum / dart_g->num_server - mean * mean; + stddev = sqrt(variance); + + // normalSum = (sum - (mean*(double)dart_g->num_server))/stddev; + // normalSqrtSum = sqrt_sum-2*sum*mean+((sum*sum)/(double)dart_g->num_server); + // normalMean = normalSum / (double)dart_g->num_server; + // normalVariance = normalSqrtSum/dart_g->num_server - normalMean * normalMean; + // normalStdDev = sqrt(normalVariance); + + if (round == 1) + println("[DART Key Distribution] STDDEV = %.3f for %d servers and %d keys in total.", stddev, + dart_g->num_server, word_count * size); + } + +/* =========================== Delete From Index ========================= */ +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + timer_start(&timer); + + for (i = 0; i < word_count; i++) { + timer_start(&detailed_timer); + int data = i; + PDC_Client_delete_obj_ref_from_dart(hash_algo, input_word_list[i], input_word_list[i], ref_type, + (uint64_t)data); + timer_pause(&detailed_timer); + if (round == 1) + println("[Client_Side_Delete] Time to delete key %s for both prefix index and suffix index = " + "%d microseconds", + input_word_list[i], timer_delta_us(&detailed_timer)); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + timer_pause(&timer); + + if (rank == 0) { + + delete_throughput = (double)((double)(total_word_count) / (double)timer_delta_ms(&timer)); + if (round == 1) + println("[Client_Side_Delete_Throughput] %.3f ops/ms", delete_throughput); + } + + } // end round loop + + // done: + + if (PDCcont_close(cont) < 0) + printf("fail to close container %lld\n", cont); + + if (PDCprop_close(cont_prop) < 0) + printf("Fail to close property @ line %d\n", __LINE__); + + if (PDCclose(pdc) < 0) + printf("fail to close PDC\n"); + +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return 0; +} diff --git a/src/tests/data_server_read.c b/src/tests/data_server_read.c index b57855a58..f56ac5f09 100644 --- a/src/tests/data_server_read.c +++ b/src/tests/data_server_read.c @@ -88,7 +88,7 @@ main(int argc, char **argv) ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to read data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to read data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/data_server_read_multi.c b/src/tests/data_server_read_multi.c index 2081547e2..9d6cdafda 100644 --- a/src/tests/data_server_read_multi.c +++ b/src/tests/data_server_read_multi.c @@ -197,7 +197,7 @@ main(int argc, char **argv) if (rank == 0) { printf( - "Total time read %d ts data each %luMB with %d ranks: %.6f, meta %.2f, wait %.2f, sleep %.2f\n", + "Total time read %d ts data each %luMB with %d ranks: %.5e, meta %.2f, wait %.2f, sleep %.2f\n", ntimestep, size_MB, size, total_elapsed / 1000000.0, total_meta_sec, total_wait_sec, sleepseconds * ntimestep); fflush(stdout); diff --git a/src/tests/data_server_read_vpic_multits.c b/src/tests/data_server_read_vpic_multits.c index 6222c1725..f260e8cfb 100644 --- a/src/tests/data_server_read_vpic_multits.c +++ b/src/tests/data_server_read_vpic_multits.c @@ -322,7 +322,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); #endif if (rank == 0) - printf("Timestep %d: query time %.4f, read time %.4f, wait time %.4f, compute time %.4f\n", ts, + printf("Timestep %d: query time %.5e, read time %.5e, wait time %.5e, compute time %.5e\n", ts, query_time, read_time, wait_time, true_sleep_time); } // end of for ts diff --git a/src/tests/data_server_read_vpic_spatial_multits.c b/src/tests/data_server_read_vpic_spatial_multits.c index 6b3c5a5ff..e667736cf 100644 --- a/src/tests/data_server_read_vpic_spatial_multits.c +++ b/src/tests/data_server_read_vpic_spatial_multits.c @@ -326,7 +326,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); #endif if (rank == 0) - printf("Timestep %d: query time %.4f, read time %.4f, wait time %.4f, compute time %.4f\n", ts, + printf("Timestep %d: query time %.5e, read time %.5e, wait time %.5e, compute time %.5e\n", ts, query_time, read_time, wait_time, true_sleep_time); } // end of for ts diff --git a/src/tests/data_server_write.c b/src/tests/data_server_write.c index b9c46527e..174d780fe 100644 --- a/src/tests/data_server_write.c +++ b/src/tests/data_server_write.c @@ -124,7 +124,7 @@ main(int argc, char **argv) ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to write data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to write data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/data_server_write_multi.c b/src/tests/data_server_write_multi.c index 541365069..fd5a02478 100644 --- a/src/tests/data_server_write_multi.c +++ b/src/tests/data_server_write_multi.c @@ -224,7 +224,7 @@ main(int argc, char **argv) if (rank == 0) { printf( - "Total time write %d ts data each %luMB with %d ranks: %.6f, meta %.2f, wait %.2f, sleep %.2f\n", + "Total time write %d ts data each %luMB with %d ranks: %.5e, meta %.2f, wait %.2f, sleep %.2f\n", ntimestep, size_MB, size, total_elapsed / 1000000.0, total_meta_sec, total_wait_sec, sleepseconds * ntimestep); fflush(stdout); diff --git a/src/tests/data_server_write_vpic_multits.c b/src/tests/data_server_write_vpic_multits.c index 6729c759f..6a943bb58 100644 --- a/src/tests/data_server_write_vpic_multits.c +++ b/src/tests/data_server_write_vpic_multits.c @@ -346,7 +346,7 @@ main(int argc, char **argv) } if (rank == 0) - printf("Timestep %d: create time %.6f, query time %.6f, write time %.6f, wait time %.6f\n", ts, + printf("Timestep %d: create time %.5e, query time %.5e, write time %.5e, wait time %.5e\n", ts, create_time, query_time, write_time, wait_time); } diff --git a/src/tests/delete_obj_scale.c b/src/tests/delete_obj_scale.c index ceff24141..e97aa409f 100644 --- a/src/tests/delete_obj_scale.c +++ b/src/tests/delete_obj_scale.c @@ -191,7 +191,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to create %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to create %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/helper/JuliaHelper.jl b/src/tests/helper/JuliaHelper.jl new file mode 100644 index 000000000..24ec92721 --- /dev/null +++ b/src/tests/helper/JuliaHelper.jl @@ -0,0 +1,105 @@ +module JuliaHelper + +export test_embedded_julia +export generate_attribute_occurrences +export generate_incremental_associations + +using Distributions + + """ + test_embedded_julia(input::Int64, input2::Int64, intput3::Int64) + + Test function for embedding Julia in C. + + # Arguments + - `input::Int64`: The first input + - `input2::Int64`: The second input + - `intput3::Int64`: The third input + + # Returns + - `result::Vector{Int64}`: A vector containing the inputs plus 1, 2, and 3, respectively. + """ + function test_embedded_julia(input::Int64, input2::Int64, intput3::Int64) + result = [input, input + 1, input2 + 2, intput3 + 3] + return result + end # end function test_embedded_julia + + """ + generate_attribute_occurrences(num_attributes::Int64, num_objects::Int64, distribution::String, s::Float64=1.0) + + Generate an array of the number of objects for each attribute, where the number of objects for each attribute is a random value. + + # Arguments + - `num_attributes::Int64`: The total number of attributes + - `num_objects::Int64`: The total number of objects + - `distribution::String`: The distribution to use for generating the number of objects for each attribute. Valid values are "uniform", "pareto", "normal", and "exponential". + - `s::Float64=1.0`: The shape parameter for the distribution. This parameter is only used for the Pareto, Normal, and Exponential distributions. + + # Returns + - `attribute_objects::Vector{Int64}`: An array of the number of objects for each attribute. The array length is equal to the number of attributes. + """ + function generate_attribute_occurrences(num_attributes::Int64, num_objects::Int64, distribution::String, s::Float64=1.0) + if distribution == "uniform" + dist = Multinomial(num_objects, fill(1.0/num_attributes, num_attributes)) + occurrences = rand(dist, 1) + elseif distribution in ["pareto", "normal", "exponential"] + if distribution == "pareto" + dist = Pareto(s, 1.0) # Pareto distribution with shape parameter s and scale 1.0 + elseif distribution == "normal" + dist = Normal(0.0, s) + elseif distribution == "exponential" + dist = Exponential(s) + end + occurrences = rand(dist, num_attributes) + occurrences = Int64.(round.(occurrences .* num_objects ./ sum(occurrences))) + + # Due to rounding, the sum might not be exactly num_objects. + # Add or subtract the difference to a random element. + diff = num_objects - sum(occurrences) + occurrences[rand(1:num_attributes)] += diff + else + error("Invalid distribution: " * distribution) + end + + return sort(occurrences[:]) + end # end function generate_attribute_occurrences + + """ + generate_incremental_associations(num_attributes::Int64, num_objects::Int64, total_groups::Int64) + + Generate an array of the number of objects for each attribute, where the number of objects for each attribute is an incremental value. + + # Arguments + - `num_attributes::Int64`: The total number of attributes + - `num_objects::Int64`: The total number of objects + - `num_obj_groups::Int64`: The total number of object groups + + # Returns + - `attribute_objects::Vector{Int64}`: An array of the number of objects for each attribute. The array length is equal to the number of attributes. + """ + function generate_incremental_associations(num_attributes::Int64, num_objects::Int64, num_obj_groups::Int64=0) + if num_obj_groups == 0 + num_obj_groups = num_attributes + end + + # Error checking: Ensure that the number of attributes does not exceed the total number of groups + if num_attributes > num_obj_groups + error("Number of attributes cannot be greater than the total number of groups") + end + + # Generate an array to store the number of objects for each attribute + attribute_objects = Vector{Int64}(undef, num_attributes) + + # calculate the number of objects within each group + increment_value = ceil(Int64, num_objects / num_obj_groups) + + # Assign an incremental number of objects to each attribute + for i = 1:num_attributes + # Ensure the number of objects does not exceed the total number of objects + attribute_objects[i] = min(i * increment_value, num_objects) + end + + return attribute_objects + end # end function generate_incremental_associations + +end # module MyModule \ No newline at end of file diff --git a/src/tests/helper/include/julia_helper_loader.h b/src/tests/helper/include/julia_helper_loader.h new file mode 100644 index 000000000..8d19db036 --- /dev/null +++ b/src/tests/helper/include/julia_helper_loader.h @@ -0,0 +1,134 @@ +#ifndef JULIA_HELPER_LOADER_H +#define JULIA_HELPER_LOADER_H + +/** + * This is a helper library to load Julia modules and run Julia functions from C. + * It is now used in the PDC C tests. We may consider moving it to PDC core in the future. + * + * When using this library, please make sure you have the following environment variables set: + * export PDC_JULIA_MODULE_DIR=/path/to/julia/modules + * + * Also, on NERSC machines, please make sure you perform: + * module load julia + * + * If you need to add any new functions, please add them to julia_helper.jl and then add the corresponding + * C function here as the Julia function caller. + * You may refer to the following link for more information: + * https://docs.julialang.org/en/v1/manual/embedding/index.html + * + * For thread safety, please make sure you call init_julia(), julia function caller and close_julia() in the + * same C thread. + * + * If you need to call share variables between multiple threads, the best practice is to start your threads in + * Julia and then call the Julia functions from Julia threads. Or you can also call the C functions from Julia + * threads using ccall(). For calling C from Julia, please refer to the following link: + * https://docs.julialang.org/en/v1/manual/calling-c-and-fortran-code/index.html + * + * An example of calling the Julia C API from a thread started by Julia itself: + * + * #include + * JULIA_DEFINE_FAST_TLS + * + * double c_func(int i) + * { + * printf("[C %08x] i = %d\n", pthread_self(), i); + * + * // Call the Julia sqrt() function to compute the square root of i, and return it + * jl_function_t *sqrt = jl_get_function(jl_base_module, "sqrt"); + * jl_value_t* arg = jl_box_int32(i); + * double ret = jl_unbox_float64(jl_call1(sqrt, arg)); + * + * return ret; + * } + * + * int main() + * { + * jl_init(); + * + * // Define a Julia function func() that calls our c_func() defined in C above + * jl_eval_string("func(i) = ccall(:c_func, Float64, (Int32,), i)"); + * + * // Call func() multiple times, using multiple threads to do so + * jl_eval_string("println(Threads.threadpoolsize())"); + * jl_eval_string("use(i) = println(\"[J $(Threads.threadid())] i = $(i) -> $(func(i))\")"); + * jl_eval_string("Threads.@threads for i in 1:5 use(i) end"); + * + * jl_atexit_hook(0); + * } + * + * @file julia_helper_loader.h + * @author Wei Zhang + */ + +#include +#include +#include + +typedef struct { + char **julia_modules; + int num_modules; +} jl_module_list_t; +typedef struct { + jl_value_t **args; + int32_t nargs; +} jl_fn_args_t; + +/** + * @brief initialize the julia runtime with a list of julia modules + * + * @param modules the list of modules to load + */ +void init_julia(jl_module_list_t *modules); + +/** + * @brief run a julia function with the specified arguments + * + * @param mod_name the name of the module to load + * @param fun_name the name of the function to run + * @param args the arguments to pass to the function + * @return jl_value_t* the return value of the function + */ +jl_value_t *run_jl_function(const char *mod_name, const char *fun_name, jl_fn_args_t *args); + +/** + * @brief run a julia function with the specified arguments and get an int64_t array and its length + * + * @param mod_name the name of the module to load + * @param fun_name the name of the function to run + * @param args the arguments to pass to the function + * @param arr_ptr the pointer to the int64_t array to be returned + * @param arr_len the length of the int64_t array to be returned + */ +void run_jl_get_int64_array(const char *mod_name, const char *fun_name, jl_fn_args_t *args, int64_t **arr_ptr, + size_t *arr_len); + +/** + * @brief run a julia function with the specified arguments and get a float64_t array and its length + * + * @param mod_name the name of the module to load + * @param fun_name the name of the function to run + * @param args the arguments to pass to the function + * @param arr_ptr the pointer to the float64_t array to be returned + * @param arr_len the length of the float64_t array to be returned + */ +void run_jl_get_float64_array(const char *mod_name, const char *fun_name, jl_fn_args_t *args, + double **arr_ptr, size_t *arr_len); + +/** + * @brief run a julia function with the specified arguments and get a string array and its length + * + * @param mod_name the name of the module to load + * @param fun_name the name of the function to run + * @param args the arguments to pass to the function + * @param arr_ptr the pointer to the string array to be returned + * @param arr_len the length of the string array to be returned + */ +void run_jl_get_string_array(const char *mod_name, const char *fun_name, jl_fn_args_t *args, char ***arr_ptr, + size_t *arr_len); + +/** + * @brief exit the julia runtime + */ +void close_julia(); + +#endif // JULIA_HELPER_LOADER_H \ No newline at end of file diff --git a/src/tests/helper/julia_helper_loader.c b/src/tests/helper/julia_helper_loader.c new file mode 100644 index 000000000..887051e46 --- /dev/null +++ b/src/tests/helper/julia_helper_loader.c @@ -0,0 +1,118 @@ +#include "julia_helper_loader.h" + +void +jl_load_module(const char *mod_name) +{ + /* get julia helper directory */ + const char *julia_module_dir = getenv("PDC_JULIA_MODULE_DIR"); + if (julia_module_dir == NULL || strlen(julia_module_dir) == 0) { + // try to get it from PWD + printf("[PDC_JL_HELPER] Warning: PDC_JULIA_MODULE_DIR is not set, fallback to PWD!\n"); + julia_module_dir = getenv("PWD"); + } + + if (julia_module_dir == NULL || strlen(julia_module_dir) == 0) { + // No way to find julia module directory + printf("[PDC_JL_HELPER] Error: Not able to find Julia module directory!\n"); + exit(-1); + } + printf("[PDC_JL_HELPER] Julia module directory: %s\n", julia_module_dir); + + /* get julia helper path */ + char *julia_module_path = malloc(strlen(julia_module_dir) + strlen(mod_name) + 5); + strcpy(julia_module_path, julia_module_dir); + strcat(julia_module_path, "/"); + strcat(julia_module_path, mod_name); + strcat(julia_module_path, ".jl"); + printf("[PDC_JL_HELPER] Julia module path: %s\n", julia_module_path); + /* get include command */ + char include_cmd[strlen(julia_module_path) + 30]; + sprintf(include_cmd, "Base.include(Main, \"%s\")", julia_module_path); + /* get using command */ + char using_cmd[strlen(mod_name) + 15]; + sprintf(using_cmd, "using Main.%s", mod_name); + + jl_eval_string(include_cmd); + jl_eval_string(using_cmd); + + printf("[PDC_JL_HELPER] Loaded module %s\n", mod_name); +} + +void +init_julia(jl_module_list_t *modules) +{ + /* required: setup the Julia context */ + jl_init(); + + /* load JuliaHelper module */ + for (int i = 0; i < modules->num_modules; ++i) { + jl_load_module(modules->julia_modules[i]); + } +} + +jl_value_t * +run_jl_function(const char *mod_name, const char *fun_name, jl_fn_args_t *args) +{ + char module_eval_cmd[strlen(mod_name) + 15]; + sprintf(module_eval_cmd, "Main.%s", mod_name); + jl_module_t * JuliaModule = (jl_module_t *)jl_eval_string(module_eval_cmd); + jl_function_t *jl_func = jl_get_function(JuliaModule, fun_name); + jl_value_t * result = jl_call(jl_func, args->args, args->nargs); + printf("[PDC_JL_HELPER] Function called: %s.%s\n", mod_name, fun_name); + return result; +} + +void +run_jl_get_int64_array(const char *mod_name, const char *fun_name, jl_fn_args_t *args, int64_t **arr_ptr, + size_t *arr_len) +{ + jl_value_t *ret = run_jl_function(mod_name, fun_name, args); + JL_GC_PUSH1(&ret); + jl_array_t *ret_array = (jl_array_t *)ret; + int64_t * data = (int64_t *)jl_array_data(ret_array); + *arr_ptr = data; + *arr_len = jl_array_len(ret_array); + JL_GC_POP(); +} + +void +run_jl_get_float64_array(const char *mod_name, const char *fun_name, jl_fn_args_t *args, double **arr_ptr, + size_t *arr_len) +{ + jl_value_t *ret = run_jl_function(mod_name, fun_name, args); + JL_GC_PUSH1(&ret); + jl_array_t *ret_array = (jl_array_t *)ret; + double * data = (double *)jl_array_data(ret_array); + *arr_ptr = data; + *arr_len = jl_array_len(ret_array); + JL_GC_POP(); +} + +void +run_jl_get_string_array(const char *mod_name, const char *fun_name, jl_fn_args_t *args, char ***arr_ptr, + size_t *arr_len) +{ + jl_value_t *ret = run_jl_function(mod_name, fun_name, args); + JL_GC_PUSH1(&ret); + jl_array_t *ret_array = (jl_array_t *)ret; + size_t length = jl_array_len(ret); + char ** strings = malloc(length * sizeof(char *)); + + for (size_t i = 0; i < length; ++i) { + jl_value_t *julia_str = jl_arrayref(ret_array, i); + const char *c_str = jl_string_ptr(julia_str); + size_t c_str_length = jl_string_len(julia_str); + strings[i] = malloc((c_str_length + 1) * sizeof(char)); + strncpy(strings[i], c_str, c_str_length); + strings[i][c_str_length] = '\0'; + } + *arr_ptr = strings; + *arr_len = length; + JL_GC_POP(); +} + +void +close_julia() +{ + jl_atexit_hook(0); +} \ No newline at end of file diff --git a/src/tests/kvtag_add_get.c b/src/tests/kvtag_add_get.c index 97eadffed..ad06b506b 100644 --- a/src/tests/kvtag_add_get.c +++ b/src/tests/kvtag_add_get.c @@ -33,13 +33,14 @@ int main() { - pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; - pdc_kvtag_t kvtag1, kvtag2, kvtag3; - char * v1 = "value1"; - int v2 = 2; - double v3 = 3.45; - void * value1, *value2, *value3; - psize_t value_size; + pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; + pdc_kvtag_t kvtag1, kvtag2, kvtag3; + char * v1 = "value1"; + int v2 = 2; + double v3 = 3.45; + pdc_var_type_t type1, type2, type3; + void * value1, *value2, *value3; + psize_t value_size; // create a pdc pdc = PDCinit("pdc"); @@ -88,62 +89,66 @@ main() kvtag1.name = "key1string"; kvtag1.value = (void *)v1; + kvtag1.type = PDC_STRING; kvtag1.size = strlen(v1) + 1; kvtag2.name = "key2int"; kvtag2.value = (void *)&v2; + kvtag2.type = PDC_INT; kvtag2.size = sizeof(int); kvtag3.name = "key3double"; kvtag3.value = (void *)&v3; + kvtag3.type = PDC_DOUBLE; kvtag3.size = sizeof(double); - if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.size) < 0) + if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.type, kvtag1.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.size) < 0) + if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.type, kvtag2.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.size) < 0) + if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.type, kvtag3.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", kvtag1.name, (char *)value1); - if (PDCobj_get_tag(obj2, kvtag2.name, (void *)&value2, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, kvtag2.name, (void *)&value2, (void *)&type2, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%d] from o2\n", kvtag2.name, *(int *)value2); - if (PDCobj_get_tag(obj2, kvtag3.name, (void *)&value3, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, kvtag3.name, (void *)&value3, (void *)&type3, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%f] from o2\n", kvtag3.name, *(double *)value3); - if (PDCtag_delete(obj1, kvtag1.name) < 0) + if (PDCobj_del_tag(obj1, kvtag1.name) < 0) printf("fail to delete a kvtag from o1\n"); else printf("successfully deleted a kvtag [%s] from o1\n", kvtag1.name); v1 = "New Value After Delete"; kvtag1.value = (void *)v1; + kvtag1.type = PDC_STRING; kvtag1.size = strlen(v1) + 1; - if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.size) < 0) + if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.type, kvtag1.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); /* PDC_free_kvtag(&value1); */ - if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", kvtag1.name, (char *)value1); diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 7b719ec96..a682e30ee 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -220,7 +220,8 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value v = i + my_obj_s; for (j = 0; j < n_attr; j++) { sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", v, j); - if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], tag_value_len + 1) < 0) + if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], PDC_STRING, tag_value_len + 1) < + 0) printf("fail to add a kvtag to o%" PRIu64 "\n", v); } } @@ -238,12 +239,13 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value void get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, uint64_t n_attr, void **tag_values, uint64_t *value_size) { - uint64_t i; - char tag_name[256]; + uint64_t i; + char tag_name[256]; + pdc_var_type_t tag_type; for (i = 0; i < n_attr; i++) { sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", obj_name_v, i); - if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[i], (void *)&value_size[i]) < 0) + if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[i], &tag_type, (void *)&value_size[i]) < 0) printf("fail to get a kvtag from o%" PRIu64 "\n", obj_name_v); } } @@ -293,12 +295,12 @@ check_and_release_query_result(uint64_t n_query, uint64_t my_obj, uint64_t my_ob } } free(values); - // close objects - for (i = 0; i < my_obj; i++) { - v = i + my_obj_s; - if (PDCobj_close(obj_ids[i]) < 0) - printf("fail to close object o%" PRIu64 "\n", v); - } + // FIXME: close objects. This is currently commented off to save node hours for benchmarks. + // for (i = 0; i < my_obj; i++) { + // v = i + my_obj_s; + // if (PDCobj_close(obj_ids[i]) < 0) + // printf("fail to close object o%" PRIu64 "\n", v); + // } } void @@ -399,12 +401,12 @@ main(int argc, char *argv[]) if (my_rank == 0) { printf("Iteration %" PRIu64 " : Objects: %" PRIu64 - " , Time: %.4f sec. Object throughput in this iteration: " - "%.4f .\n", + " , Time: %.5e sec. Object throughput in this iteration: " + "%.5e .\n", k, n_obj_incr, step_elapse, ((double)n_obj_incr) / step_elapse); printf("Overall %" PRIu64 " : Objects: %" PRIu64 - " , Time: %.4f sec. Overall object throughput: " - "%.4f .\n", + " , Time: %.5e sec. Overall object throughput: " + "%.5e .\n", k, total_object_count, total_object_time, ((double)total_object_count) / total_object_time); } @@ -424,10 +426,10 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) { printf("Iteration %" PRIu64 " : Tags: %" PRIu64 - " , Time: %.4f sec. Tag throughput in this iteration: %.4f .\n", + " , Time: %.5e sec. Tag throughput in this iteration: %.5e .\n", k, n_obj_incr * n_attr, step_elapse, (double)(n_obj_incr * n_attr) / step_elapse); printf("Overall %" PRIu64 " : Tags: %" PRIu64 - " , Time: %.4f sec. Overall tag throughput: %.4f .\n", + " , Time: %.5e sec. Overall tag throughput: %.5e .\n", k, total_tag_count, total_tag_time, ((double)total_tag_count) / total_tag_time); } @@ -448,12 +450,12 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) { printf("Iteration %" PRIu64 " : Queries: %" PRIu64 - " , Time: %.4f sec. Query throughput in this iteration: " - "%.4f .\n", + " , Time: %.5e sec. Query throughput in this iteration: " + "%.5e .\n", k, n_query * n_attr, step_elapse, (double)(n_query * n_attr) / step_elapse); printf("Overall %" PRIu64 " : Queries: %" PRIu64 - " , Time: %.4f sec. Overall query throughput: " - "%.4f .\n", + " , Time: %.5e sec. Overall query throughput: " + "%.5e .\n", k, total_query_count, total_query_time, ((double)total_query_count) / total_query_time); } @@ -468,13 +470,13 @@ main(int argc, char *argv[]) if (my_rank == 0) { printf("Final Report: \n"); - printf("[Final Report 1] Servers: %" PRIu64 " , Clients: %" PRIu64 " , C/S ratio: %.4f \n", n_servers, + printf("[Final Report 1] Servers: %" PRIu64 " , Clients: %" PRIu64 " , C/S ratio: %.5e \n", n_servers, n_clients, (double)n_clients / (double)n_servers); printf("[Final Report 2] Iterations: %" PRIu64 " , Objects: %" PRIu64 " , Tags/Object: %" PRIu64 " , Queries/Iteration: " "%" PRIu64 " , \n", k, total_object_count, n_attr, n_query); - printf("[Final Report 3] Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", + printf("[Final Report 3] Object throughput: %.5e , Tag Throughput: %.5e , Query Throughput: %.5e ,", (double)total_object_count / total_object_time, (double)(total_object_count * n_attr) / total_tag_time, (double)(total_query_count * n_attr) / total_query_time); @@ -486,7 +488,9 @@ main(int argc, char *argv[]) free(tag_values); free(obj_ids); - closePDC(pdc, cont_prop, cont, obj_prop); + // FIXME: the following is currently commented off to reduce node hours taken by time-consuming resource + // releasing procedure. + // closePDC(pdc, cont_prop, cont, obj_prop); done: #ifdef ENABLE_MPI diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index 280eda25b..dc14c597e 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -76,7 +76,8 @@ main(int argc, char *argv[]) double stime, total_time, percent_time; pdc_kvtag_t kvtag; void ** values; - size_t value_size; + pdc_var_type_t value_type; + size_t value_size; #ifdef ENABLE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); @@ -167,6 +168,7 @@ main(int argc, char *argv[]) // Add tags kvtag.name = "Group"; kvtag.value = (void *)&v; + kvtag.type = PDC_INT; kvtag.size = sizeof(int); #ifdef ENABLE_MPI @@ -175,7 +177,7 @@ main(int argc, char *argv[]) #endif for (i = 0; i < my_add_tag; i++) { v = i + my_add_tag_s; - if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) printf("fail to add a kvtag to o%d\n", i + my_obj_s); if (i % tag_1percent == 0) { @@ -208,7 +210,8 @@ main(int argc, char *argv[]) stime = MPI_Wtime(); #endif for (i = 0; i < my_query; i++) { - if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_size) < 0) + if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, + (void *)&value_size) < 0) printf("fail to get a kvtag from o%d\n", i + my_query_s); if (i % query_1percent == 0) { diff --git a/src/tests/kvtag_get.c b/src/tests/kvtag_get.c index a8fcf70f1..7ca1e7c0e 100644 --- a/src/tests/kvtag_get.c +++ b/src/tests/kvtag_get.c @@ -33,9 +33,10 @@ int main() { - pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; - pdc_kvtag_t *value1, *value2, *value3; - psize_t value_size; + pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; + pdc_kvtag_t * value1, *value2, *value3; + pdc_var_type_t type1, type2, type3; + psize_t value_size; // create a pdc pdc = PDCinit("pdc"); @@ -82,17 +83,17 @@ main() else printf("Fail to create object @ line %d!\n", __LINE__); - if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", value1->name, (char *)value1->value); - if (PDCobj_get_tag(obj2, "key2int", (void *)&value2, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, "key2int", (void *)&value2, (void *)&type2, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%d] from o2\n", value2->name, *(int *)value2->value); - if (PDCobj_get_tag(obj2, "key3double", (void *)&value3, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, "key3double", (void *)&value3, (void *)&type3, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%f] from o2\n", value3->name, @@ -100,7 +101,7 @@ main() PDC_free_kvtag(&value1); - if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", value1->name, (char *)value1->value); diff --git a/src/tests/kvtag_query.c b/src/tests/kvtag_query.c index 88e6621d7..cf1e80dcb 100644 --- a/src/tests/kvtag_query.c +++ b/src/tests/kvtag_query.c @@ -91,32 +91,35 @@ main() kvtag1.name = "key1string"; kvtag1.value = (void *)v1; + kvtag1.type = PDC_STRING; kvtag1.size = strlen(v1) + 1; kvtag2.name = "key2int"; kvtag2.value = (void *)&v2; + kvtag2.type = PDC_INT; kvtag2.size = sizeof(int); kvtag3.name = "key3double"; kvtag3.value = (void *)&v3; + kvtag3.type = PDC_DOUBLE; kvtag3.size = sizeof(double); - if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.size) < 0) + if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.type, kvtag1.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj1, kvtag2.name, kvtag2.value, kvtag2.size) < 0) + if (PDCobj_put_tag(obj1, kvtag2.name, kvtag2.value, kvtag2.type, kvtag2.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.size) < 0) + if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.type, kvtag2.size) < 0) printf("fail to add a kvtag to o2\n"); else printf("successfully added a kvtag to o2\n"); - if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.size) < 0) + if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.type, kvtag3.size) < 0) printf("fail to add a kvtag to o2\n"); else printf("successfully added a kvtag to o2\n"); diff --git a/src/tests/kvtag_query_mpi.c b/src/tests/kvtag_query_mpi.c new file mode 100644 index 000000000..22c619c97 --- /dev/null +++ b/src/tests/kvtag_query_mpi.c @@ -0,0 +1,207 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#include +#include +#include +#include +#include +#include "pdc.h" +#include "pdc_client_connect.h" + +int +assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) +{ + if (rank > size || my_count == NULL || my_start == NULL) { + printf("assign_work_to_rank(): Error with input!\n"); + return -1; + } + if (nwork < size) { + if (rank < nwork) + *my_count = 1; + else + *my_count = 0; + (*my_start) = rank * (*my_count); + } + else { + (*my_count) = nwork / size; + (*my_start) = rank * (*my_count); + + // Last few ranks may have extra work + if (rank >= size - nwork % size) { + (*my_count)++; + (*my_start) += (rank - (size - nwork % size)); + } + } + + return 1; +} + +void +print_usage(char *name) +{ + printf("%s n_obj n_query\n", name); +} + +int +main(int argc, char *argv[]) +{ + pdcid_t pdc, cont_prop, cont, obj_prop; + pdcid_t * obj_ids; + int n_obj, n_add_tag, my_obj, my_obj_s, my_add_tag, my_add_tag_s; + int proc_num, my_rank, i, v, iter, round; + char obj_name[128]; + double stime, total_time; + pdc_kvtag_t kvtag; + uint64_t * pdc_ids; + int nres, ntotal; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &proc_num); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#endif + + if (argc < 3) { + if (my_rank == 0) + print_usage(argv[0]); + goto done; + } + n_obj = atoi(argv[1]); + round = atoi(argv[2]); + n_add_tag = n_obj / 100; + + // create a pdc + pdc = PDCinit("pdc"); + + // create a container property + cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + printf("Fail to create container property @ line %d!\n", __LINE__); + + // create a container + cont = PDCcont_create("c1", cont_prop); + if (cont <= 0) + printf("Fail to create container @ line %d!\n", __LINE__); + + // create an object property + obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); + if (obj_prop <= 0) + printf("Fail to create object property @ line %d!\n", __LINE__); + + // Create a number of objects, add at least one tag to that object + assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); + if (my_rank == 0) + printf("I will create %d obj\n", my_obj); + obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + for (i = 0; i < my_obj; i++) { + sprintf(obj_name, "obj%d", my_obj_s + i); + obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); + if (obj_ids[i] <= 0) { + printf("Fail to create object @ line %d!\n", __LINE__); + goto done; + } + } + + if (my_rank == 0) + printf("Created %d objects\n", n_obj); + fflush(stdout); + + // Add tags + kvtag.name = "Group"; + kvtag.value = (void *)&v; + kvtag.type = PDC_INT; + kvtag.size = sizeof(int); + + for (iter = 0; iter < round; iter++) { + assign_work_to_rank(my_rank, proc_num, n_add_tag, &my_add_tag, &my_add_tag_s); + + v = iter; + for (i = 0; i < my_add_tag; i++) { + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) { + printf("fail to add a kvtag to o%d\n", i + my_obj_s); + goto done; + } + } + + if (my_rank == 0) + printf("Rank %d: Added a kvtag to %d objects\n", my_rank, my_add_tag); + fflush(stdout); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + n_add_tag *= 2; + } + + n_add_tag = n_obj / 100; + + for (iter = 0; iter < round; iter++) { + v = iter; + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + + if (PDC_Client_query_kvtag_mpi(&kvtag, &nres, &pdc_ids, MPI_COMM_WORLD) < 0) { + printf("fail to query kvtag [%s] with rank %d\n", kvtag.name, my_rank); + break; + } + + if (nres != n_add_tag) + printf("Rank %d: query result %d doesn't match expected %d\n", my_rank, nres, n_add_tag); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + if (my_rank == 0) + printf("Total time to query %d objects with tag: %.5e\n", nres, total_time); + fflush(stdout); + n_add_tag *= 2; + } + + // close a container + if (PDCcont_close(cont) < 0) + printf("fail to close container c1\n"); + + // close an object property + if (PDCprop_close(obj_prop) < 0) + printf("Fail to close property @ line %d\n", __LINE__); + + // close a container property + if (PDCprop_close(cont_prop) < 0) + printf("Fail to close property @ line %d\n", __LINE__); + + // close pdc + if (PDCclose(pdc) < 0) + printf("fail to close PDC\n"); +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return 0; +} diff --git a/src/tests/kvtag_query_scale.c b/src/tests/kvtag_query_scale.c index 80ca2a07c..99336cebc 100644 --- a/src/tests/kvtag_query_scale.c +++ b/src/tests/kvtag_query_scale.c @@ -61,7 +61,17 @@ assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) void print_usage(char *name) { - printf("%s n_obj n_query\n", name); + printf("%s n_obj n_round n_selectivity is_using_dart\n", name); + printf("Summary: This test will create n_obj objects, and add n_selectivity tags to each object. Then it " + "will " + "perform n_round point-to-point queries against the tags, each query from each client should get " + "a whole result set.\n"); + printf("Parameters:\n"); + printf(" n_obj: number of objects\n"); + printf(" n_round: number of rounds, it can be the total number of tags too, as each round will perform " + "one query against one tag\n"); + printf(" n_selectivity: selectivity, on a 100 scale. \n"); + printf(" is_using_dart: 1 for using dart, 0 for not using dart\n"); } int @@ -70,7 +80,7 @@ main(int argc, char *argv[]) pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t * obj_ids; int n_obj, n_add_tag, my_obj, my_obj_s, my_add_tag, my_add_tag_s; - int proc_num, my_rank, i, v, iter, round; + int proc_num, my_rank, i, v, iter, round, selectivity, is_using_dart, query_type; char obj_name[128]; double stime, total_time; pdc_kvtag_t kvtag; @@ -83,14 +93,17 @@ main(int argc, char *argv[]) MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - if (argc < 3) { + if (argc < 6) { if (my_rank == 0) print_usage(argv[0]); goto done; } - n_obj = atoi(argv[1]); - round = atoi(argv[2]); - n_add_tag = n_obj / 100; + n_obj = atoi(argv[1]); + round = atoi(argv[2]); + selectivity = atoi(argv[3]); + is_using_dart = atoi(argv[4]); + query_type = atoi(argv[5]); + n_add_tag = n_obj * selectivity / 100; // create a pdc pdc = PDCinit("pdc"); @@ -114,6 +127,7 @@ main(int argc, char *argv[]) assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); if (my_rank == 0) printf("I will create %d obj\n", my_obj); + obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); for (i = 0; i < my_obj; i++) { sprintf(obj_name, "obj%d", my_obj_s + i); @@ -126,59 +140,79 @@ main(int argc, char *argv[]) printf("Created %d objects\n", n_obj); fflush(stdout); + char *attr_name_per_rank = gen_random_strings(1, 6, 8, 26)[0]; // Add tags - kvtag.name = "Group"; + kvtag.name = attr_name_per_rank; kvtag.value = (void *)&v; + kvtag.type = PDC_INT; kvtag.size = sizeof(int); - for (iter = 0; iter < round; iter++) { - assign_work_to_rank(my_rank, proc_num, n_add_tag, &my_add_tag, &my_add_tag_s); - - v = iter; - for (i = 0; i < my_add_tag; i++) { - if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) - printf("fail to add a kvtag to o%d\n", i + my_obj_s); + char key[32]; + char value[32]; + char exact_query[48]; + + dart_object_ref_type_t ref_type = REF_PRIMARY_ID; + dart_hash_algo_t hash_algo = DART_HASH; + + assign_work_to_rank(my_rank, proc_num, n_add_tag, &my_add_tag, &my_add_tag_s); + + // This is for adding #rounds tags to the objects. + for (i = 0; i < my_add_tag; i++) { + for (iter = 0; iter < round; iter++) { + v = iter; + sprintf(value, "%d", v); + if (is_using_dart) { + if (PDC_Client_insert_obj_ref_into_dart(hash_algo, kvtag.name, value, ref_type, + (uint64_t)obj_ids[i]) < 0) { + printf("fail to add a kvtag to o%d\n", i + my_obj_s); + } + } + else { + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) { + printf("fail to add a kvtag to o%d\n", i + my_obj_s); + } + } } - if (my_rank == 0) - printf("Rank %d: Added a kvtag to %d objects\n", my_rank, my_add_tag); - fflush(stdout); + println("Rank %d: Added %d kvtag to the %d th object\n", my_rank, round, i); + } #ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(MPI_COMM_WORLD); #endif - n_add_tag *= 2; - } - - kvtag.name = "Group"; + kvtag.name = attr_name_per_rank; kvtag.value = (void *)&v; + kvtag.type = PDC_INT; kvtag.size = sizeof(int); - for (iter = 0; iter < round; iter++) { - v = iter; - #ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - stime = MPI_Wtime(); + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); #endif - if (PDC_Client_query_kvtag_col(&kvtag, &nres, &pdc_ids) < 0) { - printf("fail to query kvtag [%s] with rank %d\n", kvtag.name, my_rank); - break; + for (iter = 0; iter < round; iter++) { + v = iter; + if (is_using_dart) { + sprintf(value, "%ld", v); + sprintf(exact_query, "%s=%s", kvtag.name, value); + PDC_Client_search_obj_ref_through_dart(hash_algo, exact_query, ref_type, &nres, &pdc_ids); + } + else { + if (PDC_Client_query_kvtag(&kvtag, &nres, &pdc_ids) < 0) { + printf("fail to query kvtag [%s] with rank %d\n", kvtag.name, my_rank); + break; + } } + } #ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - MPI_Reduce(&nres, &ntotal, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); - total_time = MPI_Wtime() - stime; -#endif - - if (my_rank == 0) - printf("Total time to query %d objects with tag: %.4f\n", ntotal, total_time); - fflush(stdout); - } + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; + if (my_rank == 0) + println("Total time to query %d objects with tag: %.5f", ntotal, total_time); +#endif // close a container if (PDCcont_close(cont) < 0) printf("fail to close container c1\n"); diff --git a/src/tests/kvtag_query_scale_col.c b/src/tests/kvtag_query_scale_col.c new file mode 100644 index 000000000..78d4b26b5 --- /dev/null +++ b/src/tests/kvtag_query_scale_col.c @@ -0,0 +1,395 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "pdc_client_connect.h" + +int +assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) +{ + if (rank > size || my_count == NULL || my_start == NULL) { + printf("assign_work_to_rank(): Error with input!\n"); + return -1; + } + if (nwork < size) { + if (rank < nwork) + *my_count = 1; + else + *my_count = 0; + (*my_start) = rank * (*my_count); + } + else { + (*my_count) = nwork / size; + (*my_start) = rank * (*my_count); + + // Last few ranks may have extra work + if (rank >= size - nwork % size) { + (*my_count)++; + (*my_start) += (rank - (size - nwork % size)); + } + } + + return 1; +} + +void +print_usage(char *name) +{ + printf("%s n_obj n_round n_selectivity is_using_dart query_type comm_type\n", name); + printf("Summary: This test will create n_obj objects, and add n_selectivity tags to each object. Then it " + "will " + "perform n_round collective queries against the tags, each query from each client should get " + "a whole result set.\n"); + printf("Parameters:\n"); + printf(" n_obj: number of objects\n"); + printf(" n_round: number of rounds, it can be the total number of tags too, as each round will perform " + "one query against one tag\n"); + printf(" n_selectivity: selectivity, on a 100 scale. \n"); + printf(" is_using_dart: 1 for using dart, 0 for not using dart\n"); + printf(" query_type: 0 for exact, 1 for prefix, 2 for suffix, 3 for infix\n"); + printf(" comm_type: 0 for point-to-point, 1 for collective\n"); +} + +perr_t +prepare_container(pdcid_t *pdc, pdcid_t *cont_prop, pdcid_t *cont, pdcid_t *obj_prop, int my_rank) +{ + perr_t ret_value = FAIL; + // create a pdc + *pdc = PDCinit("pdc"); + + // create a container property + *cont_prop = PDCprop_create(PDC_CONT_CREATE, *pdc); + if (*cont_prop <= 0) { + printf("[Client %d] Fail to create container property @ line %d!\n", my_rank, __LINE__); + goto done; + } + // create a container + *cont = PDCcont_create("c1", *cont_prop); + if (*cont <= 0) { + printf("[Client %d] Fail to create container @ line %d!\n", my_rank, __LINE__); + goto done; + } + + // create an object property + *obj_prop = PDCprop_create(PDC_OBJ_CREATE, *pdc); + if (*obj_prop <= 0) { + printf("[Client %d] Fail to create object property @ line %d!\n", my_rank, __LINE__); + goto done; + } + + ret_value = SUCCEED; +done: + return ret_value; +} + +perr_t +creating_objects(pdcid_t **obj_ids, int my_obj, int my_obj_s, pdcid_t cont, pdcid_t obj_prop, int my_rank) +{ + perr_t ret_value = FAIL; + char obj_name[128]; + int64_t timestamp = get_timestamp_ms(); + *obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + for (int i = 0; i < my_obj; i++) { + sprintf(obj_name, "obj%" PRId64 "%d", timestamp, my_obj_s + i); + (*obj_ids)[i] = PDCobj_create(cont, obj_name, obj_prop); + if ((*obj_ids)[i] <= 0) { + printf("[Client %d] Fail to create object @ line %d!\n", my_rank, __LINE__); + goto done; + } + } + ret_value = SUCCEED; +done: + return ret_value; +} + +int +main(int argc, char *argv[]) +{ + pdcid_t pdc, cont_prop, cont, obj_prop; + pdcid_t * obj_ids; + int n_obj, my_obj, my_obj_s; + int proc_num, my_rank, i, v, iter, round, selectivity, is_using_dart, query_type, comm_type; + double stime, total_time; + pdc_kvtag_t kvtag; + uint64_t * pdc_ids; + int nres, ntotal; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &proc_num); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#endif + + if (argc < 7) { + if (my_rank == 0) + print_usage(argv[0]); + goto done; + } + n_obj = atoi(argv[1]); + round = atoi(argv[2]); + selectivity = atoi(argv[3]); + is_using_dart = atoi(argv[4]); // 0 for no index, 1 for using dart. + query_type = atoi(argv[5]); // 0 for exact, 1 for prefix, 2 for suffix, 3 for infix, + // 4 for num_exact, 5 for num_range + comm_type = atoi(argv[6]); // 0 for point-to-point, 1 for collective + + // prepare container + if (prepare_container(&pdc, &cont_prop, &cont, &obj_prop, my_rank) < 0) { + println("fail to prepare container @ line %d", __LINE__); + goto done; + } + // Create a number of objects, add at least one tag to that object + assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); + + if (my_rank == 0) { + println("Each client will create about %d obj", my_obj); + } + + // creating objects + creating_objects(&obj_ids, my_obj, my_obj_s, cont, obj_prop, my_rank); + + if (my_rank == 0) + println("All clients created %d objects", n_obj); + + dart_object_ref_type_t ref_type = REF_PRIMARY_ID; + dart_hash_algo_t hash_algo = DART_HASH; + + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); + + // This is for adding #rounds tags to the objects. + // Each rank will add #rounds tags to #my_obj objects. + // With the selectivity, we should be able to control how many objects will be attached with the #round + // tags. So that, each of these #round tags can roughly the same selectivity. + int my_obj_after_selectivity = my_obj * selectivity / 100; + for (i = 0; i < my_obj_after_selectivity; i++) { + for (iter = 0; iter < round; iter++) { + char attr_name[64]; + char tag_value[64]; + snprintf(attr_name, 63, "%d%dattr_name%d%d", iter, iter, iter, iter); + snprintf(tag_value, 63, "%d%dtag_value%d%d", iter, iter, iter, iter); + kvtag.name = strdup(attr_name); + kvtag.value = (void *)strdup(tag_value); + kvtag.type = PDC_STRING; + kvtag.size = (strlen(tag_value) + 1) * sizeof(char); + if (is_using_dart) { + if (PDC_Client_insert_obj_ref_into_dart(hash_algo, kvtag.name, kvtag.value, ref_type, + (uint64_t)obj_ids[i]) < 0) { + printf("fail to add a kvtag to o%d\n", i + my_obj_s); + } + } + else { + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) { + printf("fail to add a kvtag to o%d\n", i + my_obj_s); + } + } + free(kvtag.name); + free(kvtag.value); + } + if (my_rank == 0) { + println("Rank %d: Added %d kvtag to the %d / %d th object, I'm applying selectivity %d to %d " + "objects.\n", + my_rank, round, i + 1, my_obj_after_selectivity, selectivity, my_obj); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; + + if (my_rank == 0) { + println("[TAG Creation] Rank %d: Added %d kvtag to %d objects, time: %.5f ms", my_rank, round, my_obj, + total_time * 1000.0); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + + // For the queries, we issue #round queries. + // The selectivity of each exact query should be #selectivity / 100 * #n_obj. + // Namely, if you have 1M objects, selectivity is 10, then each query should return 100K objects. + for (comm_type = 1; comm_type >= 0; comm_type--) { + for (query_type = 0; query_type < 4; query_type++) { + perr_t ret_value; + if (comm_type == 0 && is_using_dart == 0) + round = 2; + int round_total = 0; + for (iter = -1; iter < round; iter++) { // -1 is for warm up +#ifdef ENABLE_MPI + if (iter == 0) { + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); + } +#endif + char attr_name[64]; + char tag_value[64]; + snprintf(attr_name, 63, "%d%dattr_name%d%d", iter, iter, iter, iter); + snprintf(tag_value, 63, "%d%dtag_value%d%d", iter, iter, iter, iter); + + kvtag.name = strdup(attr_name); + kvtag.value = (void *)strdup(tag_value); + kvtag.type = PDC_STRING; + kvtag.size = (strlen(tag_value) + 1) * sizeof(char); + + query_gen_input_t input; + query_gen_output_t output; + input.base_tag = &kvtag; + input.key_query_type = query_type; + input.value_query_type = query_type; + input.affix_len = 4; + + gen_query_key_value(&input, &output); + + if (is_using_dart) { + char *query_string = gen_query_str(&output); + ret_value = (comm_type == 0) + ? PDC_Client_search_obj_ref_through_dart(hash_algo, query_string, + ref_type, &nres, &pdc_ids) + : PDC_Client_search_obj_ref_through_dart_mpi( + hash_algo, query_string, ref_type, &nres, &pdc_ids, MPI_COMM_WORLD); + } + else { + kvtag.name = output.key_query; + kvtag.value = output.value_query; + ret_value = (comm_type == 0) + ? PDC_Client_query_kvtag(&kvtag, &nres, &pdc_ids) + : PDC_Client_query_kvtag_mpi(&kvtag, &nres, &pdc_ids, MPI_COMM_WORLD); + } + if (ret_value < 0) { + printf("fail to query kvtag [%s] with rank %d\n", kvtag.name, my_rank); + break; + } + round_total += nres; + free(kvtag.name); + free(kvtag.value); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + // MPI_Reduce(&round_total, &ntotal, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; + + if (my_rank == 0) { + char *query_type_str = "EXACT"; + if (query_type == 1) + query_type_str = "PREFIX"; + else if (query_type == 2) + query_type_str = "SUFFIX"; + else if (query_type == 3) + query_type_str = "INFIX"; + println("[%s Client %s Query with%sINDEX] %d rounds with %d results, time: %.5f ms", + comm_type == 0 ? "Single" : "Multi", query_type_str, + is_using_dart == 0 ? " NO " : " DART ", round, round_total, total_time * 1000.0); + } +#endif + } + } + + if (my_rank == 0) { + println("Rank %d: All queries are done.", my_rank); + report_avg_server_profiling_rst(); + } + + // delete all tags + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); + + my_obj_after_selectivity = my_obj * selectivity / 100; + for (i = 0; i < my_obj_after_selectivity; i++) { + for (iter = 0; iter < round; iter++) { + char attr_name[64]; + char tag_value[64]; + snprintf(attr_name, 63, "%d%dattr_name%d%d", iter, iter, iter, iter); + snprintf(tag_value, 63, "%d%dtag_value%d%d", iter, iter, iter, iter); + kvtag.name = strdup(attr_name); + kvtag.value = (void *)strdup(tag_value); + kvtag.type = PDC_STRING; + kvtag.size = (strlen(tag_value) + 1) * sizeof(char); + if (is_using_dart) { + PDC_Client_delete_obj_ref_from_dart(hash_algo, kvtag.name, (char *)kvtag.value, ref_type, + (uint64_t)obj_ids[i]); + } + else { + PDCobj_del_tag(obj_ids[i], kvtag.name); + } + free(kvtag.name); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; + if (my_rank == 0) { + println("[TAG Deletion] Rank %d: Deleted %d kvtag from %d objects, time: %.5f ms", my_rank, round, + my_obj, total_time * 1000.0); + } + + // close a container + if (PDCcont_close(cont) < 0) { + if (my_rank == 0) { + printf("fail to close container c1\n"); + } + } + else { + if (my_rank == 0) + printf("successfully close container c1\n"); + } + + // close an object property + if (PDCprop_close(obj_prop) < 0) { + if (my_rank == 0) + printf("Fail to close property @ line %d\n", __LINE__); + } + else { + if (my_rank == 0) + printf("successfully close object property\n"); + } + + // close a container property + if (PDCprop_close(cont_prop) < 0) { + if (my_rank == 0) + printf("Fail to close property @ line %d\n", __LINE__); + } + else { + if (my_rank == 0) + printf("successfully close container property\n"); + } + + // close pdc + if (PDCclose(pdc) < 0) { + if (my_rank == 0) + printf("fail to close PDC\n"); + } +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return 0; +} diff --git a/src/tests/list_all.c b/src/tests/list_all.c index 66a282030..508d23db6 100644 --- a/src/tests/list_all.c +++ b/src/tests/list_all.c @@ -187,7 +187,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to create %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to create %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/obj_lock.c b/src/tests/obj_lock.c index bf7b365cc..8a3ea71d3 100644 --- a/src/tests/obj_lock.c +++ b/src/tests/obj_lock.c @@ -118,7 +118,7 @@ main(int argc, char **argv) total_lock_overhead = elapsed / 1000000.0; if (rank == 0) { - printf("Total lock overhead : %.6f\n", total_lock_overhead); + printf("Total lock overhead : %.5e\n", total_lock_overhead); } #ifdef ENABLE_MPI @@ -142,7 +142,7 @@ main(int argc, char **argv) total_lock_overhead = elapsed / 1000000.0; if (rank == 0) { - printf("Total lock release overhead: %.6f\n", total_lock_overhead); + printf("Total lock release overhead: %.5e\n", total_lock_overhead); } // close object diff --git a/src/tests/obj_map.c b/src/tests/obj_map.c index 88d7a8333..2efae2e43 100644 --- a/src/tests/obj_map.c +++ b/src/tests/obj_map.c @@ -168,7 +168,7 @@ main(int argc, char **argv) ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; - printf("Total map overhead : %.6f\n", ht_total_sec); + printf("Total map overhead : %.5e\n", ht_total_sec); fflush(stdout); gettimeofday(&ht_total_start, 0); @@ -182,7 +182,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; - printf("Total unmap overhead : %.6f\n", ht_total_sec); + printf("Total unmap overhead : %.5e\n", ht_total_sec); // close a container if (PDCcont_close(cont_id) < 0) diff --git a/src/tests/obj_round_robin_io_all.c b/src/tests/obj_round_robin_io_all.c index 05a5c0dcc..4070107a7 100644 --- a/src/tests/obj_round_robin_io_all.c +++ b/src/tests/obj_round_robin_io_all.c @@ -108,9 +108,9 @@ main(int argc, char **argv) my_data_size *= dims[i]; } - mydata = (char **)malloc(size * WRITE_REQ_SIZE); + mydata = (char **)malloc(2 * sizeof(char *)); mydata[0] = (char *)malloc(my_data_size * type_size); - mydata[1] = mydata[0] + my_data_size * type_size; + mydata[1] = (char *)malloc(my_data_size * type_size); offset = (uint64_t *)malloc(sizeof(uint64_t) * ndim); mysize = (uint64_t *)malloc(sizeof(uint64_t)); @@ -453,6 +453,8 @@ main(int argc, char **argv) free(obj1_list); free(obj2_list); free(data_read); + free(mydata[0]); + free(mydata[1]); free(mydata); free(offset); free(mysize); diff --git a/src/tests/obj_tags.c b/src/tests/obj_tags.c index 2651cf64e..5906e3dd9 100644 --- a/src/tests/obj_tags.c +++ b/src/tests/obj_tags.c @@ -42,9 +42,10 @@ main(int argc, char **argv) dims[0] = 64; dims[1] = 3; dims[2] = 4; - char tag_value[128], tag_value2[128], *tag_value_ret; - char cont_name[128], obj_name1[128], obj_name2[128]; - psize_t value_size; + char tag_value[128], tag_value2[128], *tag_value_ret; + char cont_name[128], obj_name1[128], obj_name2[128]; + pdc_var_type_t value_type; + psize_t value_size; strcpy(tag_value, "some tag value"); strcpy(tag_value2, "some tag value 2 is longer"); @@ -119,30 +120,30 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_put_tag(obj1, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCobj_put_tag(obj1, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 1\n"); ret_value = 1; } - ret = PDCobj_put_tag(obj1, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCobj_put_tag(obj1, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 1\n"); ret_value = 1; } - ret = PDCobj_put_tag(obj2, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCobj_put_tag(obj2, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 2\n"); ret_value = 1; } - ret = PDCobj_put_tag(obj2, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCobj_put_tag(obj2, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 2\n"); ret_value = 1; } - ret = PDCobj_get_tag(obj1, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj1, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 1\n"); ret_value = 1; @@ -153,7 +154,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_get_tag(obj1, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj1, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 1\n"); ret_value = 1; @@ -164,7 +165,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_get_tag(obj2, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj2, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 2\n"); ret_value = 1; @@ -175,7 +176,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_get_tag(obj2, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj2, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 2\n"); ret_value = 1; diff --git a/src/tests/obj_transformation.c b/src/tests/obj_transformation.c index 1e9fe2036..a9e2a23d6 100644 --- a/src/tests/obj_transformation.c +++ b/src/tests/obj_transformation.c @@ -272,7 +272,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to map with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to map with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -321,7 +321,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to lock with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to lock with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -384,7 +384,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to update data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to update data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } #ifdef ENABLE_MPI diff --git a/src/tests/pdc_transforms_lib.c b/src/tests/pdc_transforms_lib.c index f1a83f742..6397bdc70 100644 --- a/src/tests/pdc_transforms_lib.c +++ b/src/tests/pdc_transforms_lib.c @@ -6,21 +6,26 @@ * >> pdc_public.h * * typedef enum { - * PDC_UNKNOWN = -1, - * PDC_INT = 0, - * PDC_FLOAT = 1, - * PDC_DOUBLE = 2, - * PDC_STRING = 3, - * PDC_COMPOUND = 4, - * PDC_ENUM = 5, - * PDC_ARRAY = 6, - * PDC_UINT = 7, - * PDC_INT64 = 8, - * PDC_UINT64 = 9, - * PDC_INT16 = 10, - * PDC_INT8 = 11, - * NCLASSES = 12 - * } PDC_var_type_t; + * PDC_UNKNOWN = -1, * error * + * PDC_INT = 0, * integer types (identical to int32_t) * + * PDC_FLOAT = 1, * floating-point types * + * PDC_DOUBLE = 2, * double types * + * PDC_CHAR = 3, * character types * + * PDC_STRING = 4, * string types * + * PDC_BOOLEAN = 5, * boolean types * + * PDC_SHORT = 6, * short types * + * PDC_UINT = 7, * unsigned integer types (identical to uint32_t) * + * PDC_INT64 = 8, * 64-bit integer types * + * PDC_UINT64 = 9, * 64-bit unsigned integer types * + * PDC_INT16 = 10, * 16-bit integer types * + * PDC_INT8 = 11, * 8-bit integer types * + * PDC_UINT8 = 12, * 8-bit unsigned integer types * + * PDC_UINT16 = 13, * 16-bit unsigned integer types * + * PDC_LONG = 14, * long types * + * PDC_VOID_PTR = 15, * void pointer type * + * PDC_SIZE_T = 16, * size_t type * + * TYPE_COUNT = 17 * this is the number of var types and has to be the last * + * } pdc_c_var_type_t; */ static int diff --git a/src/tests/query_data.c b/src/tests/query_data.c index c7c28999f..62112f6ed 100644 --- a/src/tests/query_data.c +++ b/src/tests/query_data.c @@ -142,7 +142,7 @@ main(int argc, char **argv) ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to write data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to write data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/query_vpic.c b/src/tests/query_vpic.c index b41377c8c..5b3a43305 100644 --- a/src/tests/query_vpic.c +++ b/src/tests/query_vpic.c @@ -44,7 +44,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); double get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); printf(" Query results:\n"); if (sel.nhits < 500) @@ -63,12 +63,12 @@ main(void) gettimeofday(&pdc_timer_end, 0); double get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); printf("Query result energy data (%" PRIu64 " hits):\n", sel.nhits); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] > energy_hi0 || energy_data[i] < energy_lo0) { - printf("Error with result %" PRIu64 ": %.4f\n", i, energy_data[i]); + printf("Error with result %" PRIu64 ": %.5e\n", i, energy_data[i]); } } printf("Verified: all correct!\n"); diff --git a/src/tests/query_vpic_bin_sds1_nopreload.c b/src/tests/query_vpic_bin_sds1_nopreload.c index f741f1634..c2ce6e8d2 100644 --- a/src/tests/query_vpic_bin_sds1_nopreload.c +++ b/src/tests/query_vpic_bin_sds1_nopreload.c @@ -79,7 +79,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -95,7 +95,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); fflush(stdout); } diff --git a/src/tests/query_vpic_bin_sds1_preload.c b/src/tests/query_vpic_bin_sds1_preload.c index eff619513..b6e3dd5a3 100644 --- a/src/tests/query_vpic_bin_sds1_preload.c +++ b/src/tests/query_vpic_bin_sds1_preload.c @@ -92,7 +92,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -108,7 +108,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); fflush(stdout); } diff --git a/src/tests/query_vpic_bin_sds_nopreload.c b/src/tests/query_vpic_bin_sds_nopreload.c index 2e6cea44f..d4a89dd44 100644 --- a/src/tests/query_vpic_bin_sds_nopreload.c +++ b/src/tests/query_vpic_bin_sds_nopreload.c @@ -98,7 +98,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -114,7 +114,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); fflush(stdout); } diff --git a/src/tests/query_vpic_bin_sds_preload.c b/src/tests/query_vpic_bin_sds_preload.c index 940bee35d..c0606f66d 100644 --- a/src/tests/query_vpic_bin_sds_preload.c +++ b/src/tests/query_vpic_bin_sds_preload.c @@ -94,7 +94,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -110,7 +110,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); fflush(stdout); } diff --git a/src/tests/query_vpic_exyz_nopreload.c b/src/tests/query_vpic_exyz_nopreload.c index b03167529..d3593b90a 100644 --- a/src/tests/query_vpic_exyz_nopreload.c +++ b/src/tests/query_vpic_exyz_nopreload.c @@ -111,7 +111,7 @@ main(int argc, char **argv) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -127,7 +127,7 @@ main(int argc, char **argv) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] < energy_lo) { diff --git a/src/tests/query_vpic_exyz_preload.c b/src/tests/query_vpic_exyz_preload.c index 0fe32d6e5..80313d58d 100644 --- a/src/tests/query_vpic_exyz_preload.c +++ b/src/tests/query_vpic_exyz_preload.c @@ -95,7 +95,7 @@ main(int argc, char **argv) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -109,7 +109,7 @@ main(int argc, char **argv) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); fflush(stdout); } diff --git a/src/tests/query_vpic_multi.c b/src/tests/query_vpic_multi.c index 3bb013769..4704b3e1d 100644 --- a/src/tests/query_vpic_multi.c +++ b/src/tests/query_vpic_multi.c @@ -50,7 +50,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Querying Energy in [%.2f, %.2f]\n", energy_lo0, energy_hi0); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -62,12 +62,12 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); printf("Query result energy data (%" PRIu64 " hits):\n", sel.nhits); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] > energy_hi0 || energy_data[i] < energy_lo0) { - printf("Error with result %" PRIu64 ": %.4f\n", i, energy_data[i]); + printf("Error with result %" PRIu64 ": %.5e\n", i, energy_data[i]); } } printf("Verified: all correct!\n"); diff --git a/src/tests/query_vpic_multi_nopreload.c b/src/tests/query_vpic_multi_nopreload.c index d8debbbbb..dc0da4494 100644 --- a/src/tests/query_vpic_multi_nopreload.c +++ b/src/tests/query_vpic_multi_nopreload.c @@ -50,7 +50,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Querying Energy in [%.2f, %.2f]\n", energy_lo0, energy_hi0); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -62,12 +62,12 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); printf("Query result energy data (%" PRIu64 " hits):\n", sel.nhits); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] > energy_hi0 || energy_data[i] < energy_lo0) { - printf("Error with result %" PRIu64 ": %.4f\n", i, energy_data[i]); + printf("Error with result %" PRIu64 ": %.5e\n", i, energy_data[i]); } } printf("Verified: all correct!\n"); diff --git a/src/tests/query_vpic_multi_nopreload1.c b/src/tests/query_vpic_multi_nopreload1.c index 1cd7c56b1..f2c73f68b 100644 --- a/src/tests/query_vpic_multi_nopreload1.c +++ b/src/tests/query_vpic_multi_nopreload1.c @@ -50,7 +50,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Querying Energy in [%.2f, %.2f]\n", energy_lo0, energy_hi0); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -62,12 +62,12 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); printf("Query result energy data (%" PRIu64 " hits):\n", sel.nhits); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] > energy_hi0 || energy_data[i] < energy_lo0) { - printf("Error with result %" PRIu64 ": %.4f\n", i, energy_data[i]); + printf("Error with result %" PRIu64 ": %.5e\n", i, energy_data[i]); } } printf("Verified: all correct!\n"); diff --git a/src/tests/query_vpic_multi_preload.c b/src/tests/query_vpic_multi_preload.c index 3bb013769..4704b3e1d 100644 --- a/src/tests/query_vpic_multi_preload.c +++ b/src/tests/query_vpic_multi_preload.c @@ -50,7 +50,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Querying Energy in [%.2f, %.2f]\n", energy_lo0, energy_hi0); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -62,12 +62,12 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); printf("Query result energy data (%" PRIu64 " hits):\n", sel.nhits); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] > energy_hi0 || energy_data[i] < energy_lo0) { - printf("Error with result %" PRIu64 ": %.4f\n", i, energy_data[i]); + printf("Error with result %" PRIu64 ": %.5e\n", i, energy_data[i]); } } printf("Verified: all correct!\n"); diff --git a/src/tests/read_obj.c b/src/tests/read_obj.c index aa1fc3c3c..e688219f3 100644 --- a/src/tests/read_obj.c +++ b/src/tests/read_obj.c @@ -208,7 +208,7 @@ main(int argc, char **argv) write_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); if (rank == 0) { - printf("Time to process write data with %d ranks: %.6f\n", size, write_time); + printf("Time to process write data with %d ranks: %.5e\n", size, write_time); fflush(stdout); } @@ -259,7 +259,7 @@ main(int argc, char **argv) write_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); if (rank == 0) { - printf("Time to process read data with %d ranks: %.6f\n", size, write_time); + printf("Time to process read data with %d ranks: %.5e\n", size, write_time); fflush(stdout); } diff --git a/src/tests/read_obj_shared.c b/src/tests/read_obj_shared.c index 4b0058dbd..6f56965d8 100644 --- a/src/tests/read_obj_shared.c +++ b/src/tests/read_obj_shared.c @@ -163,7 +163,7 @@ main(int argc, char **argv) offset[0] = rank * my_data_size; local_offset[0] = 0; mysize[0] = my_data_size; - printf("rank %d offset = %lu, length = %lu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); + printf("rank %d offset = %llu, length = %llu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); local_region = PDCregion_create(ndim, local_offset, mysize); global_region = PDCregion_create(ndim, offset, mysize); @@ -220,7 +220,7 @@ main(int argc, char **argv) offset[0] = rank * my_data_size; local_offset[0] = 0; mysize[0] = my_data_size; - printf("rank %d offset = %lu, length = %lu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); + printf("rank %d offset = %llu, length = %llu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); local_region = PDCregion_create(ndim, local_offset, mysize); global_region = PDCregion_create(ndim, offset, mysize); diff --git a/src/tests/region_transfer.c b/src/tests/region_transfer.c index 67a2880f2..3a3b04547 100644 --- a/src/tests/region_transfer.c +++ b/src/tests/region_transfer.c @@ -57,7 +57,7 @@ main(int argc, char **argv) int *data = (int *)malloc(sizeof(int) * BUF_LEN); int *data_read = (int *)malloc(sizeof(int) * BUF_LEN); - dims[0] = BUF_LEN; + dims[0] = PDC_SIZE_UNLIMITED; #ifdef ENABLE_MPI MPI_Init(&argc, &argv); diff --git a/src/tests/region_transfer_all_append.c b/src/tests/region_transfer_all_append.c index 84b116b73..28ebfb2c6 100644 --- a/src/tests/region_transfer_all_append.c +++ b/src/tests/region_transfer_all_append.c @@ -61,10 +61,10 @@ main(int argc, char **argv) #endif if (argc >= 2) { - start_method = atoi(argv[0]); + start_method = atoi(argv[1]); } if (argc >= 3) { - wait_method = atoi(argv[1]); + wait_method = atoi(argv[2]); } data = (int **)malloc(sizeof(int *) * OBJ_NUM); @@ -76,40 +76,32 @@ main(int argc, char **argv) data[i] = data[i - 1] + BUF_LEN; data_read[i] = data_read[i - 1] + BUF_LEN; } - printf("testing region_transfer_all_append, start_method = %d, wait_method = %d\n", start_method, - wait_method); + + if (rank == 0) + printf("testing region_transfer_all_append, start_method = %d, wait_method = %d\n", start_method, + wait_method); dims[0] = BUF_LEN; // create a pdc pdc = PDCinit("pdc"); - printf("create a new pdc\n"); // create a container property cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); - if (cont_prop > 0) { - printf("Create a container property\n"); - } - else { + if (cont_prop <= 0) { printf("Fail to create container property @ line %d!\n", __LINE__); ret_value = 1; } // create a container sprintf(cont_name, "c%d", rank); cont = PDCcont_create(cont_name, cont_prop); - if (cont > 0) { - printf("Create a container c1\n"); - } - else { + if (cont <= 0) { printf("Fail to create container @ line %d!\n", __LINE__); ret_value = 1; } // create an object property obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); - if (obj_prop > 0) { - printf("Create an object property\n"); - } - else { + if (obj_prop <= 0) { printf("Fail to create object property @ line %d!\n", __LINE__); ret_value = 1; } @@ -150,10 +142,7 @@ main(int argc, char **argv) } sprintf(obj_name, "o%d_%d", i, rank); obj[i] = PDCobj_create(cont, obj_name, obj_prop); - if (obj[i] > 0) { - printf("Create an object o1\n"); - } - else { + if (obj[i] <= 0) { printf("Fail to create object @ line %d!\n", __LINE__); ret_value = 1; } @@ -162,10 +151,7 @@ main(int argc, char **argv) offset[0] = 0; offset_length[0] = BUF_LEN / REQ_SIZE; reg = PDCregion_create(1, offset, offset_length); - if (reg > 0) { - printf("Create local region\n"); - } - else { + if (reg <= 0) { printf("Fail to create region @ line %d!\n", __LINE__); ret_value = 1; } @@ -183,10 +169,7 @@ main(int argc, char **argv) offset[0] = j * (BUF_LEN / REQ_SIZE); offset_length[0] = BUF_LEN / REQ_SIZE; reg_global = PDCregion_create(1, offset, offset_length); - if (reg_global > 0) { - printf("Create global region\n"); - } - else { + if (reg_global <= 0) { printf("Fail to create region @ line %d!\n", __LINE__); ret_value = 1; } @@ -196,9 +179,9 @@ main(int argc, char **argv) printf("fail to close global region @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully closed global region @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully closed global region @ line %d\n", __LINE__); */ + /* } */ } } @@ -264,26 +247,23 @@ main(int argc, char **argv) printf("fail to close object o1 @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully close object o1 @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully close object o1 @ line %d\n", __LINE__); */ + /* } */ } if (PDCregion_close(reg) < 0) { printf("fail to close local region @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully closed local region @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully closed local region @ line %d\n", __LINE__); */ + /* } */ for (i = 0; i < OBJ_NUM; ++i) { sprintf(obj_name, "o%d_%d", i, rank); obj[i] = PDCobj_open(obj_name, pdc); - if (obj[i] > 0) { - printf("Create an object o1\n"); - } - else { + if (obj[i] <= 0) { printf("Fail to create object @ line %d!\n", __LINE__); ret_value = 1; } @@ -299,10 +279,7 @@ main(int argc, char **argv) offset[0] = j * (BUF_LEN / REQ_SIZE); offset_length[0] = BUF_LEN / REQ_SIZE; reg_global = PDCregion_create(1, offset, offset_length); - if (reg_global > 0) { - printf("Create global region\n"); - } - else { + if (reg_global <= 0) { printf("Fail to create region @ line %d!\n", __LINE__); ret_value = 1; } @@ -312,9 +289,9 @@ main(int argc, char **argv) printf("fail to close global region @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully closed global region @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully closed global region @ line %d\n", __LINE__); */ + /* } */ } } if (start_method) { @@ -380,9 +357,9 @@ main(int argc, char **argv) printf("fail to close object o1 @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully close object o1 @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully close object o1 @ line %d\n", __LINE__); */ + /* } */ } // Check if data written previously has been correctly read. @@ -399,10 +376,7 @@ main(int argc, char **argv) for (i = 0; i < OBJ_NUM; ++i) { sprintf(obj_name, "o%d_%d", i, rank); obj[i] = PDCobj_open(obj_name, pdc); - if (obj[i] > 0) { - printf("Open an object @ line %d!\n", __LINE__); - } - else { + if (obj[i] <= 0) { printf("Fail to open object @ line %d!\n", __LINE__); ret_value = 1; } @@ -411,10 +385,7 @@ main(int argc, char **argv) offset[0] = 0; offset_length[0] = BUF_LEN; reg = PDCregion_create(1, offset, offset_length); - if (reg > 0) { - printf("Create local region\n"); - } - else { + if (reg <= 0) { printf("Fail to create region @ line %d!\n", __LINE__); ret_value = 1; } @@ -422,10 +393,7 @@ main(int argc, char **argv) offset[0] = 0; offset_length[0] = BUF_LEN; reg_global = PDCregion_create(1, offset, offset_length); - if (reg_global > 0) { - printf("Create global region\n"); - } - else { + if (reg_global <= 0) { printf("Fail to create region @ line %d!\n", __LINE__); ret_value = 1; } @@ -438,9 +406,9 @@ main(int argc, char **argv) printf("fail to close global region @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully closed global region @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully closed global region @ line %d\n", __LINE__); */ + /* } */ if (start_method) { ret = PDCregion_transfer_start_all(transfer_request, OBJ_NUM); @@ -503,9 +471,9 @@ main(int argc, char **argv) printf("fail to close object o1 @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully close object o1 @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully close object o1 @ line %d\n", __LINE__); */ + /* } */ } // Check if data written previously has been correctly read. @@ -523,19 +491,16 @@ main(int argc, char **argv) printf("fail to close local region @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully local region @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully local region @ line %d\n", __LINE__); */ + /* } */ // Now we rewrite the whole object and check its values. // open object for (i = 0; i < OBJ_NUM; ++i) { sprintf(obj_name, "o%d_%d", i, rank); obj[i] = PDCobj_open(obj_name, pdc); - if (obj[i] > 0) { - printf("Create an object o1\n"); - } - else { + if (obj[i] <= 0) { printf("Fail to create object @ line %d!\n", __LINE__); ret_value = 1; } @@ -550,19 +515,13 @@ main(int argc, char **argv) offset[0] = 0; offset_length[0] = BUF_LEN; reg = PDCregion_create(1, offset, offset_length); - if (reg > 0) { - printf("Create local region\n"); - } - else { + if (reg <= 0) { printf("Fail to create region @ line %d!\n", __LINE__); ret_value = 1; } reg_global = PDCregion_create(1, offset, offset_length); - if (reg_global > 0) { - printf("Create global region\n"); - } - else { + if (reg_global <= 0) { printf("Fail to create region @ line %d!\n", __LINE__); ret_value = 1; } @@ -630,17 +589,17 @@ main(int argc, char **argv) printf("fail to close local region @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully local region @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully local region @ line %d\n", __LINE__); */ + /* } */ if (PDCregion_close(reg_global) < 0) { printf("fail to close global region @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully global region @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully global region @ line %d\n", __LINE__); */ + /* } */ // close object for (i = 0; i < OBJ_NUM; ++i) { @@ -648,19 +607,16 @@ main(int argc, char **argv) printf("fail to close object o1 @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully close object o1 @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully close object o1 @ line %d\n", __LINE__); */ + /* } */ } // open object for (i = 0; i < OBJ_NUM; ++i) { sprintf(obj_name, "o%d_%d", i, rank); obj[i] = PDCobj_open(obj_name, pdc); - if (obj[i] > 0) { - printf("Create an object o1\n"); - } - else { + if (obj[i] <= 0) { printf("Fail to create object @ line %d!\n", __LINE__); ret_value = 1; } @@ -669,19 +625,13 @@ main(int argc, char **argv) offset[0] = 0; offset_length[0] = BUF_LEN; reg = PDCregion_create(1, offset, offset_length); - if (reg > 0) { - printf("Create local region\n"); - } - else { + if (reg <= 0) { printf("Fail to create region @ line %d!\n", __LINE__); ret_value = 1; } reg_global = PDCregion_create(1, offset, offset_length); - if (reg_global > 0) { - printf("Create global region\n"); - } - else { + if (reg_global <= 0) { printf("Fail to create region @ line %d!\n", __LINE__); ret_value = 1; } @@ -749,17 +699,17 @@ main(int argc, char **argv) printf("fail to close local region @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully local region @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully local region @ line %d\n", __LINE__); */ + /* } */ if (PDCregion_close(reg_global) < 0) { printf("fail to close global region @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully global region @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully global region @ line %d\n", __LINE__); */ + /* } */ // close object for (i = 0; i < OBJ_NUM; ++i) { @@ -767,9 +717,9 @@ main(int argc, char **argv) printf("fail to close object o1 @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully close object o1 @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully close object o1 @ line %d\n", __LINE__); */ + /* } */ } for (j = 0; j < OBJ_NUM; ++j) { @@ -787,25 +737,25 @@ main(int argc, char **argv) printf("fail to close container c1 @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully close container c1 @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully close container c1 @ line %d\n", __LINE__); */ + /* } */ // close a object property if (PDCprop_close(obj_prop) < 0) { printf("Fail to close property @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully close object property @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully close object property @ line %d\n", __LINE__); */ + /* } */ // close a container property if (PDCprop_close(cont_prop) < 0) { printf("Fail to close property @ line %d\n", __LINE__); ret_value = 1; } - else { - printf("successfully close container property @ line %d\n", __LINE__); - } + /* else { */ + /* printf("successfully close container property @ line %d\n", __LINE__); */ + /* } */ free(data); free(data_read); free(obj); @@ -815,6 +765,14 @@ main(int argc, char **argv) printf("fail to close PDC @ line %d\n", __LINE__); ret_value = 1; } + + if (rank == 0) { + if (ret_value == 0) + printf("Test succeed!\n"); + else + printf("ERROR: Test failed!\n"); + } + #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/src/tests/region_transfer_query.c b/src/tests/region_transfer_query.c index e2c7dcfe6..dede54aaa 100644 --- a/src/tests/region_transfer_query.c +++ b/src/tests/region_transfer_query.c @@ -45,9 +45,9 @@ main(int argc, char **argv) int rank = 0, size = 1, i; int ret_value = 0; - uint64_t offset[3], offset_length[3], local_offset[1]; + uint64_t offset[3], offset_length[3]; //, local_offset[1]; uint64_t dims[1]; - local_offset[0] = 0; + // local_offset[0] = 0; offset[0] = 0; offset[1] = 2; offset[2] = 5; diff --git a/src/tests/search_obj.c b/src/tests/search_obj.c index 2701d729d..465a48b6c 100644 --- a/src/tests/search_obj.c +++ b/src/tests/search_obj.c @@ -233,7 +233,7 @@ main(int argc, char **argv) ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("searched %10d ... %.2f\n", i * size, ht_total_sec); + printf("searched %10d ... %.5e\n", i * size, ht_total_sec); fflush(stdout); } @@ -251,8 +251,8 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - /* printf("Time to full query %d obj/rank with %d ranks: %.6f\n\n\n", count, size, ht_total_sec); */ - printf("Time to partial query %d obj/rank with %d ranks: %.6f\n\n\n", count, size, ht_total_sec); + /* printf("Time to full query %d obj/rank with %d ranks: %.5e\n\n\n", count, size, ht_total_sec); */ + printf("Time to partial query %d obj/rank with %d ranks: %.5e\n\n\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/search_obj_scale.c b/src/tests/search_obj_scale.c index b964c6bc2..88506ffc7 100644 --- a/src/tests/search_obj_scale.c +++ b/src/tests/search_obj_scale.c @@ -180,7 +180,7 @@ main(int argc, char **argv) ht_total_end.tv_usec - ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; - printf("%10d queried ... %.2fs\n", i * size, ht_total_sec); + printf("%10d queried ... %.5es\n", i * size, ht_total_sec); fflush(stdout); } } @@ -193,7 +193,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to create %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to create %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/stat_obj.c b/src/tests/stat_obj.c index da8a5f927..6a068edc7 100644 --- a/src/tests/stat_obj.c +++ b/src/tests/stat_obj.c @@ -207,7 +207,7 @@ main(int argc, char **argv) ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("stated %10d ... %.2f\n", i * size, ht_total_sec); + printf("stated %10d ... %.5e\n", i * size, ht_total_sec); fflush(stdout); } @@ -225,7 +225,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to stat %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to stat %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/update_obj.c b/src/tests/update_obj.c index a589725ce..b63082361 100644 --- a/src/tests/update_obj.c +++ b/src/tests/update_obj.c @@ -211,7 +211,7 @@ main(int argc, char **argv) ht_total_end.tv_usec - ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; - printf("updated %10d ... %.2f\n", i * size, ht_total_sec); + printf("updated %10d ... %.5e\n", i * size, ht_total_sec); fflush(stdout); } } @@ -225,7 +225,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to update %d obj/rank with %d ranks: %.6f\n\n\n", count, size, ht_total_sec); + printf("Time to update %d obj/rank with %d ranks: %.5e\n\n\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/vpicio.c b/src/tests/vpicio.c index 1a6692628..081376f43 100644 --- a/src/tests/vpicio.c +++ b/src/tests/vpicio.c @@ -241,7 +241,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Obj create time: %.2f\n", t1 - t0); + printf("Obj create time: %.5e\n", t1 - t0); } #endif @@ -290,7 +290,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("Transfer create time: %.2f\n", t0 - t1); + printf("Transfer create time: %.5e\n", t0 - t1); } #endif @@ -339,7 +339,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Transfer start time: %.2f\n", t1 - t0); + printf("Transfer start time: %.5e\n", t1 - t0); } #endif @@ -388,7 +388,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("Transfer wait time: %.2f\n", t0 - t1); + printf("Transfer wait time: %.5e\n", t0 - t1); } #endif @@ -438,7 +438,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Transfer close time: %.2f\n", t1 - t0); + printf("Transfer close time: %.5e\n", t1 - t0); } #endif diff --git a/src/tests/vpicio_mts.c b/src/tests/vpicio_mts.c index 110febbfd..4851d934f 100644 --- a/src/tests/vpicio_mts.c +++ b/src/tests/vpicio_mts.c @@ -76,7 +76,7 @@ main(int argc, char **argv) uint64_t *offset_remote; uint64_t *mysize; double t0, t1; - int steps = 1, sleeptime = 0; + uint64_t steps = 1, sleeptime = 0; pdcid_t transfer_request_x, transfer_request_y, transfer_request_z, transfer_request_px, transfer_request_py, transfer_request_pz, transfer_request_id1, transfer_request_id2; @@ -95,8 +95,8 @@ main(int argc, char **argv) sleeptime = atoi(argv[3]); } if (rank == 0) - printf("Writing %" PRIu64 " number of particles for %d steps with %d clients.\n", numparticles, steps, - size); + printf("Writing %" PRIu64 " number of particles for %llu steps with %d clients.\n", numparticles, + steps, size); dims[0] = numparticles; @@ -199,7 +199,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("\n#Step %d\n", i); + printf("\n#Step %llu\n", i); } #endif PDCprop_set_obj_time_step(obj_prop_xx, i); @@ -258,7 +258,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Obj create time: %.2f\n", t1 - t0); + printf("Obj create time: %.5e\n", t1 - t0); } #endif @@ -309,7 +309,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("Transfer create time: %.2f\n", t0 - t1); + printf("Transfer create time: %.5e\n", t0 - t1); } #endif @@ -358,7 +358,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Transfer start time: %.2f\n", t1 - t0); + printf("Transfer start time: %.5e\n", t1 - t0); } #endif @@ -407,7 +407,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("Transfer wait time: %.2f\n", t0 - t1); + printf("Transfer wait time: %.5e\n", t0 - t1); } #endif @@ -456,7 +456,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Transfer close time: %.2f\n", t1 - t0); + printf("Transfer close time: %.5e\n", t1 - t0); } #endif @@ -498,13 +498,13 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("Obj close time: %.2f\n", t0 - t1); + printf("Obj close time: %.5e\n", t0 - t1); } #endif if (i != steps - 1) { sleep(sleeptime); if (rank == 0) { - printf("Sleep time: %d.00\n", sleeptime); + printf("Sleep time: %llu.00\n", sleeptime); } } } // End for steps diff --git a/src/tests/vpicio_v2.c b/src/tests/vpicio_v2.c index ee3f3512a..89839352b 100644 --- a/src/tests/vpicio_v2.c +++ b/src/tests/vpicio_v2.c @@ -299,7 +299,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to map with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to map with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -348,7 +348,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to lock with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to lock with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -408,7 +408,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to update data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to update data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/write_obj_shared.c b/src/tests/write_obj_shared.c index 9fa01107f..d8796bb61 100644 --- a/src/tests/write_obj_shared.c +++ b/src/tests/write_obj_shared.c @@ -209,7 +209,7 @@ main(int argc, char **argv) write_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); if (rank == 0) { - printf("Time to lock and release data with %d ranks: %.6f\n", size, write_time); + printf("Time to lock and release data with %d ranks: %.5e\n", size, write_time); fflush(stdout); } done: diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt new file mode 100644 index 000000000..c7f43576f --- /dev/null +++ b/src/tools/CMakeLists.txt @@ -0,0 +1,33 @@ +project(${PDC_SOURCE_DIR}) +cmake_minimum_required (VERSION 3.0) + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${PDC_INCLUDES_BUILD_TIME} + ${PROJECT_BINARY_DIR} + ${PDC_SOURCE_DIR} + ${PDC_SOURCE_DIR}/src/client_api/include + $ENV{HOME}/Sandbox/c-blosc/blosc + $ENV{HOME}/include +) + +set(PROGRAMS + pdc_import + pdc_export + pdc_ls + ) + +add_library(cjson cjson/cJSON.c) + +foreach(program ${PROGRAMS}) + add_executable(${program} ${program}.c) + target_link_libraries(${program} pdc) + target_link_libraries(${program} cjson) +endforeach(program) + + +# ******************************************* +# Add the HDF5 library for pdc-neon +# ******************************************* +FIND_LIBRARY(HDF5_LIBRARY NAMES hdf5_debug PATHS $ENV{HOME}/lib) diff --git a/tools/cjson/cJSON.c b/src/tools/cjson/cJSON.c similarity index 100% rename from tools/cjson/cJSON.c rename to src/tools/cjson/cJSON.c diff --git a/tools/cjson/cJSON.h b/src/tools/cjson/cJSON.h similarity index 100% rename from tools/cjson/cJSON.h rename to src/tools/cjson/cJSON.h diff --git a/tools/pdc_export.c b/src/tools/pdc_export.c similarity index 96% rename from tools/pdc_export.c rename to src/tools/pdc_export.c index e46c17f2a..3614fa9fb 100644 --- a/tools/pdc_export.c +++ b/src/tools/pdc_export.c @@ -7,8 +7,9 @@ #include #include #include "hdf5.h" +#include "pdc_generic.h" -#define ENABLE_MPI 1 +// #define ENABLE_MPI 1 #ifdef ENABLE_MPI #include "mpi.h" @@ -240,45 +241,11 @@ get_data_type(int data_type) if (data_type == -1) { return "PDC_UNKNOWN"; } - else if (data_type == 0) { - return "PDC_INT"; - } - else if (data_type == 1) { - return "PDC_FLOAT"; - } - else if (data_type == 2) { - return "PDC_DOUBLE"; - } - else if (data_type == 3) { - return "PDC_CHAR"; - } - else if (data_type == 4) { - return "PDC_COMPOUND"; - } - else if (data_type == 5) { - return "PDC_ENUM"; - } - else if (data_type == 6) { - return "PDC_ARRAY"; - } - else if (data_type == 7) { - return "PDC_UINT"; - } - else if (data_type == 8) { - return "PDC_INT64"; - } - else if (data_type == 9) { - return "PDC_UINT64"; - } - else if (data_type == 10) { - return "PDC_INT16"; - } - else if (data_type == 11) { - return "PDC_INT16"; - } - else { + char *result = get_enum_name_by_dtype(data_type); + if (result == NULL) { return "NULL"; } + return result; } char * @@ -725,7 +692,7 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[]) strcpy(only_file_name, last_slash + 1); char file_name[strlen(cur_metadata->obj_name) + 1]; - char buf[20]; + // char buf[20]; sprintf(buf, "%d", cur_metadata->cont_id); char *cur_path = (char *)malloc(sizeof(char) * strlen(cur_metadata->obj_name) + strlen(buf) + 1); strcpy(cur_path, buf); @@ -737,7 +704,7 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[]) printf("%s\n", token); strcat(cur_path, "/"); strcat(cur_path, token); - hid_t cur_group_id = H5Gopen(file_id, cur_path, H5P_DEFAULT); + cur_group_id = H5Gopen(file_id, cur_path, H5P_DEFAULT); if (cur_group_id < 0) { cur_group_id = H5Gcreate(file_id, cur_path, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } @@ -759,15 +726,15 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[]) hsize_t dtype_size = H5Tget_size(data_type); void * data_buf = malloc(buf_size * dtype_size); // if data side is large, need to write in batches - uint64_t offset[10], size[10]; + uint64_t region_offset[10], region_size[10]; for (int i = 0; i < cur_metadata->ndim; i++) { - offset[i] = 0; - size[i] = (cur_metadata->dims)[i]; + region_offset[i] = 0; + region_size[i] = (cur_metadata->dims)[i]; } - // size[0] *= dtype_size; + // region_size[0] *= dtype_size; - pdcid_t local_region = PDCregion_create(cur_metadata->ndim, offset, size); - pdcid_t remote_region = PDCregion_create(cur_metadata->ndim, offset, size); + pdcid_t local_region = PDCregion_create(cur_metadata->ndim, region_offset, region_size); + pdcid_t remote_region = PDCregion_create(cur_metadata->ndim, region_offset, region_size); pdcid_t local_obj_id = PDCobj_open(cur_metadata->obj_name, pdc_id_g); pdcid_t transfer_request = PDCregion_transfer_create(data_buf, PDC_READ, local_obj_id, local_region, remote_region); diff --git a/tools/pdc_import.c b/src/tools/pdc_import.c similarity index 89% rename from tools/pdc_import.c rename to src/tools/pdc_import.c index 9e12d3a84..63827d950 100644 --- a/tools/pdc_import.c +++ b/src/tools/pdc_import.c @@ -1,8 +1,10 @@ #include #include #include +#include +#include -#define ENABLE_MPI 1 +// #define ENABLE_MPI 1 #ifdef ENABLE_MPI #include "mpi.h" @@ -10,13 +12,14 @@ #include "hdf5.h" #include "pdc.h" -#include "pdc_client_server_common.h" -#include "pdc_client_connect.h" +// #include "pdc_client_server_common.h" +// #include "pdc_client_connect.h" #define MAX_NAME 1024 #define MAX_FILES 2500 #define MAX_FILENAME_LEN 64 #define MAX_TAG_SIZE 8192 +#define TAG_LEN_MAX 2048 typedef struct ArrayList { int length; @@ -80,40 +83,10 @@ int ndset_g = 0; /* FILE *summary_fp_g; */ int max_tag_size_g = 0; pdcid_t pdc_id_g = 0, cont_prop_g = 0, cont_id_g = 0, obj_prop_g = 0; -struct timeval write_timer_start_g; -struct timeval write_timer_end_g; +struct timespec write_timer_start_g, write_timer_end_g; struct ArrayList *container_names; int overwrite = 0; -int -add_tag(char *str) -{ - int str_len = 0; - if (NULL == str || NULL == tags_ptr_g) { - fprintf(stderr, "%s - input str is NULL!", __func__); - return 0; - } - else if (tag_size_g + str_len >= MAX_TAG_SIZE) { - fprintf(stderr, "%s - tags_ptr_g overflow!", __func__); - return 0; - } - - // Remove the trailing ',' - if (*str == '}' || *str == ')' || *str == ']') { - if (*(--tags_ptr_g) != ',') { - tags_ptr_g++; - } - } - - str_len = strlen(str); - strncpy(tags_ptr_g, str, str_len); - - tag_size_g += str_len; - tags_ptr_g += str_len; - - return str_len; -} - int main(int argc, char **argv) { @@ -246,9 +219,8 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); #endif - struct timeval pdc_timer_start; - struct timeval pdc_timer_end; - gettimeofday(&pdc_timer_start, 0); + struct timespec pdc_timer_start, pdc_timer_end; + clock_gettime(CLOCK_MONOTONIC, &pdc_timer_start); for (i = 0; i < my_count; i++) { filename = my_filenames[i]; @@ -272,7 +244,8 @@ main(int argc, char **argv) #endif // Checkpoint all metadata after import each hdf5 file if (rank == 0) { - PDC_Client_all_server_checkpoint(); + // FIXME: this should be replaced by a function in public headers. + // PDC_Client_all_server_checkpoint(); } /* printf("%s, %d\n", filename, max_tag_size_g); */ /* printf("\n\n======================\nNumber of datasets: %d\n", ndset_g); */ @@ -285,8 +258,10 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); #endif - gettimeofday(&pdc_timer_end, 0); - double write_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); + clock_gettime(CLOCK_MONOTONIC, &pdc_timer_end); + double write_time = + (pdc_timer_end.tv_sec - pdc_timer_start.tv_sec) * 1e9 + + (pdc_timer_end.tv_nsec - pdc_timer_start.tv_nsec); // calculate duration in nanoseconds #ifdef ENABLE_MPI MPI_Reduce(&ndset_g, &total_dset, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); @@ -294,7 +269,8 @@ main(int argc, char **argv) total_dset = ndset_g; #endif if (rank == 0) { - printf("Import %d datasets with %d ranks took %.2f seconds.\n", total_dset, size, write_time); + printf("Import %d datasets with %d ranks took %.2f seconds.\n", total_dset, size, + write_time / 1e9); } } @@ -376,8 +352,8 @@ scan_group(hid_t gid, int level, char *app_name) // create container - check if container exists first // create a container create_cont = 1; - for (int i = 0; i < container_names->length; i++) { - if (strcmp(container_names->items[i], group_name) == 0) { + for (int ctn = 0; ctn < container_names->length; ctn++) { + if (strcmp(container_names->items[ctn], group_name) == 0) { create_cont = 0; break; } @@ -423,7 +399,7 @@ do_dset(hid_t did, char *name, char *app_name) char * obj_name; int name_len, i; hsize_t ndim, dims[10]; - uint64_t offset[10], size[10]; + uint64_t region_offset[10], region_size[10]; void * buf; struct pdc_region_info obj_region; @@ -439,7 +415,6 @@ do_dset(hid_t did, char *name, char *app_name) H5Iget_name(did, ds_name, TAG_LEN_MAX); memset(dset_name_g, 0, TAG_LEN_MAX); strcpy(dset_name_g, ds_name); - /* add_tag(ds_name); */ // dset_name_g has the full path to the dataset, e.g. /group/subgroup/dset_name // substract the actual dset name with following @@ -459,7 +434,6 @@ do_dset(hid_t did, char *name, char *app_name) */ sid = H5Dget_space(did); /* the dimensions of the dataset (not shown) */ tid = H5Dget_type(did); - /* add_tag(",DT:"); */ pdcid_t cur_obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); @@ -479,12 +453,12 @@ do_dset(hid_t did, char *name, char *app_name) PDCprop_set_obj_app_name(obj_prop_g, app_name); pdcid_t obj_id = PDCobj_open(ds_name, pdc_id_g); - if (check > 0) { - if (!overwrite) { - return; - } + + if (!overwrite) { + return; } else { + // FIXME: should open the object here instead of creating a new one. obj_id = PDCobj_create(cont_id_g, ds_name, obj_prop_g); } if (obj_id <= 0) { @@ -493,8 +467,6 @@ do_dset(hid_t did, char *name, char *app_name) do_dtype(tid, did, 0); - add_tag(","); - ndset_g++; /* if (ndset_g > 10) { */ /* return; */ @@ -539,18 +511,18 @@ do_dset(hid_t did, char *name, char *app_name) scan_attrs(did, obj_id); - pdc_metadata_t *meta = NULL; - obj_region.ndim = ndim; + // pdc_metadata_t *meta = NULL; + obj_region.ndim = ndim; for (i = 0; i < ndim; i++) { - offset[i] = 0; - size[i] = dims[i]; + region_offset[i] = 0; + region_size[i] = dims[i]; } - size[0] *= dtype_size; - obj_region.offset = offset; - obj_region.size = size; + region_size[0] *= dtype_size; + obj_region.offset = region_offset; + obj_region.size = region_size; if (ndset_g == 1) - gettimeofday(&write_timer_start_g, 0); + clock_gettime(CLOCK_MONOTONIC, &write_timer_start_g); /* PDC_Client_query_metadata_name_timestep(dset_name_g, 0, &meta); */ /* if (meta == NULL) */ @@ -558,20 +530,22 @@ do_dset(hid_t did, char *name, char *app_name) /* else */ /* PDC_Client_write(meta, &obj_region, buf); */ - pdcid_t local_region = PDCregion_create(ndim, offset, size); - pdcid_t remote_region = PDCregion_create(ndim, offset, size); + pdcid_t local_region = PDCregion_create(ndim, region_offset, region_size); + pdcid_t remote_region = PDCregion_create(ndim, region_offset, region_size); pdcid_t transfer_request = PDCregion_transfer_create(buf, PDC_WRITE, obj_id, local_region, remote_region); PDCregion_transfer_start(transfer_request); PDCregion_transfer_wait(transfer_request); // PDC_Client_write_id(obj_id, &obj_region, buf); if (ndset_g % 100 == 0) { - gettimeofday(&write_timer_end_g, 0); - double elapsed_time = PDC_get_elapsed_time_double(&write_timer_start_g, &write_timer_end_g); - printf("Importer%2d: Finished written 100 objects, took %.2f, my total %d\n", rank, elapsed_time, - ndset_g); + clock_gettime(CLOCK_MONOTONIC, &write_timer_end_g); + double elapsed_time = + (write_timer_end_g.tv_sec - write_timer_start_g.tv_sec) * 1e9 + + (write_timer_end_g.tv_nsec - write_timer_start_g.tv_nsec); // calculate duration in nanoseconds; + printf("Importer%2d: Finished written 100 objects, took %.2f, my total %d\n", rank, + elapsed_time / 1e9, ndset_g); fflush(stdout); - gettimeofday(&write_timer_start_g, 0); + clock_gettime(CLOCK_MONOTONIC, &write_timer_start_g); } free(buf); @@ -597,7 +571,7 @@ do_dtype(hid_t tid, hid_t oid, int is_compound) hsize_t dims[8], ndim; char * mem_name; char * attr_string[100], new_string[TAG_LEN_MAX], tmp_str[TAG_LEN_MAX]; - hsize_t size, attr_len; + hsize_t attr_size, attr_len; hid_t mem_type; hid_t atype, aspace, naive_type; H5T_class_t t_class, compound_class; @@ -606,11 +580,11 @@ do_dtype(hid_t tid, hid_t oid, int is_compound) /* puts(" Invalid datatype.\n"); */ } else { - size = H5Tget_size(tid); - /* printf(" Datasize %3d, type", size); */ + attr_size = H5Tget_size(tid); + /* printf(" Datasize %3d, type", attr_size); */ /* * Each class has specific properties that can be - * retrieved, e.g., size, byte order, exponent, etc. + * retrieved, e.g., attr_size, byte order, exponent, etc. */ if (t_class == H5T_INTEGER) { /* puts(" 'H5T_INTEGER'."); */ @@ -633,7 +607,7 @@ do_dtype(hid_t tid, hid_t oid, int is_compound) } else if (t_class == H5T_ARRAY) { if (is_compound == 0) { - tag_size_g += size; + tag_size_g += attr_size; } ndim = H5Tget_array_ndims(tid); H5Tget_array_dims2(tid, dims); @@ -646,13 +620,14 @@ do_dtype(hid_t tid, hid_t oid, int is_compound) } else if (t_class == H5T_ENUM) { /* puts(" 'H5T_ENUM'."); */ - return PDC_ENUM; + return PDC_INT; } else { printf("PDC does not support this data type yet.\n"); return PDC_UNKNOWN; } } + return PDC_UNKNOWN; } /* @@ -698,12 +673,16 @@ scan_attrs(hid_t oid, pdcid_t obj_id) void do_attr(hid_t aid, pdcid_t obj_id) { - ssize_t len; - hid_t atype; - hid_t aspace; - char buf[MAX_NAME] = {0}; - char read_buf[TAG_LEN_MAX] = {0}; - pdc_kvtag_t kvtag1; + ssize_t len; + hid_t atype; + hid_t aspace; + char buf[MAX_NAME] = {0}; + char read_buf[TAG_LEN_MAX] = {0}; + // pdc_kvtag_t kvtag1; + char * tag_name; + void * tag_value; + pdc_var_type_t value_type; + size_t tag_size; /* * Get the name of the attribute. @@ -717,15 +696,15 @@ do_attr(hid_t aid, pdcid_t obj_id) atype = H5Aget_type(aid); H5Aread(aid, atype, read_buf); - kvtag1.name = buf; - kvtag1.value = (void *)read_buf; + tag_name = buf; + tag_value = (void *)read_buf; if (atype == H5T_STRING) { - kvtag1.size = strlen(read_buf) + 1; + tag_size = strlen(read_buf) + 1; } else { - kvtag1.size = H5Tget_size(atype); + tag_size = H5Tget_size(atype); } - PDCobj_put_tag(obj_id, kvtag1.name, kvtag1.value, kvtag1.size); + PDCobj_put_tag(obj_id, tag_name, tag_value, value_type, tag_size); /* * Get attribute information: dataspace, data type @@ -885,4 +864,4 @@ do_plist(hid_t pid) } /* ... and so on for other dataset properties ... */ -} \ No newline at end of file +} diff --git a/tools/pdc_ls.c b/src/tools/pdc_ls.c similarity index 96% rename from tools/pdc_ls.c rename to src/tools/pdc_ls.c index 9d3db32fe..47fecbcd3 100644 --- a/tools/pdc_ls.c +++ b/src/tools/pdc_ls.c @@ -129,6 +129,7 @@ main(int argc, char *argv[]) DIR * d; struct dirent *dir; d = opendir(argv[1]); + if (d) { while ((dir = readdir(d)) != NULL) { if (strstr(dir->d_name, "metadata_checkpoint.")) { @@ -201,45 +202,11 @@ get_data_type(int data_type) if (data_type == -1) { return "PDC_UNKNOWN"; } - else if (data_type == 0) { - return "PDC_INT"; - } - else if (data_type == 1) { - return "PDC_FLOAT"; - } - else if (data_type == 2) { - return "PDC_DOUBLE"; - } - else if (data_type == 3) { - return "PDC_CHAR"; - } - else if (data_type == 4) { - return "PDC_COMPOUND"; - } - else if (data_type == 5) { - return "PDC_ENUM"; - } - else if (data_type == 6) { - return "PDC_ARRAY"; - } - else if (data_type == 7) { - return "PDC_UINT"; - } - else if (data_type == 8) { - return "PDC_INT64"; - } - else if (data_type == 9) { - return "PDC_UINT64"; - } - else if (data_type == 10) { - return "PDC_INT16"; - } - else if (data_type == 11) { - return "PDC_INT16"; - } - else { + char *result = get_enum_name_by_dtype(data_type); + if (result == NULL) { return "NULL"; } + return result; } char * @@ -720,7 +687,7 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[]) } } else { - char buf[12]; + // char buf[12]; sprintf(buf, "%d", cur_metadata->obj_id); reti = regexec(®ex, buf, 0, NULL, 0); if (!reti) { @@ -758,7 +725,7 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[]) } } else { - char buf[12]; + // char buf[12]; sprintf(buf, "%d", cur_metadata->obj_id); reti = regexec(®ex, buf, 0, NULL, 0); if (!reti) { @@ -801,7 +768,8 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[]) cJSON_AddStringToObject(region_info_json, "storage_loc", cur_region->storage_location); cJSON_AddNumberToObject(region_info_json, "offset", cur_region->offset); cJSON_AddNumberToObject(region_info_json, "num_dims", cur_region->ndim); - dims[cur_region->ndim]; + // FIXME: statement with no effect. what did we expect to do here? + // dims[cur_region->ndim]; for (int i = 0; i < (cur_metadata->ndim); i++) { dims[i] = (cur_region->start)[i]; } @@ -835,7 +803,8 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[]) fp = stdout; } if (list_names) { - cJSON *all_names_json = cJSON_CreateStringArray(obj_names->items, obj_names->length); + cJSON *all_names_json = + cJSON_CreateStringArray((const char *const *)obj_names->items, obj_names->length); cJSON_AddItemToObject(output, "all_obj_names", all_names_json); } if (list_ids) { diff --git a/src/utils/pdc_interface.c b/src/utils/pdc_interface.c index 4cca6b0bc..727b3ee96 100644 --- a/src/utils/pdc_interface.c +++ b/src/utils/pdc_interface.c @@ -25,6 +25,8 @@ #include "pdc_malloc.h" #include "pdc_id_pkg.h" #include "pdc_interface.h" +#include "pdc_cont_pkg.h" +#include "pdc_cont.h" #include #include @@ -79,7 +81,7 @@ PDC_register_type(PDC_type_t type_id, PDC_free_t free_func) /* Initialize the type */ if (NULL == (pdc_id_list_g->PDC_id_type_list_g)[type_id]) { /* Allocate the type information for new type */ - if (NULL == (type_ptr = PDC_CALLOC(struct PDC_id_type))) + if (NULL == (type_ptr = PDC_CALLOC(1, struct PDC_id_type))) PGOTO_ERROR(FAIL, "ID type allocation failed"); (pdc_id_list_g->PDC_id_type_list_g)[type_id] = type_ptr; } diff --git a/src/utils/pdc_malloc.c b/src/utils/pdc_malloc.c deleted file mode 100644 index 40379d400..000000000 --- a/src/utils/pdc_malloc.c +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#include -#include -#include "pdc_malloc.h" -#include "pdc_private.h" - -void * -PDC_malloc(size_t size) -{ - void *ret_value; - - FUNC_ENTER(NULL); - - assert(size); - - if (size) - ret_value = malloc(size); - else - ret_value = NULL; - - FUNC_LEAVE(ret_value); -} - -void * -PDC_calloc(size_t size) -{ - void *ret_value; - - FUNC_ENTER(NULL); - - assert(size); - - if (size) - ret_value = calloc(1, size); - else - ret_value = NULL; - - FUNC_LEAVE(ret_value); -} - -void * -PDC_free(void *mem) -{ - void *ret_value = NULL; - - FUNC_ENTER(NULL); - - if (mem) { - free(mem); - } - - FUNC_LEAVE(ret_value); -} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index f9915b221..b14402393 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,5 +1,24 @@ cmake_minimum_required (VERSION 2.8.12) +# Setup cmake policies. +foreach(p + CMP0012 + CMP0013 + CMP0014 + CMP0022 # CMake 2.8.12 + CMP0025 # CMake 3.0 + CMP0053 # CMake 3.1 + CMP0054 # CMake 3.1 + CMP0074 # CMake 3.12 + CMP0075 # CMake 3.12 + CMP0083 # CMake 3.14 + CMP0093 # CMake 3.15 + ) + if(POLICY ${p}) + cmake_policy(SET ${p} NEW) + endif() +endforeach() + project(PDC_VOL C) include_directories( @@ -51,18 +70,67 @@ option(USE_SYSTEM_HDF5 "Use system-installed HDF5." ON) endif() endif() +option(USE_SYSTEM_OPENMP "Use system-installed OpenMP." ON) +if(USE_SYSTEM_OPENMP) + find_package(OpenMP REQUIRED) + if(OPENMP_FOUND) + add_definitions(-DENABLE_OPENMP=1) + set(ENABLE_OPENMP 1) + set(OPENMP_LIBRARIES "${OpenMP_C_LIBRARIES}") + else() + message(FATAL_ERROR "OpenMP not found") + endif() +endif() + + add_definitions(-DENABLE_MPI=1) add_library(cjson cjson/cJSON.c) -set(PROGRAMS - pdc_ls - pdc_import - pdc_export - ) -foreach(program ${PROGRAMS}) - add_executable(${program} ${program}.c) - target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) - target_link_libraries(${program} cjson) -endforeach(program) +# set(PROGRAMS +# pdc_ls +# pdc_import +# pdc_export +# ) + +# foreach(program ${PROGRAMS}) +# add_executable(${program} ${program}.c) +# target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) +# target_link_libraries(${program} pdc) +# target_link_libraries(${program} cjson) +# target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR}) +# endforeach(program) + + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -fopenmp -DNDEBUG") +# Find LibTIFF +option(USE_LIB_TIFF "Enable LibTiff." ON) +if(USE_LIB_TIFF) + find_package(TIFF REQUIRED) + if(TIFF_FOUND) + set(LLSM_LIB_SOURCE + llsm/parallelReadTiff.c + llsm/csvReader.c + llsm/pdc_list.c + ) + # Add the LibTIFF include directory to the include path + include_directories(${TIFF_INCLUDE_DIRS}) + add_library(llsm_tiff ${LLSM_LIB_SOURCE}) + target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS}) + target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES}) + target_link_libraries(llsm_tiff PUBLIC ${TIFF_LIBRARIES}) + target_include_directories(llsm_tiff PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/llsm) + + + add_executable(llsm_importer llsm_importer.c) + target_link_libraries(llsm_importer ${PDC_EXT_LIB_DEPENDENCIES}) + target_link_libraries(llsm_importer pdc) + target_link_libraries(llsm_importer cjson) + target_link_libraries(llsm_importer ${TIFF_LIBRARIES}) + target_link_libraries(llsm_importer llsm_tiff) + target_include_directories(llsm_importer PUBLIC ${PDC_INCLUDE_DIR}) + else() + message(WARNING "LibTiff not found, ignore building the executables which requires LibTiff support.") + endif() +endif() \ No newline at end of file diff --git a/tools/LLSM_IMPORTER.md b/tools/LLSM_IMPORTER.md new file mode 100644 index 000000000..42ded7682 --- /dev/null +++ b/tools/LLSM_IMPORTER.md @@ -0,0 +1,76 @@ +# LLSM_Importer Tutorial + +This is a tutorial for you to run llsm_importer on Perlmutter supercomputer at NERSC. + +## Prerequisite + +Before building and installing LLSM_importer tool, you need to make sure you install PDC correctly. Check out the latest update on the `develop` branch of `PDC`. Please refer to [Proactive Data Containers (PDC) Installation Guide](../README.md) + +Once you finish all the steps in the installation guide above, you should have environment variable `$WORK_SPACE` defined. + +## Installation + +To build and install LLSM_importer, you need to download libtiff 4.4.0 first. + +```bash +cd $WORK_SPACE/source +wget https://download.osgeo.org/libtiff/tiff-4.4.0.tar.gz +tar zxvf tiff-4.4.0.tar.gz +cd tiff-4.4.0 +./configure --prefix=$WORK_SPACE/install/tiff-4.4.0 +make -j 32 install +``` + +Now you should have libtiff 4.4.0 installed and you need to include the path to the library to your environment variables: + +```bash +echo "export TIFF_DIR=$WORK_SPACE/install/tiff-4.4.0" +echo 'export LD_LIBRARY_PATH=$TIFF_DIR/lib:$LD_LIBRARY_PATH' +echo 'export PATH=$TIFF_DIR/include:$TIFF_DIR/lib:$PATH' + +echo "export TIFF_DIR=$WORK_SPACE/install/tiff-4.4.0" >> $WORK_SPACE/pdc_env.sh +echo 'export LD_LIBRARY_PATH=$TIFF_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh +echo 'export PATH=$TIFF_DIR/include:$TIFF_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh +``` + +Copy the 3 export commands on your screen and run them, and next time if you need to rebuild the llsm_importer program, you can run `$WORK_SPACE/pdc_env.sh` again in advance! + +Now, time to build llsm_importer program. + +```bash +mkdir -p $WORK_SPACE/source/pdc/tools/build +cd $WORK_SPACE/source/pdc/tools/build + +cmake ../ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DPDC_DIR=$PDC_DIR -DUSE_LIB_TIFF=ON -DUSE_SYSTEM_HDF5=ON -DUSE_SYSTEM_OPENMP=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR/tools/ -DCMAKE_C_COMPILER=cc + +make -j 32 +``` + +After this, you should be able to see `llsm_importer` artifact under your `$WORK_SPACE/source/pdc/tools/build` directory. + +## Running LLSM_importer + +First, locate the llsm_importer script + +```bash +cd $WORK_SPACE/source/pdc/scripts/llsm_importer +``` + +Modify the template script `template.sh`. + +Change `EXECPATH` to where your `pdc_server.exe` is installed +Change `TOOLPATH` to where your `llsm_importer` artifact is. + +Change `LLSM_DATA_PATH` to where your sample dataset is. For exmaple, + +```bash +LLSM_DATA_PATH=/pscratch/sd/w/wzhang5/data/llsm/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1 +``` + +Note: you may download the sample dataset from the this [link](https://drive.google.com/file/d/19hH7v58iF_QBJ985ajwLD86MMseBH-YR/view?usp=sharing). It is provided with the courtesy of [Advanced BioImaging Center at UC Berkeley](https://mcb.berkeley.edu/faculty/cdb/upadhyayulas). + +Now, run `gen_script.sh` to generate scripts for different settings with various number of servers. + +After this, enter into any directory named with a number, and submit the job with `sbatch` command. + +Note: This program is still under development and changes will be made available from time to time. Please always use the develop branch for a stable version of this llsm_importer tool. \ No newline at end of file diff --git a/tools/llsm/csvReader.c b/tools/llsm/csvReader.c new file mode 100644 index 000000000..c6623a287 --- /dev/null +++ b/tools/llsm/csvReader.c @@ -0,0 +1,395 @@ +#include "csvReader.h" + +char csv_delimiter = ','; +char csv_quote = '\"'; +char csv_escape = '\\'; +char csv_newline = '\n'; + +void +csv_set_delimiter(char delimiter) +{ + csv_delimiter = delimiter; +} + +void +csv_set_quote(char quote) +{ + csv_quote = quote; +} + +void +csv_set_escape(char escape) +{ + csv_escape = escape; +} + +void +csv_set_newline(char newline) +{ + csv_newline = newline; +} + +csv_header_t * +csv_parse_header(char *line, char *field_types) +{ + csv_header_t *first_header = NULL; + csv_header_t *last_header = NULL; + char * token = NULL; + char * saveptr = NULL; + int field_index = 0; + int in_quotes = 0; + int value_start = 0; + int i = 0; + + for (int i = 0; line[i] != csv_newline; ++i) { + if (line[i] == csv_quote) { + in_quotes = !in_quotes; + } + else if (!in_quotes && (line[i] == csv_delimiter || line[i + 1] == csv_newline)) { + // Allocate memory for the header struct + csv_header_t *header = (csv_header_t *)malloc(sizeof(csv_header_t)); + if (header == NULL) { + return NULL; + } + // Remove quotes and spaces from the field name + header->field_name = strndup(line + value_start, i - value_start + (line[i + 1] == csv_newline)); + + // Set the field index + header->field_index = field_index; + + // Set the field type + if (field_types != NULL) { + header->field_type = field_types[field_index]; + } + else { + header->field_type = 's'; + } + + // Set the next pointer to NULL + header->next = NULL; + + // Add the header to the linked list + if (first_header == NULL) { + first_header = header; + last_header = header; + } + else { + last_header->next = header; + last_header = header; + } + + value_start = i + 1; + field_index++; + } + } + + return first_header; +} + +csv_row_t * +csv_parse_row(char *line, csv_header_t *header) +{ + csv_cell_t * first_cell = NULL; + csv_cell_t * last_cell = NULL; + csv_header_t *current_header = header; + char * token = NULL; + char * saveptr = NULL; + int field_index = 0; + int in_quotes = 0; + int value_start = 0; + int i = 0; + + for (int i = 0; line[i] != csv_newline; ++i) { + if (line[i] == csv_quote) { + in_quotes = !in_quotes; + } + else if (!in_quotes && (line[i] == csv_delimiter || line[i + 1] == csv_newline)) { + // Allocate memory for the cell struct + csv_cell_t *cell = (csv_cell_t *)malloc(sizeof(csv_cell_t)); + if (cell == NULL) { + return NULL; + } + + // Set the field name + cell->header = current_header; + + // Set the field value + cell->field_value = strndup(line + value_start, i - value_start + (line[i + 1] == csv_newline)); + + // Set the next pointer to NULL + cell->next = NULL; + + // Add the cell to the linked list + if (first_cell == NULL) { + first_cell = cell; + last_cell = cell; + } + else { + last_cell->next = cell; + last_cell = cell; + } + + value_start = i + 1; + field_index++; + current_header = current_header->next; + } + } + csv_row_t *row = (csv_row_t *)malloc(sizeof(csv_row_t)); + row->first_cell = first_cell; + row->next = NULL; + return row; +} + +csv_cell_t * +csv_get_field_value_by_name(csv_row_t *row, csv_header_t *header, char *field_name) +{ + csv_cell_t *cell = row->first_cell; + while (cell != NULL) { + if (strcmp(cell->header->field_name, field_name) == 0) { + return cell; + } + cell = cell->next; + } + return NULL; +} + +csv_cell_t * +csv_get_field_value_by_index(csv_row_t *row, csv_header_t *header, int field_index) +{ + csv_cell_t *cell = row->first_cell; + while (cell != NULL) { + if (cell->header->field_index == field_index) { + return cell; + } + cell = cell->next; + } + return NULL; +} + +csv_table_t * +csv_parse_file(char *file_name, char *field_types) +{ + FILE *fp = fopen(file_name, "r"); + if (fp == NULL) { + return NULL; + } + + // Allocate memory for the table struct + csv_table_t *table = (csv_table_t *)malloc(sizeof(csv_table_t)); + if (table == NULL) { + return NULL; + } + + // Read the first line of the file + char * line = NULL; + size_t len = 0; + ssize_t read = getline(&line, &len, fp); + + // Parse the header + table->first_header = csv_parse_header(line, field_types); + + // Parse the rows + csv_row_t *first_row = NULL; + csv_row_t *last_row = NULL; + while ((read = getline(&line, &len, fp)) != -1) { + // Allocate memory for the row struct + csv_row_t *row = csv_parse_row(line, table->first_header); + if (row == NULL) { + return NULL; + } + + // Add the row to the linked list + if (first_row == NULL) { + first_row = row; + last_row = row; + } + else { + last_row->next = row; + last_row = row; + } + } + + table->first_row = first_row; + + return table; +} + +csv_table_t * +csv_parse_list(PDC_LIST *list, char *field_types) +{ + csv_table_t *table = (csv_table_t *)malloc(sizeof(csv_table_t)); + if (table == NULL) { + return NULL; + } + int num_file_read = 0; + csv_row_t *first_row = NULL; + csv_row_t *last_row = NULL; + + PDC_LIST_ITERATOR *iter = pdc_list_iterator_new(list); + while (pdc_list_iterator_has_next(iter)) { + char *line = (char *)pdc_list_iterator_next(iter); + if (num_file_read == 0) { + table->first_header = csv_parse_header(line, field_types); + } + else { + // Allocate memory for the row struct + csv_row_t *row = csv_parse_row(line, table->first_header); + if (row == NULL) { + return NULL; + } + + // Add the row to the linked list + if (first_row == NULL) { + first_row = row; + last_row = row; + } + else { + last_row->next = row; + last_row = row; + } + } + num_file_read++; + } + + table->first_row = first_row; + + return table; +} + +void +csv_free_header(csv_header_t *header) +{ + csv_header_t *current_header = header; + csv_header_t *next_header = NULL; + while (current_header != NULL) { + next_header = current_header->next; + free(current_header->field_name); + free(current_header); + current_header = next_header; + } +} + +void +csv_free_row(csv_row_t *row) +{ + csv_row_t *current_row = row; + csv_row_t *next_row = NULL; + while (current_row != NULL) { + next_row = current_row->next; + csv_free_cell(current_row->first_cell); + free(current_row); + current_row = next_row; + } +} + +void +csv_free_cell(csv_cell_t *cell) +{ + csv_cell_t *current_cell = cell; + csv_cell_t *next_cell = NULL; + while (current_cell != NULL) { + next_cell = current_cell->next; + free(current_cell->field_value); + free(current_cell); + current_cell = next_cell; + } +} + +void +csv_free_table(csv_table_t *table) +{ + csv_free_header(table->first_header); + csv_free_row(table->first_row); + free(table); +} + +void +csv_print_header(csv_header_t *header) +{ + csv_header_t *current_header = header; + while (current_header != NULL) { + printf("%s", current_header->field_name); + if (current_header->next != NULL) { + printf(", "); + } + current_header = current_header->next; + } + printf("\n"); +} + +void +csv_print_row(csv_row_t *row, int with_key) +{ + csv_cell_t *current_cell = row->first_cell; + while (current_cell != NULL) { + csv_print_cell(current_cell, with_key); + if (current_cell->next != NULL) { + printf(", "); + } + if (with_key) { + printf("\n"); + } + current_cell = current_cell->next; + } + printf("\n"); +} + +void +csv_print_cell(csv_cell_t *cell, int with_key) +{ + if (with_key) { + printf("%s: ", cell->header->field_name); + } + switch (cell->header->field_type) { + case 'i': + printf("%ld", strtol(cell->field_value, NULL, 10)); + break; + + case 'f': + printf("%f", strtod(cell->field_value, NULL)); + break; + + case 's': + printf("%s", cell->field_value); + break; + + default: + printf("%s", cell->field_value); + break; + } +} + +void +csv_print_table(csv_table_t *table) +{ + csv_print_header(table->first_header); + csv_row_t *current_row = table->first_row; + while (current_row != NULL) { + csv_print_row(current_row, 0); + current_row = current_row->next; + } +} + +int +csv_get_num_rows(csv_table_t *table) +{ + int num_rows = 0; + csv_row_t *current_row = table->first_row; + while (current_row != NULL) { + num_rows++; + current_row = current_row->next; + } + return num_rows; +} + +int +csv_get_num_fields(csv_table_t *table) +{ + int num_fields = 0; + csv_header_t *current_header = table->first_header; + while (current_header != NULL) { + num_fields++; + current_header = current_header->next; + } + return num_fields; +} \ No newline at end of file diff --git a/tools/llsm/csvReader.h b/tools/llsm/csvReader.h new file mode 100644 index 000000000..d5aa87aaf --- /dev/null +++ b/tools/llsm/csvReader.h @@ -0,0 +1,190 @@ +#ifndef CSVREADER_H +#define CSVREADER_H + +#include +#include +#include + +#include "pdc_list.h" + +typedef struct csv_header_t { + char * field_name; + int field_index; + char field_type; + struct csv_header_t *next; +} csv_header_t; + +typedef struct csv_cell_t { + char * field_value; + csv_header_t * header; + struct csv_cell_t *next; +} csv_cell_t; + +typedef struct csv_row_t { + csv_cell_t * first_cell; + struct csv_row_t *next; +} csv_row_t; + +typedef struct csv_table_t { + csv_header_t *first_header; + csv_row_t * first_row; +} csv_table_t; + +/** + * @brief This function sets the delimiter for the CSV file. The default is a comma. + * @param delimiter The delimiter to use. + */ +void csv_set_delimiter(char delimiter); + +/** + * @brief This function sets the quote character for the CSV file. The default is a double quote. + * @param quote The quote character to use. + */ +void csv_set_quote(char quote); + +/** + * @brief This function sets the escape character for the CSV file. The default is a backslash. + * @param escape The escape character to use. + */ +void csv_set_escape(char escape); + +/** + * @brief This function sets the newline character for the CSV file. The default is a newline. + * @param newline The newline character to use. + */ +void csv_set_newline(char newline); + +/** + * @brief This function parses a CSV header line and returns a linked list of csv_header_t structs. The header + * string may contain quotes and spaces + * @param line The CSV header line to parse. + * @param field_types A string of field types. The field types are 's' for string, 'i' for long integer, 'f' + * for float, and 'd' for double. If this is NULL, all fields are assumed to be strings. + * + * @return A pointer to the first csv_header_t struct in the linked list. + */ +csv_header_t *csv_parse_header(char *line, char *field_types); + +/** + * @brief This function parse a CSV row line and returns a linked list of csv_cell_t structs. The row string + * may contain quotes and spaces + * @param line The CSV row line to parse. + * @param header A pointer to the first csv_header_t struct in the linked list. + * + * @return A pointer to the csv_row_t struct. The value in the csv_cell should be + * free of quotes or spaces. + */ +csv_row_t *csv_parse_row(char *line, csv_header_t *header); + +/** + * @brief This function returns the string value of a field for a given row string. The row string may contain + * quotes and spaces + * @param row The CSV row to look for. + * @param header A pointer to the first csv_header_t struct in the linked list. + * @param field_name The name of the field to get the value for. + * + * @return A pointer to the csv_cell struct of the field. The value in the csv_cell should be free of quotes + * or spaces. + */ +csv_cell_t *csv_get_field_value_by_name(csv_row_t *row, csv_header_t *header, char *field_name); + +/** + * @brief This function returns the string value of a field for a given row string. The row string may contain + * quotes and spaces + * @param row The CSV row to look for. + * @param header A pointer to the first csv_header_t struct in the linked list. + * @param field_index The index of the field to get the value for. + * + * @return A pointer to the csv_cell struct of the field. The value in the csv_cell should be free of quotes + * or spaces. + */ +csv_cell_t *csv_get_field_value_by_index(csv_row_t *row, csv_header_t *header, int field_index); + +/** + * @brief This function parses a CSV file and returns a csv_table_t struct. + * @param file_name The name of the CSV file to parse. + * @param field_types A string of field types. The field types are 's' for string, 'i' for long integer, 'f' + * for float, and 'd' for double. If this is NULL, all fields are assumed to be strings. + * + * @return A pointer to the csv_table_t struct. + */ +csv_table_t *csv_parse_file(char *file_name, char *field_types); + +/** + * @brief This function parses a PDC_LIST of strings as a CSV file and returns a csv_table_t struct. + * @param list A PDC_LIST of strings to parse. + * @param field_types A string of field types. The field types are 's' for string, 'i' for long integer, 'f' + * for float, and 'd' for double. If this is NULL, all fields are assumed to be strings. + * + * @return A pointer to the csv_table_t struct. + */ +csv_table_t *csv_parse_list(PDC_LIST *list, char *field_types); + +/** + * @brief This function frees the memory allocated for a csv_table_t struct. + * @param table A pointer to the csv_table_t struct to free. + */ +void csv_free_table(csv_table_t *table); + +/** + * @brief This function frees the memory allocated for a csv_header_t struct. + * @param header A pointer to the csv_header_t struct to free. + */ +void csv_free_header(csv_header_t *header); + +/** + * @brief This function frees the memory allocated for a csv_row_t struct. + * @param row A pointer to the csv_row_t struct to free. + */ +void csv_free_row(csv_row_t *row); + +/** + * @brief This function frees the memory allocated for a csv_cell_t struct. + * @param cell A pointer to the csv_cell_t struct to free. + */ +void csv_free_cell(csv_cell_t *cell); + +/** + * @brief This function prints the contents of a csv_table_t struct. + * @param table A pointer to the csv_table_t struct to print. + */ +void csv_print_table(csv_table_t *table); + +/** + * @brief This function prints the contents of a csv_header_t struct. + * @param header A pointer to the csv_header_t struct to print. + */ +void csv_print_header(csv_header_t *header); + +/** + * @brief This function prints the contents of a csv_row_t struct. + * @param row A pointer to the csv_row_t struct to print. + * @param with_key A flag to indicate whether to print the key or not. + */ + +void csv_print_row(csv_row_t *row, int with_key); + +/** + * @brief This function prints the contents of a csv_cell_t struct. + * @param cell A pointer to the csv_cell_t struct to print. + * @param with_key A flag to indicate whether to print the key or not. + */ +void csv_print_cell(csv_cell_t *cell, int with_key); + +/** + * @brief This function returns the number of rows in a csv_table_t struct. + * @param table A pointer to the csv_table_t struct. + * + * @return The number of rows in the table. + */ +int csv_get_num_rows(csv_table_t *table); + +/** + * @brief This function returns the number of fields in a csv_table_t struct. + * @param table A pointer to the csv_table_t struct. + * + * @return The number of fields in the table. + */ +int csv_get_num_fields(csv_table_t *table); + +#endif // CSVREADER_H \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c new file mode 100644 index 000000000..cc4026ac9 --- /dev/null +++ b/tools/llsm/parallelReadTiff.c @@ -0,0 +1,818 @@ +#include "parallelReadTiff.h" +#include "tiffio.h" + +#define ENABLE_OPENMP + +#ifdef ENABLE_OPENMP +#include "omp.h" +#endif + +#define CREATE_ARRAY(result_var, type, ndim, dim) \ + do { \ + size_t i = 0, dim_prod = 1; \ + for (i = 0; i < (ndim); i++) { \ + dim_prod *= (dim)[i]; \ + } \ + result_var = (void *)malloc(dim_prod * sizeof(type)); \ + } while (0) + +void +DummyHandler(const char *module, const char *fmt, va_list ap) +{ + // ignore errors and warnings +} + +// Backup method in case there are errors reading strips +void +readTiffParallelBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (z - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; + + printf("numWorkers %d\n", numWorkers); + + int32_t w; +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (w = 0; w < numWorkers; w++) { + + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:threadError | Thread %d: File \"%s\" cannot be opened\n", w, fileName); + + void *buffer = malloc(x * bytes); + for (int64_t dir = startSlice + (w * batchSize); dir < startSlice + ((w + 1) * batchSize); dir++) { + if (dir >= z + startSlice) + break; + + int counter = 0; + while (!TIFFSetDirectory(tif, (uint64_t)dir) && counter < 3) { + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n", w, fileName, dir, + counter + 1); + counter++; + } + + for (int64_t i = 0; i < y; i++) { + TIFFReadScanline(tif, buffer, i, 0); + if (!flipXY) { + memcpy(tiff + ((i * x) * bytes), buffer, x * bytes); + continue; + } + // loading the data into a buffer + switch (bits) { + case 8: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint8_t *)buffer)[j]; + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint16_t *)buffer)[j]; + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((float *)buffer)[j]; + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((double *)buffer)[j]; + } + break; + } + } + } + free(buffer); + TIFFClose(tif); + } +} + +void +readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (z - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; + + printf("numWorkers %d\n", numWorkers); + + uint16_t compressed = 1; + TIFF * tif = TIFFOpen(fileName, "r"); + TIFFGetField(tif, TIFFTAG_COMPRESSION, &compressed); + + int32_t w; + uint8_t errBak = 0; + uint8_t err = 0; + char errString[10000]; + if (compressed > 1 || z < 32768) { + TIFFClose(tif); +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (w = 0; w < numWorkers; w++) { + + uint8_t outCounter = 0; + TIFF * tif = TIFFOpen(fileName, "r"); + while (!tif) { + tif = TIFFOpen(fileName, "r"); + if (outCounter == 3) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + err = 1; + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); + } + continue; + } + outCounter++; + } + + void *buffer = malloc(x * stripSize * bytes); + for (int64_t dir = startSlice + (w * batchSize); dir < startSlice + ((w + 1) * batchSize); + dir++) { + if (dir >= z + startSlice || err) + break; + + uint8_t counter = 0; + while (!TIFFSetDirectory(tif, (uint64_t)dir) && counter < 3) { + counter++; + if (counter == 3) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + err = 1; + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); + } + } + } + if (err) + break; + for (int64_t i = 0; i * stripSize < y; i++) { + + // loading the data into a buffer + int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize * x * bytes); + if (cBytes < 0) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + errBak = 1; + err = 1; + sprintf(errString, "Thread %d: Strip %ld cannot be read\n", w, i); + } + break; + } + if (!flipXY) { + memcpy(tiff + ((i * stripSize * x) * bytes), buffer, cBytes); + continue; + } + switch (bits) { + case 8: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((uint8_t *)buffer)[j + (k * x)]; + } + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((uint16_t *)buffer)[j + (k * x)]; + } + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((float *)buffer)[j + (k * x)]; + } + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((double *)buffer)[j + (k * x)]; + } + } + break; + } + } + } + free(buffer); + TIFFClose(tif); + } + } + else { + uint64_t stripsPerDir = (uint64_t)ceil((double)y / (double)stripSize); +#ifdef _WIN32 + int fd = open(fileName, O_RDONLY | O_BINARY); +#else + int fd = open(fileName, O_RDONLY); +#endif + if (fd == -1) + printf("disk:threadError | File \"%s\" cannot be opened from Disk\n", fileName); + + if (!tif) + printf("tiff:threadError | File \"%s\" cannot be opened\n", fileName); + uint64_t offset = 0; + uint64_t *offsets = NULL; + TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); + uint64_t *byteCounts = NULL; + TIFFGetField(tif, TIFFTAG_STRIPBYTECOUNTS, &byteCounts); + if (!offsets || !byteCounts) + printf("tiff:threadError | Could not get offsets or byte counts from the tiff file\n"); + offset = offsets[0]; + uint64_t fOffset = offsets[stripsPerDir - 1] + byteCounts[stripsPerDir - 1]; + uint64_t zSize = fOffset - offset; + TIFFSetDirectory(tif, 1); + TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); + uint64_t gap = offsets[0] - fOffset; + + lseek(fd, offset, SEEK_SET); + + TIFFClose(tif); + uint64_t curr = 0; + uint64_t bytesRead = 0; + // TESTING + // Not sure if we will need to read in chunks like for ImageJ + for (uint64_t i = 0; i < z; i++) { + bytesRead = read(fd, tiff + curr, zSize); + curr += bytesRead; + lseek(fd, gap, SEEK_CUR); + } + close(fd); + uint64_t size = x * y * z * (bits / 8); + void * tiffC = malloc(size); + memcpy(tiffC, tiff, size); +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (uint64_t k = 0; k < z; k++) { + for (uint64_t j = 0; j < x; j++) { + for (uint64_t i = 0; i < y; i++) { + switch (bits) { + case 8: + ((uint8_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint8_t *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 16: + ((uint16_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint16_t *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 32: + ((float *)tiff)[i + (j * y) + (k * x * y)] = + ((float *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 64: + ((double *)tiff)[i + (j * y) + (k * x * y)] = + ((double *)tiffC)[j + (i * x) + (k * x * y)]; + break; + } + } + } + } + free(tiffC); + } + if (err) { + if (errBak) + readTiffParallelBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); + else + printf("tiff:threadError %s\n", errString); + } +} + +// Backup method in case there are errors reading strips +void +readTiffParallel2DBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (y - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; + + printf("numWorkers %d\n", numWorkers); + + int32_t w; +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (w = 0; w < numWorkers; w++) { + + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:threadError | Thread %d: File \"%s\" cannot be opened\n", w, fileName); + + void *buffer = malloc(x * bytes); + for (int64_t dir = startSlice + (w * batchSize); dir < startSlice + ((w + 1) * batchSize); dir++) { + if (dir >= z + startSlice) + break; + + int counter = 0; + while (!TIFFSetDirectory(tif, (uint64_t)0) && counter < 3) { + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n", w, fileName, dir, + counter + 1); + counter++; + } + + for (int64_t i = (w * batchSize); i < ((w + 1) * batchSize); i++) { + if (i >= y) + break; + TIFFReadScanline(tif, buffer, i, 0); + if (!flipXY) { + memcpy(tiff + ((i * x) * bytes), buffer, x * bytes); + continue; + } + // loading the data into a buffer + switch (bits) { + case 8: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint8_t *)buffer)[j]; + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint16_t *)buffer)[j]; + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((float *)buffer)[j]; + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((double *)buffer)[j]; + } + break; + } + } + } + free(buffer); + TIFFClose(tif); + } +} + +void +readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + uint64_t stripsPerDir = (uint64_t)ceil((double)y / (double)stripSize); + int32_t batchSize = (stripsPerDir - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; + + int32_t w; + uint8_t err = 0; + uint8_t errBak = 0; + char errString[10000]; + + printf("numWorkers %d\n", numWorkers); + +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (w = 0; w < numWorkers; w++) { + + uint8_t outCounter = 0; + TIFF * tif = TIFFOpen(fileName, "r"); + while (!tif) { + tif = TIFFOpen(fileName, "r"); + if (outCounter == 3) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + err = 1; + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); + } + continue; + } + outCounter++; + } + + void *buffer = malloc(x * stripSize * bytes); + + uint8_t counter = 0; + while (!TIFFSetDirectory(tif, 0) && counter < 3) { + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n", w, fileName, 0, + counter + 1); + counter++; + if (counter == 3) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + err = 1; + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); + } + } + } + for (int64_t i = (w * batchSize); i < (w + 1) * batchSize; i++) { + if (i * stripSize >= y || err) + break; + // loading the data into a buffer + int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize * x * bytes); + if (cBytes < 0) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + errBak = 1; + err = 1; + sprintf(errString, "Thread %d: Strip %ld cannot be read\n", w, i); + } + break; + } + if (!flipXY) { + memcpy(tiff + ((i * stripSize * x) * bytes), buffer, cBytes); + continue; + } + switch (bits) { + case 8: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((uint8_t *)buffer)[j + (k * x)]; + } + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((uint16_t *)buffer)[j + (k * x)]; + } + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((float *)buffer)[j + (k * x)]; + } + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((double *)buffer)[j + (k * x)]; + } + } + break; + } + } + free(buffer); + TIFFClose(tif); + } + + if (err) { + if (errBak) + readTiffParallel2DBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); + else + printf("tiff:threadError %s\n", errString); + } +} + +// Reading images saved by ImageJ +void +readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY) +{ +#ifdef _WIN32 + int fd = open(fileName, O_RDONLY | O_BINARY); +#else + int fd = open(fileName, O_RDONLY); +#endif + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:threadError | File \"%s\" cannot be opened\n", fileName); + uint64_t offset = 0; + uint64_t *offsets = NULL; + TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); + if (offsets) + offset = offsets[0]; + + TIFFClose(tif); + lseek(fd, offset, SEEK_SET); + uint64_t bytes = bits / 8; + // #pragma omp parallel for + /* + for(uint64_t i = 0; i < z; i++){ + uint64_t cOffset = x*y*bytes*i; + //pread(fd,tiff+cOffset,x*y*bytes,offset+cOffset); + read(fd,tiff+cOffset,x*y*bytes); + }*/ + uint64_t chunk = 0; + uint64_t tBytes = x * y * z * bytes; + uint64_t bytesRead; + uint64_t rBytes = tBytes; + if (tBytes < INT_MAX) + bytesRead = read(fd, tiff, tBytes); + else { + while (chunk < tBytes) { + rBytes = tBytes - chunk; + if (rBytes > INT_MAX) + bytesRead = read(fd, tiff + chunk, INT_MAX); + else + bytesRead = read(fd, tiff + chunk, rBytes); + chunk += bytesRead; + } + } + close(fd); + // Swap endianess for types greater than 8 bits + // TODO: May need to change later because we may not always need to swap + if (bits > 8) { +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (uint64_t i = 0; i < x * y * z; i++) { + switch (bits) { + case 16: + //((uint16_t*)tiff)[i] = ((((uint16_t*)tiff)[i] & 0xff) >> 8) | (((uint16_t*)tiff)[i] << + // 8); + //((uint16_t*)tiff)[i] = bswap_16(((uint16_t*)tiff)[i]); + ((uint16_t *)tiff)[i] = + ((((uint16_t *)tiff)[i] << 8) & 0xff00) | ((((uint16_t *)tiff)[i] >> 8) & 0x00ff); + break; + case 32: + //((num & 0xff000000) >> 24) | ((num & 0x00ff0000) >> 8) | ((num & 0x0000ff00) << 8) | + //(num << 24) + //((float*)tiff)[i] = bswap_32(((float*)tiff)[i]); + ((uint32_t *)tiff)[i] = ((((uint32_t *)tiff)[i] << 24) & 0xff000000) | + ((((uint32_t *)tiff)[i] << 8) & 0x00ff0000) | + ((((uint32_t *)tiff)[i] >> 8) & 0x0000ff00) | + ((((uint32_t *)tiff)[i] >> 24) & 0x000000ff); + break; + case 64: + //((double*)tiff)[i] = bswap_64(((double*)tiff)[i]); + ((uint64_t *)tiff)[i] = ((((uint64_t *)tiff)[i] << 56) & 0xff00000000000000UL) | + ((((uint64_t *)tiff)[i] << 40) & 0x00ff000000000000UL) | + ((((uint64_t *)tiff)[i] << 24) & 0x0000ff0000000000UL) | + ((((uint64_t *)tiff)[i] << 8) & 0x000000ff00000000UL) | + ((((uint64_t *)tiff)[i] >> 8) & 0x00000000ff000000UL) | + ((((uint64_t *)tiff)[i] >> 24) & 0x0000000000ff0000UL) | + ((((uint64_t *)tiff)[i] >> 40) & 0x000000000000ff00UL) | + ((((uint64_t *)tiff)[i] >> 56) & 0x00000000000000ffUL); + break; + } + } + } + // Find a way to do this in-place without making a copy + if (flipXY) { + uint64_t size = x * y * z * (bits / 8); + void * tiffC = malloc(size); + memcpy(tiffC, tiff, size); +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (uint64_t k = 0; k < z; k++) { + for (uint64_t j = 0; j < x; j++) { + for (uint64_t i = 0; i < y; i++) { + switch (bits) { + case 8: + ((uint8_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint8_t *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 16: + ((uint16_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint16_t *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 32: + ((float *)tiff)[i + (j * y) + (k * x * y)] = + ((float *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 64: + ((double *)tiff)[i + (j * y) + (k * x * y)] = + ((double *)tiffC)[j + (i * x) + (k * x * y)]; + break; + } + } + } + } + free(tiffC); + } +} + +uint8_t +isImageJIm(TIFF *tif) +{ + if (!tif) + return 0; + char *tiffDesc = NULL; + if (TIFFGetField(tif, TIFFTAG_IMAGEDESCRIPTION, &tiffDesc)) { + if (strstr(tiffDesc, "ImageJ")) { + return 1; + } + } + return 0; +} + +uint64_t +imageJImGetZ(TIFF *tif) +{ + if (!tif) + return 0; + char *tiffDesc = NULL; + if (TIFFGetField(tif, TIFFTAG_IMAGEDESCRIPTION, &tiffDesc)) { + if (strstr(tiffDesc, "ImageJ")) { + char *nZ = strstr(tiffDesc, "images="); + if (nZ) { + nZ += 7; + char *temp; + return strtol(nZ, &temp, 10); + } + } + } + return 0; +} + +void +get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, uint64_t *y, uint64_t *z, + uint64_t *bits, uint64_t *startSlice, uint64_t *stripSize, uint64_t *is_imageJ, + uint64_t *imageJ_Z) +{ + TIFFSetWarningHandler(DummyHandler); + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:inputError | File \"%s\" cannot be opened", fileName); + + TIFFGetField(tif, TIFFTAG_IMAGEWIDTH, x); + TIFFGetField(tif, TIFFTAG_IMAGELENGTH, y); + + if (strip_range == NULL) { + uint16_t s = 0, m = 0, t = 1; + while (TIFFSetDirectory(tif, t)) { + s = t; + t *= 8; + if (s > t) { + t = 65535; + printf("Number of slices > 32768\n"); + break; + } + } + while (s != t) { + m = (s + t + 1) / 2; + if (TIFFSetDirectory(tif, m)) { + s = m; + } + else { + if (m > 0) + t = m - 1; + else + t = m; + } + } + *z = s + 1; + } + else { + if (strip_range->length != 2) { + printf("tiff:inputError | Input range is not 2"); + } + else { + *startSlice = (uint64_t)(*(strip_range->range)) - 1; + *z = (uint64_t)(*(strip_range->range + 1)) - startSlice[0]; + if (!TIFFSetDirectory(tif, startSlice[0] + z[0] - 1) || !TIFFSetDirectory(tif, startSlice[0])) { + printf("tiff:rangeOutOfBound | Range is out of bounds"); + } + } + } + + *is_imageJ = isImageJIm(tif); + *imageJ_Z = imageJImGetZ(tif); + if (*is_imageJ) { + *is_imageJ = 1; + *imageJ_Z = imageJImGetZ(tif); + if (*imageJ_Z) + *z = *imageJ_Z; + } + + TIFFGetField(tif, TIFFTAG_BITSPERSAMPLE, bits); + TIFFGetField(tif, TIFFTAG_ROWSPERSTRIP, stripSize); + TIFFClose(tif); +} + +void * +_get_tiff_array(int bits, int ndim, size_t *dims) +{ + void *tiff = NULL; + if (bits == 8) { + CREATE_ARRAY(tiff, uint8_t, ndim, dims); + } + else if (bits == 16) { + CREATE_ARRAY(tiff, uint16_t, ndim, dims); + } + else if (bits == 32) { + CREATE_ARRAY(tiff, float, ndim, dims); + } + else if (bits == 64) { + CREATE_ARRAY(tiff, double, ndim, dims); + } + return tiff; +} + +void +_TIFF_load(char *fileName, uint8_t isImageJIm, uint64_t x, uint64_t y, uint64_t z, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY, int ndim, size_t *dims, void **tiff_ptr) +{ + if (tiff_ptr == NULL) { + printf("tiff:dataTypeError, Data type not suppported\n"); + } + *tiff_ptr = _get_tiff_array(bits, ndim, dims); + // Case for ImageJ + if (isImageJIm) { + readTiffParallelImageJ(x, y, z, fileName, *tiff_ptr, bits, startSlice, stripSize, flipXY); + } + // Case for 2D + else if (z <= 1) { + readTiffParallel2D(x, y, z, fileName, *tiff_ptr, bits, startSlice, stripSize, flipXY); + } + // Case for 3D + else { + readTiffParallel(x, y, z, fileName, *tiff_ptr, bits, startSlice, stripSize, flipXY); + } +} + +void +parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, + image_info_t **image_info) +{ + uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0, stripeSize = 1, is_imageJ = 0, imageJ_Z = 0; + + get_tiff_info(fileName, strip_range, &x, &y, &z, &bits, &startSlice, &stripeSize, &is_imageJ, &imageJ_Z); + + int ndim = 3; + uint64_t dims[ndim]; + dims[0] = flipXY ? y : x; + dims[1] = flipXY ? x : y; + dims[2] = z; + + *image_info = (image_info_t *)malloc(sizeof(image_info_t)); + (*image_info)->x = dims[0]; + (*image_info)->y = dims[1]; + (*image_info)->z = dims[2]; + (*image_info)->bits = bits; + (*image_info)->startSlice = startSlice; + (*image_info)->stripeSize = stripeSize; + (*image_info)->is_imageJ = is_imageJ; + (*image_info)->imageJ_Z = imageJ_Z; + (*image_info)->tiff_size = dims[0] * dims[1] * dims[2] * (bits / 8); + + _TIFF_load(fileName, is_imageJ, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, + (void **)&((*image_info)->tiff_ptr)); +} \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.h b/tools/llsm/parallelReadTiff.h new file mode 100644 index 000000000..081640110 --- /dev/null +++ b/tools/llsm/parallelReadTiff.h @@ -0,0 +1,34 @@ +#ifndef PARALLELREADTIFF_H +#define PARALLELREADTIFF_H + +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + uint64_t *range; + size_t length; +} parallel_tiff_range_t; + +typedef struct { + uint64_t x; + uint64_t y; + uint64_t z; + uint64_t bits; + uint64_t startSlice; + uint64_t stripeSize; + uint64_t is_imageJ; + uint64_t imageJ_Z; + void * tiff_ptr; + size_t tiff_size; +} image_info_t; + +void parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, + image_info_t **image_info_t); + +#endif // PARALLELREADTIFF_H \ No newline at end of file diff --git a/tools/llsm/pdc_list.c b/tools/llsm/pdc_list.c new file mode 100644 index 000000000..95c6e59e7 --- /dev/null +++ b/tools/llsm/pdc_list.c @@ -0,0 +1,165 @@ +#include "pdc_list.h" + +PDC_LIST * +pdc_list_new() +{ + return pdc_list_create(100, 2.0); +} + +PDC_LIST * +pdc_list_create(size_t initial_capacity, double expansion_factor) +{ + // Allocate memory for the list struct. + PDC_LIST *list = (PDC_LIST *)malloc(sizeof(PDC_LIST)); + if (list == NULL) { + return NULL; + } + + // Allocate memory for the array of items. + list->items = (void **)malloc(initial_capacity * sizeof(void *)); + if (list->items == NULL) { + free(list); + return NULL; + } + + // Initialize the list fields. + list->item_count = 0; + list->capacity = initial_capacity; + list->expansion_factor = expansion_factor; + + return list; +} + +void +pdc_list_destroy(PDC_LIST *list) +{ + if (list == NULL) { + return; + } + + // Free all allocated memory for each item. + for (size_t i = 0; i < list->item_count; i++) { + free(list->items[i]); + } + + // Free the array of items and the list struct. + free(list->items); + free(list); +} + +void +pdc_list_add(PDC_LIST *list, void *item) +{ + if (list == NULL || item == NULL) { + return; + } + + // Expand the array of items if necessary. + if (list->item_count >= list->capacity) { + list->capacity *= list->expansion_factor; + list->items = (void **)realloc(list->items, list->capacity * sizeof(void *)); + if (list->items == NULL) { + return; + } + } + + // Add the new item to the end of the array. + list->items[list->item_count++] = item; +} + +void * +pdc_list_get(PDC_LIST *list, size_t index) +{ + if (list == NULL || index >= list->item_count) { + return NULL; + } + + // Return a pointer to the item at the given index. + return list->items[index]; +} + +size_t +pdc_list_size(PDC_LIST *list) +{ + if (list == NULL) { + return 0; + } + + // Return the number of items in the list. + return list->item_count; +} + +void +pdc_list_set_expansion_factor(PDC_LIST *list, double expansion_factor) +{ + if (list == NULL) { + return; + } + + // Set the new expansion factor for the list. + list->expansion_factor = expansion_factor; +} + +double +pdc_list_get_expansion_factor(PDC_LIST *list) +{ + if (list == NULL) { + return 0; + } + + // Return the current expansion factor for the list. + return list->expansion_factor; +} + +PDC_LIST_ITERATOR * +pdc_list_iterator_new(PDC_LIST *list) +{ + if (list == NULL) { + return NULL; + } + + // Allocate memory for the iterator struct. + PDC_LIST_ITERATOR *iterator = (PDC_LIST_ITERATOR *)malloc(sizeof(PDC_LIST_ITERATOR)); + if (iterator == NULL) { + return NULL; + } + + // Initialize the iterator fields. + iterator->list = list; + iterator->index = 0; + + return iterator; +} + +void +pdc_list_iterator_destroy(PDC_LIST_ITERATOR *iterator) +{ + if (iterator == NULL) { + return; + } + + // Free the iterator struct. + free(iterator); +} + +void * +pdc_list_iterator_next(PDC_LIST_ITERATOR *iterator) +{ + if (iterator == NULL) { + return NULL; + } + + // Return the next item in the list. + return pdc_list_get(iterator->list, iterator->index++); +} + +int +pdc_list_iterator_has_next(PDC_LIST_ITERATOR *iterator) +{ + if (iterator == NULL) { + return 0; + } + + // Return true if there are more items in the list. + return iterator->index < pdc_list_size(iterator->list); +} \ No newline at end of file diff --git a/tools/llsm/pdc_list.h b/tools/llsm/pdc_list.h new file mode 100644 index 000000000..aa71e6124 --- /dev/null +++ b/tools/llsm/pdc_list.h @@ -0,0 +1,110 @@ +#ifndef PDC_LIST_H +#define PDC_LIST_H + +#include + +/** + * A generic list data structure that stores a variable number of items of any type. + */ +typedef struct { + void **items; // Pointer to the array of items. + size_t item_count; // Number of items in the list. + size_t capacity; // Capacity of the array of items. + double expansion_factor; // Factor by which the capacity is expanded. +} PDC_LIST; + +/** + * A generic iterator for iterating over the items in a PDC_LIST. + */ +typedef struct { + PDC_LIST *list; // The list being iterated over. + size_t index; // The index of the next item to be returned. +} PDC_LIST_ITERATOR; + +/** + * Creates a new PDC_LIST with default initial capacity 100 and default expansion factor 2.0. + * @return A pointer to the new PDC_LIST. + */ +PDC_LIST *pdc_list_new(); + +/** + * Creates a new PDC_LIST with the given initial capacity and expansion factor. + * @param initial_capacity The initial capacity of the list. + * @param expansion_factor The factor by which the capacity is expanded when the list is full. + * + * @return A pointer to the new PDC_LIST. + */ +PDC_LIST *pdc_list_create(size_t initial_capacity, double expansion_factor); + +/** + * Destroys the given PDC_LIST and frees all allocated memory. + * @param list The PDC_LIST to destroy. + */ +void pdc_list_destroy(PDC_LIST *list); + +/** + * Adds the given item to the end of the given PDC_LIST. + * @param list The PDC_LIST to add the item to. + * @param item The item to add to the PDC_LIST. + * + */ +void pdc_list_add(PDC_LIST *list, void *item); + +/** + * Gets the item at the given index in the given PDC_LIST. + * @param list The PDC_LIST to get the item from. + * @param index The index of the item to get. + * + * @return A pointer to the item at the given index. + */ +void *pdc_list_get(PDC_LIST *list, size_t index); + +/** + * Sets the item at the given index in the given PDC_LIST. + * @param list The PDC_LIST to set the item in. + * + * @return The number of items in the list. + */ +size_t pdc_list_size(PDC_LIST *list); + +/** + * Sets the expansion factor for the given PDC_LIST. + * @param list The PDC_LIST to set the expansion factor for. + * @param expansion_factor The factor by which the capacity is expanded when the list is full. + */ +void pdc_list_set_expansion_factor(PDC_LIST *list, double expansion_factor); + +/** + * Gets the expansion factor for the given PDC_LIST. + * @param list The PDC_LIST to get the expansion factor for. + */ +double pdc_list_get_expansion_factor(PDC_LIST *list); + +/** + * Creates a new PDC_LIST_ITERATOR for the given PDC_LIST. + * @param list The PDC_LIST to create the iterator for. + * @return A pointer to the new PDC_LIST_ITERATOR. + */ +PDC_LIST_ITERATOR *pdc_list_iterator_new(PDC_LIST *list); + +/** + * Destroys the given PDC_LIST_ITERATOR and frees all allocated memory. + * @param iterator The PDC_LIST_ITERATOR to destroy. + */ +void pdc_list_iterator_destroy(PDC_LIST_ITERATOR *iterator); + +/** + * Returns the next item in the PDC_LIST_ITERATOR. + * @param iterator The PDC_LIST_ITERATOR to get the next item from. + * @return A pointer to the next item in the PDC_LIST_ITERATOR. + */ +void *pdc_list_iterator_next(PDC_LIST_ITERATOR *iterator); + +/** + * Returns true if the PDC_LIST_ITERATOR has more items. + * @param iterator The PDC_LIST_ITERATOR to check. + * @return True if the PDC_LIST_ITERATOR has more items. + */ +int pdc_list_iterator_has_next(PDC_LIST_ITERATOR *iterator); + +#endif // PDC_LIST_H \ No newline at end of file diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c new file mode 100644 index 000000000..ea5097278 --- /dev/null +++ b/tools/llsm_importer.c @@ -0,0 +1,392 @@ +#include +#include +#include +#include +#include + +#ifndef ENABLE_MPI +#define ENABLE_MPI +#endif + +#ifdef ENABLE_MPI +#include "mpi.h" +// #undef ENABLE_MPI +#endif + +#include "pdc.h" +// #include "pdc_client_server_common.h" +// #include "pdc_client_connect.h" + +#include "llsm/parallelReadTiff.h" +#include "llsm/pdc_list.h" +#include "llsm/csvReader.h" +#include + +typedef struct llsm_importer_args_t { + char * directory_path; + csv_header_t *csv_header; +} llsm_importer_args_t; + +int rank = 0, size = 1; + +pdcid_t pdc_id_g = 0, cont_prop_g = 0, cont_id_g = 0, obj_prop_g = 0; +struct timespec ts; + +double +getDoubleTimestamp() +{ + clock_gettime(CLOCK_MONOTONIC, &ts); + double timestamp = (double)ts.tv_sec + (double)ts.tv_nsec / 1e9; + return timestamp; +} + +int +parse_console_args(int argc, char *argv[], char **file_name) +{ + int c, parse_code = -1; + + while ((c = getopt(argc, argv, "f:")) != -1) { + switch (c) { + case 'f': + *file_name = optarg; + parse_code = 0; + break; + default: + fprintf(stderr, "Usage: %s [-f filename]\n", argv[0]); + parse_code = -1; + exit(EXIT_FAILURE); + } + } + return parse_code; +} + +void +import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) +{ + double duration, start; + + start = getDoubleTimestamp(); // start timing the operation + + obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); + + psize_t ndims = 3; + uint64_t offsets[3] = {0, 0, 0}; + // FIXME: we should support uint64_t. + uint64_t dims[3] = {image_info->x, image_info->y, image_info->z}; + + // psize_t ndims = 1; + // uint64_t offsets[1] = {0}; + // // FIXME: we should support uint64_t. + // uint64_t dims[1] = {image_info->x * image_info->y * image_info->z}; + + // FIXME: we should change the ndims parameter to psize_t type. + PDCprop_set_obj_dims(obj_prop_g, (PDC_int_t)ndims, dims); + pdc_var_type_t pdc_type = PDC_UNKNOWN; + switch (image_info->bits) { + case 8: + pdc_type = PDC_UINT8; + break; + case 16: + pdc_type = PDC_UINT16; + break; + case 32: + pdc_type = PDC_FLOAT; + break; + case 64: + pdc_type = PDC_DOUBLE; + break; + default: + printf("Error: unsupported data type.\n"); + exit(-1); + } + PDCprop_set_obj_type(obj_prop_g, pdc_type); + PDCprop_set_obj_time_step(obj_prop_g, 0); + PDCprop_set_obj_user_id(obj_prop_g, getuid()); + PDCprop_set_obj_app_name(obj_prop_g, "LLSM"); + + // uint64_t *offsets = (uint64_t *)malloc(sizeof(uint64_t) * ndims); + // uint64_t *num_bytes = (uint64_t *)malloc(sizeof(uint64_t) * ndims); + // for (int i = 0; i < ndims; i++) { + // offsets[i] = 0; + // num_bytes[i] = dims[i] * image_info->bits / 8; + // } + + // create object + // FIXME: There are many attributes currently in one file name, + // and we should do some research to see what would be a good object name for each image. + pdcid_t cur_obj_g = PDCobj_create(cont_id_g, fileName_cell->field_value, obj_prop_g); + + // write data to object + pdcid_t local_region = PDCregion_create(ndims, offsets, dims); + pdcid_t remote_region = PDCregion_create(ndims, offsets, dims); + pdcid_t transfer_request = + PDCregion_transfer_create(image_info->tiff_ptr, PDC_WRITE, cur_obj_g, local_region, remote_region); + PDCregion_transfer_start(transfer_request); + PDCregion_transfer_wait(transfer_request); + + duration = getDoubleTimestamp() - start; // end timing the operation and calculate duration in nanoseconds + + printf("[Rank %4d] Region_Transfer %s_[%d_Bytes] Done! Time taken: %.4f seconds\n", rank, + fileName_cell->field_value, image_info->tiff_size, duration); + + // add metadata tags based on the csv row + csv_cell_t *cell = fileName_cell; + while (cell != NULL) { + char *field_name = cell->header->field_name; + char data_type = cell->header->field_type; + char *field_value = cell->field_value; + switch (data_type) { + case 'i': + int ivalue = atoi(field_value); + PDCobj_put_tag(cur_obj_g, field_name, &ivalue, PDC_INT, sizeof(int)); + break; + case 'f': + double fvalue = atof(field_value); + PDCobj_put_tag(cur_obj_g, field_name, &fvalue, PDC_DOUBLE, sizeof(double)); + break; + case 's': + PDCobj_put_tag(cur_obj_g, field_name, field_value, PDC_STRING, strlen(field_value)); + break; + default: + break; + } + cell = cell->next; + } + + // add extra metadata tags based on the image_info struct + PDCobj_put_tag(cur_obj_g, "x", &(image_info->x), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "y", &(image_info->y), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "z", &(image_info->z), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "bits", &(image_info->bits), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "startSlice", &(image_info->startSlice), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "stripeSize", &(image_info->stripeSize), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "is_imageJ", &(image_info->is_imageJ), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "imageJ_Z", &(image_info->imageJ_Z), PDC_UINT64, sizeof(uint64_t)); + + // close object + PDCobj_close(cur_obj_g); + + // get timing + duration = getDoubleTimestamp() - start; // end timing the operation calculate duration in nanoseconds + + printf("[Rank %4d] Create_object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, + duration); + + // free memory + // free(offsets); + // free(num_bytes); + // PDCregion_close(local_region); + // PDCregion_close(remote_region); + // PDCregion_transfer_close(transfer_request); + PDCprop_close(obj_prop_g); +} + +void +on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) +{ + csv_print_row(row, 1); + + char *dirname = strdup(llsm_args->directory_path); + char filepath[256]; + // calling tiff loading process. + image_info_t *image_info = NULL; + int i = 0; + double duration, start; + // Filepath,Filename,StageX_um_,StageY_um_,StageZ_um_,ObjectiveX_um_,ObjectiveY_um_,ObjectiveZ_um_ + + // get the file name from the csv row + csv_cell_t *fileName_cell = csv_get_field_value_by_name(row, llsm_args->csv_header, "Filename"); + + // check if the path ends with a forward slash + if (dirname[strlen(dirname) - 1] != '/') { + strcat(dirname, "/"); // add a forward slash to the end of the path + } + + strcpy(filepath, dirname); // copy the directory path to the file path + strcat(filepath, fileName_cell->field_value); // concatenate the file name to the file path + + start = getDoubleTimestamp(); // start timing the operation + + parallel_TIFF_load(filepath, 1, NULL, &image_info); + + duration = getDoubleTimestamp() - start; // end timing the operation and calculate duration in nanoseconds + + printf("[Rank %4d] Read %s Done! Time taken: %.4f seconds\n", rank, filepath, duration); + + if (image_info == NULL || image_info->tiff_ptr == NULL) { + return; + } + + printf("first few bytes "); + for (i = 0; i < 10; i++) { + printf("%d ", ((uint8_t *)image_info->tiff_ptr)[i]); + } + printf("\n"); + + // import the image to PDC + import_to_pdc(image_info, fileName_cell); + + // free the image info + free(image_info->tiff_ptr); + free(image_info); + free(dirname); +} + +void +read_txt(char *txtFileName, PDC_LIST *list, int *max_row_length) +{ + FILE *file = fopen(txtFileName, "r"); + + int row_length = 0; + + if (file == NULL) { + printf("Error: could not open file %s\n", txtFileName); + return; + } + char buffer[1024]; + // Read the lines of the file + while (fgets(buffer, sizeof(buffer), file)) { + pdc_list_add(list, strdup(buffer)); + if (row_length < strlen(buffer)) { + row_length = strlen(buffer); + } + } + + fclose(file); + + // Find the maximum row length + *max_row_length = row_length + 5; +} + +int +main(int argc, char *argv[]) +{ + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + char * file_name = NULL; + PDC_LIST * list = pdc_list_new(); + char * csv_line = NULL; + int num_row_read = 0; + csv_header_t * csv_header = NULL; + csv_row_t * csv_row = NULL; + llsm_importer_args_t *llsm_args = NULL; + int bcast_count = 512; + double duration = 0, start = 0; + char csv_field_types[] = {'s', 's', 'f', 'f', 'f', 'f', 'f', 'f'}; + // parse console argument + int parse_code = parse_console_args(argc, argv, &file_name); + if (parse_code) { + return parse_code; + } + char *directory_path = dirname(strdup(file_name)); + + // print file name for validating purpose + printf("Filename: %s\n", file_name ? file_name : "(none)"); + printf("Directory: %s\n", directory_path ? directory_path : "(none)"); + + // create a pdc + pdc_id_g = PDCinit("pdc"); + + // create a container property + cont_prop_g = PDCprop_create(PDC_CONT_CREATE, pdc_id_g); + if (cont_prop_g <= 0) + printf("Fail to create container property @ line %d!\n", __LINE__); + + // create a container + cont_id_g = PDCcont_create("c1", cont_prop_g); + if (cont_id_g <= 0) + printf("Fail to create container @ line %d!\n", __LINE__); + + // Rank 0 reads the filename list and distribute data to other ranks + if (rank == 0) { + read_txt(file_name, list, &bcast_count); + // print bcast_count + printf("bcast_count: %d \n", bcast_count); + +#ifdef ENABLE_MPI + // broadcast the number of lines + int num_lines = pdc_list_size(list); + MPI_Bcast(&num_lines, 1, MPI_INT, 0, MPI_COMM_WORLD); + // broadcast the bcast_count + MPI_Bcast(&bcast_count, 1, MPI_INT, 0, MPI_COMM_WORLD); + // broadcast the file names + PDC_LIST_ITERATOR *iter = pdc_list_iterator_new(list); + while (pdc_list_iterator_has_next(iter)) { + char *csv_line = (char *)pdc_list_iterator_next(iter); + MPI_Bcast(csv_line, bcast_count, MPI_CHAR, 0, MPI_COMM_WORLD); + } +#endif + } + else { +#ifdef ENABLE_MPI + // other ranks receive the number of files + int num_lines; + MPI_Bcast(&num_lines, 1, MPI_INT, 0, MPI_COMM_WORLD); + // receive the bcast_count + MPI_Bcast(&bcast_count, 1, MPI_INT, 0, MPI_COMM_WORLD); + // receive the file names + int i; + for (i = 0; i < num_lines; i++) { + csv_line = (char *)malloc(bcast_count * sizeof(char)); + MPI_Bcast(csv_line, bcast_count, MPI_CHAR, 0, MPI_COMM_WORLD); + pdc_list_add(list, csv_line); + } +#endif + } + // parse the csv + csv_table_t *csv_table = csv_parse_list(list, csv_field_types); + if (csv_table == NULL) { + printf("Fail to parse csv file @ line %d!\n", __LINE__); + return -1; + } + llsm_args = (llsm_importer_args_t *)malloc(sizeof(llsm_importer_args_t)); + llsm_args->directory_path = directory_path; + llsm_args->csv_header = csv_table->first_header; + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + start = MPI_Wtime(); +#else + start = getDoubleTimestamp(); +#endif + // go through the csv table + csv_row_t *current_row = csv_table->first_row; + while (current_row != NULL) { + if (num_row_read % size == rank) { + on_csv_row(current_row, llsm_args); + } + num_row_read++; + current_row = current_row->next; + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + duration = MPI_Wtime() - start; +#else + duration = getDoubleTimestamp() - start; +#endif + + if (rank == 0) { + printf("[Completion Time] LLSM IMPORTER FINISHES! Time taken: %.4f seconds\n", rank, duration); + } + // free memory for csv table + csv_free_table(csv_table); + + // close the container + PDCcont_close(cont_id_g); + // close the container property + PDCprop_close(cont_prop_g); + // close the pdc + PDCclose(pdc_id_g); + +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return 0; +}