diff --git a/.github/codeql-config.yml b/.github/codeql-config.yml new file mode 100644 index 000000000..7adbf6792 --- /dev/null +++ b/.github/codeql-config.yml @@ -0,0 +1,31 @@ +# Query filters to include or exclude specific queries +query-filters: + - exclude: + # See: https://codeql.github.com/codeql-query-help/cpp/cpp-short-global-name/ + id: cpp/short-global-name + - exclude: + # See: https://codeql.github.com/codeql-query-help/cpp/cpp-commented-out-code/ + id: cpp/commented-out-code + - exclude: + # See: https://codeql.github.com/codeql-query-help/cpp/cpp-poorly-documented-function/ + id: cpp/poorly-documented-function + - exclude: + # See: https://codeql.github.com/codeql-query-help/cpp/cpp-trivial-switch/ + id: cpp/trivial-switch + - exclude: + # See: https://codeql.github.com/codeql-query-help/cpp/cpp-irregular-enum-init/ + id: cpp/irregular-enum-init + - exclude: + # See: https://codeql.github.com/codeql-query-help/cpp/cpp-guarded-free/ + id: cpp/guarded-free + +# Directories to scan for vulnerabilities +paths: + - src + +# Directories and files to ignore during the scan +paths-ignore: + - scripts + - examples + - docs + - benchmark diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..ac3271f01 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,119 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Advanced" + +env: + CMAKE_POLICY_VERSION_MINIMUM: 3.5 + +on: + push: + branches: [ "pdc_codeql" ] + pull_request: + branches: [ "pdc_codeql" ] + schedule: + - cron: '33 23 * * 2' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: actions + build-mode: none + - language: c-cpp + build-mode: manual + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install libopenmpi-dev libhdf5-dev uuid-dev cmake + + - name: Build PDC Dependencies + run: | + # libfabric + wget https://github.com/ofiwg/libfabric/archive/refs/tags/v1.12.1.tar.gz + tar xf v1.12.1.tar.gz + cd libfabric-1.12.1 + ./autogen.sh + ./configure --disable-usnic --disable-mrail --disable-rstream --disable-perf --disable-efa --disable-psm2 --disable-psm --disable-verbs --disable-shm --disable-static --disable-silent-rules + make -j2 && sudo make install + cd .. + + # Mercury + git clone --recursive https://github.com/mercury-hpc/mercury.git + cd mercury + git checkout v2.2.0 + mkdir build && cd build + cmake ../ -DCMAKE_C_COMPILER=gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DMERCURY_USE_CHECKSUMS=OFF -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + make -j2 && sudo make install + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + config-file: ./.github/codeql-config.yml + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + queries: security-extended,security-and-quality + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + - name: Build PDC + shell: bash + run: | + mkdir build && cd build + cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DPDC_SERVER_CACHE=ON -DBUILD_TESTING=ON -DPDC_ENABLE_MPI=ON -DCMAKE_C_COMPILER=mpicc -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + make -j2 && sudo make install + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/examples/bdcats.c b/examples/bdcats.c index cd1f1e95f..e7c93c114 100644 --- a/examples/bdcats.c +++ b/examples/bdcats.c @@ -33,7 +33,8 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -82,6 +83,11 @@ main(int argc, char **argv) LOG_INFO("Writing %" PRIu64 " number of particles with %d clients.\n", numparticles, size); } + if (numparticles > MAX_PARTICLES) { + LOG_ERRROR("numparticles exceeds max size\n"); + goto done; + } + x = (float *)malloc(numparticles * sizeof(float)); y = (float *)malloc(numparticles * sizeof(float)); z = (float *)malloc(numparticles * sizeof(float)); @@ -463,6 +469,7 @@ main(int argc, char **argv) free(offset_remote); free(mysize); +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/examples/bdcats_batch.c b/examples/bdcats_batch.c index e69aef296..1bdb74ff9 100644 --- a/examples/bdcats_batch.c +++ b/examples/bdcats_batch.c @@ -33,8 +33,9 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 -#define N_OBJS 8 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) +#define N_OBJS 8 double uniform_random_number() @@ -115,6 +116,11 @@ main(int argc, char **argv) sleep_time, timestep, numparticles, test_method); } + if (numparticles > MAX_PARTICLES) { + LOG_ERRROR("numparticles exceeds max size\n"); + goto done; + } + x = (float *)malloc(numparticles * sizeof(float)); y = (float *)malloc(numparticles * sizeof(float)); z = (float *)malloc(numparticles * sizeof(float)); @@ -710,6 +716,8 @@ main(int argc, char **argv) free(pz); free(id1); free(id2); + +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/examples/bdcats_old.c b/examples/bdcats_old.c index 77c45a8f1..f2108264e 100644 --- a/examples/bdcats_old.c +++ b/examples/bdcats_old.c @@ -33,7 +33,8 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -390,6 +391,7 @@ main(int argc, char **argv) free(offset_remote); free(mysize); +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/examples/read_write_col_perf.c b/examples/read_write_col_perf.c index 7aa15998a..c688e583c 100644 --- a/examples/read_write_col_perf.c +++ b/examples/read_write_col_perf.c @@ -33,7 +33,11 @@ #include "pdc.h" #include "pdc_analysis.h" -#define BUF_LEN 128 +#define BUF_LEN 128 +#define MAX_DATA_SIZE 1000000 +#define GOTO_DONE_ERROR() \ + ret_value = 1; \ + goto done; int main(int argc, char **argv) @@ -73,18 +77,18 @@ main(int argc, char **argv) ndim = 3; } else { - LOG_INFO("usage: ./read_write_perf n_objects data_size1 datasize2 datasize3"); + LOG_INFO("usage: ./read_write_perf n_objects data_size1 datasize2 datasize3\n"); } local_offset[0] = 0; data_size_array[0] = atoi(argv[2]); if (ndim == 1) { - offset[0] = rank * data_size_array[0] * 1048576; + offset[0] = (uint64_t)rank * (uint64_t)data_size_array[0] * 1048576; offset_length[0] = data_size_array[0] * 1048576; data_size = data_size_array[0] * 1048576; } else { - offset[0] = rank * data_size_array[0]; + offset[0] = (uint64_t)rank * (uint64_t)data_size_array[0]; offset_length[0] = data_size_array[0]; data_size = data_size_array[0]; } @@ -113,6 +117,12 @@ main(int argc, char **argv) dims[2] = offset_length[2]; } n_objects = atoi(argv[1]); + + if (data_size > MAX_DATA_SIZE) { + LOG_ERROR("data_size exceeds max size\n"); + GOTO_DONE_ERROR(); + } + int *data = (int *)malloc(sizeof(int) * data_size); char hostname[256]; @@ -135,27 +145,26 @@ main(int argc, char **argv) // create a container property cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); if (cont_prop <= 0) { - LOG_ERROR("Failed to create container property"); - ret_value = 1; + GOTO_DONE_ERROR(); } // create a container sprintf(cont_name, "c%d", rank); cont = PDCcont_create(cont_name, cont_prop); if (cont <= 0) { - LOG_ERROR("Failed to create container"); - ret_value = 1; + LOG_ERROR("Failed to create container\n"); + GOTO_DONE_ERROR(); } // create an object property obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); if (obj_prop <= 0) { - LOG_ERROR("Failed to create object property"); - ret_value = 1; + LOG_ERROR("Failed to create object property\n"); + GOTO_DONE_ERROR(); } ret = PDCprop_set_obj_type(obj_prop, PDC_INT); if (ret != SUCCEED) { - LOG_ERROR("Failed to set obj type"); - ret_value = 1; + LOG_ERROR("Failed to set obj type\n"); + GOTO_DONE_ERROR(); } PDCprop_set_obj_dims(obj_prop, ndim, dims); PDCprop_set_obj_user_id(obj_prop, getuid()); @@ -177,8 +186,8 @@ main(int argc, char **argv) #endif if (obj1 <= 0) { - LOG_ERROR("Failed to create object"); - ret_value = 1; + LOG_ERROR("Failed to create object\n"); + GOTO_DONE_ERROR(); } reg = PDCregion_create(ndim, local_offset, offset_length); reg_global = PDCregion_create(ndim, offset, offset_length); @@ -187,45 +196,45 @@ main(int argc, char **argv) transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global); if (transfer_request == 0) { - LOG_INFO("PDCregion_transfer_create failed"); - ret_value = 1; + LOG_INFO("PDCregion_transfer_create failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); ret = PDCregion_transfer_start(transfer_request); write_reg_transfer_start_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_start failed"); - ret_value = 1; + LOG_INFO("PDCregion_transfer_start failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); ret = PDCregion_transfer_wait(transfer_request); write_reg_transfer_wait_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_wait failed"); - ret_value = 1; + LOG_INFO("PDCregion_transfer_wait failed\n"); + GOTO_DONE_ERROR(); } ret = PDCregion_transfer_close(transfer_request); if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_close failed"); - ret_value = 1; + LOG_INFO("PDCregion_transfer_close failed\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg) < 0) { - LOG_ERROR("Failed to close local region"); - ret_value = 1; + LOG_ERROR("Failed to close local region\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg_global) < 0) { - LOG_ERROR("Failed to close global region"); - ret_value = 1; + LOG_ERROR("Failed to close global region\n"); + GOTO_DONE_ERROR(); } if (PDCobj_close(obj1) < 0) { - LOG_ERROR("Failed to close object o1"); - ret_value = 1; + LOG_ERROR("Failed to close object o1\n"); + GOTO_DONE_ERROR(); } } @@ -235,8 +244,8 @@ main(int argc, char **argv) sprintf(obj_name1, "o1_%d", i); obj1 = PDCobj_open(obj_name1, pdc); if (obj1 <= 0) { - LOG_ERROR("Failed to open object"); - ret_value = 1; + LOG_ERROR("Failed to open object\n"); + GOTO_DONE_ERROR(); } reg = PDCregion_create(ndim, local_offset, offset_length); @@ -247,8 +256,8 @@ main(int argc, char **argv) transfer_request = PDCregion_transfer_create(data, PDC_READ, obj1, reg, reg_global); if (transfer_request == 0) { - LOG_INFO("PDCregion_transfer_create failed"); - ret_value = 1; + LOG_INFO("PDCregion_transfer_create failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); @@ -256,8 +265,8 @@ main(int argc, char **argv) read_reg_transfer_start_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_start failed"); - exit(-1); + LOG_INFO("PDCregion_transfer_start failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); @@ -265,35 +274,35 @@ main(int argc, char **argv) read_reg_transfer_wait_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_wait failed"); - ret_value = 1; + LOG_INFO("PDCregion_transfer_wait failed\n"); + GOTO_DONE_ERROR(); } ret = PDCregion_transfer_close(transfer_request); if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_close failed"); - ret_value = 1; + LOG_INFO("PDCregion_transfer_close failed\n"); + GOTO_DONE_ERROR(); } if (ret != SUCCEED) { - LOG_INFO("PDCbuf_obj_unmap failed"); - ret_value = 1; + LOG_INFO("PDCbuf_obj_unmap failed\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg) < 0) { - LOG_ERROR("Failed to close local region"); - ret_value = 1; + LOG_ERROR("Failed to close local region\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg_global) < 0) { - LOG_ERROR("Failed to close global region"); - ret_value = 1; + LOG_ERROR("Failed to close global region\n"); + GOTO_DONE_ERROR(); } if (PDCobj_close(obj1) < 0) { - LOG_ERROR("Failed to close object o1"); - ret_value = 1; + LOG_ERROR("Failed to close object o1\n"); + GOTO_DONE_ERROR(); } } @@ -301,23 +310,23 @@ main(int argc, char **argv) // close a container if (PDCcont_close(cont) < 0) { - LOG_ERROR("Failed to close container c1"); - ret_value = 1; + LOG_ERROR("Failed to close container c1\n"); + GOTO_DONE_ERROR(); } // close a object property if (PDCprop_close(obj_prop) < 0) { - LOG_ERROR("Failed to close property"); - ret_value = 1; + LOG_ERROR("Failed to close property\n"); + GOTO_DONE_ERROR(); } // close a container property if (PDCprop_close(cont_prop) < 0) { - LOG_ERROR("Failed to close property"); - ret_value = 1; + LOG_ERROR("Failed to close property\n"); + GOTO_DONE_ERROR(); } // close pdc if (PDCclose(pdc) < 0) { - LOG_ERROR("Failed to close PDC"); - ret_value = 1; + LOG_ERROR("Failed to close PDC\n"); + GOTO_DONE_ERROR(); } #ifdef ENABLE_MPI MPI_Reduce(&write_reg_transfer_start_time, &start, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); @@ -345,6 +354,8 @@ main(int argc, char **argv) } free(data); + +done: #endif #ifdef ENABLE_MPI diff --git a/examples/read_write_perf.c b/examples/read_write_perf.c index 923fdae1a..03d839fa6 100644 --- a/examples/read_write_perf.c +++ b/examples/read_write_perf.c @@ -33,7 +33,11 @@ #include "pdc.h" #include "pdc_analysis.h" -#define BUF_LEN 128 +#define BUF_LEN 128 +#define MAX_DATA_SIZE 1000000 +#define GOTO_DONE_ERROR() \ + ret_value = 1; \ + goto done; int main(int argc, char **argv) @@ -67,7 +71,7 @@ main(int argc, char **argv) ndim = 3; } else { - LOG_INFO("usage: ./read_write_perf n_objects data_size1 datasize2 datasize3"); + LOG_INFO("usage: ./read_write_perf n_objects data_size1 datasize2 datasize3\n"); } local_offset[0] = 0; @@ -101,6 +105,12 @@ main(int argc, char **argv) data_size *= (data_size_array[1] * data_size_array[2] * 1048576); } n_objects = atoi(argv[1]); + + if (data_size > MAX_DATA_SIZE) { + LOG_ERROR("data_size exceeds max size\n"); + GOTO_DONE_ERROR(); + } + int *data = (int *)malloc(sizeof(int) * data_size); memcpy(dims, offset_length, sizeof(uint64_t) * ndim); @@ -129,27 +139,27 @@ main(int argc, char **argv) // create a container property cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); if (cont_prop <= 0) { - LOG_ERROR("Failed to create container property"); - ret_value = 1; + LOG_ERROR("Failed to create container property\n"); + GOTO_DONE_ERROR(); } // create a container sprintf(cont_name, "c%d", rank); cont = PDCcont_create(cont_name, cont_prop); if (cont <= 0) { - LOG_ERROR("Failed to create container"); - ret_value = 1; + LOG_ERROR("Failed to create container\n"); + GOTO_DONE_ERROR(); } // create an object property obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); if (obj_prop <= 0) { - LOG_ERROR("Failed to create object property"); - ret_value = 1; + LOG_ERROR("Failed to create object property\n"); + GOTO_DONE_ERROR(); } ret = PDCprop_set_obj_type(obj_prop, PDC_INT); if (ret != SUCCEED) { - LOG_ERROR("Failed to set obj type"); - ret_value = 1; + LOG_ERROR("Failed to set obj type\n"); + GOTO_DONE_ERROR(); } PDCprop_set_obj_dims(obj_prop, ndim, dims); PDCprop_set_obj_user_id(obj_prop, getuid()); @@ -165,8 +175,8 @@ main(int argc, char **argv) sprintf(obj_name1, "o1_%d_%d", rank, i); obj1 = PDCobj_create(cont, obj_name1, obj_prop); if (obj1 <= 0) { - LOG_ERROR("Failed to create object"); - ret_value = 1; + LOG_ERROR("Failed to create object\n"); + GOTO_DONE_ERROR(); } reg = PDCregion_create(ndim, offset, offset_length); @@ -177,8 +187,8 @@ main(int argc, char **argv) transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global); if (transfer_request == 0) { - LOG_INFO("PDCregion_transfer_create failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_create failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); @@ -186,8 +196,8 @@ main(int argc, char **argv) write_reg_transfer_start_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_start failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_start failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); @@ -195,29 +205,29 @@ main(int argc, char **argv) write_reg_transfer_wait_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_wait failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_wait failed\n"); + GOTO_DONE_ERROR(); } ret = PDCregion_transfer_close(transfer_request); if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_close failed"); - ret_value = 1; + LOG_INFO("PDCregion_transfer_close failed\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg) < 0) { - LOG_ERROR("Failed to close local region"); - ret_value = 1; + LOG_ERROR("Failed to close local region\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg_global) < 0) { - LOG_ERROR("Failed to close global region"); - ret_value = 1; + LOG_ERROR("Failed to close global region\n"); + GOTO_DONE_ERROR(); } if (PDCobj_close(obj1) < 0) { - LOG_ERROR("Failed to close object o1"); - ret_value = 1; + LOG_ERROR("Failed to close object o1\n"); + GOTO_DONE_ERROR(); } } @@ -227,8 +237,8 @@ main(int argc, char **argv) sprintf(obj_name1, "o1_%d_%d", rank, i); obj1 = PDCobj_open(obj_name1, pdc); if (obj1 <= 0) { - LOG_ERROR("Failed to open object"); - ret_value = 1; + LOG_ERROR("Failed to open object\n"); + GOTO_DONE_ERROR(); } reg = PDCregion_create(ndim, local_offset, offset_length); @@ -239,8 +249,8 @@ main(int argc, char **argv) transfer_request = PDCregion_transfer_create(data, PDC_READ, obj1, reg, reg_global); if (transfer_request == 0) { - LOG_INFO("PDCregion_transfer_create failed"); - ret_value = 1; + LOG_INFO("PDCregion_transfer_create failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); @@ -248,7 +258,7 @@ main(int argc, char **argv) read_reg_transfer_start_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_start failed"); + LOG_INFO("PDCregion_transfer_start failed\n"); exit(-1); } @@ -257,35 +267,35 @@ main(int argc, char **argv) read_reg_transfer_wait_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_wait failed"); - ret_value = 1; + LOG_INFO("PDCregion_transfer_wait failed\n"); + GOTO_DONE_ERROR(); } ret = PDCregion_transfer_close(transfer_request); if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_close failed"); - ret_value = 1; + LOG_INFO("PDCregion_transfer_close failed\n"); + GOTO_DONE_ERROR(); } if (ret != SUCCEED) { - LOG_INFO("PDCbuf_obj_unmap failed"); - ret_value = 1; + LOG_INFO("PDCbuf_obj_unmap failed\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg) < 0) { - LOG_ERROR("Failed to close local region"); - ret_value = 1; + LOG_ERROR("Failed to close local region\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg_global) < 0) { - LOG_ERROR("Failed to close global region"); - ret_value = 1; + LOG_ERROR("Failed to close global region\n"); + GOTO_DONE_ERROR(); } if (PDCobj_close(obj1) < 0) { - LOG_ERROR("Failed to close object o1"); - ret_value = 1; + LOG_ERROR("Failed to close object o1\n"); + GOTO_DONE_ERROR(); } } @@ -293,23 +303,23 @@ main(int argc, char **argv) // close a container if (PDCcont_close(cont) < 0) { - LOG_ERROR("Failed to close container c1"); - ret_value = 1; + LOG_ERROR("Failed to close container c1\n"); + GOTO_DONE_ERROR(); } // close a object property if (PDCprop_close(obj_prop) < 0) { - LOG_ERROR("Failed to close property"); - ret_value = 1; + LOG_ERROR("Failed to close property\n"); + GOTO_DONE_ERROR(); } // close a container property if (PDCprop_close(cont_prop) < 0) { - LOG_ERROR("Failed to close property"); - ret_value = 1; + LOG_ERROR("Failed to close property\n"); + GOTO_DONE_ERROR(); } // close pdc if (PDCclose(pdc) < 0) { - LOG_ERROR("Failed to close PDC"); - ret_value = 1; + LOG_ERROR("Failed to close PDC\n"); + GOTO_DONE_ERROR(); } #ifdef ENABLE_MPI MPI_Reduce(&write_reg_transfer_start_time, &start, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); @@ -339,6 +349,7 @@ main(int argc, char **argv) free(data); #endif +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/examples/vpicio.c b/examples/vpicio.c index b55603924..9ddf0d89c 100644 --- a/examples/vpicio.c +++ b/examples/vpicio.c @@ -33,7 +33,8 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -560,6 +561,8 @@ main(int argc, char **argv) free(pz); free(id1); free(id2); + +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/examples/vpicio_batch.c b/examples/vpicio_batch.c index 216627791..f386e6ffd 100644 --- a/examples/vpicio_batch.c +++ b/examples/vpicio_batch.c @@ -33,8 +33,9 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 -#define N_OBJS 8 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) +#define N_OBJS 8 double uniform_random_number() @@ -819,6 +820,8 @@ main(int argc, char **argv) free(pz); free(id1); free(id2); + +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/examples/vpicio_old.c b/examples/vpicio_old.c index 30c7e2316..5c394adf4 100644 --- a/examples/vpicio_old.c +++ b/examples/vpicio_old.c @@ -33,7 +33,8 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -92,6 +93,11 @@ main(int argc, char **argv) dims[0] = numparticles; + if (numparticles > MAX_PARTICLES) { + LOG_ERRROR("numparticles exceeds max size\n"); + goto done; + } + x = (float *)malloc(numparticles * sizeof(float)); y = (float *)malloc(numparticles * sizeof(float)); z = (float *)malloc(numparticles * sizeof(float)); @@ -548,6 +554,7 @@ main(int argc, char **argv) free(id1); free(id2); +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index 50e299565..fb88915c6 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -1380,7 +1380,7 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po } memset(hostname, 0, sizeof(hostname)); gethostname(hostname, sizeof(hostname)); - sprintf(na_info_string, "%s://%s:%d", hg_transport, hostname, port); + snprintf(na_info_string, sizeof(na_info_string), "%s://%s:%d", hg_transport, hostname, port); if (pdc_client_mpi_rank_g == 0) { LOG_INFO("==PDC_CLIENT: using %.7s\n", na_info_string); fflush(stdout); @@ -1572,9 +1572,9 @@ PDC_Client_init() // Get up tmp dir env var tmp_dir = getenv("PDC_TMPDIR"); if (tmp_dir == NULL) - strcpy(pdc_client_tmp_dir_g, "./pdc_tmp"); + strncpy(pdc_client_tmp_dir_g, "./pdc_tmp", sizeof(pdc_client_tmp_dir_g)); else - strcpy(pdc_client_tmp_dir_g, tmp_dir); + strncpy(pdc_client_tmp_dir_g, tmp_dir, sizeof(pdc_client_tmp_dir_g)); // Get debug environment var char *is_debug_env = getenv("PDC_DEBUG"); @@ -4698,7 +4698,7 @@ PDC_Client_get_data_from_server_shm_cb(const struct hg_cb_info *callback_info) } /* open the shared memory segment as if it was a file */ - shm_fd = shm_open(shm_addr, O_RDONLY, 0666); + shm_fd = shm_open(shm_addr, O_RDONLY, 0644); if (shm_fd == -1) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: - Shared memory open failed [%s]!", pdc_client_mpi_rank_g, shm_addr); @@ -4811,7 +4811,7 @@ PDC_Client_data_server_read_check(int server_id, uint32_t client_id, pdc_metadat shm_addr = lookup_args.ret_string; /* open the shared memory segment as if it was a file */ - shm_fd = shm_open(shm_addr, O_RDONLY, 0666); + shm_fd = shm_open(shm_addr, O_RDONLY, 0644); if (shm_fd == -1) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Shared memory open failed [%s]!", pdc_client_mpi_rank_g, shm_addr); @@ -5172,7 +5172,7 @@ PDC_Client_data_server_write(struct pdc_request *request) rnd); /* create the shared memory segment as if it was a file */ - request->shm_fd = shm_open(request->shm_addr, O_CREAT | O_RDWR, 0666); + request->shm_fd = shm_open(request->shm_addr, O_CREAT | O_RDWR, 0644); if (request->shm_fd == -1) PGOTO_ERROR(FAIL, "==PDC_CLIENT: Shared memory creation with shm_open failed"); @@ -6034,7 +6034,7 @@ PDC_Client_complete_read_request(int nbuf, struct pdc_request *req) for (i = 0; i < nbuf; i++) { /* open the shared memory segment as if it was a file */ - req->shm_fd_arr[i] = shm_open(req->shm_addr_arr[i], O_RDONLY, 0666); + req->shm_fd_arr[i] = shm_open(req->shm_addr_arr[i], O_RDONLY, 0644); if (req->shm_fd_arr[i] == -1) { LOG_ERROR("==PDC_CLIENT[%d]: Shared memory open failed [%s]!\n", pdc_client_mpi_rank_g, req->shm_addr_arr[i]); @@ -6183,38 +6183,7 @@ PDC_Client_all_server_checkpoint() fflush(stdout); FUNC_LEAVE(ret_value); } -/* -perr_t -PDC_Client_attach_metadata_to_local_obj(const char *obj_name, uint64_t obj_id, uint64_t cont_id, - uint32_t data_server_id, struct _pdc_obj_info *obj_info) -{ - perr_t ret_value = SUCCEED; - - FUNC_ENTER(NULL); - obj_info->metadata = (pdc_metadata_t *)calloc(1, sizeof(pdc_metadata_t)); - ((pdc_metadata_t *)obj_info->metadata)->user_id = obj_info->obj_pt->user_id; - if (NULL != obj_info->obj_pt->app_name) - strcpy(((pdc_metadata_t *)obj_info->metadata)->app_name, obj_info->obj_pt->app_name); - if (NULL != obj_name) - strcpy(((pdc_metadata_t *)obj_info->metadata)->obj_name, obj_name); - ((pdc_metadata_t *)obj_info->metadata)->time_step = obj_info->obj_pt->time_step; - ((pdc_metadata_t *)obj_info->metadata)->obj_id = obj_id; - ((pdc_metadata_t *)obj_info->metadata)->cont_id = cont_id; - ((pdc_metadata_t *)obj_info->metadata)->data_server_id = data_server_id; - if (NULL != obj_info->obj_pt->tags) - strcpy(((pdc_metadata_t *)obj_info->metadata)->tags, obj_info->obj_pt->tags); - if (NULL != obj_info->obj_pt->data_loc) - - strcpy(((pdc_metadata_t *)obj_info->metadata)->data_location, obj_info->obj_pt->data_loc); - ((pdc_metadata_t *)obj_info->metadata)->ndim = obj_info->obj_pt->obj_prop_pub->ndim; - if (NULL != obj_info->obj_pt->obj_prop_pub->dims) - memcpy(((pdc_metadata_t *)obj_info->metadata)->dims, obj_info->obj_pt->obj_prop_pub->dims, - sizeof(uint64_t) * obj_info->obj_pt->obj_prop_pub->ndim); - - FUNC_LEAVE(ret_value); -} -*/ perr_t PDC_Client_send_client_shm_info(uint32_t server_id, char *shm_addr, uint64_t size) { @@ -6624,7 +6593,7 @@ PDC_Client_query_multi_storage_info(int nobj, char **obj_names, region_storage_m obj_names_by_server = (char ***)calloc(sizeof(char **), pdc_server_num_g); n_obj_name_by_server = (int *)calloc(sizeof(int), pdc_server_num_g); obj_names_server_seq_mapping = (int **)calloc(sizeof(int *), pdc_server_num_g); - obj_names_server_seq_mapping_1d = (int *)calloc(sizeof(int), nobj * pdc_server_num_g); + obj_names_server_seq_mapping_1d = (int *)calloc(sizeof(int), (size_t)nobj * (size_t)pdc_server_num_g); for (i = 0; i < pdc_server_num_g; i++) { obj_names_by_server[i] = (char **)calloc(sizeof(char *), nobj); obj_names_server_seq_mapping[i] = obj_names_server_seq_mapping_1d + i * nobj; diff --git a/src/api/pdc_obj/pdc_obj.c b/src/api/pdc_obj/pdc_obj.c index 18e62382e..25210dc53 100644 --- a/src/api/pdc_obj/pdc_obj.c +++ b/src/api/pdc_obj/pdc_obj.c @@ -205,7 +205,8 @@ PDC_Client_attach_metadata_to_local_obj(const char *obj_name, uint64_t obj_id, u if (NULL != obj_info->obj_pt->app_name) strcpy(((pdc_metadata_t *)obj_info->metadata)->app_name, obj_info->obj_pt->app_name); if (NULL != obj_name) - strcpy(((pdc_metadata_t *)obj_info->metadata)->obj_name, obj_name); + strncpy(((pdc_metadata_t *)obj_info->metadata)->obj_name, obj_name, + sizeof(((pdc_metadata_t *)obj_info->metadata)->obj_name)); ((pdc_metadata_t *)obj_info->metadata)->time_step = obj_info->obj_pt->time_step; ((pdc_metadata_t *)obj_info->metadata)->obj_id = obj_id; ((pdc_metadata_t *)obj_info->metadata)->cont_id = cont_id; @@ -867,15 +868,21 @@ PDCprop_set_obj_dims(pdcid_t obj_prop, PDC_int_t ndim, uint64_t *dims) info = PDC_find_id(obj_prop); if (info == NULL) PGOTO_ERROR(FAIL, "cannot locate object property ID"); + prop = (struct _pdc_obj_prop *)(info->obj_ptr); if (ndim > (PDC_int_t)prop->obj_prop_pub->ndim) { if (prop->obj_prop_pub->ndim > 0) { - free(prop->obj_prop_pub->dims); + PDC_free(prop->obj_prop_pub->dims); + prop->obj_prop_pub->dims = NULL; } prop->obj_prop_pub->dims = (uint64_t *)malloc(ndim * sizeof(uint64_t)); prop->obj_prop_pub->ndim = ndim; } - memcpy(prop->obj_prop_pub->dims, dims, ndim * sizeof(uint64_t)); + + if (prop->obj_prop_pub->dims != NULL) + memcpy(prop->obj_prop_pub->dims, dims, ndim * sizeof(uint64_t)); + else + PGOTO_ERROR(FAIL, "prop->obj_prop->dims was NULL"); done: fflush(stdout); diff --git a/src/commons/collections/libhl/rbtree.c b/src/commons/collections/libhl/rbtree.c index 14ad28302..acb2d1433 100644 --- a/src/commons/collections/libhl/rbtree.c +++ b/src/commons/collections/libhl/rbtree.c @@ -851,7 +851,7 @@ rbt_remove(rbt_t *rbt, void *k, size_t klen, void **v) else { (*n)->parent->right = (*n)->left; } - if (n && *n && (*n)->left) { + if ((*n)->left) { (*n)->left->parent = *node; } } @@ -862,26 +862,13 @@ rbt_remove(rbt_t *rbt, void *k, size_t klen, void **v) else { (*n)->parent->left = (*n)->right; } - if (n && *n && (*n)->right) { + if ((*n)->right) { (*n)->right->parent = *node; } } - if (n && *n) { - rbt_free_node(rbt, n, &prev_value, v); - } + rbt_free_node(rbt, n, &prev_value, v); - // free((*n)->key); - // (*n)->key = NULL; - // if (v) - // *v = prev_value; - // else if (rbt->free_value_cb) { - // rbt->free_value_cb(prev_value); - // (&prev_value)[0] = NULL; - // } - - // free((*n)); - // *n = NULL; rbt->size = rbt->size - 1; return 0; } @@ -905,21 +892,8 @@ rbt_remove(rbt_t *rbt, void *k, size_t klen, void **v) } } - if (node && *node) { - rbt_free_node(rbt, node, &((*node)->value), v); - } + rbt_free_node(rbt, node, &((*node)->value), v); - // if (v) - // *v = (*node)->value; - // else if (rbt->free_value_cb) { - // rbt->free_value_cb((*node)->value); - // (*node)->value = NULL; - // } - - // free((*node)->key); - // (*node)->key = NULL; - // free((*node)); - // *node = NULL; rbt->size = rbt->size - 1; return 0; } @@ -933,20 +907,8 @@ rbt_remove(rbt_t *rbt, void *k, size_t klen, void **v) (*node)->parent->right = NULL; } - if (node && *node) { - rbt_free_node(rbt, node, &((*node)->value), v); - } - // if (v) - // *v = (*node)->value; - // else if (rbt->free_value_cb && (*node)->value) { - // rbt->free_value_cb((*node)->value); - // (*node)->value = NULL; - // } + rbt_free_node(rbt, node, &((*node)->value), v); - // free(node->key); - // node->key = NULL; - // // free(node); - // // (&node)[0] = NULL; rbt->size = rbt->size - 1; return 0; } diff --git a/src/commons/collections/pdc_set_test.c b/src/commons/collections/pdc_set_test.c index 694d8370d..ed75aa57b 100644 --- a/src/commons/collections/pdc_set_test.c +++ b/src/commons/collections/pdc_set_test.c @@ -52,7 +52,7 @@ main(int argc, char **argv) } // iterate through all values - SetIterator *it; + SetIterator *it = NULL; set_iterate(set, it); while (set_iter_has_more(it)) { uint64_t *value = set_iter_next(it); diff --git a/src/commons/index/dart/dart_core.c b/src/commons/index/dart/dart_core.c index 9c54a5ba4..30b7e06c1 100644 --- a/src/commons/index/dart/dart_core.c +++ b/src/commons/index/dart/dart_core.c @@ -274,8 +274,8 @@ get_reconciled_vnode_id_with_power_of_two_choice_rehashing_2(DART *dart, uint64_ int ir_idx = (int)ceil((double)(dart->alphabet_size / 2)); // We also calculate the region start position. - uint64_t region_start = ((((int)word[0] + ir_idx) % dart->alphabet_size)) * - region_size; // ((reconciled_vnode_idx)/region_size) * (region_size); + uint64_t region_start = + (((uint64_t)word[0] + (uint64_t)ir_idx) % (uint64_t)(dart->alphabet_size)) * (uint64_t)region_size; // Finally, the reconciled vnode index is calculated. // reconciled_vnode_idx = (0 + region_start + region_offset) % dart->num_vnode; reconciled_vnode_idx = (reconciled_vnode_idx + region_start + region_offset) % dart->num_vnode; diff --git a/src/commons/index/dart/index/idioms/idioms_local_index.c b/src/commons/index/dart/index/idioms/idioms_local_index.c index 16c0d31eb..8e780d46a 100644 --- a/src/commons/index/dart/index/idioms/idioms_local_index.c +++ b/src/commons/index/dart/index/idioms/idioms_local_index.c @@ -236,7 +236,7 @@ idioms_local_index_create(IDIOMS_t *idioms, IDIOMS_md_idx_record_t *idx_record) idioms->server_id_g, key, idx_record->value, timer_delta_us(&index_timer)); char value_str[64]; if (idx_record->type == PDC_STRING) { - snprintf(value_str, 64, "%s", idx_record->value); + snprintf(value_str, 64, "%s", (char *)(idx_record->value)); } else if (is_PDC_UINT(idx_record->type)) { snprintf(value_str, 64, "%" PRIu64, *((uint64_t *)idx_record->value)); diff --git a/src/commons/index/dart/index/idioms/idioms_persistence.c b/src/commons/index/dart/index/idioms/idioms_persistence.c index e1a00f1f8..2d9b465fe 100644 --- a/src/commons/index/dart/index/idioms/idioms_persistence.c +++ b/src/commons/index/dart/index/idioms/idioms_persistence.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include "bulki_serde.h" /****************************/ @@ -195,7 +197,8 @@ dump_attr_root_tree(art_tree *art, char *dir_path, char *base_name, uint32_t ser // and this is why do we need to differentiate the file name by the server ID. sprintf(file_name, "%s/%s_%" PRIu32 "_%" PRIu64 ".bin", dir_path, base_name, serverID, *vid); LOG_INFO("Writing index to file_name: %s\n", file_name); - FILE *stream = fopen(file_name, "wb"); + int fd = open(file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644); + FILE *stream = fdopen(fd, "wb"); BULKI_serialize_to_file(bulki, stream); } return rst; @@ -208,7 +211,8 @@ dump_dart_info(DART *dart, char *dir_path, uint32_t serverID) char file_name[1024]; sprintf(file_name, "%s/%s.bin", dir_path, "dart_info"); LOG_INFO("Writing DART info to file_name: %s\n", file_name); - FILE * stream = fopen(file_name, "wb"); + int fd = open(file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644); + FILE * stream = fdopen(fd, "wb"); BULKI_Entity *dart_ent = empty_Bent_Array_Entity(); BULKI_ENTITY_append_BULKI_Entity(dart_ent, BULKI_ENTITY(&(dart->alphabet_size), 1, PDC_INT, PDC_CLS_ITEM)); @@ -355,10 +359,6 @@ read_attr_name_node(IDIOMS_t *idioms, char *dir_path, char *base_name, uint32_t char file_name[1024]; sprintf(file_name, "%s/%s_%" PRIu32 "_%" PRIu64 ".bin", dir_path, base_name, serverID, vnode_id); - // check file existence - if (access(file_name, F_OK) == -1) { - return FAIL; - } FILE *stream = fopen(file_name, "rb"); if (stream == NULL) { return FAIL; diff --git a/src/commons/logging/pdc_logger.c b/src/commons/logging/pdc_logger.c index 86618222b..9bf6851c6 100644 --- a/src/commons/logging/pdc_logger.c +++ b/src/commons/logging/pdc_logger.c @@ -2,6 +2,8 @@ #include #include #include +#include +#include void setLogFile(PDC_LogLevel level, const char *fileName) @@ -20,7 +22,8 @@ setLogFile(PDC_LogLevel level, const char *fileName) else { strncpy(logFilenames[level], fileName, sizeof(logFilenames[level]) - 1); logFilenames[level][sizeof(logFilenames[level]) - 1] = '\0'; - logFiles[level] = fopen(fileName, "a"); + int fd = open(fileName, O_WRONLY | O_CREAT | O_APPEND, 0644); + logFiles[level] = fdopen(fd, "a"); } } else { @@ -45,17 +48,21 @@ rotate_log_file(PDC_LogLevel level) logFiles[level] = NULL; } - char newFilename[MAX_LOG_FILE_NAME_LENGTH]; - char timeStr[20]; - time_t rawtime = time(NULL); - struct tm *timeinfo = localtime(&rawtime); + char newFilename[MAX_LOG_FILE_NAME_LENGTH]; + char timeStr[20]; + time_t rawtime = time(NULL); + struct tm timeinfo; - strftime(timeStr, 20, "%Y%m%d%H:%M:%S", timeinfo); + // Use localtime_r for thread safety + localtime_r(&rawtime, &timeinfo); + + strftime(timeStr, sizeof(timeStr), "%Y%m%d%H:%M:%S", &timeinfo); newFilename[strlen(newFilename) - 1] = '\0'; // Remove trailing newline snprintf(newFilename, MAX_LOG_FILE_NAME_LENGTH, "%s_%s", logFilenames[level], timeStr); rename(logFilenames[level], newFilename); - logFiles[level] = fopen(logFilenames[level], "a"); + int fd = open(logFilenames[level], O_WRONLY | O_CREAT | O_APPEND, 0644); + logFiles[level] = fdopen(fd, "a"); } static FILE * diff --git a/src/commons/serde/bulki/bulki_serde_test.c b/src/commons/serde/bulki/bulki_serde_test.c index 9f1f1076f..e8580e54f 100644 --- a/src/commons/serde/bulki/bulki_serde_test.c +++ b/src/commons/serde/bulki/bulki_serde_test.c @@ -1,6 +1,9 @@ #include "bulki_serde.h" #include "pdc_logger.h" +#include +#include + int test_base_type() { @@ -41,14 +44,16 @@ test_base_type() void * buffer = BULKI_serialize(bulki, &size); // Do some I/O if you like - FILE *fp = fopen("test_bulki.bin", "wb"); + int fd = open("test_bulki.bin", O_WRONLY | O_CREAT | O_TRUNC, 0644); + FILE *fp = fdopen(fd, "wb"); fwrite(buffer, 1, size, fp); fclose(fp); BULKI_free(bulki, 1); // read the file and deserialize - fp = fopen("test_bulki.bin", "rb"); + fd = open("test_bulki.bin", O_RDONLY | O_CREAT | O_TRUNC, 0644); + fp = fdopen(fd, "rb"); fseek(fp, 0, SEEK_END); long fsize = ftell(fp); fseek(fp, 0, SEEK_SET); /* same as rewind(f); */ @@ -331,7 +336,8 @@ bulki_small_json_serialization_test() BULKI_put(dataset, key5, value5); - FILE *fp = fopen("dataset.bin", "w"); + int fd = open("dataset.bin", O_WRONLY | O_CREAT | O_TRUNC, 0644); + FILE *fp = fdopen(fd, "w"); BULKI_serialize_to_file(dataset, fp); // fclose(fp); // Free the memory diff --git a/src/commons/utils/Readme.md b/src/commons/utils/Readme.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/commons/utils/pdc_timing.c b/src/commons/utils/pdc_timing.c index 0adea269a..bfec86658 100644 --- a/src/commons/utils/pdc_timing.c +++ b/src/commons/utils/pdc_timing.c @@ -556,13 +556,14 @@ inline void PDC_get_time_str(char *cur_time) { struct timespec ts; + struct tm tm_info; assert(cur_time); clock_gettime(CLOCK_REALTIME, &ts); - sprintf(cur_time, "%04d-%02d-%02d %02d:%02d:%02d.%06ld", 1900 + localtime(&ts.tv_sec)->tm_year, - localtime(&ts.tv_sec)->tm_mon + 1, localtime(&ts.tv_sec)->tm_mday, localtime(&ts.tv_sec)->tm_hour, - localtime(&ts.tv_sec)->tm_min, localtime(&ts.tv_sec)->tm_sec, ts.tv_nsec / 1000); + localtime_r(&ts.tv_sec, &tm_info); + sprintf(cur_time, "%04d-%02d-%02d %02d:%02d:%02d.%06ld", 1900 + tm_info.tm_year, tm_info.tm_mon + 1, + tm_info.tm_mday, tm_info.tm_hour, tm_info.tm_min, tm_info.tm_sec, ts.tv_nsec / 1000); return; -} +} \ No newline at end of file diff --git a/src/commons/utils/query_utils.c b/src/commons/utils/query_utils.c index c1a02372e..87ea9ce31 100644 --- a/src/commons/utils/query_utils.c +++ b/src/commons/utils/query_utils.c @@ -91,16 +91,26 @@ gen_query_key_value(query_gen_input_t *input, query_gen_output_t *output) char *temp_value = NULL; value_ptr_len = _gen_affix_for_token((char *)input->base_tag->value, input->value_query_type, affix_len, &temp_value); - value_ptr = (char *)calloc(value_ptr_len + 3, sizeof(char)); - value_ptr[0] = '"'; - strcat(value_ptr, temp_value); - value_ptr[value_ptr_len + 1] = '"'; - value_ptr[value_ptr_len + 2] = '\0'; - if (value_ptr_len == 0) { + if (value_ptr_len == 0 || temp_value == NULL) { LOG_ERROR("Failed to generate value query!\n"); return; } + + // Allocate space for: '"' + temp_value + '"' + '\0' + value_ptr = (char *)calloc(value_ptr_len + 3, sizeof(char)); + if (!value_ptr) { + LOG_ERROR("Memory allocation failed for value_ptr"); + return; + } + + int written = snprintf(value_ptr, value_ptr_len + 3, "\"%s\"", temp_value); + if (written < 0 || written >= value_ptr_len + 3) { + LOG_ERROR("Value string formatting failed or was truncated"); + free(value_ptr); + value_ptr = NULL; + return; + } } else { if (is_PDC_INT(input->base_tag->type)) { @@ -146,11 +156,25 @@ gen_query_key_value(query_gen_input_t *input, query_gen_output_t *output) char * gen_query_str(query_gen_output_t *query_gen_output) { - char *final_query_str = - (char *)calloc(query_gen_output->key_query_len + query_gen_output->value_query_len + 2, sizeof(char)); - strcat(final_query_str, query_gen_output->key_query); - strcat(final_query_str, "="); - strcat(final_query_str, query_gen_output->value_query); + // Allocate space for the final query string: + size_t total_len = query_gen_output->key_query_len + query_gen_output->value_query_len + 2; + char * final_query_str = (char *)calloc(total_len + 1, sizeof(char)); + + if (!final_query_str) { + LOG_ERROR("Memory allocation failed for final_query_str"); + return NULL; + } + + // Build the query string using snprintf for safety + int written = snprintf(final_query_str, total_len + 1, "%s=%s", query_gen_output->key_query, + query_gen_output->value_query); + + if (written < 0 || (size_t)written >= total_len) { + LOG_ERROR("Failed to generate query string or it was truncated"); + free(final_query_str); + return NULL; + } + return final_query_str; } diff --git a/src/commons/utils/string_utils.c b/src/commons/utils/string_utils.c index aa810d3b7..b98c845ec 100644 --- a/src/commons/utils/string_utils.c +++ b/src/commons/utils/string_utils.c @@ -318,11 +318,9 @@ gen_random_strings(int count, int minlen, int maxlen, int alphabet_size) len = len < minlen ? minlen : len; // Ensure at least minlen character char *str = (char *)calloc(len + 1, sizeof(char)); for (i = 0; i < len; i++) { - int randnum = rand(); - if (randnum < 0) - randnum *= -1; - char chr = VISIBLE_ALPHABET[randnum % abc_size]; - str[i] = chr; + unsigned int randnum = (unsigned int)rand(); + char chr = VISIBLE_ALPHABET[randnum % abc_size]; + str[i] = chr; } str[len] = '\0'; // Null-terminate the string result[c] = str; diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index b9fc8cea3..31aca0dbf 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -2680,25 +2680,23 @@ HG_TEST_RPC_CB(region_release, handle) hg_return_t hg_ret; region_lock_in_t in; region_lock_out_t out; - const struct hg_info * hg_info = NULL; - data_server_region_t * target_obj; - int error = 0; - int dirty_reg = 0; + const struct hg_info * hg_info = NULL; + data_server_region_t * target_obj = NULL; + int error = 0; + int dirty_reg = 0; hg_size_t size, size2; void * data_buf; struct pdc_region_info * server_region; - region_list_t * elt, *request_region, *tmp, *elt_tmp; + region_list_t * elt = NULL, *request_region = NULL, *tmp = NULL, *elt_tmp = NULL; struct region_lock_update_bulk_args *lock_update_bulk_args = NULL; struct buf_map_release_bulk_args * buf_map_bulk_args = NULL, *obj_map_bulk_args = NULL; hg_bulk_t lock_local_bulk_handle = HG_BULK_NULL; hg_bulk_t remote_bulk_handle = HG_BULK_NULL; - struct pdc_region_info * remote_reg_info; - region_buf_map_t * eltt, *eltt2, *eltt_tmp; - hg_uint32_t /*k, m, */ remote_count; + struct pdc_region_info * remote_reg_info = NULL; + region_buf_map_t * eltt = NULL, *eltt2 = NULL, *eltt_tmp = NULL; + hg_uint32_t remote_count; void ** data_ptrs_to = NULL; size_t * data_size_to = NULL; - // size_t type_size = 0; - // size_t dims[4] = {0, 0, 0, 0}; #ifdef PDC_TIMING double start, end; #endif @@ -4737,7 +4735,7 @@ remove_relative_dirs(char *workingDir, char *application) k = strlen(workingDir); if ((appName[0] == '.') && (appName[1] == '/')) appName += 2; - sprintf(&workingDir[k], "/%s", appName); + snprintf(&workingDir[k], PATH_MAX, "/%s", appName); ret_value = strdup(workingDir); @@ -4760,7 +4758,7 @@ PDC_find_in_path(char *workingDir, char *application) while (next) { *next++ = 0; - sprintf(checkPath, "%s/%s", pathVar, application); + snprintf(checkPath, sizeof(checkPath), "%s/%s", pathVar, application); if (stat(checkPath, &fileStat) == 0) { PGOTO_DONE(strdup(checkPath)); } @@ -4768,7 +4766,7 @@ PDC_find_in_path(char *workingDir, char *application) next = strchr(pathVar, colon); } if (application[0] == '.') { - sprintf(checkPath, "%s/%s", workingDir, application); + snprintf(checkPath, sizeof(checkPath), "%s/%s", workingDir, application); if (stat(checkPath, &fileStat) == 0) { char *foundPath = strrchr(checkPath, '/'); char *appName = foundPath + 1; @@ -4787,7 +4785,7 @@ PDC_find_in_path(char *workingDir, char *application) if (chdir(workingDir) != 0) { LOG_ERROR("Check dir failed\n"); } - sprintf(&checkPath[offset], "/%s", appName); + snprintf(&checkPath[offset], PATH_MAX, "/%s", appName); PGOTO_DONE(strdup(checkPath)); } } @@ -6526,7 +6524,7 @@ PDC_create_shm_segment_ind(uint64_t size, char *shm_addr, void **buf) srand(time(0)); while (retry < PDC_MAX_TRIAL_NUM) { snprintf(shm_addr, ADDR_MAX, "/PDCshm%d", rand()); - shm_fd = shm_open(shm_addr, O_CREAT | O_RDWR, 0666); + shm_fd = shm_open(shm_addr, O_CREAT | O_RDWR, 0644); if (shm_fd != -1) break; retry++; @@ -6566,7 +6564,7 @@ PDC_create_shm_segment(region_list_t *region) /* create the shared memory segment as if it was a file */ retry = 0; while (retry < PDC_MAX_TRIAL_NUM) { - region->shm_fd = shm_open(region->shm_addr, O_CREAT | O_RDWR, 0666); + region->shm_fd = shm_open(region->shm_addr, O_CREAT | O_RDWR, 0644); if (region->shm_fd != -1) break; retry++; diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 7ec555db9..dfdcecbcd 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -357,7 +357,8 @@ PDC_Server_write_addr_to_file(char **addr_strings, int n) // write to file snprintf(config_fname, ADDR_MAX, "%s%s", pdc_server_tmp_dir_g, pdc_server_cfg_name_g); - FILE *na_config = fopen(config_fname, "w+"); + int fd = open(config_fname, O_RDWR | O_CREAT | O_TRUNC, 0644); + FILE *na_config = fdopen(fd, "w+"); if (!na_config) { LOG_ERROR("Could not open config file from: %s\n", config_fname); goto done; @@ -1262,10 +1263,12 @@ PDC_Server_checkpoint() fflush(stdout); } + int fd; if (use_tmpfs) - file = fopen(checkpoint_file_local, "w+"); + fd = open(checkpoint_file_local, O_RDWR | O_CREAT | O_TRUNC, 0644); else - file = fopen(checkpoint_file, "w+"); + fd = open(checkpoint_file, O_RDWR | O_CREAT | O_TRUNC, 0644); + file = fdopen(fd, "w+"); if (file == NULL) { LOG_ERROR("==PDC_SERVER[%d]: Checkpoint file open error", pdc_server_rank_g); diff --git a/src/server/pdc_server_metadata.c b/src/server/pdc_server_metadata.c index 465433242..d6c6a9daa 100644 --- a/src/server/pdc_server_metadata.c +++ b/src/server/pdc_server_metadata.c @@ -700,33 +700,34 @@ PDC_Server_add_tag_metadata(metadata_add_tag_in_t *in, metadata_add_tag_out_t *o // obj_name change is done through client with delete and add operation. if (in->new_tag != NULL && in->new_tag[0] != 0 && !(in->new_tag[0] == ' ' && in->new_tag[1] == 0)) { - // add a ',' to separate different tags - target->tags[strlen(target->tags) + 1] = 0; - target->tags[strlen(target->tags)] = ','; - strcat(target->tags, in->new_tag); - out->ret = 1; + size_t tag_len = strlen(target->tags); + size_t space_left = TAG_LEN_MAX - tag_len; + + if (space_left <= 1) { + PGOTO_ERROR(FAIL, "Not enough space to append comma"); + } + + int written = snprintf(target->tags + tag_len, space_left, ",%s", in->new_tag); + if (written < 0 || (size_t)written >= space_left) { + PGOTO_ERROR(FAIL, "Not enough space in target->tags to append new_tag with snprintf"); + } } else - out->ret = -1; + out->ret = FAIL; } // end if (target != NULL) else { // Object not found for deletion request - LOG_ERROR("==PDC_SERVER: add tag target not found 1!\n"); - out->ret = -1; + PGOTO_ERROR(FAIL, "==PDC_SERVER: add tag target not found 1!\n"); } } // end if lookup_value != NULL else { - LOG_ERROR("==PDC_SERVER: add tag target not found 2!\n"); - out->ret = -1; + PGOTO_ERROR(FAIL, "==PDC_SERVER: add tag target not found 2!\n"); } } // if (metadata_hash_table_g != NULL) else { - LOG_ERROR("==PDC_SERVER: metadata_hash_table_g not initilized!\n"); - ret_value = FAIL; - out->ret = -1; - goto done; + PGOTO_ERROR(FAIL, "==PDC_SERVER: metadata_hash_table_g not initilized!\n"); } if (ret_value != SUCCEED) { @@ -825,10 +826,19 @@ PDC_Server_update_metadata(metadata_update_in_t *in, metadata_update_out_t *out) strcpy(target->data_location, in->new_metadata.data_location); if (in->new_metadata.tags[0] != 0 && !(in->new_metadata.tags[0] == ' ' && in->new_metadata.tags[1] == 0)) { - // add a ',' to separate different tags - target->tags[strlen(target->tags) + 1] = 0; - target->tags[strlen(target->tags)] = ','; - strcat(target->tags, in->new_metadata.tags); + size_t tag_len = strlen(target->tags); + size_t space_left = TAG_LEN_MAX - tag_len; + + if (space_left <= 1) { + PGOTO_ERROR(FAIL, "Not enough space to append comma"); + } + + int written = snprintf(target->tags + tag_len, space_left, ",%s", in->new_metadata.tags); + if (written < 0 || (size_t)written >= space_left) { + PGOTO_ERROR( + FAIL, + "Not enough space in target->tags to append new_metadata.tags with snprintf"); + } } if (in->new_metadata.current_state != 0) { target->transform_state = in->new_metadata.current_state; @@ -2553,16 +2563,21 @@ PDC_Server_container_add_tags(uint64_t cont_id, char *tags) FUNC_ENTER(NULL); ret_value = PDC_Server_find_container_by_id(cont_id, &cont_entry); - if (cont_entry != NULL) { + if (cont_entry != NULL && tags != NULL) { + size_t tag_len = strlen(cont_entry->tags); + size_t space_left = TAG_LEN_MAX - tag_len; + + if (space_left <= 1) { + PGOTO_ERROR(FAIL, "Not enough space in cont_entry->tags to append new tags!"); + } - if (tags != NULL) { - strcat(cont_entry->tags, tags); + int written = snprintf(cont_entry->tags + tag_len, space_left, "%s", tags); + if (written < 0 || (size_t)written >= space_left) { + PGOTO_ERROR(FAIL, "Truncated tag append in cont_entry->tags!"); } } else { - LOG_ERROR("==PDC_SERVER[%d]: container %" PRIu64 " not found!\n", pdc_server_rank_g, cont_id); - ret_value = FAIL; - goto done; + PGOTO_ERROR(FAIL, "==PDC_SERVER[%d]: container %" PRIu64 " not found!\n", pdc_server_rank_g, cont_id); } done: diff --git a/src/server/pdc_server_region/pdc_server_data.c b/src/server/pdc_server_region/pdc_server_data.c index 35d000b5a..71d1629f5 100644 --- a/src/server/pdc_server_region/pdc_server_data.c +++ b/src/server/pdc_server_region/pdc_server_data.c @@ -126,7 +126,7 @@ static int server_open_storage(char *storage_location, pdcid_t obj_id) { fill_storage_path(storage_location, obj_id); - return open(storage_location, O_RDWR | O_CREAT, 0666); + return open(storage_location, O_RDWR | O_CREAT, 0644); } /* @@ -416,9 +416,9 @@ PDC_Server_register_obj_region_by_pointer(data_server_region_t **new_obj_reg_ptr } if (new_obj_reg->fd < 0) { new_obj_reg->close_flag = close_flag; - new_obj_reg->fd = open(new_obj_reg->storage_location, O_RDWR | O_CREAT, 0666); + new_obj_reg->fd = open(new_obj_reg->storage_location, O_RDWR | O_CREAT, 0644); if (new_obj_reg->fd < 0) { - goto done; + PGOTO_ERROR(FAIL, "Failed to open file"); } } } @@ -1211,7 +1211,7 @@ PDC_Data_Server_buf_map(const struct hg_info *info, buf_map_in_t *in, region_lis LOG_INFO("storage_location is %s\n", storage_location); } #endif - new_obj_reg->fd = open(storage_location, O_RDWR | O_CREAT, 0666); + new_obj_reg->fd = open(storage_location, O_RDWR | O_CREAT, 0644); if (new_obj_reg->fd == -1) { LOG_ERROR("==PDC_SERVER[%d]: open %s failed\n", pdc_server_rank_g, storage_location); goto done; @@ -1907,7 +1907,8 @@ PDC_Server_cache_region_to_BB(region_list_t *region) if (pdc_cache_file_ptr_g == NULL) { char *bb_data_path = getenv("PDC_BB_LOC"); if (bb_data_path != NULL) { - sprintf(pdc_cache_file_path_g, "%s/PDCcacheBB.%d", bb_data_path, pdc_server_rank_g); + snprintf(pdc_cache_file_path_g, sizeof(pdc_cache_file_path_g), "%s/PDCcacheBB.%d", bb_data_path, + pdc_server_rank_g); } else { char *user_specified_data_path = getenv("PDC_DATA_LOC"); @@ -1918,12 +1919,14 @@ PDC_Server_cache_region_to_BB(region_list_t *region) if (bb_data_path == NULL) bb_data_path = "."; } - sprintf(pdc_cache_file_path_g, "%s/PDCcacheBB.%d", bb_data_path, pdc_server_rank_g); + snprintf(pdc_cache_file_path_g, sizeof(pdc_cache_file_path_g), "%s/PDCcacheBB.%d", bb_data_path, + pdc_server_rank_g); LOG_ERROR("==PDC_SERVER[%d]: No PDC_BB_LOC specified, use [%s]!\n", pdc_server_rank_g, bb_data_path); } - pdc_cache_file_ptr_g = fopen(pdc_cache_file_path_g, "ab"); + int fd = open(pdc_cache_file_path_g, O_WRONLY | O_CREAT | O_TRUNC, 0644); + pdc_cache_file_ptr_g = fdopen(fd, "wb"); if (NULL == pdc_cache_file_ptr_g) { LOG_ERROR("==PDC_SERVER[%d]: fopen failed [%s]\n", pdc_server_rank_g, pdc_cache_file_path_g); ret_value = FAIL; @@ -1957,9 +1960,9 @@ PDC_Server_cache_region_to_BB(region_list_t *region) #endif // Prepare update - strcpy(region->storage_location, pdc_cache_file_path_g); + strncpy(region->storage_location, pdc_cache_file_path_g, sizeof(region->storage_location)); region->offset = offset; - strcpy(region->cache_location, pdc_cache_file_path_g); + strncpy(region->cache_location, pdc_cache_file_path_g, sizeof(region->cache_location)); region->cache_offset = offset; // Update storage meta @@ -2442,9 +2445,9 @@ PDC_Server_get_storage_location_of_region_mpi(region_list_t *regions_head) // Now server_id has all the data in all_requests, find all storage regions that overlaps with it // equivalent to storage metadadata searching if (server_id == (uint32_t)pdc_server_rank_g) { - send_buf = (update_region_storage_meta_bulk_t *)calloc(sizeof(update_region_storage_meta_bulk_t), - pdc_server_size_g * nrequest_per_server * - PDC_MAX_OVERLAP_REGION_NUM); + send_buf = (update_region_storage_meta_bulk_t *)calloc( + sizeof(update_region_storage_meta_bulk_t), + (size_t)pdc_server_size_g * (size_t)nrequest_per_server * PDC_MAX_OVERLAP_REGION_NUM); // All participants are querying the same object, so obj_ids are the same // Search one by one @@ -2653,7 +2656,7 @@ PDC_Server_data_write_from_shm(region_list_t *region_list_head) region_elt->data_size = PDC_get_region_size(region_elt); // Open shared memory and map to data buf - region_elt->shm_fd = shm_open(region_elt->shm_addr, O_RDONLY, 0666); + region_elt->shm_fd = shm_open(region_elt->shm_addr, O_RDONLY, 0644); if (region_elt->shm_fd == -1) { LOG_ERROR("==PDC_SERVER[%d]: Shared memory open failed [%s]!\n", pdc_server_rank_g, region_elt->shm_addr); @@ -4168,7 +4171,8 @@ PDC_Server_posix_one_file_io(region_list_t *region_list_head) // Open current file as binary and append only, it is guarenteed that only current // server process access this file, so no lock is needed. - fp_write = fopen(region_elt->storage_location, "ab"); + int fd = open(region_elt->storage_location, O_WRONLY | O_CREAT | O_APPEND, 0666); + fp_write = fdopen(fd, "ab"); n_fopen_g++; #ifdef ENABLE_TIMING @@ -4430,7 +4434,8 @@ PDC_Server_data_write_out(uint64_t obj_id, struct pdc_region_info *region_info, } request_region->ndim = region_info->ndim; request_region->unit_size = unit; - strcpy(request_region->storage_location, region->storage_location); + strncpy(request_region->storage_location, region->storage_location, + sizeof(request_region->storage_location)); #ifdef ENABLE_TIMING struct timeval pdc_timer_start, pdc_timer_end; double write_total_sec; @@ -5622,7 +5627,7 @@ PDC_Server_add_client_shm_to_cache(int cnt, void *buf_cp) new_region->data_size = storage_metas[i].size; // Open shared memory and map to data buf - new_region->shm_fd = shm_open(new_region->shm_addr, O_RDONLY, 0666); + new_region->shm_fd = shm_open(new_region->shm_addr, O_RDONLY, 0644); if (new_region->shm_fd == -1) { LOG_ERROR("==PDC_SERVER[%d]: Shared memory open failed [%s]!\n", pdc_server_rank_g, new_region->shm_addr); @@ -8195,15 +8200,15 @@ perr_t add_storage_region_to_buf(void **in_buf, uint64_t *buf_alloc, uint64_t *buf_off, const region_list_t *region) { perr_t ret_value = SUCCEED; - void * buf = *in_buf; uint64_t my_size, tmp_size; if (in_buf == NULL || *in_buf == NULL || region == NULL || buf_alloc == NULL || buf_off == NULL || region->storage_location[0] == '\0') { - LOG_ERROR("==PDC_SERVER[%d]: ERROR! NULL input!\n", pdc_server_rank_g); - goto done; + PGOTO_ERROR(FAIL, "==PDC_SERVER[%d]: ERROR! NULL input!\n", pdc_server_rank_g); } + void *buf = *in_buf; + my_size = 4 + strlen(region->storage_location) + 1 + sizeof(region_info_transfer_t) + 20; if (region->region_hist != NULL) { my_size += (8 * (region->region_hist->nbin) * 3); diff --git a/src/server/pdc_server_region/pdc_server_region_transfer.c b/src/server/pdc_server_region/pdc_server_region_transfer.c index ab9b5b1ee..62799d3a8 100644 --- a/src/server/pdc_server_region/pdc_server_region_transfer.c +++ b/src/server/pdc_server_region/pdc_server_region_transfer.c @@ -308,7 +308,7 @@ PDC_Server_transfer_request_io(uint64_t obj_id, int obj_ndim, const uint64_t *ob /* LOG_ERROR("Rank %d, write to offset %llu, size %llu\n", server_rank, region_info->offset[0], * region_info->size[0]); */ - fd = open(storage_location, O_RDWR | O_CREAT, 0666); + fd = open(storage_location, O_RDWR | O_CREAT, 0644); if (region_info->ndim == 1) { lseek(fd, region_info->offset[0] * unit, SEEK_SET); io_size = region_info->size[0] * unit; diff --git a/src/tests/bdcats.c b/src/tests/bdcats.c index 228aa1b2e..3033341a6 100644 --- a/src/tests/bdcats.c +++ b/src/tests/bdcats.c @@ -33,7 +33,8 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -82,6 +83,11 @@ main(int argc, char **argv) LOG_INFO("Writing %" PRIu64 " number of particles with %d clients.\n", numparticles, size); } + if (numparticles > MAX_PARTICLES) { + LOG_ERROR("numparticles exceeds max size\n"); + goto done; + } + x = (float *)malloc(numparticles * sizeof(float)); y = (float *)malloc(numparticles * sizeof(float)); z = (float *)malloc(numparticles * sizeof(float)); @@ -431,6 +437,7 @@ main(int argc, char **argv) free(offset_remote); free(mysize); +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/src/tests/bdcats_old.c b/src/tests/bdcats_old.c index 7570988a4..e52facafe 100644 --- a/src/tests/bdcats_old.c +++ b/src/tests/bdcats_old.c @@ -33,7 +33,8 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -78,6 +79,11 @@ main(int argc, char **argv) LOG_INFO("Writing %" PRIu64 " number of particles with %d clients.\n", numparticles, size); } + if (numparticles > MAX_PARTICLES) { + LOG_ERROR("numparticles exceeds max size\n"); + goto done; + } + x = (float *)malloc(numparticles * sizeof(float)); y = (float *)malloc(numparticles * sizeof(float)); z = (float *)malloc(numparticles * sizeof(float)); @@ -390,6 +396,7 @@ main(int argc, char **argv) free(offset_remote); free(mysize); +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/src/tests/bdcats_v2.c b/src/tests/bdcats_v2.c index e945cdeaf..b72b9032b 100644 --- a/src/tests/bdcats_v2.c +++ b/src/tests/bdcats_v2.c @@ -33,7 +33,8 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -467,6 +468,7 @@ main(int argc, char **argv) free(offset_remote); free(mysize); +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/src/tests/dart_algo_sim.c b/src/tests/dart_algo_sim.c index e3514da27..2c9aef866 100644 --- a/src/tests/dart_algo_sim.c +++ b/src/tests/dart_algo_sim.c @@ -306,11 +306,9 @@ gen_random_strings_with_cb(int count, int minlen, int maxlen, int alphabet_size, len = len < minlen ? minlen : len; char *str = (char *)calloc(len + 1, sizeof(len)); for (i = 0; i < len; i++) { - int randnum = rand(); - if (randnum < 0) - randnum *= -1; - char chr = (char)((randnum % alphabet_size) + 65); - str[i] = chr; + unsigned int randnum = (unsigned int)rand(); + char chr = (char)((randnum % alphabet_size) + 65); + str[i] = chr; } result[c] = str; if (insert_cb != NULL && search_cb != NULL) { @@ -329,49 +327,59 @@ char ** read_words_from_text(const char *fileName, int *word_count, int **req_count, int prefix_len, insert_key_cb insert_cb, search_key_cb search_cb) { - - FILE *file = fopen(fileName, "r"); /* should check the result */ + FILE *file = fopen(fileName, "r"); if (file == NULL) { - println("File not available\n"); + fprintf(stderr, "File not available: %s\n", fileName); exit(4); } - int lines_allocated = 128; - int max_line_len = 512; - - int i; - int line_count = 0; - char *line = (char *)malloc(sizeof(char) * (2 * max_line_len)); - char *word = (char *)malloc(sizeof(char) * (max_line_len)); - for (i = 0; 1; i++) { - int j; - if (fgets(line, max_line_len - 1, file) == NULL) { - break; + + int max_line_len = 512; + char *line = malloc(2 * max_line_len); + char *word = malloc(max_line_len); + + if (!line || !word) { + fprintf(stderr, "Memory allocation failed\n"); + fclose(file); + exit(1); + } + + int line_count = 0; + + while (fgets(line, max_line_len, file) != NULL) { + int j = strlen(line) - 1; + while (j >= 0 && (line[j] == '\n' || line[j] == '\r')) { + line[j--] = '\0'; + } + + if (strchr(line, '\n') == NULL && !feof(file)) { + fprintf(stderr, "Input line too long at line %d, skipping.\n", line_count + 1); + int ch; + while ((ch = fgetc(file)) != '\n' && ch != EOF) + ; + continue; } - /* Get rid of CR or LF at end of line */ - for (j = strlen(line) - 1; j >= 0 && (line[j] == '\n' || line[j] == '\r'); j--) - ; - line[j + 1] = '\0'; int rc; - sscanf(line, "%d %s", &rc, word); - // (*req_count)[line_count]=rc; + if (sscanf(line, "%d %511s", &rc, word) != 2) { + fprintf(stderr, "Malformed line at line %d: '%s'\n", line_count + 1, line); + continue; + } - if (insert_cb != NULL && search_cb != NULL) { + if (insert_cb && search_cb) { insert_cb(word, prefix_len); - - int sch = 0; - for (sch = 0; sch < rc; sch++) { + for (int sch = 0; sch < rc; sch++) { search_cb(word, prefix_len); } } + line_count++; } + free(line); free(word); - *word_count = line_count; - //*total_word_count = i; - fclose(file); + + *word_count = line_count; return NULL; } diff --git a/src/tests/dart_attr_dist_test.c b/src/tests/dart_attr_dist_test.c index a619d36c8..3f2b0339d 100644 --- a/src/tests/dart_attr_dist_test.c +++ b/src/tests/dart_attr_dist_test.c @@ -212,8 +212,8 @@ main(int argc, char *argv[]) val = i + 23456; pct = 0; int num_obj_per_attr = attr_2_obj_array[i]; - sprintf(key, "k%ld", i + 12345); - sprintf(value, "v%ld", val); + sprintf(key, "k%d", i + 12345); + sprintf(value, "v%d", val); println("attaching attribute #%d [%s:%s] to %d objects", i, key, value, num_obj_per_attr); for (j = 0; j < num_obj_per_attr; j++) { // each attribute is attached to a specific number of objects, and this is how we make up @@ -251,9 +251,9 @@ main(int argc, char *argv[]) dart_hash_algo_t hash_algo = DART_HASH; for (i = 0; i < arr_len; i++) { - sprintf(key, "k%ld", i + 12345); + sprintf(key, "k%d", i + 12345); val = i + 23456; - sprintf(value, "v%ld", val); + sprintf(value, "v%d", val); pct = 0; for (j = 0; j < attr_2_obj_array[i]; j++) { if (j % size == rank) { @@ -295,7 +295,7 @@ main(int argc, char *argv[]) int rest_count1 = 0; timer_start(&timer_obj); for (k = 0; k < q_repeat_count; k++) { - sprintf(key, "k%ld", i + 12345); + sprintf(key, "k%d", i + 12345); val = i + 23456; kvtag.name = key; @@ -335,8 +335,8 @@ main(int argc, char *argv[]) int rest_count1 = 0; timer_start(&timer_dart); for (k = 0; k < q_repeat_count; k++) { - sprintf(key, "k%ld", i + 12345); - sprintf(value, "v%ld", i + 23456); + sprintf(key, "k%d", i + 12345); + sprintf(value, "v%d", i + 23456); sprintf(exact_query, "%s=%s", key, value); // DART query methods diff --git a/src/tests/dart_test.c b/src/tests/dart_test.c index baee6b791..257065667 100644 --- a/src/tests/dart_test.c +++ b/src/tests/dart_test.c @@ -54,9 +54,9 @@ char ** read_words_from_text(const char *fileName, int *word_count, int *total_word_count, int mpi_rank) { - FILE *file = fopen(fileName, "r"); /* should check the result */ + FILE *file = fopen(fileName, "r"); if (file == NULL) { - println("File not available\n"); + LOG_ERROR("File not available\n"); exit(4); } int lines_allocated = 128; @@ -73,15 +73,10 @@ read_words_from_text(const char *fileName, int *word_count, int *total_word_coun #ifdef ENABLE_MPI if (i % (mpi_rank + 1) != 0) { - // char *trash_skip_buf=(char *)calloc(max_line_len, sizeof(char)); char trash_skip_buf[512]; if (fgets(trash_skip_buf, max_line_len - 1, file) == NULL) { - // free(trash_skip_buf); break; } - // println("skip '%s'", trash_skip_buf); - // free(trash_skip_buf); - // i++; continue; } #endif @@ -243,7 +238,7 @@ main(int argc, char **argv) timer_pause(&detailed_timer); if (round == 1) println("[Client_Side_Insert] Time to insert key %s for both prefix index and suffix index = " - "%d microseconds", + "%f microseconds", input_word_list[i], timer_delta_us(&detailed_timer)); } @@ -282,7 +277,7 @@ main(int argc, char **argv) char **query_input_list = (char **)calloc(word_count, sizeof(char *)); for (i = 0; i < word_count; i++) { query_input_list[i] = (char *)calloc((strlen(input_word_list[i]) + 1), sizeof(char)); - strcpy(query_input_list[i], input_word_list[i]); + strncpy(query_input_list[i], input_word_list[i], strlen(input_word_list[i]) + 1); } #ifdef ENABLE_MPI @@ -295,13 +290,13 @@ main(int argc, char **argv) int data = i; char *key = query_input_list[i]; char query_str[100]; - sprintf(query_str, "%s=%s", key, key); + snprintf(query_str, 100 - 1, "%s=%s", key, key); uint64_t *out; int rest_count = 0; PDC_Client_search_obj_ref_through_dart(hash_algo, query_str, ref_type, &rest_count, &out); timer_pause(&detailed_timer); if (round == 1) - println("[Client_Side_Exact] Time to search '%s' and get %d results = %d microseconds for " + println("[Client_Side_Exact] Time to search '%s' and get %d results = %f microseconds for " "rank %d", query_str, rest_count, timer_delta_us(&detailed_timer), rank); } @@ -323,7 +318,7 @@ main(int argc, char **argv) server_abstract.id = srv_cnt; dart_retrieve_server_info_cb(&server_abstract); if (round == 1) - println("[DART Load Balance 1] Server %d has query requests = %d", srv_cnt, + println("[DART Load Balance 1] Server %d has query requests = %" PRId64, srv_cnt, server_abstract.request_count); } } @@ -332,7 +327,7 @@ main(int argc, char **argv) for (i = 0; i < word_count; i++) { query_input_list[i] = (char *)calloc((strlen(input_word_list[i]) + 1), sizeof(char)); - strcpy(query_input_list[i], input_word_list[i]); + strncpy(query_input_list[i], input_word_list[i], (strlen(input_word_list[i]) + 1)); } #ifdef ENABLE_MPI @@ -348,13 +343,13 @@ main(int argc, char **argv) key[4] = '\0'; // trim to prefix of 4. } char query_str[80]; - sprintf(query_str, "%s*=%s*", key, key); + snprintf(query_str, 80 - 1, "%s*=%s*", key, key); uint64_t *out; int rest_count = 0; PDC_Client_search_obj_ref_through_dart(hash_algo, query_str, ref_type, &rest_count, &out); timer_pause(&detailed_timer); if (round == 1) - println("[Client_Side_Prefix] Time to search '%s' and get %d results = %d microseconds for " + println("[Client_Side_Prefix] Time to search '%s' and get %d results = %f microseconds for " "rank %d", query_str, rest_count, timer_delta_us(&detailed_timer), rank); } @@ -376,7 +371,7 @@ main(int argc, char **argv) server_abstract.id = srv_cnt; dart_retrieve_server_info_cb(&server_abstract); if (round == 1) - println("[DART Load Balance 2] Server %d has query requests = %d", srv_cnt, + println("[DART Load Balance 2] Server %d has query requests = %" PRId64, srv_cnt, server_abstract.request_count); } } @@ -384,7 +379,7 @@ main(int argc, char **argv) /* =============== Suffix Query testing ======================= */ for (i = 0; i < word_count; i++) { query_input_list[i] = (char *)calloc((strlen(input_word_list[i]) + 1), sizeof(char)); - strcpy(query_input_list[i], input_word_list[i]); + strncpy(query_input_list[i], input_word_list[i], (strlen(input_word_list[i]) + 1)); } #ifdef ENABLE_MPI @@ -401,13 +396,13 @@ main(int argc, char **argv) key = &key[key_len - 4]; // trim to suffix of 4. } char query_str[80]; - sprintf(query_str, "*%s=*%s", key, key); + snprintf(query_str, 80 - 1, "*%s=*%s", key, key); uint64_t *out; int rest_count = 0; PDC_Client_search_obj_ref_through_dart(hash_algo, query_str, ref_type, &rest_count, &out); timer_pause(&detailed_timer); if (round == 1) - println("[Client_Side_Suffix] Time to search '%s' and get %d results = %d microseconds for " + println("[Client_Side_Suffix] Time to search '%s' and get %d results = %f microseconds for " "rank %d", query_str, rest_count, timer_delta_us(&detailed_timer), rank); } @@ -429,7 +424,7 @@ main(int argc, char **argv) server_abstract.id = srv_cnt; dart_retrieve_server_info_cb(&server_abstract); if (round == 1) - println("[DART Load Balance 3] Server %d has query requests = %d", srv_cnt, + println("[DART Load Balance 3] Server %d has query requests = %" PRId64, srv_cnt, server_abstract.request_count); } } @@ -437,7 +432,7 @@ main(int argc, char **argv) /* =============== Infix Query testing ======================= */ for (i = 0; i < word_count; i++) { query_input_list[i] = (char *)calloc((strlen(input_word_list[i]) + 1), sizeof(char)); - strcpy(query_input_list[i], input_word_list[i]); + strncpy(query_input_list[i], input_word_list[i], (strlen(input_word_list[i]) + 1)); } #ifdef ENABLE_MPI @@ -457,13 +452,13 @@ main(int argc, char **argv) key[4] = '\0'; // trim to prefix of 4. } char query_str[80]; - sprintf(query_str, "*%s*=*%s*", key, key); + snprintf(query_str, 80 - 1, "*%s*=*%s*", key, key); uint64_t *out; int rest_count = 0; PDC_Client_search_obj_ref_through_dart(hash_algo, query_str, ref_type, &rest_count, &out); timer_pause(&detailed_timer); if (round == 1) - println("[Client_Side_Infix] Time to search '%s' and get %d results = %d microseconds for " + println("[Client_Side_Infix] Time to search '%s' and get %d results = %f microseconds for " "rank %d", query_str, rest_count, timer_delta_us(&detailed_timer), rank); } @@ -485,7 +480,7 @@ main(int argc, char **argv) server_abstract.id = srv_cnt; dart_retrieve_server_info_cb(&server_abstract); if (round == 1) - println("[DART Load Balance 4] Server %d has query requests = %d", srv_cnt, + println("[DART Load Balance 4] Server %d has query requests = %" PRId64, srv_cnt, server_abstract.request_count); } } @@ -504,8 +499,9 @@ main(int argc, char **argv) dart_retrieve_server_info_cb(&server_abstract); int64_t num_request = server_abstract.request_count; if (round == 1) - println("[DART Load Balance All] The total number of query requests on server %d = %d", - srv_cnt, num_request); + println( + "[DART Load Balance All] The total number of query requests on server %d = %" PRId64, + srv_cnt, num_request); sum += (double)num_request; sqrt_sum += (double)((double)num_request * (double)num_request); } @@ -533,7 +529,7 @@ main(int argc, char **argv) dart_retrieve_server_info_cb(&server_abstract); int64_t num_indexed_word = server_abstract.indexed_word_count / 2; if (round == 1) - println("[DART Key Distribution] Server %d has %d words indexed", srv_cnt, + println("[DART Key Distribution] Server %d has %" PRIu64 " words indexed", srv_cnt, num_indexed_word); sum += (double)num_indexed_word; sqrt_sum += (double)((double)num_indexed_word * (double)num_indexed_word); @@ -568,7 +564,7 @@ main(int argc, char **argv) timer_pause(&detailed_timer); if (round == 1) println("[Client_Side_Delete] Time to delete key %s for both prefix index and suffix index = " - "%d microseconds", + "%f microseconds", input_word_list[i], timer_delta_us(&detailed_timer)); } diff --git a/src/tests/helper/julia_helper_loader.c b/src/tests/helper/julia_helper_loader.c index 56d255688..127f7c08a 100644 --- a/src/tests/helper/julia_helper_loader.c +++ b/src/tests/helper/julia_helper_loader.c @@ -8,13 +8,13 @@ jl_load_module(const char *mod_name) const char *julia_module_dir = getenv("PDC_JULIA_MODULE_DIR"); if (julia_module_dir == NULL || strlen(julia_module_dir) == 0) { // try to get it from PWD - LOG_WARNING("[PDC_JL_HELPER] Warning: PDC_JULIA_MODULE_DIR is not set, fallback to PWD!\n"); + LOG_WARNING("[PDC_JL_HELPER] PDC_JULIA_MODULE_DIR is not set, fallback to PWD!\n"); julia_module_dir = getenv("PWD"); } if (julia_module_dir == NULL || strlen(julia_module_dir) == 0) { // No way to find julia module directory - LOG_ERROR("[PDC_JL_HELPER] Error: Not able to find Julia module directory!\n"); + LOG_ERROR("[PDC_JL_HELPER] Not able to find Julia module directory!\n"); exit(-1); } LOG_INFO("[PDC_JL_HELPER] Julia module directory: %s\n", julia_module_dir); diff --git a/src/tests/kvtag_query.c b/src/tests/kvtag_query.c index 3f11bae79..807f8cb07 100644 --- a/src/tests/kvtag_query.c +++ b/src/tests/kvtag_query.c @@ -196,7 +196,7 @@ main(int argc, char *argv[]) for (iter = 0; iter < round; iter++) { v = iter; if (is_using_dart) { - sprintf(value, "%ld", v); + sprintf(value, "%d", v); sprintf(exact_query, "%s=%s", kvtag.name, value); #ifdef ENABLE_MPI PDC_Client_search_obj_ref_through_dart_mpi(hash_algo, exact_query, ref_type, &nres, &pdc_ids, diff --git a/src/tests/llsm_idioms_bench.c b/src/tests/llsm_idioms_bench.c index 8f447f5d5..bfbfebada 100644 --- a/src/tests/llsm_idioms_bench.c +++ b/src/tests/llsm_idioms_bench.c @@ -101,11 +101,11 @@ create_objects(pdcid_t **obj_ids, int my_csv_rows, int csv_expand_factor, pdcid_ char obj_name[128]; int64_t timestamp = get_timestamp_us(); - *obj_ids = (pdcid_t *)calloc(my_csv_rows * csv_expand_factor, sizeof(pdcid_t)); + *obj_ids = (pdcid_t *)calloc((size_t)my_csv_rows * (size_t)csv_expand_factor, sizeof(pdcid_t)); for (int i = 0; i < my_csv_rows; i++) { // create `csv_expansion_factor` data objects for each csv row. for (int obj_idx = 0; obj_idx < csv_expand_factor; obj_idx++) { - sprintf(obj_name, "obj%" PRId64 "%d", timestamp, obj_created); + sprintf(obj_name, "obj%" PRId64 "%zu", timestamp, obj_created); pdcid_t obj_id = PDCobj_create(cont, obj_name, obj_prop); (*obj_ids)[obj_created] = obj_id; @@ -243,7 +243,7 @@ csv_tags_on_objects(pdcid_t *obj_ids, char ***csv_data, char **csv_header, int n char new_iter_value[30]; sprintf(new_iter_value, "Scan_Iter_%04d", j); char new_iter_tok[10]; - sprintf(new_iter_tok, "%04dt.tif", j); + snprintf(new_iter_tok, sizeof(new_iter_tok), "%04dt.tif", j); char extra_attr_name[100]; char extra_attr_value[200]; @@ -253,9 +253,9 @@ csv_tags_on_objects(pdcid_t *obj_ids, char ***csv_data, char **csv_header, int n char *attr_value = strdup(row[col_idx]); if (strstr(attr_value, "Scan_Iter_0000")) { char *start = strstr(attr_value, "Scan_Iter_0000"); - strncpy(start, new_iter_value, strlen(new_iter_value)); + strncpy(start, new_iter_value, PDC_MIN(strlen(new_iter_value), strlen(start))); char *zerot = strstr(attr_value, "0000t.tif"); - strncpy(zerot, new_iter_tok, strlen(new_iter_tok)); + strncpy(zerot, new_iter_tok, PDC_MIN(strlen(new_iter_tok), strlen(zerot))); if (startsWith(attr_name, "Filepath")) { sprintf(extra_attr_value, "%s", attr_value); @@ -484,7 +484,7 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) { - println("[Object Creation] Rank %d/%d: Created %d objects, time: %.5f ms", my_rank, proc_num, + println("[Object Creation] Rank %d/%d: Created %zu objects, time: %.5f ms", my_rank, proc_num, obj_created, total_time * 1000.0); } @@ -504,7 +504,7 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) { - println("[Tag Creation] Rank %d/%d: Added %d tags for %d objects, time: %.5f ms", my_rank, proc_num, + println("[Tag Creation] Rank %d/%d: Added %zu tags for %zu objects, time: %.5f ms", my_rank, proc_num, tags_added, obj_created, total_time * 1000.0); } @@ -579,7 +579,7 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) { - println("[TAG Deletion] Rank %d/%d: Deleted %d kvtag from %d objects, time: %.5f ms", my_rank, + println("[TAG Deletion] Rank %d/%d: Deleted %zu kvtag from %zu objects, time: %.5f ms", my_rank, proc_num, tags_deleted, obj_created, total_time * 1000.0); } diff --git a/src/tests/obj_transformation.c b/src/tests/obj_transformation.c index d6f8b2791..223a3afa6 100644 --- a/src/tests/obj_transformation.c +++ b/src/tests/obj_transformation.c @@ -33,7 +33,8 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -543,6 +544,7 @@ main(int argc, char **argv) free(id1); free(id2); +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/src/tests/query_data.c b/src/tests/query_data.c index 22db011a7..01e9f05cf 100644 --- a/src/tests/query_data.c +++ b/src/tests/query_data.c @@ -119,7 +119,7 @@ main(int argc, char **argv) region.ndim = ndim; region.offset = (uint64_t *)malloc(sizeof(uint64_t) * ndim); region.size = (uint64_t *)malloc(sizeof(uint64_t) * ndim); - region.offset[0] = rank * my_data_count; + region.offset[0] = (uint64_t)rank * my_data_count; region.size[0] = my_data_count; mydata = (int *)malloc(my_data_count); diff --git a/src/tests/read_obj.c b/src/tests/read_obj.c index 73f6198f2..2bb93b6f7 100644 --- a/src/tests/read_obj.c +++ b/src/tests/read_obj.c @@ -66,7 +66,7 @@ main(int argc, char **argv) return ret_value; } - sprintf(obj_name, "%s_%d", argv[1], rank); + snprintf(obj_name, sizeof(obj_name), "%s_%d", argv[1], rank); size_MB = atoi(argv[2]); diff --git a/src/tests/read_obj_shared.c b/src/tests/read_obj_shared.c index f75eb6c51..8d062ad39 100644 --- a/src/tests/read_obj_shared.c +++ b/src/tests/read_obj_shared.c @@ -62,7 +62,7 @@ main(int argc, char **argv) return ret_value; } - sprintf(obj_name, "%s", argv[1]); + snprintf(obj_name, sizeof(obj_name), "%s", argv[1]); size_MB = atoi(argv[2]); diff --git a/src/tests/read_write_col_perf.c b/src/tests/read_write_col_perf.c index d9d0d0ee1..95432c84b 100644 --- a/src/tests/read_write_col_perf.c +++ b/src/tests/read_write_col_perf.c @@ -35,7 +35,11 @@ #include "pdc_timing.h" -#define BUF_LEN 128 +#define BUF_LEN 128 +#define MAX_DATA_SIZE 1000000 +#define GOTO_DONE_ERROR() \ + ret_value = 1; \ + goto done; int main(int argc, char **argv) @@ -75,18 +79,18 @@ main(int argc, char **argv) ndim = 3; } else { - LOG_JUST_PRINT("usage: ./read_write_perf n_objects data_size1 datasize2 datasize3"); + LOG_JUST_PRINT("usage: ./read_write_perf n_objects data_size1 datasize2 datasize3\n"); } local_offset[0] = 0; data_size_array[0] = atoi(argv[2]); if (ndim == 1) { - offset[0] = rank * data_size_array[0] * 1048576; + offset[0] = (uint64_t)rank * (uint64_t)data_size_array[0] * 1048576; offset_length[0] = data_size_array[0] * 1048576; data_size = data_size_array[0] * 1048576; } else { - offset[0] = rank * data_size_array[0]; + offset[0] = (uint64_t)rank * (uint64_t)data_size_array[0]; offset_length[0] = data_size_array[0]; data_size = data_size_array[0]; } @@ -95,7 +99,7 @@ main(int argc, char **argv) local_offset[1] = 0; offset[1] = 0; data_size_array[1] = atoi(argv[3]); - offset_length[1] = data_size_array[1] * 1048576; + offset_length[1] = (uint64_t)data_size_array[1] * 1048576; data_size *= offset_length[1]; dims[1] = offset_length[1]; } @@ -108,14 +112,19 @@ main(int argc, char **argv) local_offset[2] = 0; offset[2] = 0; data_size_array[2] = atoi(argv[4]); - offset_length[2] = data_size_array[2] * 1048576; + offset_length[2] = (uint64_t)data_size_array[2] * 1048576; data_size *= offset_length[2] * offset_length[1]; dims[1] = offset_length[1]; dims[2] = offset_length[2]; } n_objects = atoi(argv[1]); - int *data = (int *)malloc(sizeof(int) * data_size); + + if (data_size > MAX_DATA_SIZE) { + GOTO_DONE_ERROR(); + } + + int *data = (int *)malloc(sizeof(int) * (size_t)data_size); char hostname[256]; gethostname(hostname, 256); @@ -138,27 +147,27 @@ main(int argc, char **argv) // create a container property cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); if (cont_prop <= 0) { - LOG_ERROR("Failed to create container property"); - ret_value = 1; + LOG_ERROR("Failed to create container property\n"); + GOTO_DONE_ERROR(); } // create a container sprintf(cont_name, "c%d", rank); cont = PDCcont_create(cont_name, cont_prop); if (cont <= 0) { - LOG_ERROR("Failed to create container"); - ret_value = 1; + LOG_ERROR("Failed to create container\n"); + GOTO_DONE_ERROR(); } // create an object property obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); if (obj_prop <= 0) { - LOG_ERROR("Failed to create object property"); - ret_value = 1; + LOG_ERROR("Failed to create object property\n"); + GOTO_DONE_ERROR(); } ret = PDCprop_set_obj_type(obj_prop, PDC_INT); if (ret != SUCCEED) { - LOG_ERROR("Failed to set obj type"); - ret_value = 1; + LOG_ERROR("Failed to set obj type\n"); + GOTO_DONE_ERROR(); } PDCprop_set_obj_dims(obj_prop, ndim, dims); PDCprop_set_obj_user_id(obj_prop, getuid()); @@ -180,8 +189,8 @@ main(int argc, char **argv) obj1 = PDCobj_create(cont, obj_name1, obj_prop); #endif if (obj1 <= 0) { - LOG_ERROR("Failed to create object"); - ret_value = 1; + LOG_ERROR("Failed to create object\n"); + GOTO_DONE_ERROR(); } reg = PDCregion_create(ndim, local_offset, offset_length); reg_global = PDCregion_create(ndim, offset, offset_length); @@ -190,45 +199,45 @@ main(int argc, char **argv) transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global); if (transfer_request == 0) { - LOG_INFO("PDCregion_transfer_create failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_create failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); ret = PDCregion_transfer_start(transfer_request); write_reg_transfer_start_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_start failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_start failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); ret = PDCregion_transfer_wait(transfer_request); write_reg_transfer_wait_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_wait failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_wait failed\n"); + GOTO_DONE_ERROR(); } ret = PDCregion_transfer_close(transfer_request); if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_close failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_close failed\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg) < 0) { - LOG_ERROR("Failed to close local region"); - ret_value = 1; + LOG_ERROR("Failed to close local region\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg_global) < 0) { - LOG_ERROR("Failed to close global region"); - ret_value = 1; + LOG_ERROR("Failed to close global region\n"); + GOTO_DONE_ERROR(); } if (PDCobj_close(obj1) < 0) { - LOG_ERROR("Failed to close object o1"); - ret_value = 1; + LOG_ERROR("Failed to close object o1\n"); + GOTO_DONE_ERROR(); } } #ifdef PDC_TIMING @@ -238,8 +247,8 @@ main(int argc, char **argv) sprintf(obj_name1, "o1_%d", i); obj1 = PDCobj_open(obj_name1, pdc); if (obj1 <= 0) { - LOG_ERROR("Failed to open object"); - ret_value = 1; + LOG_ERROR("Failed to open object\n"); + GOTO_DONE_ERROR(); } reg = PDCregion_create(ndim, local_offset, offset_length); @@ -250,8 +259,8 @@ main(int argc, char **argv) transfer_request = PDCregion_transfer_create(data, PDC_READ, obj1, reg, reg_global); if (transfer_request == 0) { - LOG_ERROR("PDCregion_transfer_create failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_create failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); @@ -259,8 +268,8 @@ main(int argc, char **argv) read_reg_transfer_start_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_ERROR("PDCregion_transfer_start failed"); - exit(-1); + LOG_ERROR("PDCregion_transfer_start failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); @@ -268,35 +277,35 @@ main(int argc, char **argv) read_reg_transfer_wait_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_ERROR("PDCregion_transfer_wait failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_wait failed\n"); + GOTO_DONE_ERROR(); } ret = PDCregion_transfer_close(transfer_request); if (ret != SUCCEED) { - LOG_ERROR("PDCregion_transfer_close failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_close failed\n"); + GOTO_DONE_ERROR(); } if (ret != SUCCEED) { - LOG_ERROR("PDCbuf_obj_unmap failed"); - ret_value = 1; + LOG_ERROR("PDCbuf_obj_unmap failed\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg) < 0) { - LOG_ERROR("Failed to close local region"); - ret_value = 1; + LOG_ERROR("Failed to close local region\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg_global) < 0) { - LOG_ERROR("Failed to close global region"); - ret_value = 1; + LOG_ERROR("Failed to close global region\n"); + GOTO_DONE_ERROR(); } if (PDCobj_close(obj1) < 0) { - LOG_ERROR("Failed to close object o1"); - ret_value = 1; + LOG_ERROR("Failed to close object o1\n"); + GOTO_DONE_ERROR(); } } #ifdef PDC_TIMING @@ -304,23 +313,23 @@ main(int argc, char **argv) #endif // close a container if (PDCcont_close(cont) < 0) { - LOG_ERROR("Failed to close container c1"); - ret_value = 1; + LOG_ERROR("Failed to close container c1\n"); + GOTO_DONE_ERROR(); } // close a object property if (PDCprop_close(obj_prop) < 0) { - LOG_ERROR("Failed to close property"); - ret_value = 1; + LOG_ERROR("Failed to close property\n"); + GOTO_DONE_ERROR(); } // close a container property if (PDCprop_close(cont_prop) < 0) { - LOG_ERROR("Failed to close property"); - ret_value = 1; + LOG_ERROR("Failed to close property\n"); + GOTO_DONE_ERROR(); } // close pdc if (PDCclose(pdc) < 0) { - LOG_ERROR("Failed to close PDC"); - ret_value = 1; + LOG_ERROR("Failed to close PDC\n"); + GOTO_DONE_ERROR(); } #ifdef ENABLE_MPI MPI_Reduce(&write_reg_transfer_start_time, &start, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); @@ -352,5 +361,7 @@ main(int argc, char **argv) #ifdef ENABLE_MPI MPI_Finalize(); #endif + +done: return ret_value; } diff --git a/src/tests/read_write_perf.c b/src/tests/read_write_perf.c index 152140d5f..bda075dad 100644 --- a/src/tests/read_write_perf.c +++ b/src/tests/read_write_perf.c @@ -35,7 +35,11 @@ #include "pdc_timing.h" -#define BUF_LEN 128 +#define BUF_LEN 128 +#define MAX_DATA_SIZE 1000000 +#define GOTO_DONE_ERROR() \ + ret_value = 1; \ + goto done; int main(int argc, char **argv) @@ -69,7 +73,7 @@ main(int argc, char **argv) ndim = 3; } else { - LOG_INFO("usage: ./read_write_perf n_objects data_size1 datasize2 datasize3"); + LOG_JUST_PRINT("usage: ./read_write_perf n_objects data_size1 datasize2 datasize3\n"); } local_offset[0] = 0; @@ -103,6 +107,12 @@ main(int argc, char **argv) data_size *= (data_size_array[1] * data_size_array[2] * 1048576); } n_objects = atoi(argv[1]); + + if (data_size > MAX_DATA_SIZE) { + LOG_ERROR("data_size exceeds max size\n"); + GOTO_DONE_ERROR(); + } + int *data = (int *)malloc(sizeof(int) * data_size); memcpy(dims, offset_length, sizeof(uint64_t) * ndim); @@ -116,7 +126,7 @@ main(int argc, char **argv) if (rank == 0) { LOG_INFO("number of dimensions in this test is %d\n", ndim); LOG_INFO("data size = %llu\n", (long long unsigned)data_size); - LOG_ERROR("First dim has size %d\n", data_size_array[0]); + LOG_INFO("First dim has size %d\n", data_size_array[0]); if (ndim >= 2) { LOG_INFO("second dim has size %d\n", data_size_array[1]); } @@ -131,27 +141,27 @@ main(int argc, char **argv) // create a container property cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); if (cont_prop <= 0) { - LOG_ERROR("Failed to create container property"); - ret_value = 1; + LOG_ERROR("Failed to create container property\n"); + GOTO_DONE_ERROR(); } // create a container sprintf(cont_name, "c%d", rank); cont = PDCcont_create(cont_name, cont_prop); if (cont <= 0) { - LOG_ERROR("Failed to create container"); - ret_value = 1; + LOG_ERROR("Failed to create container\n"); + GOTO_DONE_ERROR(); } // create an object property obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); if (obj_prop <= 0) { - LOG_ERROR("Failed to create object property"); - ret_value = 1; + LOG_ERROR("Failed to create object property\n"); + GOTO_DONE_ERROR(); } ret = PDCprop_set_obj_type(obj_prop, PDC_INT); if (ret != SUCCEED) { - LOG_ERROR("Failed to set obj type"); - ret_value = 1; + LOG_ERROR("Failed to set obj type\n"); + GOTO_DONE_ERROR(); } PDCprop_set_obj_dims(obj_prop, ndim, dims); PDCprop_set_obj_user_id(obj_prop, getuid()); @@ -168,7 +178,7 @@ main(int argc, char **argv) obj1 = PDCobj_create(cont, obj_name1, obj_prop); if (obj1 <= 0) { LOG_ERROR("Failed to create object"); - ret_value = 1; + GOTO_DONE_ERROR(); } reg = PDCregion_create(ndim, offset, offset_length); @@ -179,8 +189,8 @@ main(int argc, char **argv) transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global); if (transfer_request == 0) { - LOG_ERROR("PDCregion_transfer_create failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_create failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); @@ -188,8 +198,8 @@ main(int argc, char **argv) write_reg_transfer_start_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_ERROR("PDCregion_transfer_start failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_start failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); @@ -197,29 +207,29 @@ main(int argc, char **argv) write_reg_transfer_wait_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_ERROR("PDCregion_transfer_wait failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_wait failed\n"); + GOTO_DONE_ERROR(); } ret = PDCregion_transfer_close(transfer_request); if (ret != SUCCEED) { - LOG_ERROR("PDCregion_transfer_close failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_close failed\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg) < 0) { - LOG_ERROR("Failed to close local region"); - ret_value = 1; + LOG_ERROR("Failed to close local region\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg_global) < 0) { - LOG_ERROR("Failed to close global region"); - ret_value = 1; + LOG_ERROR("Failed to close global region\n"); + GOTO_DONE_ERROR(); } if (PDCobj_close(obj1) < 0) { - LOG_ERROR("Failed to close object o1"); - ret_value = 1; + LOG_ERROR("Failed to close object o1\n"); + GOTO_DONE_ERROR(); } } #ifdef PDC_TIMING @@ -230,8 +240,8 @@ main(int argc, char **argv) sprintf(obj_name1, "o1_%d_%d", rank, i); obj1 = PDCobj_open(obj_name1, pdc); if (obj1 <= 0) { - LOG_ERROR("Failed to open object"); - ret_value = 1; + LOG_ERROR("Failed to open object\n"); + GOTO_DONE_ERROR(); } reg = PDCregion_create(ndim, local_offset, offset_length); @@ -242,8 +252,8 @@ main(int argc, char **argv) transfer_request = PDCregion_transfer_create(data, PDC_READ, obj1, reg, reg_global); if (transfer_request == 0) { - LOG_INFO("PDCregion_transfer_create failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_create failed\n"); + GOTO_DONE_ERROR(); } start = MPI_Wtime(); @@ -251,7 +261,7 @@ main(int argc, char **argv) read_reg_transfer_start_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_INFO("PDCregion_transfer_start failed"); + LOG_ERROR("PDCregion_transfer_start failed\n"); exit(-1); } @@ -260,35 +270,35 @@ main(int argc, char **argv) read_reg_transfer_wait_time += MPI_Wtime() - start; if (ret != SUCCEED) { - LOG_ERROR("PDCregion_transfer_wait failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_wait failed\n"); + GOTO_DONE_ERROR(); } ret = PDCregion_transfer_close(transfer_request); if (ret != SUCCEED) { - LOG_ERROR("PDCregion_transfer_close failed"); - ret_value = 1; + LOG_ERROR("PDCregion_transfer_close failed\n"); + GOTO_DONE_ERROR(); } if (ret != SUCCEED) { - LOG_ERROR("PDCbuf_obj_unmap failed"); - ret_value = 1; + LOG_ERROR("PDCbuf_obj_unmap failed\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg) < 0) { - LOG_ERROR("Failed to close local region"); - ret_value = 1; + LOG_ERROR("Failed to close local region\n"); + GOTO_DONE_ERROR(); } if (PDCregion_close(reg_global) < 0) { - LOG_ERROR("Failed to close global region"); - ret_value = 1; + LOG_ERROR("Failed to close global region\n"); + GOTO_DONE_ERROR(); } if (PDCobj_close(obj1) < 0) { - LOG_ERROR("Failed to close object o1"); - ret_value = 1; + LOG_ERROR("Failed to close object o1\n"); + GOTO_DONE_ERROR(); } } #ifdef PDC_TIMING @@ -296,23 +306,23 @@ main(int argc, char **argv) #endif // close a container if (PDCcont_close(cont) < 0) { - LOG_ERROR("Failed to close container c1"); - ret_value = 1; + LOG_ERROR("Failed to close container c1\n"); + GOTO_DONE_ERROR(); } // close a object property if (PDCprop_close(obj_prop) < 0) { - LOG_ERROR("Failed to close property"); - ret_value = 1; + LOG_ERROR("Failed to close property\n"); + GOTO_DONE_ERROR(); } // close a container property if (PDCprop_close(cont_prop) < 0) { - LOG_ERROR("Failed to close property"); - ret_value = 1; + LOG_ERROR("Failed to close property\n"); + GOTO_DONE_ERROR(); } // close pdc if (PDCclose(pdc) < 0) { - LOG_ERROR("Failed to close PDC"); - ret_value = 1; + LOG_ERROR("Failed to close PDC\n"); + GOTO_DONE_ERROR(); } #ifdef ENABLE_MPI MPI_Reduce(&write_reg_transfer_start_time, &start, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); @@ -342,6 +352,7 @@ main(int argc, char **argv) free(data); #endif +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/src/tests/vpicio.c b/src/tests/vpicio.c index f0bf64f92..255b3323b 100644 --- a/src/tests/vpicio.c +++ b/src/tests/vpicio.c @@ -33,7 +33,8 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -96,6 +97,11 @@ main(int argc, char **argv) dims[0] = numparticles * size; + if (numparticles > MAX_PARTICLES) { + LOG_ERROR("numparticles exceeds max size\n"); + goto done; + } + x = (float *)malloc(numparticles * sizeof(float)); y = (float *)malloc(numparticles * sizeof(float)); z = (float *)malloc(numparticles * sizeof(float)); @@ -597,6 +603,8 @@ main(int argc, char **argv) free(pz); free(id1); free(id2); + +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/src/tests/vpicio_mts.c b/src/tests/vpicio_mts.c index 4f5a8705e..4f81667b3 100644 --- a/src/tests/vpicio_mts.c +++ b/src/tests/vpicio_mts.c @@ -34,7 +34,8 @@ #include "pdc.h" #include "pdc_timing.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -92,6 +93,11 @@ main(int argc, char **argv) dims[0] = numparticles * size; + if (numparticles > MAX_PARTICLES) { + LOG_ERROR("numparticles exceeds max size\n"); + goto done; + } + x = (float *)malloc(numparticles * sizeof(float)); y = (float *)malloc(numparticles * sizeof(float)); z = (float *)malloc(numparticles * sizeof(float)); @@ -409,6 +415,8 @@ main(int argc, char **argv) free(pz); free(id1); free(id2); + +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/src/tests/vpicio_old.c b/src/tests/vpicio_old.c index fe2786b52..8a461fec8 100644 --- a/src/tests/vpicio_old.c +++ b/src/tests/vpicio_old.c @@ -33,7 +33,8 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -92,6 +93,11 @@ main(int argc, char **argv) dims[0] = numparticles; + if (numparticles > MAX_PARTICLES) { + LOG_ERROR("numparticles exceeds max size\n"); + goto done; + } + x = (float *)malloc(numparticles * sizeof(float)); y = (float *)malloc(numparticles * sizeof(float)); z = (float *)malloc(numparticles * sizeof(float)); @@ -548,6 +554,7 @@ main(int argc, char **argv) free(id1); free(id2); +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/src/tests/vpicio_v2.c b/src/tests/vpicio_v2.c index ff0347fbc..95cde8a6a 100644 --- a/src/tests/vpicio_v2.c +++ b/src/tests/vpicio_v2.c @@ -33,7 +33,8 @@ #include #include "pdc.h" -#define NPARTICLES 8388608 +#define NPARTICLES 8388608 +#define MAX_PARTICLES (NPARTICLES * 2) double uniform_random_number() @@ -571,6 +572,7 @@ main(int argc, char **argv) free(id1); free(id2); +done: #ifdef ENABLE_MPI MPI_Finalize(); #endif diff --git a/src/tests/write_obj_shared.c b/src/tests/write_obj_shared.c index 51125a1bd..6e3184602 100644 --- a/src/tests/write_obj_shared.c +++ b/src/tests/write_obj_shared.c @@ -66,7 +66,7 @@ main(int argc, char **argv) return ret_value; } - sprintf(obj_name, "%s", argv[1]); + snprintf(obj_name, sizeof(obj_name), "%s", argv[1]); size_MB = atoi(argv[2]);