From 117f604f3c48b8b88f1d9a2f03eefae3d45d6741 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 31 Jul 2023 15:49:29 +0800 Subject: [PATCH 01/42] fix(logs): fix logs in FATAL level not take effect (#1573) https://github.com/apache/incubator-pegasus/issues/1572 If the logs are called in FATAL level, the process should exit. --- src/utils/simple_logger.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index f982670186..f6a21b16d1 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -112,6 +112,10 @@ void screen_logger::dsn_logv(const char *file, } vprintf(fmt, args); printf("\n"); + + if (dsn_unlikely(log_level >= LOG_LEVEL_FATAL)) { + dsn_coredump(); + } } void screen_logger::flush() { ::fflush(stdout); } @@ -287,6 +291,10 @@ void simple_logger::dsn_log(const char *file, printf("%s\n", str); } + if (dsn_unlikely(log_level >= LOG_LEVEL_FATAL)) { + dsn_coredump(); + } + if (++_lines >= 200000) { create_log_file(); } From 4b0ea26506b40fdabf6c1753d7446ef032d3c658 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 31 Jul 2023 16:37:33 +0800 Subject: [PATCH 02/42] fix(slog): add an option to exit the process when find slog error (#1574) https://github.com/apache/incubator-pegasus/issues/1572 Add an option to make it possible to exit the process and leave the corrupted slog and replicas to be handled by the administrator when open slog failed. --- src/replica/replica_stub.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/replica/replica_stub.cpp b/src/replica/replica_stub.cpp index f828585562..e9bbab7a4b 100644 --- a/src/replica/replica_stub.cpp +++ b/src/replica/replica_stub.cpp @@ -132,6 +132,14 @@ DSN_DEFINE_bool(replication, verbose_commit_log_on_start, false, "whether to print verbose log when commit mutation when starting the server"); +DSN_DEFINE_bool( + replication, + crash_on_slog_error, + false, + "whether to exit the process while fail to open slog. If true, the process will exit and leave " + "the corrupted slog and replicas to be handled by the administrator. If false, the process " + "will continue, and remove the slog and move all the replicas to corresponding error " + "directories"); DSN_DEFINE_uint32(replication, max_concurrent_manual_emergency_checkpointing_count, 10, @@ -708,6 +716,9 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f if (err == ERR_OK) { LOG_INFO("replay shared log succeed, time_used = {} ms", finish_time - start_time); } else { + if (FLAGS_crash_on_slog_error) { + LOG_FATAL("replay shared log failed, err = {}, please check the error details", err); + } LOG_ERROR("replay shared log failed, err = {}, time_used = {} ms, clear all logs ...", err, finish_time - start_time); From 8cad63e1d153684327786134caf512818e58c850 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 8 Aug 2023 11:46:32 +0800 Subject: [PATCH 03/42] chore(github): free disk space for github workflows (#1577) https://github.com/apache/incubator-pegasus/issues/1576 Free more disk space for github workflows in the following ways: - remove unnecessary packages and directories with reduced disk space up to 11GB; - support updating options such as `disk_min_available_space_ratio` by env `TEST_OPTS` for some tests; - reduce `disk_min_available_space_ratio` from 10 to 5 for some failed tests due to `SPACE_INSUFFICIENT`. --- .github/workflows/free_disk_space.sh | 70 +++++++++++++++++++ .github/workflows/lint_and_test_cpp.yaml | 67 ++++++++++++++++-- src/common/test/config-test.ini | 1 + src/common/test/run.sh | 21 +++++- src/meta/test/config-test.ini | 1 + src/meta/test/run.sh | 21 +++++- src/replica/backup/test/config-test.ini | 1 + src/replica/backup/test/run.sh | 20 +++++- src/replica/bulk_load/test/config-test.ini | 3 + src/replica/bulk_load/test/run.sh | 20 +++++- src/replica/duplication/test/config-test.ini | 3 + src/replica/duplication/test/run.sh | 20 +++++- src/replica/split/test/config-test.ini | 3 + src/replica/split/test/run.sh | 20 +++++- src/replica/storage/simple_kv/config.ini | 1 + src/replica/storage/simple_kv/run.sh | 19 +++++ .../storage/simple_kv/test/case-000.ini | 1 + .../storage/simple_kv/test/case-001.ini | 1 + .../storage/simple_kv/test/case-002.ini | 1 + .../storage/simple_kv/test/case-003.ini | 1 + .../storage/simple_kv/test/case-004.ini | 1 + .../storage/simple_kv/test/case-005.ini | 1 + .../storage/simple_kv/test/case-006.ini | 1 + .../storage/simple_kv/test/case-100.ini | 1 + .../storage/simple_kv/test/case-101.ini | 1 + .../storage/simple_kv/test/case-102.ini | 1 + .../storage/simple_kv/test/case-103.ini | 1 + .../storage/simple_kv/test/case-104.ini | 1 + .../storage/simple_kv/test/case-105.ini | 1 + .../storage/simple_kv/test/case-106.ini | 1 + .../storage/simple_kv/test/case-107.ini | 1 + .../storage/simple_kv/test/case-108.ini | 1 + .../storage/simple_kv/test/case-109.ini | 1 + .../storage/simple_kv/test/case-200.ini | 1 + .../storage/simple_kv/test/case-201.ini | 1 + .../storage/simple_kv/test/case-202-0.ini | 1 + .../storage/simple_kv/test/case-202-1.ini | 1 + .../storage/simple_kv/test/case-203-0.ini | 1 + .../storage/simple_kv/test/case-204.ini | 1 + .../storage/simple_kv/test/case-205.ini | 1 + .../storage/simple_kv/test/case-206.ini | 1 + .../storage/simple_kv/test/case-207.ini | 1 + .../storage/simple_kv/test/case-208.ini | 1 + .../storage/simple_kv/test/case-209.ini | 1 + .../storage/simple_kv/test/case-210.ini | 1 + .../storage/simple_kv/test/case-211.ini | 1 + .../storage/simple_kv/test/case-212.ini | 1 + .../storage/simple_kv/test/case-213.ini | 1 + .../storage/simple_kv/test/case-214.ini | 1 + .../storage/simple_kv/test/case-215.ini | 1 + .../storage/simple_kv/test/case-216.ini | 1 + .../storage/simple_kv/test/case-300-0.ini | 1 + .../storage/simple_kv/test/case-300-1.ini | 1 + .../storage/simple_kv/test/case-300-2.ini | 1 + .../storage/simple_kv/test/case-301.ini | 1 + .../storage/simple_kv/test/case-302.ini | 1 + .../storage/simple_kv/test/case-303.ini | 1 + .../storage/simple_kv/test/case-304.ini | 1 + .../storage/simple_kv/test/case-305.ini | 1 + .../storage/simple_kv/test/case-306.ini | 1 + .../storage/simple_kv/test/case-307.ini | 1 + .../storage/simple_kv/test/case-400.ini | 1 + .../storage/simple_kv/test/case-401.ini | 1 + .../storage/simple_kv/test/case-402.ini | 1 + .../storage/simple_kv/test/case-600.ini | 1 + .../storage/simple_kv/test/case-601.ini | 1 + .../storage/simple_kv/test/case-602.ini | 1 + .../storage/simple_kv/test/case-603.ini | 1 + src/replica/storage/simple_kv/test/config.ini | 1 + src/replica/storage/simple_kv/test/run.sh | 20 ++++++ src/replica/test/config-test.ini | 3 + src/replica/test/run.sh | 21 +++++- src/server/test/config.ini | 1 + src/server/test/run.sh | 21 +++++- src/test/function_test/run.sh | 2 +- src/utils/filesystem.cpp | 2 +- src/utils/simple_logger.cpp | 1 + 77 files changed, 399 insertions(+), 16 deletions(-) create mode 100755 .github/workflows/free_disk_space.sh diff --git a/.github/workflows/free_disk_space.sh b/.github/workflows/free_disk_space.sh new file mode 100755 index 0000000000..87f0cdef8e --- /dev/null +++ b/.github/workflows/free_disk_space.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +echo "==============================================================================" +echo "Freeing up disk space on Github workflows" +echo "==============================================================================" + +echo "Before freeing, the space of each disk:" +df -h + +echo "Listing 100 largest packages ..." +dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 + +echo "Removing large packages ..." +apt-get -s autoremove +apt-get remove -y openjdk-11-jre-headless + +echo "After removing large packages, the space of each disk:" +df -h + +echo "Listing directories ..." +mount +ls -lrt / +du -csh /* +du -csh /__e/*/* +du -csh /__t/*/* +du -csh /__w/*/* +ls -lrt /github +du -csh /github/* +du -csh /opt/* +du -csh /usr/local/* +du -csh /usr/local/lib/* +du -csh /usr/local/share/* +du -csh /usr/share/* +echo "AGENT_TOOLSDIRECTORY is $AGENT_TOOLSDIRECTORY" + +echo "Removing large directories ..." +rm -rf /__t/CodeQL +rm -rf /__t/PyPy +rm -rf /__t/Python +rm -rf /__t/Ruby +rm -rf /__t/go +rm -rf /__t/node +rm -rf /opt/ghc +rm -rf /usr/local/.ghcup +rm -rf /usr/local/graalvm +rm -rf /usr/local/lib/android +rm -rf /usr/local/lib/node_modules +rm -rf /usr/local/share/boost +rm -rf /usr/local/share/chromium +rm -rf /usr/local/share/powershell +rm -rf /usr/share/dotnet + +echo "After freeing, the space of each disk:" +df -h diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 5c320eda5a..38d8cc53fe 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -49,8 +49,8 @@ on: env: # Update the options to reduce the consumption of the disk space ONEBOX_OPTS: disk_min_available_space_ratio=5 - TEST_OPTS: throttle_test_medium_value_kb=10,throttle_test_large_value_kb=25 - + TEST_OPTS: disk_min_available_space_ratio=5,throttle_test_medium_value_kb=10,throttle_test_large_value_kb=25 + jobs: cpp_clang_format_linter: name: Lint @@ -70,6 +70,9 @@ jobs: image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }} steps: - uses: actions/checkout@v3 + - name: Free Disk Space (Ubuntu) + run: | + .github/workflows/free_disk_space.sh - uses: dorny/paths-filter@v2 id: changes with: @@ -120,6 +123,9 @@ jobs: path: | /github/home/.ccache key: release_ccache + - name: Free Disk Space (Ubuntu) + run: | + .github/workflows/free_disk_space.sh - uses: dorny/paths-filter@v2 id: changes with: @@ -132,25 +138,36 @@ jobs: - name: Unpack prebuilt third-parties if: steps.changes.outputs.thirdparty == 'false' run: | + rm -f /root/thirdparties-src.zip unzip /root/thirdparties-bin.zip -d ./thirdparty rm -f /root/thirdparties-bin.zip + find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + rm -rf ./thirdparty/hadoop-bin/share/doc + rm -rf ./thirdparty/zookeeper-bin/docs - name: Rebuild third-parties if: steps.changes.outputs.thirdparty == 'true' working-directory: thirdparty # Build thirdparties and leave some necessary libraries and source run: | + rm -f /root/thirdparties-src.zip mkdir build cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* + find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + ../scripts/download_hadoop.sh hadoop-bin ../scripts/download_zk.sh zookeeper-bin + rm -rf hadoop-bin/share/doc + rm -rf zookeeper-bin/docs - name: Compilation run: | ccache -p ccache -z ./run.sh build --test --skip_thirdparty -j $(nproc) -t release ccache -s + - name: Clear Build Files + run: | + find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - name: Pack Server run: | ./run.sh pack_server @@ -164,7 +181,6 @@ jobs: mv thirdparty/hadoop-bin ./ mv thirdparty/zookeeper-bin ./ rm -rf thirdparty - find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + tar -zcvhf release__builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - name: Upload Artifact uses: actions/upload-artifact@v3 @@ -252,6 +268,9 @@ jobs: path: | /github/home/.ccache key: asan_ccache + - name: Free Disk Space (Ubuntu) + run: | + .github/workflows/free_disk_space.sh - uses: dorny/paths-filter@v2 id: changes with: @@ -264,31 +283,41 @@ jobs: - name: Unpack prebuilt third-parties if: steps.changes.outputs.thirdparty == 'false' run: | + rm -f /root/thirdparties-src.zip unzip /root/thirdparties-bin.zip -d ./thirdparty rm -f /root/thirdparties-bin.zip + find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + rm -rf ./thirdparty/hadoop-bin/share/doc + rm -rf ./thirdparty/zookeeper-bin/docs - name: Rebuild third-parties if: steps.changes.outputs.thirdparty == 'true' working-directory: thirdparty # Build thirdparties and leave some necessary libraries and source run: | + rm -f /root/thirdparties-src.zip mkdir build cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* + find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + ../scripts/download_hadoop.sh hadoop-bin ../scripts/download_zk.sh zookeeper-bin + rm -rf hadoop-bin/share/doc + rm -rf zookeeper-bin/docs - name: Compilation run: | ccache -p ccache -z ./run.sh build --test --sanitizer address --skip_thirdparty --disable_gperf -j $(nproc) ccache -s + - name: Clear Build Files + run: | + find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - name: Tar files run: | mv thirdparty/hadoop-bin ./ mv thirdparty/zookeeper-bin ./ rm -rf thirdparty - find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + tar -zcvhf release_address_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - name: Upload Artifact uses: actions/upload-artifact@v3 @@ -393,31 +422,41 @@ jobs: # - name: Unpack prebuilt third-parties # if: steps.changes.outputs.thirdparty == 'false' # run: | +# rm -f /root/thirdparties-src.zip # unzip /root/thirdparties-bin.zip -d ./thirdparty # rm -f /root/thirdparties-bin.zip +# find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + +# rm -rf ./thirdparty/hadoop-bin/share/doc +# rm -rf ./thirdparty/zookeeper-bin/docs # - name: Rebuild third-parties # if: steps.changes.outputs.thirdparty == 'true' # working-directory: thirdparty # # Build thirdparties and leave some necessary libraries and source # run: | +# rm -f /root/thirdparties-src.zip # mkdir build # cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ # cmake --build build/ -j $(nproc) # rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* +# find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + # ../scripts/download_hadoop.sh hadoop-bin # ../scripts/download_zk.sh zookeeper-bin +# rm -rf hadoop-bin/share/doc +# rm -rf zookeeper-bin/docs # - name: Compilation # run: | # ccache -p # ccache -z # ./run.sh build --test --sanitizer undefined --skip_thirdparty --disable_gperf -j $(nproc) # ccache -s +# - name: Clear Build Files +# run: | +# find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + # - name: Tar files # run: | # mv thirdparty/hadoop-bin ./ # mv thirdparty/zookeeper-bin ./ # rm -rf thirdparty -# find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + # tar -zcvhf release_undefined_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin # - name: Upload Artifact # uses: actions/upload-artifact@v3 @@ -470,6 +509,9 @@ jobs: # options: --cap-add=SYS_PTRACE # steps: # - uses: actions/checkout@v3 +# - name: Free Disk Space (Ubuntu) +# run: | +# .github/workflows/free_disk_space.sh # - name: Unpack prebuilt third-parties # run: | # unzip /root/thirdparties-bin.zip -d ./thirdparty @@ -503,6 +545,9 @@ jobs: path: | /github/home/.ccache key: jemalloc_ccache + - name: Free Disk Space (Ubuntu) + run: | + .github/workflows/free_disk_space.sh - uses: dorny/paths-filter@v2 id: changes with: @@ -515,25 +560,36 @@ jobs: - name: Unpack prebuilt third-parties if: steps.changes.outputs.thirdparty == 'false' run: | + rm -f /root/thirdparties-src.zip unzip /root/thirdparties-bin.zip -d ./thirdparty rm -f /root/thirdparties-bin.zip + find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + rm -rf ./thirdparty/hadoop-bin/share/doc + rm -rf ./thirdparty/zookeeper-bin/docs - name: Rebuild third-parties if: steps.changes.outputs.thirdparty == 'true' working-directory: thirdparty # Build thirdparties and leave some necessary libraries and source run: | + rm -f /root/thirdparties-src.zip mkdir build cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* + find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + ../scripts/download_hadoop.sh hadoop-bin ../scripts/download_zk.sh zookeeper-bin + rm -rf hadoop-bin/share/doc + rm -rf zookeeper-bin/docs - name: Compilation run: | ccache -p ccache -z ./run.sh build --test --skip_thirdparty -j $(nproc) -t release --use_jemalloc ccache -s + - name: Clear Build Files + run: | + find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - name: Pack Server run: ./run.sh pack_server -j - name: Pack Tools @@ -543,7 +599,6 @@ jobs: mv thirdparty/hadoop-bin ./ mv thirdparty/zookeeper-bin ./ rm -rf thirdparty - find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + tar -zcvhf release_jemalloc_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - name: Upload Artifact uses: actions/upload-artifact@v3 diff --git a/src/common/test/config-test.ini b/src/common/test/config-test.ini index 774ac82cd3..2775e06bd4 100644 --- a/src/common/test/config-test.ini +++ b/src/common/test/config-test.ini @@ -92,6 +92,7 @@ rpc_message_header_format = dsn rpc_timeout_milliseconds = 5000 [replication] +disk_min_available_space_ratio = 10 cluster_name = master-cluster [duplication-group] diff --git a/src/common/test/run.sh b/src/common/test/run.sh index ce89cc4eac..affb98db8b 100755 --- a/src/common/test/run.sh +++ b/src/common/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -31,6 +31,25 @@ exit_if_fail() { fi } +if [ -n ${TEST_OPTS} ]; then + if [ ! -f ./config-test.ini ]; then + echo "./config-test.ini does not exists !" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi + ./dsn_replication_common_test exit_if_fail $? "run unit test failed" diff --git a/src/meta/test/config-test.ini b/src/meta/test/config-test.ini index 26bf34f759..d0a8656567 100644 --- a/src/meta/test/config-test.ini +++ b/src/meta/test/config-test.ini @@ -106,6 +106,7 @@ only_move_primary = false cold_backup_disabled = false [replication] +disk_min_available_space_ratio = 10 cluster_name = master-cluster duplication_enabled = true diff --git a/src/meta/test/run.sh b/src/meta/test/run.sh index ad104aa6b4..207a9c7f80 100755 --- a/src/meta/test/run.sh +++ b/src/meta/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -28,6 +28,25 @@ if [ -z "${REPORT_DIR}" ]; then REPORT_DIR="." fi +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi + ./clear.sh output_xml="${REPORT_DIR}/dsn.meta.test.1.xml" GTEST_OUTPUT="xml:${output_xml}" ./dsn.meta.test diff --git a/src/replica/backup/test/config-test.ini b/src/replica/backup/test/config-test.ini index b24a8ff146..ad6433590a 100644 --- a/src/replica/backup/test/config-test.ini +++ b/src/replica/backup/test/config-test.ini @@ -51,6 +51,7 @@ partitioned = true name = replica_long [replication] +disk_min_available_space_ratio = 10 cluster_name = master-cluster [duplication-group] diff --git a/src/replica/backup/test/run.sh b/src/replica/backup/test/run.sh index 996196944e..73aa02be9a 100755 --- a/src/replica/backup/test/run.sh +++ b/src/replica/backup/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -23,6 +23,24 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi ./dsn_replica_backup_test diff --git a/src/replica/bulk_load/test/config-test.ini b/src/replica/bulk_load/test/config-test.ini index d9d8fd9daa..7bdc530aa8 100644 --- a/src/replica/bulk_load/test/config-test.ini +++ b/src/replica/bulk_load/test/config-test.ini @@ -91,6 +91,9 @@ rpc_call_channel = RPC_CHANNEL_TCP rpc_message_header_format = dsn rpc_timeout_milliseconds = 5000 +[replication] +disk_min_available_space_ratio = 10 + [block_service.local_service] type = local_service args = diff --git a/src/replica/bulk_load/test/run.sh b/src/replica/bulk_load/test/run.sh index 91ba506453..7b060300d2 100755 --- a/src/replica/bulk_load/test/run.sh +++ b/src/replica/bulk_load/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -23,6 +23,24 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi ./dsn_replica_bulk_load_test diff --git a/src/replica/duplication/test/config-test.ini b/src/replica/duplication/test/config-test.ini index 5eb6b16342..a06a16552a 100644 --- a/src/replica/duplication/test/config-test.ini +++ b/src/replica/duplication/test/config-test.ini @@ -50,6 +50,9 @@ partitioned = true [threadpool.THREAD_POOL_REPLICATION_LONG] name = replica_long +[replication] +disk_min_available_space_ratio = 10 + [duplication-group] master-cluster = 1 slave-cluster = 2 diff --git a/src/replica/duplication/test/run.sh b/src/replica/duplication/test/run.sh index c72b65788f..cf9d4b11ef 100755 --- a/src/replica/duplication/test/run.sh +++ b/src/replica/duplication/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -23,6 +23,24 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi ./dsn_replica_dup_test diff --git a/src/replica/split/test/config-test.ini b/src/replica/split/test/config-test.ini index ca0550f482..86279cd5a6 100644 --- a/src/replica/split/test/config-test.ini +++ b/src/replica/split/test/config-test.ini @@ -86,3 +86,6 @@ allow_inline = false rpc_call_channel = RPC_CHANNEL_TCP rpc_message_header_format = dsn rpc_timeout_milliseconds = 5000 + +[replication] +disk_min_available_space_ratio = 10 diff --git a/src/replica/split/test/run.sh b/src/replica/split/test/run.sh index 397d6f7a82..71820b0939 100755 --- a/src/replica/split/test/run.sh +++ b/src/replica/split/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -23,6 +23,24 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi ./dsn_replica_split_test diff --git a/src/replica/storage/simple_kv/config.ini b/src/replica/storage/simple_kv/config.ini index e1b1013e07..ad9ec33715 100644 --- a/src/replica/storage/simple_kv/config.ini +++ b/src/replica/storage/simple_kv/config.ini @@ -152,6 +152,7 @@ max_replica_count = 3 stateful = true [replication] +disk_min_available_space_ratio = 10 prepare_timeout_ms_for_secondaries = 10000 prepare_timeout_ms_for_potential_secondaries = 20000 diff --git a/src/replica/storage/simple_kv/run.sh b/src/replica/storage/simple_kv/run.sh index 6f0b84f183..7669b4daf9 100755 --- a/src/replica/storage/simple_kv/run.sh +++ b/src/replica/storage/simple_kv/run.sh @@ -29,6 +29,25 @@ if [ ! -f dsn.replication.simple_kv ]; then exit 1 fi +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config.ini" ]; then + echo "./config.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config.ini + done +fi + ./clear.sh echo "running dsn.replication.simple_kv for 20 seconds ..." diff --git a/src/replica/storage/simple_kv/test/case-000.ini b/src/replica/storage/simple_kv/test/case-000.ini index 76896bdd84..a05a95e53f 100644 --- a/src/replica/storage/simple_kv/test/case-000.ini +++ b/src/replica/storage/simple_kv/test/case-000.ini @@ -155,6 +155,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-001.ini b/src/replica/storage/simple_kv/test/case-001.ini index d5b5339f0c..4f214ef80b 100644 --- a/src/replica/storage/simple_kv/test/case-001.ini +++ b/src/replica/storage/simple_kv/test/case-001.ini @@ -155,6 +155,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-002.ini b/src/replica/storage/simple_kv/test/case-002.ini index 6beb5a0246..d75bbc1013 100644 --- a/src/replica/storage/simple_kv/test/case-002.ini +++ b/src/replica/storage/simple_kv/test/case-002.ini @@ -155,6 +155,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-003.ini b/src/replica/storage/simple_kv/test/case-003.ini index b6865db7ad..8d60d50d66 100644 --- a/src/replica/storage/simple_kv/test/case-003.ini +++ b/src/replica/storage/simple_kv/test/case-003.ini @@ -155,6 +155,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-004.ini b/src/replica/storage/simple_kv/test/case-004.ini index 6dcbb398ce..5c74f7e281 100644 --- a/src/replica/storage/simple_kv/test/case-004.ini +++ b/src/replica/storage/simple_kv/test/case-004.ini @@ -155,6 +155,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-005.ini b/src/replica/storage/simple_kv/test/case-005.ini index 7446967ddb..4c8c84b324 100644 --- a/src/replica/storage/simple_kv/test/case-005.ini +++ b/src/replica/storage/simple_kv/test/case-005.ini @@ -155,6 +155,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-006.ini b/src/replica/storage/simple_kv/test/case-006.ini index beb1272fcd..18e782b779 100644 --- a/src/replica/storage/simple_kv/test/case-006.ini +++ b/src/replica/storage/simple_kv/test/case-006.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-100.ini b/src/replica/storage/simple_kv/test/case-100.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-100.ini +++ b/src/replica/storage/simple_kv/test/case-100.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-101.ini b/src/replica/storage/simple_kv/test/case-101.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-101.ini +++ b/src/replica/storage/simple_kv/test/case-101.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-102.ini b/src/replica/storage/simple_kv/test/case-102.ini index 9b0c33ea71..8c35047783 100644 --- a/src/replica/storage/simple_kv/test/case-102.ini +++ b/src/replica/storage/simple_kv/test/case-102.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-103.ini b/src/replica/storage/simple_kv/test/case-103.ini index c57b99aa25..64a643dd0a 100644 --- a/src/replica/storage/simple_kv/test/case-103.ini +++ b/src/replica/storage/simple_kv/test/case-103.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-104.ini b/src/replica/storage/simple_kv/test/case-104.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-104.ini +++ b/src/replica/storage/simple_kv/test/case-104.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-105.ini b/src/replica/storage/simple_kv/test/case-105.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-105.ini +++ b/src/replica/storage/simple_kv/test/case-105.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-106.ini b/src/replica/storage/simple_kv/test/case-106.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-106.ini +++ b/src/replica/storage/simple_kv/test/case-106.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-107.ini b/src/replica/storage/simple_kv/test/case-107.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-107.ini +++ b/src/replica/storage/simple_kv/test/case-107.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-108.ini b/src/replica/storage/simple_kv/test/case-108.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-108.ini +++ b/src/replica/storage/simple_kv/test/case-108.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-109.ini b/src/replica/storage/simple_kv/test/case-109.ini index b674b1b40b..d95415e373 100644 --- a/src/replica/storage/simple_kv/test/case-109.ini +++ b/src/replica/storage/simple_kv/test/case-109.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-200.ini b/src/replica/storage/simple_kv/test/case-200.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-200.ini +++ b/src/replica/storage/simple_kv/test/case-200.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-201.ini b/src/replica/storage/simple_kv/test/case-201.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-201.ini +++ b/src/replica/storage/simple_kv/test/case-201.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-202-0.ini b/src/replica/storage/simple_kv/test/case-202-0.ini index 0dda324d66..a25eb9145e 100644 --- a/src/replica/storage/simple_kv/test/case-202-0.ini +++ b/src/replica/storage/simple_kv/test/case-202-0.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-202-1.ini b/src/replica/storage/simple_kv/test/case-202-1.ini index 0dda324d66..a25eb9145e 100644 --- a/src/replica/storage/simple_kv/test/case-202-1.ini +++ b/src/replica/storage/simple_kv/test/case-202-1.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-203-0.ini b/src/replica/storage/simple_kv/test/case-203-0.ini index 28af6cfe95..82d693aa9b 100644 --- a/src/replica/storage/simple_kv/test/case-203-0.ini +++ b/src/replica/storage/simple_kv/test/case-203-0.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-204.ini b/src/replica/storage/simple_kv/test/case-204.ini index 7dab9e62e6..9fde171e66 100644 --- a/src/replica/storage/simple_kv/test/case-204.ini +++ b/src/replica/storage/simple_kv/test/case-204.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-205.ini b/src/replica/storage/simple_kv/test/case-205.ini index 7dab9e62e6..9fde171e66 100644 --- a/src/replica/storage/simple_kv/test/case-205.ini +++ b/src/replica/storage/simple_kv/test/case-205.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-206.ini b/src/replica/storage/simple_kv/test/case-206.ini index 7dab9e62e6..9fde171e66 100644 --- a/src/replica/storage/simple_kv/test/case-206.ini +++ b/src/replica/storage/simple_kv/test/case-206.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-207.ini b/src/replica/storage/simple_kv/test/case-207.ini index 7dab9e62e6..9fde171e66 100644 --- a/src/replica/storage/simple_kv/test/case-207.ini +++ b/src/replica/storage/simple_kv/test/case-207.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-208.ini b/src/replica/storage/simple_kv/test/case-208.ini index 7dab9e62e6..9fde171e66 100644 --- a/src/replica/storage/simple_kv/test/case-208.ini +++ b/src/replica/storage/simple_kv/test/case-208.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-209.ini b/src/replica/storage/simple_kv/test/case-209.ini index 7dab9e62e6..9fde171e66 100644 --- a/src/replica/storage/simple_kv/test/case-209.ini +++ b/src/replica/storage/simple_kv/test/case-209.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-210.ini b/src/replica/storage/simple_kv/test/case-210.ini index a3897bb679..23784c15f0 100644 --- a/src/replica/storage/simple_kv/test/case-210.ini +++ b/src/replica/storage/simple_kv/test/case-210.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-211.ini b/src/replica/storage/simple_kv/test/case-211.ini index a3897bb679..23784c15f0 100644 --- a/src/replica/storage/simple_kv/test/case-211.ini +++ b/src/replica/storage/simple_kv/test/case-211.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-212.ini b/src/replica/storage/simple_kv/test/case-212.ini index 2b43d145aa..4582c07878 100644 --- a/src/replica/storage/simple_kv/test/case-212.ini +++ b/src/replica/storage/simple_kv/test/case-212.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-213.ini b/src/replica/storage/simple_kv/test/case-213.ini index 247a8fea5b..103a599a63 100644 --- a/src/replica/storage/simple_kv/test/case-213.ini +++ b/src/replica/storage/simple_kv/test/case-213.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-214.ini b/src/replica/storage/simple_kv/test/case-214.ini index 7dab9e62e6..9fde171e66 100644 --- a/src/replica/storage/simple_kv/test/case-214.ini +++ b/src/replica/storage/simple_kv/test/case-214.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-215.ini b/src/replica/storage/simple_kv/test/case-215.ini index 7dab9e62e6..9fde171e66 100644 --- a/src/replica/storage/simple_kv/test/case-215.ini +++ b/src/replica/storage/simple_kv/test/case-215.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-216.ini b/src/replica/storage/simple_kv/test/case-216.ini index 7dab9e62e6..9fde171e66 100644 --- a/src/replica/storage/simple_kv/test/case-216.ini +++ b/src/replica/storage/simple_kv/test/case-216.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-300-0.ini b/src/replica/storage/simple_kv/test/case-300-0.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-300-0.ini +++ b/src/replica/storage/simple_kv/test/case-300-0.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-300-1.ini b/src/replica/storage/simple_kv/test/case-300-1.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-300-1.ini +++ b/src/replica/storage/simple_kv/test/case-300-1.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-300-2.ini b/src/replica/storage/simple_kv/test/case-300-2.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-300-2.ini +++ b/src/replica/storage/simple_kv/test/case-300-2.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-301.ini b/src/replica/storage/simple_kv/test/case-301.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-301.ini +++ b/src/replica/storage/simple_kv/test/case-301.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-302.ini b/src/replica/storage/simple_kv/test/case-302.ini index 90681e6b97..38d3a831f1 100644 --- a/src/replica/storage/simple_kv/test/case-302.ini +++ b/src/replica/storage/simple_kv/test/case-302.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-303.ini b/src/replica/storage/simple_kv/test/case-303.ini index 0e0e866f8d..04932bc943 100644 --- a/src/replica/storage/simple_kv/test/case-303.ini +++ b/src/replica/storage/simple_kv/test/case-303.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-304.ini b/src/replica/storage/simple_kv/test/case-304.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-304.ini +++ b/src/replica/storage/simple_kv/test/case-304.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-305.ini b/src/replica/storage/simple_kv/test/case-305.ini index 0e0e866f8d..04932bc943 100644 --- a/src/replica/storage/simple_kv/test/case-305.ini +++ b/src/replica/storage/simple_kv/test/case-305.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-306.ini b/src/replica/storage/simple_kv/test/case-306.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-306.ini +++ b/src/replica/storage/simple_kv/test/case-306.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-307.ini b/src/replica/storage/simple_kv/test/case-307.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-307.ini +++ b/src/replica/storage/simple_kv/test/case-307.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-400.ini b/src/replica/storage/simple_kv/test/case-400.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-400.ini +++ b/src/replica/storage/simple_kv/test/case-400.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-401.ini b/src/replica/storage/simple_kv/test/case-401.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-401.ini +++ b/src/replica/storage/simple_kv/test/case-401.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-402.ini b/src/replica/storage/simple_kv/test/case-402.ini index e71811be91..0f601799a1 100644 --- a/src/replica/storage/simple_kv/test/case-402.ini +++ b/src/replica/storage/simple_kv/test/case-402.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-600.ini b/src/replica/storage/simple_kv/test/case-600.ini index beb1272fcd..18e782b779 100644 --- a/src/replica/storage/simple_kv/test/case-600.ini +++ b/src/replica/storage/simple_kv/test/case-600.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-601.ini b/src/replica/storage/simple_kv/test/case-601.ini index 0401980fc0..9a25f26cc8 100644 --- a/src/replica/storage/simple_kv/test/case-601.ini +++ b/src/replica/storage/simple_kv/test/case-601.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-602.ini b/src/replica/storage/simple_kv/test/case-602.ini index beb1272fcd..18e782b779 100644 --- a/src/replica/storage/simple_kv/test/case-602.ini +++ b/src/replica/storage/simple_kv/test/case-602.ini @@ -156,6 +156,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-603.ini b/src/replica/storage/simple_kv/test/case-603.ini index 4b171e934b..1531c037fe 100644 --- a/src/replica/storage/simple_kv/test/case-603.ini +++ b/src/replica/storage/simple_kv/test/case-603.ini @@ -158,6 +158,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/config.ini b/src/replica/storage/simple_kv/test/config.ini index 892a318259..b014b44c3f 100644 --- a/src/replica/storage/simple_kv/test/config.ini +++ b/src/replica/storage/simple_kv/test/config.ini @@ -179,6 +179,7 @@ max_replica_count = 3 stateful = true [replication] +disk_min_available_space_ratio = 10 prepare_timeout_ms_for_secondaries = 10000 prepare_timeout_ms_for_potential_secondaries = 20000 diff --git a/src/replica/storage/simple_kv/test/run.sh b/src/replica/storage/simple_kv/test/run.sh index 7737b09c96..b3daa127d1 100755 --- a/src/replica/storage/simple_kv/test/run.sh +++ b/src/replica/storage/simple_kv/test/run.sh @@ -29,6 +29,26 @@ bin=./dsn.rep_tests.simple_kv function run_single() { prefix=$1 + + if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./${prefix}.ini" ]; then + echo "./${prefix}.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./${prefix}.ini + done + fi + echo "${bin} ${prefix}.ini ${prefix}.act" ${bin} ${prefix}.ini ${prefix}.act ret=$? diff --git a/src/replica/test/config-test.ini b/src/replica/test/config-test.ini index 1a1997cc9f..9d73bb5d01 100644 --- a/src/replica/test/config-test.ini +++ b/src/replica/test/config-test.ini @@ -87,6 +87,9 @@ rpc_call_channel = RPC_CHANNEL_TCP rpc_message_header_format = dsn rpc_timeout_milliseconds = 5000 +[replication] +disk_min_available_space_ratio = 10 + [block_service.local_service] type = local_service args = diff --git a/src/replica/test/run.sh b/src/replica/test/run.sh index 6d645f3327..12152709dd 100755 --- a/src/replica/test/run.sh +++ b/src/replica/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -28,6 +28,25 @@ if [ -z "${REPORT_DIR}" ]; then REPORT_DIR="." fi +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi + ./clear.sh output_xml="${REPORT_DIR}/dsn.replica.test.1.xml" GTEST_OUTPUT="xml:${output_xml}" ./dsn.replica.test diff --git a/src/server/test/config.ini b/src/server/test/config.ini index a1125b265a..018d5b28d7 100644 --- a/src/server/test/config.ini +++ b/src/server/test/config.ini @@ -144,6 +144,7 @@ max_replica_count = 3 stateful = true [replication] +disk_min_available_space_ratio = 10 data_dirs_black_list_file = /home/mi/.pegasus_data_dirs_black_list cluster_name = onebox diff --git a/src/server/test/run.sh b/src/server/test/run.sh index 76baf02a0b..d8da4db1a6 100755 --- a/src/server/test/run.sh +++ b/src/server/test/run.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -23,6 +23,25 @@ exit_if_fail() { fi } +if [ -n ${TEST_OPTS} ]; then + if [ ! -f ./config.ini ]; then + echo "./config.ini does not exists !" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config.ini + done +fi + ./pegasus_unit_test exit_if_fail $? "run unit test failed" diff --git a/src/test/function_test/run.sh b/src/test/function_test/run.sh index 9bd561386b..947cc35f8f 100755 --- a/src/test/function_test/run.sh +++ b/src/test/function_test/run.sh @@ -26,7 +26,7 @@ fi if [ -n ${TEST_OPTS} ]; then if [ ! -f ./config.ini ]; then - echo "./config.ini does not exists !" + echo "./config.ini does not exists" exit 1 fi diff --git a/src/utils/filesystem.cpp b/src/utils/filesystem.cpp index 1954f29b5a..8bfe4cfa6f 100644 --- a/src/utils/filesystem.cpp +++ b/src/utils/filesystem.cpp @@ -694,7 +694,7 @@ bool get_disk_space_info(const std::string &path, disk_space_info &info) FAIL_POINT_INJECT_F("filesystem_get_disk_space_info", [&info](string_view str) { info.capacity = 100 * 1024 * 1024; if (str.find("insufficient") != string_view::npos) { - info.available = 5 * 1024 * 1024; + info.available = 512 * 1024; } else { info.available = 50 * 1024 * 1024; } diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index f6a21b16d1..53fc3a8892 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -41,6 +41,7 @@ #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" +#include "utils/ports.h" #include "utils/process_utils.h" #include "utils/strings.h" #include "utils/time_utils.h" From 10a54350de09fdfb1720c3d50478b4cd8989932b Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 8 Aug 2023 11:54:17 +0800 Subject: [PATCH 04/42] fix: support core dump for fatal logging in simple_logger::dsn_logv and call fatal logging exactly once for the assertion (#1580) https://github.com/apache/incubator-pegasus/issues/1579 Following fixes are included in this PR to solve the problems: - support core dump for fatal logging in `simple_logger::dsn_logv`; - call fatal logging exactly once for the assertion macros `CHECK*`; - add fail point to keep the logging test running without termination. --- src/client/replication_ddl_client.cpp | 1 - src/meta/test/backup_test.cpp | 1 + src/server/result_writer.cpp | 1 + src/utils/fmt_logging.h | 6 ++--- src/utils/simple_logger.cpp | 35 ++++++++++++++++++++++----- src/utils/simple_logger.h | 24 +++++++++--------- src/utils/test/logging.cpp | 14 +++++++++-- 7 files changed, 59 insertions(+), 23 deletions(-) diff --git a/src/client/replication_ddl_client.cpp b/src/client/replication_ddl_client.cpp index 00276e9861..ff0b3269c3 100644 --- a/src/client/replication_ddl_client.cpp +++ b/src/client/replication_ddl_client.cpp @@ -45,7 +45,6 @@ #include "common/replication.codes.h" #include "common/replication_common.h" #include "common/replication_enums.h" -#include "fmt/core.h" #include "fmt/ostream.h" #include "meta/meta_rpc_types.h" #include "runtime/api_layer1.h" diff --git a/src/meta/test/backup_test.cpp b/src/meta/test/backup_test.cpp index 70062ba693..35b88bb97e 100644 --- a/src/meta/test/backup_test.cpp +++ b/src/meta/test/backup_test.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include diff --git a/src/server/result_writer.cpp b/src/server/result_writer.cpp index d4e9ebeb74..16ec2886bb 100644 --- a/src/server/result_writer.cpp +++ b/src/server/result_writer.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include "pegasus/client.h" diff --git a/src/utils/fmt_logging.h b/src/utils/fmt_logging.h index 1f6e961207..f0c74bd293 100644 --- a/src/utils/fmt_logging.h +++ b/src/utils/fmt_logging.h @@ -57,9 +57,9 @@ #define CHECK_EXPRESSION(expression, evaluation, ...) \ do { \ if (dsn_unlikely(!(evaluation))) { \ - dlog_f(LOG_LEVEL_FATAL, "assertion expression: " #expression); \ - dlog_f(LOG_LEVEL_FATAL, __VA_ARGS__); \ - dsn_coredump(); \ + std::string assertion_info("assertion expression: [" #expression "] "); \ + assertion_info += fmt::format(__VA_ARGS__); \ + LOG_FATAL(assertion_info); \ } \ } while (false) diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index 53fc3a8892..98b1693d1a 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -38,11 +38,14 @@ #include "runtime/api_layer1.h" #include "runtime/task/task_spec.h" #include "utils/command_manager.h" +#include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/ports.h" #include "utils/process_utils.h" +#include "utils/string_conv.h" +#include "utils/string_view.h" #include "utils/strings.h" #include "utils/time_utils.h" @@ -94,6 +97,28 @@ static void print_header(FILE *fp, dsn_log_level_t log_level) log_prefixed_message_func().c_str()); } +namespace { + +inline void process_fatal_log(dsn_log_level_t log_level) +{ + if (dsn_likely(log_level < LOG_LEVEL_FATAL)) { + return; + } + + bool coredump = true; + FAIL_POINT_INJECT_NOT_RETURN_F("coredump_for_fatal_log", [&coredump](dsn::string_view str) { + CHECK(buf2bool(str, coredump), + "invalid coredump toggle for fatal log, should be true or false: {}", + str); + }); + + if (dsn_likely(coredump)) { + dsn_coredump(); + } +} + +} // anonymous namespace + screen_logger::screen_logger(bool short_header) { _short_header = short_header; } screen_logger::~screen_logger(void) {} @@ -114,9 +139,7 @@ void screen_logger::dsn_logv(const char *file, vprintf(fmt, args); printf("\n"); - if (dsn_unlikely(log_level >= LOG_LEVEL_FATAL)) { - dsn_coredump(); - } + process_fatal_log(log_level); } void screen_logger::flush() { ::fflush(stdout); } @@ -262,6 +285,8 @@ void simple_logger::dsn_logv(const char *file, printf("\n"); } + process_fatal_log(log_level); + if (++_lines >= 200000) { create_log_file(); } @@ -292,9 +317,7 @@ void simple_logger::dsn_log(const char *file, printf("%s\n", str); } - if (dsn_unlikely(log_level >= LOG_LEVEL_FATAL)) { - dsn_coredump(); - } + process_fatal_log(log_level); if (++_lines >= 200000) { create_log_file(); diff --git a/src/utils/simple_logger.h b/src/utils/simple_logger.h index 74a44d2dde..71535b26fc 100644 --- a/src/utils/simple_logger.h +++ b/src/utils/simple_logger.h @@ -46,18 +46,20 @@ class screen_logger : public logging_provider explicit screen_logger(bool short_header); ~screen_logger() override; - virtual void dsn_logv(const char *file, - const char *function, - const int line, - dsn_log_level_t log_level, - const char *fmt, - va_list args); + void dsn_logv(const char *file, + const char *function, + const int line, + dsn_log_level_t log_level, + const char *fmt, + va_list args) override; - virtual void dsn_log(const char *file, - const char *function, - const int line, - dsn_log_level_t log_level, - const char *str){}; + void dsn_log(const char *file, + const char *function, + const int line, + dsn_log_level_t log_level, + const char *str) override + { + } virtual void flush(); diff --git a/src/utils/test/logging.cpp b/src/utils/test/logging.cpp index f526dc5a28..eb3170d2a9 100644 --- a/src/utils/test/logging.cpp +++ b/src/utils/test/logging.cpp @@ -38,6 +38,7 @@ #include #include "utils/api_utilities.h" +#include "utils/fail_point.h" #include "utils/fmt_logging.h" TEST(core, logging) @@ -63,7 +64,7 @@ TEST(core, logging_big_log) big_str.c_str()); } -TEST(core, dlog_f) +TEST(core, dlog) { struct test_case { @@ -76,7 +77,16 @@ TEST(core, dlog_f) {dsn_log_level_t::LOG_LEVEL_ERROR, "\\x00%d\\x00\\x01%n/nm"}, {dsn_log_level_t::LOG_LEVEL_FATAL, "\\x00%d\\x00\\x01%n/nm"}}; + dsn::fail::setup(); + dsn::fail::cfg("coredump_for_fatal_log", "void(false)"); + for (auto test : tests) { - dlog_f(test.level, "sortkey = {}", test.str); + // Test logging_provider::dsn_log + dlog_f(test.level, "dlog_f: sortkey = {}", test.str); + + // Test logging_provider::dsn_logv + dlog(test.level, "dlog: sortkey = %s", test.str.c_str()); } + + dsn::fail::teardown(); } From fc616286b3f14b6f4ce92da95e83e74e333ba463 Mon Sep 17 00:00:00 2001 From: Pengfan Lu Date: Fri, 11 Aug 2023 10:41:47 +0800 Subject: [PATCH 05/42] Feature:Online Query and Dynamic Modification of Table-level RocksDB Options (#1511) https://github.com/apache/incubator-pegasus/issues/1488 Complete the dynamic setting function of num_levels and write_buffer_size option. I use rocksdb.write_buffer_size as a dynamically modifiable parameter and rocksdb.num_levels as a non-dynamically modifiable parameter to test my idea. --- src/base/pegasus_const.cpp | 49 +++++ src/base/pegasus_const.h | 22 +++ src/common/replica_envs.h | 6 + src/common/replication_common.cpp | 9 + src/meta/app_env_validator.cpp | 58 +++++- src/meta/app_env_validator.h | 2 + src/meta/server_state.cpp | 3 + src/meta/test/meta_app_envs_test.cpp | 24 +++ src/meta/test/meta_app_operation_test.cpp | 191 +++++++++++++++---- src/server/pegasus_server_impl.cpp | 108 ++++++++++- src/server/pegasus_server_impl.h | 9 + src/server/pegasus_server_impl_init.cpp | 47 +++-- src/server/test/pegasus_server_impl_test.cpp | 58 ++++++ 13 files changed, 520 insertions(+), 66 deletions(-) diff --git a/src/base/pegasus_const.cpp b/src/base/pegasus_const.cpp index c93c0baf58..2a788cd24d 100644 --- a/src/base/pegasus_const.cpp +++ b/src/base/pegasus_const.cpp @@ -19,6 +19,12 @@ #include "pegasus_const.h" +#include +#include +#include + +#include "utils/string_conv.h" + namespace pegasus { // should be same with items in dsn::backup_restore_constant @@ -101,4 +107,47 @@ const std::string USER_SPECIFIED_COMPACTION("user_specified_compaction"); const std::string READ_SIZE_THROTTLING("replica.read_throttling_by_size"); const std::string ROCKSDB_ALLOW_INGEST_BEHIND("rocksdb.allow_ingest_behind"); + +const std::string ROCKSDB_WRITE_BUFFER_SIZE("rocksdb.write_buffer_size"); + +const std::string ROCKSDB_NUM_LEVELS("rocksdb.num_levels"); + +const std::set ROCKSDB_DYNAMIC_OPTIONS = { + ROCKSDB_WRITE_BUFFER_SIZE, +}; +const std::set ROCKSDB_STATIC_OPTIONS = { + ROCKSDB_NUM_LEVELS, +}; + +const std::unordered_map cf_opts_setters = { + {ROCKSDB_WRITE_BUFFER_SIZE, + [](const std::string &str, rocksdb::ColumnFamilyOptions &option) -> bool { + uint64_t val = 0; + if (!dsn::buf2uint64(str, val)) { + return false; + } + option.write_buffer_size = static_cast(val); + return true; + }}, + {ROCKSDB_NUM_LEVELS, + [](const std::string &str, rocksdb::ColumnFamilyOptions &option) -> bool { + int32_t val = 0; + if (!dsn::buf2int32(str, val)) { + return false; + } + option.num_levels = val; + return true; + }}, +}; + +const std::unordered_map cf_opts_getters = { + {ROCKSDB_WRITE_BUFFER_SIZE, + [](const rocksdb::ColumnFamilyOptions &option, /*out*/ std::string &str) { + str = std::to_string(option.write_buffer_size); + }}, + {ROCKSDB_NUM_LEVELS, + [](const rocksdb::ColumnFamilyOptions &option, /*out*/ std::string &str) { + str = std::to_string(option.num_levels); + }}, +}; } // namespace pegasus diff --git a/src/base/pegasus_const.h b/src/base/pegasus_const.h index c2a47ce519..e9326dfaa9 100644 --- a/src/base/pegasus_const.h +++ b/src/base/pegasus_const.h @@ -19,7 +19,14 @@ #pragma once +#include +#include #include +#include + +namespace rocksdb { +struct ColumnFamilyOptions; +} // namespace rocksdb namespace pegasus { @@ -72,4 +79,19 @@ extern const std::string USER_SPECIFIED_COMPACTION; extern const std::string READ_SIZE_THROTTLING; extern const std::string ROCKSDB_ALLOW_INGEST_BEHIND; + +extern const std::string ROCKSDB_WRITE_BUFFER_SIZE; + +extern const std::string ROCKSDB_NUM_LEVELS; + +extern const std::set ROCKSDB_DYNAMIC_OPTIONS; + +extern const std::set ROCKSDB_STATIC_OPTIONS; + +using cf_opts_setter = std::function; +extern const std::unordered_map cf_opts_setters; + +using cf_opts_getter = + std::function; +extern const std::unordered_map cf_opts_getters; } // namespace pegasus diff --git a/src/common/replica_envs.h b/src/common/replica_envs.h index 4db367a7fa..1db2e5399f 100644 --- a/src/common/replica_envs.h +++ b/src/common/replica_envs.h @@ -28,6 +28,7 @@ #include #include +#include namespace dsn { namespace replication { @@ -64,6 +65,11 @@ class replica_envs static const std::string USER_SPECIFIED_COMPACTION; static const std::string ROCKSDB_ALLOW_INGEST_BEHIND; static const std::string UPDATE_MAX_REPLICA_COUNT; + static const std::string ROCKSDB_WRITE_BUFFER_SIZE; + static const std::string ROCKSDB_NUM_LEVELS; + + static const std::set ROCKSDB_DYNAMIC_OPTIONS; + static const std::set ROCKSDB_STATIC_OPTIONS; }; } // namespace replication diff --git a/src/common/replication_common.cpp b/src/common/replication_common.cpp index 63617797ac..1afe9d49fe 100644 --- a/src/common/replication_common.cpp +++ b/src/common/replication_common.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include "common/gpid.h" #include "common/replica_envs.h" @@ -394,6 +395,14 @@ const std::string replica_envs::USER_SPECIFIED_COMPACTION("user_specified_compac const std::string replica_envs::BACKUP_REQUEST_QPS_THROTTLING("replica.backup_request_throttling"); const std::string replica_envs::ROCKSDB_ALLOW_INGEST_BEHIND("rocksdb.allow_ingest_behind"); const std::string replica_envs::UPDATE_MAX_REPLICA_COUNT("max_replica_count.update"); +const std::string replica_envs::ROCKSDB_WRITE_BUFFER_SIZE("rocksdb.write_buffer_size"); +const std::string replica_envs::ROCKSDB_NUM_LEVELS("rocksdb.num_levels"); +const std::set replica_envs::ROCKSDB_DYNAMIC_OPTIONS = { + replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, +}; +const std::set replica_envs::ROCKSDB_STATIC_OPTIONS = { + replica_envs::ROCKSDB_NUM_LEVELS, +}; } // namespace replication } // namespace dsn diff --git a/src/meta/app_env_validator.cpp b/src/meta/app_env_validator.cpp index 41f9c299ce..7f197f5a0f 100644 --- a/src/meta/app_env_validator.cpp +++ b/src/meta/app_env_validator.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,26 @@ namespace dsn { namespace replication { +bool validate_app_envs(const std::map &envs) +{ + // only check rocksdb app envs currently + + for (const auto &it : envs) { + if (replica_envs::ROCKSDB_STATIC_OPTIONS.find(it.first) == + replica_envs::ROCKSDB_STATIC_OPTIONS.end() && + replica_envs::ROCKSDB_DYNAMIC_OPTIONS.find(it.first) == + replica_envs::ROCKSDB_DYNAMIC_OPTIONS.end()) { + continue; + } + std::string hint_message; + if (!validate_app_env(it.first, it.second, hint_message)) { + LOG_WARNING( + "app env {}={} is invaild, hint_message:{}", it.first, it.second, hint_message); + return false; + } + } + return true; +} bool validate_app_env(const std::string &env_name, const std::string &env_value, @@ -151,6 +172,36 @@ bool check_bool_value(const std::string &env_value, std::string &hint_message) return true; } +bool check_rocksdb_write_buffer_size(const std::string &env_value, std::string &hint_message) +{ + uint64_t val = 0; + + if (!dsn::buf2uint64(env_value, val)) { + hint_message = fmt::format("rocksdb.write_buffer_size cannot set this val: {}", env_value); + return false; + } + if (val < (16 << 20) || val > (512 << 20)) { + hint_message = + fmt::format("rocksdb.write_buffer_size suggest set val in range [16777216, 536870912]"); + return false; + } + return true; +} +bool check_rocksdb_num_levels(const std::string &env_value, std::string &hint_message) +{ + int32_t val = 0; + + if (!dsn::buf2int32(env_value, val)) { + hint_message = fmt::format("rocksdb.num_levels cannot set this val: {}", env_value); + return false; + } + if (val < 1 || val > 10) { + hint_message = fmt::format("rocksdb.num_levels suggest set val in range [1 , 10]"); + return false; + } + return true; +} + bool app_env_validator::validate_app_env(const std::string &env_name, const std::string &env_value, std::string &hint_message) @@ -198,6 +249,10 @@ void app_env_validator::register_all_validators() std::bind(&check_bool_value, std::placeholders::_1, std::placeholders::_2)}, {replica_envs::DENY_CLIENT_REQUEST, std::bind(&check_deny_client, std::placeholders::_1, std::placeholders::_2)}, + {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, + std::bind(&check_rocksdb_write_buffer_size, std::placeholders::_1, std::placeholders::_2)}, + {replica_envs::ROCKSDB_NUM_LEVELS, + std::bind(&check_rocksdb_num_levels, std::placeholders::_1, std::placeholders::_2)}, // TODO(zhaoliwei): not implemented {replica_envs::BUSINESS_INFO, nullptr}, {replica_envs::TABLE_LEVEL_DEFAULT_TTL, nullptr}, @@ -213,7 +268,8 @@ void app_env_validator::register_all_validators() {replica_envs::MANUAL_COMPACT_PERIODIC_TARGET_LEVEL, nullptr}, {replica_envs::MANUAL_COMPACT_PERIODIC_BOTTOMMOST_LEVEL_COMPACTION, nullptr}, {replica_envs::REPLICA_ACCESS_CONTROLLER_ALLOWED_USERS, nullptr}, - {replica_envs::REPLICA_ACCESS_CONTROLLER_RANGER_POLICIES, nullptr}}; + {replica_envs::REPLICA_ACCESS_CONTROLLER_RANGER_POLICIES, nullptr}, + }; } } // namespace replication diff --git a/src/meta/app_env_validator.h b/src/meta/app_env_validator.h index c401c39598..d963df0513 100644 --- a/src/meta/app_env_validator.h +++ b/src/meta/app_env_validator.h @@ -25,6 +25,8 @@ namespace dsn { namespace replication { +bool validate_app_envs(const std::map &envs); + bool validate_app_env(const std::string &env_name, const std::string &env_value, std::string &hint_message); diff --git a/src/meta/server_state.cpp b/src/meta/server_state.cpp index ecf66a92bc..2b6c6d9108 100644 --- a/src/meta/server_state.cpp +++ b/src/meta/server_state.cpp @@ -1151,6 +1151,9 @@ void server_state::create_app(dsn::message_ex *msg) !validate_target_max_replica_count(request.options.replica_count)) { response.err = ERR_INVALID_PARAMETERS; will_create_app = false; + } else if (!validate_app_envs(request.options.envs)) { + response.err = ERR_INVALID_PARAMETERS; + will_create_app = false; } else { zauto_write_lock l(_lock); app = get_app(request.app_name); diff --git a/src/meta/test/meta_app_envs_test.cpp b/src/meta/test/meta_app_envs_test.cpp index 255930320d..d5dc21edac 100644 --- a/src/meta/test/meta_app_envs_test.cpp +++ b/src/meta/test/meta_app_envs_test.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include "common/replica_envs.h" @@ -142,6 +143,17 @@ TEST_F(meta_app_envs_test, update_app_envs_test) {replica_envs::MANUAL_COMPACT_ONCE_TARGET_LEVEL, "80", ERR_OK, "", "80"}, {replica_envs::MANUAL_COMPACT_PERIODIC_TRIGGER_TIME, "90", ERR_OK, "", "90"}, {replica_envs::MANUAL_COMPACT_PERIODIC_TARGET_LEVEL, "100", ERR_OK, "", "100"}, + {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, + "100", + ERR_INVALID_PARAMETERS, + "rocksdb.write_buffer_size suggest set val in range [16777216, 536870912]", + "67108864"}, + {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, + "636870912", + ERR_INVALID_PARAMETERS, + "rocksdb.write_buffer_size suggest set val in range [16777216, 536870912]", + "536870912"}, + {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, "67108864", ERR_OK, "", "67108864"}, {replica_envs::MANUAL_COMPACT_PERIODIC_BOTTOMMOST_LEVEL_COMPACTION, "200", ERR_OK, @@ -192,6 +204,18 @@ TEST_F(meta_app_envs_test, update_app_envs_test) ASSERT_EQ(app->envs.at(test.env_key), test.expect_value); } } + + { + // Make sure all rocksdb options of ROCKSDB_DYNAMIC_OPTIONS are tested. + // Hint: Mainly verify the update_rocksdb_dynamic_options function. + std::map all_test_envs; + for (const auto &test : tests) { + all_test_envs[test.env_key] = test.env_value; + } + for (const auto &option : replica_envs::ROCKSDB_DYNAMIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + } } } // namespace replication diff --git a/src/meta/test/meta_app_operation_test.cpp b/src/meta/test/meta_app_operation_test.cpp index c3ddafa72c..e0afc1baa5 100644 --- a/src/meta/test/meta_app_operation_test.cpp +++ b/src/meta/test/meta_app_operation_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -68,7 +69,8 @@ class meta_app_operation_test : public meta_test_base error_code create_app_test(int32_t partition_count, int32_t replica_count, bool success_if_exist, - const std::string &app_name) + const std::string &app_name, + const std::map &envs = {}) { configuration_create_app_request create_request; configuration_create_app_response create_response; @@ -78,6 +80,7 @@ class meta_app_operation_test : public meta_test_base create_request.options.replica_count = replica_count; create_request.options.success_if_exist = success_if_exist; create_request.options.is_stateful = true; + create_request.options.envs = envs; auto result = fake_create_app(_ss.get(), create_request); fake_wait_rpc(result, create_response); @@ -362,6 +365,14 @@ TEST_F(meta_app_operation_test, create_app) // - wrong app_status dropping // - create succeed with app_status dropped // - create succeed with success_if_exist=true + // - wrong rocksdb.num_levels (< 1) + // - wrong rocksdb.num_levels (> 10) + // - wrong rocksdb.num_levels (non-digital character) + // - create app with rocksdb.num_levels (= 5) succeed + // - wrong rocksdb.write_buffer_size (< (16<<20)) + // - wrong rocksdb.write_buffer_size (> (512<<20)) + // - wrong rocksdb.write_buffer_size (non-digital character) + // - create app with rocksdb.write_buffer_size (= (32<<20)) succeed struct create_test { std::string app_name; @@ -373,43 +384,126 @@ TEST_F(meta_app_operation_test, create_app) bool success_if_exist; app_status::type before_status; error_code expected_err; - } tests[] = {{APP_NAME, -1, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 0, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, -1, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 0, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 7, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 5, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 5, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 4, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 7, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME + "_1", 4, 5, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_2", 4, 5, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_3", 4, 4, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_4", 4, 4, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_5", 4, 3, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_6", 4, 4, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 3, 2, 5, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 4, 5, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 4, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 2, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 4, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME + "_7", 4, 3, 2, 4, 3, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 1, 1, 0, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, - {APP_NAME, 4, 2, 2, 1, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, - {APP_NAME, 4, 3, 3, 2, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, - {APP_NAME + "_8", 4, 3, 3, 3, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_9", 4, 1, 1, 1, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_10", 4, 2, 1, 2, 2, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_APP_EXIST}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_CREATING, ERR_BUSY_CREATING}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_RECALLING, ERR_BUSY_CREATING}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPING, ERR_BUSY_DROPPING}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPED, ERR_OK}, - {APP_NAME, 4, 3, 2, 3, 3, true, app_status::AS_INVALID, ERR_OK}}; + std::map envs = {}; + } tests[] = { + {APP_NAME, -1, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 0, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, -1, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 0, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 7, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 5, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 5, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 4, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 7, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME + "_1", 4, 5, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_2", 4, 5, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_3", 4, 4, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_4", 4, 4, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_5", 4, 3, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_6", 4, 4, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 3, 2, 5, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 4, 5, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 4, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 2, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 4, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME + "_7", 4, 3, 2, 4, 3, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 1, 1, 0, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, + {APP_NAME, 4, 2, 2, 1, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, + {APP_NAME, 4, 3, 3, 2, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, + {APP_NAME + "_8", 4, 3, 3, 3, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_9", 4, 1, 1, 1, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_10", 4, 2, 1, 2, 2, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_APP_EXIST}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_CREATING, ERR_BUSY_CREATING}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_RECALLING, ERR_BUSY_CREATING}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPING, ERR_BUSY_DROPPING}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPED, ERR_OK}, + {APP_NAME, 4, 3, 2, 3, 3, true, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.num_levels", "0"}}}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.num_levels", "11"}}}, + {APP_NAME + "_11", + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.num_levels", "5i"}}}, + {APP_NAME + "_11", + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_OK, + {{"rocksdb.num_levels", "5"}}}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.write_buffer_size", "1000"}}}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.write_buffer_size", "1073741824"}}}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.write_buffer_size", "n33554432"}}}, + {APP_NAME + "_12", + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_OK, + {{"rocksdb.write_buffer_size", "33554432"}}}, + }; clear_nodes(); @@ -453,13 +547,30 @@ TEST_F(meta_app_operation_test, create_app) } else if (test.before_status != app_status::AS_INVALID) { update_app_status(test.before_status); } - auto err = create_app_test( - test.partition_count, test.replica_count, test.success_if_exist, test.app_name); + auto err = create_app_test(test.partition_count, + test.replica_count, + test.success_if_exist, + test.app_name, + test.envs); ASSERT_EQ(err, test.expected_err); _ms->set_node_state(nodes, true); } + { + // Make sure all rocksdb options of ROCKSDB_DYNAMIC_OPTIONS and ROCKSDB_STATIC_OPTIONS are + // tested. Hint: Mainly verify the validate_app_envs function. + std::map all_test_envs; + for (const auto &test : tests) { + all_test_envs.insert(test.envs.begin(), test.envs.end()); + } + for (const auto &option : replica_envs::ROCKSDB_DYNAMIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + for (const auto &option : replica_envs::ROCKSDB_STATIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + } // set FLAGS_min_allowed_replica_count successfully res = update_flag("min_allowed_replica_count", "2"); ASSERT_TRUE(res.is_ok()); diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index e6e9d48edc..998b4873f7 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -38,10 +38,12 @@ #include // IWYU pragma: keep #include #include +#include #include #include #include #include +#include #include "base/pegasus_key_schema.h" #include "base/pegasus_utils.h" @@ -1663,7 +1665,7 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) // We don't use `loaded_data_cf_opts` directly because pointer-typed options will // only be initialized with default values when calling 'LoadLatestOptions', see // 'rocksdb/utilities/options_util.h'. - reset_usage_scenario_options(loaded_data_cf_opts, &_table_data_cf_opts); + reset_rocksdb_options(loaded_data_cf_opts, &_table_data_cf_opts); _db_opts.allow_ingest_behind = parse_allow_ingest_behind(envs); } } else { @@ -2625,6 +2627,67 @@ pegasus_server_impl::get_restore_dir_from_env(const std::map &envs) +{ + if (envs.empty()) { + return; + } + + std::unordered_map new_options; + for (const auto &option : ROCKSDB_DYNAMIC_OPTIONS) { + const auto &find = envs.find(option); + if (find == envs.end()) { + continue; + } + + std::vector args; + // split_args example: Parse "write_buffer_size" from "rocksdb.write_buffer_size" + dsn::utils::split_args(option.c_str(), args, '.'); + CHECK_EQ(args.size(), 2); + new_options[args[1]] = find->second; + } + + // doing set option + if (!new_options.empty() && set_options(new_options)) { + LOG_INFO("Set rocksdb dynamic options success"); + } +} + +void pegasus_server_impl::set_rocksdb_options_before_creating( + const std::map &envs) +{ + if (envs.empty()) { + return; + } + + for (const auto &option : pegasus::ROCKSDB_STATIC_OPTIONS) { + const auto &find = envs.find(option); + if (find == envs.end()) { + continue; + } + + const auto &setter = cf_opts_setters.find(option); + CHECK_TRUE(setter != cf_opts_setters.end()); + if (setter->second(find->second, _data_cf_opts)) { + LOG_INFO_PREFIX("Set {} \"{}\" succeed", find->first, find->second); + } + } + + for (const auto &option : pegasus::ROCKSDB_DYNAMIC_OPTIONS) { + const auto &find = envs.find(option); + if (find == envs.end()) { + continue; + } + + const auto &setter = cf_opts_setters.find(option); + CHECK_TRUE(setter != cf_opts_setters.end()); + if (setter->second(find->second, _data_cf_opts)) { + LOG_INFO_PREFIX("Set {} \"{}\" succeed", find->first, find->second); + } + } +} + void pegasus_server_impl::update_app_envs(const std::map &envs) { update_usage_scenario(envs); @@ -2637,6 +2700,7 @@ void pegasus_server_impl::update_app_envs(const std::map &envs) { envs[ROCKSDB_ENV_USAGE_SCENARIO_KEY] = _usage_scenario; + // write_buffer_size involves random values (refer to pegasus_server_impl::set_usage_scenario), + // so it can only be taken from _data_cf_opts + envs[ROCKSDB_WRITE_BUFFER_SIZE] = std::to_string(_data_cf_opts.write_buffer_size); + + // Get Data ColumnFamilyOptions directly from _data_cf + rocksdb::ColumnFamilyDescriptor desc; + CHECK_TRUE(_data_cf->GetDescriptor(&desc).ok()); + for (const auto &option : pegasus::ROCKSDB_STATIC_OPTIONS) { + auto getter = cf_opts_getters.find(option); + CHECK_TRUE(getter != cf_opts_getters.end()); + std::string option_val; + getter->second(desc.options, option_val); + envs[option] = option_val; + } + for (const auto &option : pegasus::ROCKSDB_DYNAMIC_OPTIONS) { + if (option.compare(ROCKSDB_WRITE_BUFFER_SIZE) == 0) { + continue; + } + auto getter = cf_opts_getters.find(option); + CHECK_TRUE(getter != cf_opts_getters.end()); + std::string option_val; + getter->second(desc.options, option_val); + envs[option] = option_val; + } } void pegasus_server_impl::update_usage_scenario(const std::map &envs) @@ -3038,6 +3127,23 @@ bool pegasus_server_impl::set_usage_scenario(const std::string &usage_scenario) } } +void pegasus_server_impl::reset_rocksdb_options(const rocksdb::ColumnFamilyOptions &base_opts, + rocksdb::ColumnFamilyOptions *target_opts) +{ + LOG_INFO_PREFIX("Reset rocksdb envs options"); + // Reset rocksdb option includes two aspects: + // 1. Set usage_scenario related rocksdb options + // 2. Rocksdb option set in app envs, consists of ROCKSDB_DYNAMIC_OPTIONS and + // ROCKSDB_STATIC_OPTIONS + + // aspect 1: + reset_usage_scenario_options(base_opts, target_opts); + + // aspect 2: + target_opts->num_levels = base_opts.num_levels; + target_opts->write_buffer_size = base_opts.write_buffer_size; +} + void pegasus_server_impl::reset_usage_scenario_options( const rocksdb::ColumnFamilyOptions &base_opts, rocksdb::ColumnFamilyOptions *target_opts) { diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index d156acdce6..c9c5b90ef6 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -328,6 +329,10 @@ class pegasus_server_impl : public pegasus_read_service void update_user_specified_compaction(const std::map &envs); + void update_rocksdb_dynamic_options(const std::map &envs); + + void set_rocksdb_options_before_creating(const std::map &envs); + void update_throttling_controller(const std::map &envs); bool parse_allow_ingest_behind(const std::map &envs); @@ -359,6 +364,9 @@ class pegasus_server_impl : public pegasus_read_service void reset_usage_scenario_options(const rocksdb::ColumnFamilyOptions &base_opts, rocksdb::ColumnFamilyOptions *target_opts); + void reset_rocksdb_options(const rocksdb::ColumnFamilyOptions &base_opts, + rocksdb::ColumnFamilyOptions *target_opts); + // return true if successfully set bool set_options(const std::unordered_map &new_options); @@ -468,6 +476,7 @@ class pegasus_server_impl : public pegasus_read_service // Dynamically calculate the value of current data_cf option according to the conf module file // and usage scenario rocksdb::ColumnFamilyOptions _table_data_cf_opts; + rocksdb::BlockBasedTableOptions _tbl_opts; rocksdb::ColumnFamilyOptions _meta_cf_opts; rocksdb::ReadOptions _data_cf_rd_opts; std::string _usage_scenario; diff --git a/src/server/pegasus_server_impl_init.cpp b/src/server/pegasus_server_impl_init.cpp index 896f8ab945..c457ebd9fa 100644 --- a/src/server/pegasus_server_impl_init.cpp +++ b/src/server/pegasus_server_impl_init.cpp @@ -137,7 +137,7 @@ DSN_DEFINE_bool(pegasus.server, DSN_DEFINE_bool(pegasus.server, rocksdb_disable_table_block_cache, false, - "rocksdb tbl_opts.no_block_cache"); + "rocksdb _tbl_opts.no_block_cache"); DSN_DEFINE_bool(pegasus.server, rocksdb_enable_write_buffer_manager, false, @@ -466,12 +466,11 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) CHECK(parse_compression_types("none", _meta_cf_opts.compression_per_level), "parse rocksdb_compression_type failed."); - rocksdb::BlockBasedTableOptions tbl_opts; - tbl_opts.read_amp_bytes_per_bit = FLAGS_read_amp_bytes_per_bit; + _tbl_opts.read_amp_bytes_per_bit = FLAGS_read_amp_bytes_per_bit; if (FLAGS_rocksdb_disable_table_block_cache) { - tbl_opts.no_block_cache = true; - tbl_opts.block_restart_interval = 4; + _tbl_opts.no_block_cache = true; + _tbl_opts.block_restart_interval = 4; } else { // If block cache is enabled, all replicas on this server will share the same block cache // object. It's convenient to control the total memory used by this server, and the LRU @@ -484,7 +483,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) }); // every replica has the same block cache - tbl_opts.block_cache = _s_block_cache; + _tbl_opts.block_cache = _s_block_cache; } // FLAGS_rocksdb_limiter_max_write_megabytes_per_sec <= 0 means close the rate limit. @@ -520,7 +519,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) FLAGS_rocksdb_total_size_across_write_buffer); _s_write_buffer_manager = std::make_shared( static_cast(FLAGS_rocksdb_total_size_across_write_buffer), - tbl_opts.block_cache); + _tbl_opts.block_cache); }); _db_opts.write_buffer_manager = _s_write_buffer_manager; } @@ -541,33 +540,33 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) CHECK(index_type_item != INDEX_TYPE_STRING_MAP.end(), "[pegasus.server]rocksdb_index_type should be one among binary_search, " "hash_search, two_level_index_search or binary_search_with_first_key."); - tbl_opts.index_type = index_type_item->second; + _tbl_opts.index_type = index_type_item->second; LOG_INFO_PREFIX("rocksdb_index_type = {}", FLAGS_rocksdb_index_type); - tbl_opts.partition_filters = FLAGS_rocksdb_partition_filters; + _tbl_opts.partition_filters = FLAGS_rocksdb_partition_filters; // TODO(yingchun): clean up these useless log ? - LOG_INFO_PREFIX("rocksdb_partition_filters = {}", tbl_opts.partition_filters); + LOG_INFO_PREFIX("rocksdb_partition_filters = {}", _tbl_opts.partition_filters); - tbl_opts.metadata_block_size = FLAGS_rocksdb_metadata_block_size; - LOG_INFO_PREFIX("rocksdb_metadata_block_size = {}", tbl_opts.metadata_block_size); + _tbl_opts.metadata_block_size = FLAGS_rocksdb_metadata_block_size; + LOG_INFO_PREFIX("rocksdb_metadata_block_size = {}", _tbl_opts.metadata_block_size); - tbl_opts.cache_index_and_filter_blocks = FLAGS_rocksdb_cache_index_and_filter_blocks; + _tbl_opts.cache_index_and_filter_blocks = FLAGS_rocksdb_cache_index_and_filter_blocks; LOG_INFO_PREFIX("rocksdb_cache_index_and_filter_blocks = {}", - tbl_opts.cache_index_and_filter_blocks); + _tbl_opts.cache_index_and_filter_blocks); - tbl_opts.pin_top_level_index_and_filter = FLAGS_rocksdb_pin_top_level_index_and_filter; + _tbl_opts.pin_top_level_index_and_filter = FLAGS_rocksdb_pin_top_level_index_and_filter; LOG_INFO_PREFIX("rocksdb_pin_top_level_index_and_filter = {}", - tbl_opts.pin_top_level_index_and_filter); + _tbl_opts.pin_top_level_index_and_filter); - tbl_opts.cache_index_and_filter_blocks_with_high_priority = + _tbl_opts.cache_index_and_filter_blocks_with_high_priority = FLAGS_rocksdb_cache_index_and_filter_blocks_with_high_priority; LOG_INFO_PREFIX("rocksdb_cache_index_and_filter_blocks_with_high_priority = {}", - tbl_opts.cache_index_and_filter_blocks_with_high_priority); + _tbl_opts.cache_index_and_filter_blocks_with_high_priority); - tbl_opts.pin_l0_filter_and_index_blocks_in_cache = + _tbl_opts.pin_l0_filter_and_index_blocks_in_cache = FLAGS_rocksdb_pin_l0_filter_and_index_blocks_in_cache; LOG_INFO_PREFIX("rocksdb_pin_l0_filter_and_index_blocks_in_cache = {}", - tbl_opts.pin_l0_filter_and_index_blocks_in_cache); + _tbl_opts.pin_l0_filter_and_index_blocks_in_cache); // Bloom filter configurations. if (!FLAGS_rocksdb_disable_bloom_filter) { @@ -584,8 +583,8 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) // 50 | 0.225453 | ~0.00003 // Recommend using no more than three decimal digits after the decimal point, as in 6.667. // More details: https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter - tbl_opts.format_version = FLAGS_rocksdb_format_version; - tbl_opts.filter_policy.reset( + _tbl_opts.format_version = FLAGS_rocksdb_format_version; + _tbl_opts.filter_policy.reset( rocksdb::NewBloomFilterPolicy(FLAGS_rocksdb_bloom_filter_bits_per_key, false)); if (dsn::utils::equals(FLAGS_rocksdb_filter_type, "prefix")) { @@ -596,8 +595,8 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) } } - _data_cf_opts.table_factory.reset(NewBlockBasedTableFactory(tbl_opts)); - _meta_cf_opts.table_factory.reset(NewBlockBasedTableFactory(tbl_opts)); + _data_cf_opts.table_factory.reset(NewBlockBasedTableFactory(_tbl_opts)); + _meta_cf_opts.table_factory.reset(NewBlockBasedTableFactory(_tbl_opts)); _key_ttl_compaction_filter_factory = std::make_shared(); _data_cf_opts.compaction_filter_factory = _key_ttl_compaction_filter_factory; diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 1363d8054b..9776571ae9 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -29,7 +29,9 @@ #include #include #include +#include #include +#include #include "pegasus_const.h" #include "pegasus_server_test_base.h" @@ -95,6 +97,50 @@ class pegasus_server_impl_test : public pegasus_server_test_base ASSERT_EQ(before_count + test.expect_perf_counter_incr, after_count); } } + + void test_open_db_with_rocksdb_envs(bool is_restart) + { + struct create_test + { + std::string env_key; + std::string env_value; + std::string expect_value; + } tests[] = { + {"rocksdb.num_levels", "5", "5"}, {"rocksdb.write_buffer_size", "33554432", "33554432"}, + }; + + std::map all_test_envs; + { + // Make sure all rocksdb options of ROCKSDB_DYNAMIC_OPTIONS and ROCKSDB_STATIC_OPTIONS + // are tested. + for (const auto &test : tests) { + all_test_envs[test.env_key] = test.env_value; + } + for (const auto &option : pegasus::ROCKSDB_DYNAMIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + for (const auto &option : pegasus::ROCKSDB_STATIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + } + + start(all_test_envs); + if (is_restart) { + _server->stop(false); + start(); + } + + std::map query_envs; + _server->query_app_envs(query_envs); + for (const auto &test : tests) { + const auto &iter = query_envs.find(test.env_key); + if (iter != query_envs.end()) { + ASSERT_EQ(iter->second, test.expect_value); + } else { + ASSERT_TRUE(false) << fmt::format("query_app_envs not supported {}", test.env_key); + } + } + } }; TEST_F(pegasus_server_impl_test, test_table_level_slow_query) @@ -137,6 +183,18 @@ TEST_F(pegasus_server_impl_test, test_open_db_with_app_envs) ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario); } +TEST_F(pegasus_server_impl_test, test_open_db_with_rocksdb_envs) +{ + // Hint: Verify the set_rocksdb_options_before_creating function by boolean is_restart=false. + test_open_db_with_rocksdb_envs(false); +} + +TEST_F(pegasus_server_impl_test, test_restart_db_with_rocksdb_envs) +{ + // Hint: Verify the reset_rocksdb_options function by boolean is_restart=true. + test_open_db_with_rocksdb_envs(true); +} + TEST_F(pegasus_server_impl_test, test_stop_db_twice) { start(); From 7fe41d4bfe046ad73c203c7f6dc208d2e4d22ef1 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Fri, 11 Aug 2023 14:46:37 +0800 Subject: [PATCH 06/42] chore(github): free disk space after packaging with jemalloc in github workflows (#1581) https://github.com/apache/incubator-pegasus/issues/1576 In the workflow that builds pegasus with jemalloc, more space could be freed after packaging is tested. --- .github/workflows/lint_and_test_cpp.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 38d8cc53fe..115f8b124c 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -410,6 +410,9 @@ jobs: # path: | # /github/home/.ccache # key: ubsan_ccache +# - name: Free Disk Space (Ubuntu) +# run: | +# .github/workflows/free_disk_space.sh # - uses: dorny/paths-filter@v2 # id: changes # with: @@ -591,9 +594,13 @@ jobs: run: | find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - name: Pack Server - run: ./run.sh pack_server -j + run: | + ./run.sh pack_server -j + rm -rf pegasus-server-* - name: Pack Tools - run: ./run.sh pack_tools -j + run: | + ./run.sh pack_tools -j + rm -rf pegasus-tools-* - name: Tar files run: | mv thirdparty/hadoop-bin ./ From 72c8f2f753eba3bc06ce9c75da61510e8a35a6d2 Mon Sep 17 00:00:00 2001 From: WHBANG <38547944+WHBANG@users.noreply.github.com> Date: Fri, 11 Aug 2023 14:50:33 +0800 Subject: [PATCH 07/42] feat(security): Support to use the principal of Unix account for authentication (#1569) https://github.com/apache/incubator-pegasus/issues/1568 When kerberos authentication is enabled: - The config 'krb5_keytab' and 'krb5_principal' is not empty, which means that pegasus completes the kinit action by itself. - When the value of both is empty, the user needs to complete the kinit action by himself, and pegasus will obtain the principal of the current unix account for authentication. --- src/runtime/security/init.cpp | 10 +++++++++ src/runtime/security/kinit_context.cpp | 28 ++++++++++++++++++++++++++ src/runtime/security/kinit_context.h | 1 + 3 files changed, 39 insertions(+) diff --git a/src/runtime/security/init.cpp b/src/runtime/security/init.cpp index 36d78e567c..870dd32204 100644 --- a/src/runtime/security/init.cpp +++ b/src/runtime/security/init.cpp @@ -23,11 +23,14 @@ #include "utils/errors.h" #include "utils/flags.h" #include "utils/fmt_logging.h" +#include "utils/strings.h" namespace dsn { namespace security { +DSN_DECLARE_bool(enable_auth); DSN_DECLARE_string(krb5_config); DSN_DECLARE_string(krb5_keytab); +DSN_DECLARE_string(krb5_principal); /*** * set kerberos envs(for more details: @@ -43,6 +46,13 @@ void set_krb5_env(bool is_server) error_s init_kerberos(bool is_server) { + // When FLAGS_enable_auth is set but lacks of necessary parameters to execute kinit by itself, + // then try to obtain the principal under the current Unix account for identity + // authentication automatically. + if (FLAGS_enable_auth && utils::is_empty(FLAGS_krb5_keytab) && + utils::is_empty(FLAGS_krb5_principal)) { + return run_get_principal_without_kinit(); + } // set kerberos env set_krb5_env(is_server); diff --git a/src/runtime/security/kinit_context.cpp b/src/runtime/security/kinit_context.cpp index 08ff5eea86..7b281d5d3a 100644 --- a/src/runtime/security/kinit_context.cpp +++ b/src/runtime/security/kinit_context.cpp @@ -93,6 +93,9 @@ class kinit_context : public utils::singleton public: // implementation of 'kinit -k -t ' error_s kinit(); + // If kinit has been executed outside the program, then directly obtain the principal + // information of the unix account for permission verification. + error_s get_principal_without_kinit(); const std::string &username() const { return _user_name; } private: @@ -174,6 +177,26 @@ error_s kinit_context::kinit() return error_s::ok(); } +// obtain _principal info under the current unix account for permission verification. +error_s kinit_context::get_principal_without_kinit() +{ + // get krb5_ctx + init_krb5_ctx(); + + // acquire credential cache handle + KRB5_RETURN_NOT_OK(krb5_cc_default(_krb5_context, &_ccache), + "couldn't acquire credential cache handle"); + + // get '_principal' from '_ccache' + KRB5_RETURN_NOT_OK(krb5_cc_get_principal(_krb5_context, _ccache, &_principal), + "get principal from cache failed"); + + // get '_user_name' from '_principal' + RETURN_NOT_OK(parse_username_from_principal()); + + return error_s::ok(); +} + void kinit_context::init_krb5_ctx() { static std::once_flag once; @@ -333,6 +356,11 @@ error_s kinit_context::wrap_krb5_err(krb5_error_code krb5_err, const std::string error_s run_kinit() { return kinit_context::instance().kinit(); } +error_s run_get_principal_without_kinit() +{ + return kinit_context::instance().get_principal_without_kinit(); +} + const std::string &get_username() { return kinit_context::instance().username(); } } // namespace security } // namespace dsn diff --git a/src/runtime/security/kinit_context.h b/src/runtime/security/kinit_context.h index 05a0de3993..bb97a43f22 100644 --- a/src/runtime/security/kinit_context.h +++ b/src/runtime/security/kinit_context.h @@ -24,6 +24,7 @@ namespace dsn { namespace security { extern error_s run_kinit(); +extern error_s run_get_principal_without_kinit(); extern const std::string &get_username(); } // namespace security } // namespace dsn From c5a01c1f41e762fbfd42df6e0ea7d3ccdec60f02 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Mon, 28 Aug 2023 14:44:24 +0800 Subject: [PATCH 08/42] chore(github): free disk space for daily building on github (#1586) https://github.com/apache/incubator-pegasus/issues/1585 Free more disk space for daily building cpp by referring to https://github.com/apache/incubator-pegasus/pull/1577. Test has been run manually on [Lint and build regularly](https://github.com/ apache/incubator-pegasus/actions/workflows/regular-build.yml) for the branch [free-regular-disk-space](https://github.com/apache/ incubator-pegasus/tree/free-regular-disk-space). --- .github/workflows/regular-build.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regular-build.yml b/.github/workflows/regular-build.yml index 798b28e3a9..9e9a0d06fb 100644 --- a/.github/workflows/regular-build.yml +++ b/.github/workflows/regular-build.yml @@ -75,14 +75,27 @@ jobs: working-directory: /root run: | git clone -b ${{ github.ref_name }} --depth=1 https://github.com/apache/incubator-pegasus.git + - name: Free Disk Space (Ubuntu) + run: | + .github/workflows/free_disk_space.sh - name: Unpack prebuilt third-parties run: | + rm -f /root/thirdparties-src.zip unzip /root/thirdparties-bin.zip -d ./thirdparty rm -f /root/thirdparties-bin.zip + find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + rm -rf ./thirdparty/hadoop-bin/share/doc + rm -rf ./thirdparty/zookeeper-bin/docs - name: Compilation Pegasus - run: ./run.sh build --test --compiler ${{ matrix.compiler }} --skip_thirdparty -j $(nproc) + run: | + ./run.sh build --test --compiler ${{ matrix.compiler }} --skip_thirdparty -j $(nproc) + - name: Clear Build Files + run: | + find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - name: Packaging Server - run: ./run.sh pack_server + run: | + ./run.sh pack_server + rm -rf pegasus-server-* build_and_lint_go: name: Build and Lint Golang From be9119e75f27b9f0563acaf33f5c96edf0051307 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Mon, 28 Aug 2023 14:45:00 +0800 Subject: [PATCH 09/42] fix: fix errors in format string for logging and macro (#1588) https://github.com/apache/incubator-pegasus/issues/1587 Fix some wrong format string in logging and macro that are: - still using c-style format string, which would not print the correct messages; - calling `to_string()` method for the classes which have implemented `operator<<`. --- src/meta/meta_backup_service.cpp | 13 +++---------- src/meta/meta_data.cpp | 12 ++++-------- src/meta/partition_guardian.cpp | 14 +++++--------- src/meta/test/backup_test.cpp | 1 - src/replica/replica.cpp | 6 ++---- src/replica/replica_2pc.cpp | 6 +----- src/runtime/task/task.cpp | 4 ++-- src/server/pegasus_server_impl.cpp | 2 +- 8 files changed, 18 insertions(+), 40 deletions(-) diff --git a/src/meta/meta_backup_service.cpp b/src/meta/meta_backup_service.cpp index db2afe1d90..8a28ffaf35 100644 --- a/src/meta/meta_backup_service.cpp +++ b/src/meta/meta_backup_service.cpp @@ -1036,10 +1036,7 @@ void policy_context::sync_remove_backup_info(const backup_info &info, dsn::task_ 0, _backup_service->backup_option().meta_retry_delay_ms); } else { - CHECK(false, - "{}: we can't handle this right now, error({})", - _policy.policy_name, - err.to_string()); + CHECK(false, "{}: we can't handle this right now, error({})", _policy.policy_name, err); } }; @@ -1373,9 +1370,7 @@ void backup_service::do_add_policy(dsn::message_ex *req, _opt.meta_retry_delay_ms); return; } else { - CHECK(false, - "we can't handle this when create backup policy, err({})", - err.to_string()); + CHECK(false, "we can't handle this when create backup policy, err({})", err); } }, value); @@ -1411,9 +1406,7 @@ void backup_service::do_update_policy_to_remote_storage( 0, _opt.meta_retry_delay_ms); } else { - CHECK(false, - "we can't handle this when create backup policy, err({})", - err.to_string()); + CHECK(false, "we can't handle this when create backup policy, err({})", err); } }); } diff --git a/src/meta/meta_data.cpp b/src/meta/meta_data.cpp index 48ab6ccd3f..23dba0c929 100644 --- a/src/meta/meta_data.cpp +++ b/src/meta/meta_data.cpp @@ -159,10 +159,7 @@ bool construct_replica(meta_view view, const gpid &pid, int max_replica_count) // we put max_replica_count-1 recent replicas to last_drops, in case of the DDD-state when the // only primary dead // when add node to pc.last_drops, we don't remove it from our cc.drop_list - CHECK(pc.last_drops.empty(), - "last_drops of partition({}.{}) must be empty", - pid.get_app_id(), - pid.get_partition_index()); + CHECK(pc.last_drops.empty(), "last_drops of partition({}) must be empty", pid); for (auto iter = drop_list.rbegin(); iter != drop_list.rend(); ++iter) { if (pc.last_drops.size() + 1 >= max_replica_count) break; @@ -402,10 +399,9 @@ int config_context::collect_drop_replica(const rpc_address &node, const replica_ iter = find_from_dropped(node); if (iter == dropped.end()) { CHECK(!in_dropped, - "adjust position of existing node({}) failed, this is a bug, partition({}.{})", - node.to_string(), - config_owner->pid.get_app_id(), - config_owner->pid.get_partition_index()); + "adjust position of existing node({}) failed, this is a bug, partition({})", + node, + config_owner->pid); return -1; } return in_dropped ? 1 : 0; diff --git a/src/meta/partition_guardian.cpp b/src/meta/partition_guardian.cpp index 09b0862c4c..44e0017f69 100644 --- a/src/meta/partition_guardian.cpp +++ b/src/meta/partition_guardian.cpp @@ -102,9 +102,8 @@ void partition_guardian::reconfig(meta_view view, const configuration_update_req if (!cc->lb_actions.empty()) { const configuration_proposal_action *current = cc->lb_actions.front(); CHECK(current != nullptr && current->type != config_type::CT_INVALID, - "invalid proposal for gpid({}.{})", - gpid.get_app_id(), - gpid.get_partition_index()); + "invalid proposal for gpid({})", + gpid); // if the valid proposal is from cure if (!cc->lb_actions.is_from_balancer()) { finish_cure_proposal(view, gpid, *current); @@ -132,7 +131,7 @@ void partition_guardian::reconfig(meta_view view, const configuration_update_req CHECK(cc->record_drop_history(request.node), "node({}) has been in the dropped", - request.node.to_string()); + request.node); } }); } @@ -241,8 +240,7 @@ pc_status partition_guardian::on_missing_primary(meta_view &view, const dsn::gpi for (int i = 0; i < pc.secondaries.size(); ++i) { node_state *ns = get_node_state(*(view.nodes), pc.secondaries[i], false); - CHECK_NOTNULL( - ns, "invalid secondary address, address = {}", pc.secondaries[i].to_string()); + CHECK_NOTNULL(ns, "invalid secondary address, address = {}", pc.secondaries[i]); if (!ns->alive()) continue; @@ -607,9 +605,7 @@ pc_status partition_guardian::on_missing_secondary(meta_view &view, const dsn::g // if not emergency, only try to recover last dropped server const dropped_replica &server = cc.dropped.back(); if (is_node_alive(*view.nodes, server.node)) { - CHECK(!server.node.is_invalid(), - "invalid server address, address = {}", - server.node.to_string()); + CHECK(!server.node.is_invalid(), "invalid server address, address = {}", server.node); action.node = server.node; } diff --git a/src/meta/test/backup_test.cpp b/src/meta/test/backup_test.cpp index 35b88bb97e..70062ba693 100644 --- a/src/meta/test/backup_test.cpp +++ b/src/meta/test/backup_test.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp index be2a0177b5..f159d7699e 100644 --- a/src/replica/replica.cpp +++ b/src/replica/replica.cpp @@ -28,11 +28,10 @@ #include #include -#include +#include #include #include #include -#include #include #include "backup/replica_backup_manager.h" @@ -285,8 +284,7 @@ void replica::on_client_read(dsn::message_ex *request, bool ignore_throttling) // a small window where the state is not the latest yet if (last_committed_decree() < _primary_states.last_prepare_decree_on_new_primary) { - LOG_ERROR_PREFIX("last_committed_decree(%" PRId64 - ") < last_prepare_decree_on_new_primary(%" PRId64 ")", + LOG_ERROR_PREFIX("last_committed_decree({}) < last_prepare_decree_on_new_primary({})", last_committed_decree(), _primary_states.last_prepare_decree_on_new_primary); response_client_read(request, ERR_INVALID_STATE); diff --git a/src/replica/replica_2pc.cpp b/src/replica/replica_2pc.cpp index e0cff3868e..6138ad7e7d 100644 --- a/src/replica/replica_2pc.cpp +++ b/src/replica/replica_2pc.cpp @@ -259,11 +259,7 @@ void replica::init_prepare(mutation_ptr &mu, bool reconciliation, bool pop_all_c } mu->_tracer->set_name(fmt::format("mutation[{}]", mu->name())); - dlog(level, - "%s: mutation %s init_prepare, mutation_tid=%" PRIu64, - name(), - mu->name(), - mu->tid()); + dlog_f(level, "{}: mutation {} init_prepare, mutation_tid={}", name(), mu->name(), mu->tid()); // child should prepare mutation synchronously mu->set_is_sync_to_child(_primary_states.sync_send_write_request); diff --git a/src/runtime/task/task.cpp b/src/runtime/task/task.cpp index 6e08bd714d..7d83948f3b 100644 --- a/src/runtime/task/task.cpp +++ b/src/runtime/task/task.cpp @@ -120,8 +120,8 @@ task::task(dsn::task_code code, int hash, service_node *node) } else { auto p = get_current_node(); CHECK_NOTNULL(p, - "tasks without explicit service node " - "can only be created inside threads which is attached to specific node"); + "tasks without explicit service node can only be created " + "inside threads which is attached to specific node"); _node = p; } diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 998b4873f7..5bc03eafe9 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -2308,7 +2308,7 @@ bool pegasus_server_impl::validate_filter(::dsn::apps::filter_type::type filter_ } } default: - CHECK(false, "unsupported filter type: %d", filter_type); + CHECK(false, "unsupported filter type: {}", filter_type); } return false; } From 26b99e1e80efc40d8c1569de35070004cd72df18 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Sun, 10 Sep 2023 10:32:46 +0800 Subject: [PATCH 10/42] Revert "build(thirdparty): add thirdparty for centos6 build env (#1017)" (#1600) * Revert "build(thirdparty): add thirdparty for centos6 build env (#1017)" This reverts commit 862c824231bcc8309444e62d7b2d40523c8ea742. * push master images * run ASAN in Ubuntu 2204 * revert * 1804 * 1804 --- .github/workflows/thirdparty-regular-push.yml | 2 +- docker/pegasus-build-env/centos7/Dockerfile | 2 ++ .../pegasus-build-env/ubuntu1804/Dockerfile | 2 ++ .../pegasus-build-env/ubuntu2004/Dockerfile | 2 ++ .../pegasus-build-env/ubuntu2204/Dockerfile | 2 ++ scripts/pack_server.sh | 2 -- thirdparty/CMakeLists.txt | 32 ------------------- 7 files changed, 9 insertions(+), 35 deletions(-) diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml index 152fe15a78..4fb58b59af 100644 --- a/.github/workflows/thirdparty-regular-push.yml +++ b/.github/workflows/thirdparty-regular-push.yml @@ -144,7 +144,7 @@ jobs: fail-fast: false matrix: osversion: - - ubuntu2204 + - ubuntu1804 steps: - uses: actions/checkout@v3 - name: Set up QEMU diff --git a/docker/pegasus-build-env/centos7/Dockerfile b/docker/pegasus-build-env/centos7/Dockerfile index 5c536166c1..0e89f316b1 100644 --- a/docker/pegasus-build-env/centos7/Dockerfile +++ b/docker/pegasus-build-env/centos7/Dockerfile @@ -50,6 +50,8 @@ RUN yum -y install centos-release-scl \ lz4-devel \ bison \ flex \ + krb5-devel \ + cyrus-sasl-devel \ patch; \ yum clean all; \ rm -rf /var/cache/yum; diff --git a/docker/pegasus-build-env/ubuntu1804/Dockerfile b/docker/pegasus-build-env/ubuntu1804/Dockerfile index c1e53891af..667b64120e 100644 --- a/docker/pegasus-build-env/ubuntu1804/Dockerfile +++ b/docker/pegasus-build-env/ubuntu1804/Dockerfile @@ -50,6 +50,8 @@ RUN apt-get update -y; \ libtool \ libssl-dev \ bison \ + libkrb5-dev \ + libsasl2-dev \ maven \ flex \ python3-setuptools; \ diff --git a/docker/pegasus-build-env/ubuntu2004/Dockerfile b/docker/pegasus-build-env/ubuntu2004/Dockerfile index 6438fb03bf..604b5a0b1e 100644 --- a/docker/pegasus-build-env/ubuntu2004/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2004/Dockerfile @@ -50,6 +50,8 @@ RUN apt-get update -y; \ libtool \ libssl-dev \ bison \ + libkrb5-dev \ + libsasl2-dev \ maven \ flex; \ rm -rf /var/lib/apt/lists/* diff --git a/docker/pegasus-build-env/ubuntu2204/Dockerfile b/docker/pegasus-build-env/ubuntu2204/Dockerfile index bab697c786..0eed16c6cf 100644 --- a/docker/pegasus-build-env/ubuntu2204/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2204/Dockerfile @@ -51,6 +51,8 @@ RUN apt-get update -y; \ libtool \ libssl-dev \ bison \ + libkrb5-dev \ + libsasl2-dev \ maven \ flex; \ rm -rf /var/lib/apt/lists/* diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh index 391f7822f5..d80d6437e2 100755 --- a/scripts/pack_server.sh +++ b/scripts/pack_server.sh @@ -125,8 +125,6 @@ fi copy_file ./thirdparty/output/lib/libboost*.so.1.69.0 ${pack}/bin copy_file ./thirdparty/output/lib/libhdfs* ${pack}/bin -copy_file ./thirdparty/output/lib/libsasl*.so.* ${pack}/bin -copy_file ./thirdparty/output/lib/libcom_err*.so.* ${pack}/bin copy_file ./scripts/sendmail.sh ${pack}/bin copy_file ./src/server/config.ini ${pack}/bin copy_file ./src/server/config.min.ini ${pack}/bin diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index a0984a2cad..577d9bc6ff 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -159,35 +159,6 @@ ExternalProject_Add(thrift DEPENDS boost ) -if (NOT APPLE) - # kerberos - ExternalProject_Add(krb5 - URL ${OSS_URL_PREFIX}/krb5-1.16.1.tar.gz - http://web.mit.edu/kerberos/dist/krb5/1.16/krb5-1.16.1.tar.gz - URL_MD5 848e9b80d6aaaa798e3f3df24b83c407 - CONFIGURE_COMMAND cd src && ./configure --prefix=${TP_OUTPUT} - BUILD_COMMAND cd src && make - INSTALL_COMMAND cd src && make install - BUILD_IN_SOURCE 1 - ) - - # cyrus-sasl - ExternalProject_Add(cyrus-sasl - URL ${OSS_URL_PREFIX}/cyrus-sasl-2.1.27.tar.gz - http://www.cyrusimap.org/releases/cyrus-sasl-2.1.27.tar.gz - URL_MD5 a33820c66e0622222c5aefafa1581083 - CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} - --enable-gssapi=${TP_OUTPUT} - --enable-scram=no - --enable-digest=no - --enable-cram=no - --enable-otp=no - BUILD_COMMAND make - INSTALL_COMMAND make install - BUILD_IN_SOURCE 1 - ) -endif() - check_cxx_compiler_flag(-Wformat-overflow COMPILER_SUPPORTS_FORMAT_OVERFLOW) if (COMPILER_SUPPORTS_FORMAT_OVERFLOW) set(ZOOKEEPER_CFLAGS -Wno-error=format-overflow) @@ -212,9 +183,6 @@ ExternalProject_Add(zookeeper INSTALL_COMMAND "" BUILD_IN_SOURCE 1 ) -if (NOT APPLE) - add_dependencies(zookeeper cyrus-sasl krb5) -endif () ExternalProject_Add(libevent URL ${OSS_URL_PREFIX}/libevent-release-2.1.8-stable.tar.gz From 5489b43a887520abd7ae560f704737b76212a9cb Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 12 Sep 2023 19:08:32 +0800 Subject: [PATCH 11/42] build(thirdparty): bump fmtlib to 10.1.1 (#1605) https://github.com/apache/incubator-pegasus/issues/1604 - Add ecplicit formatters for user defined types. Because since fmtlib 8.0: > Format string compilation now requires `format` functions of `formatter` specializations for user-defined types to be `const`. See https://github.com/fmtlib/fmt/releases/tag/8.0.0 --- cmake_modules/BaseFunctions.cmake | 8 ++--- src/base/idl_utils.h | 9 ++++++ src/base/value_field.h | 3 ++ src/client/replication_ddl_client.cpp | 2 +- src/client/test/ddl_client_test.cpp | 3 +- src/client_lib/pegasus_client_impl.cpp | 1 + src/common/duplication_common.h | 3 ++ src/common/fs_manager.cpp | 4 +-- src/common/gpid.h | 4 +++ src/common/replication_enums.h | 15 ++++++++++ src/http/http_message_parser.cpp | 4 ++- src/http/http_server.cpp | 1 - src/include/pegasus/client.h | 4 +++ .../duplication/meta_duplication_service.cpp | 2 -- src/meta/meta_backup_service.h | 4 +++ src/meta/meta_options.cpp | 1 + src/meta/meta_service.cpp | 3 +- src/meta/meta_state_service_utils_impl.h | 9 +++--- src/meta/meta_state_service_zookeeper.cpp | 5 ++-- src/meta/test/meta_service_test.cpp | 3 +- src/perf_counter/perf_counter.h | 2 ++ src/replica/backup/cold_backup_context.cpp | 2 +- src/replica/duplication/mutation_batch.cpp | 2 -- src/replica/prepare_list.h | 2 ++ src/replica/replica.cpp | 2 -- src/replica/replica_learn.cpp | 2 ++ src/replica/replica_stub.cpp | 1 - src/replica/replica_stub.h | 6 +++- src/replica/replication_app_base.h | 3 +- src/replica/storage/simple_kv/test/case.cpp | 1 - src/replica/storage/simple_kv/test/case.h | 4 +++ src/replica/storage/simple_kv/test/common.h | 5 ++++ .../test/replica_disk_migrate_test.cpp | 2 -- src/runtime/rpc/asio_net_provider.cpp | 2 +- src/runtime/rpc/network.h | 3 ++ src/runtime/rpc/rpc_address.h | 7 ++++- src/runtime/rpc/rpc_host_port.h | 3 ++ src/runtime/rpc/thrift_message_parser.cpp | 3 ++ src/runtime/security/negotiation_manager.cpp | 1 - src/runtime/task/task_code.h | 11 +++++-- src/runtime/task/task_spec.h | 5 ++++ src/server/available_detector.cpp | 3 ++ src/server/compaction_filter_rule.h | 5 +++- src/server/pegasus_event_listener.cpp | 2 -- src/server/pegasus_mutation_duplicator.cpp | 2 -- src/server/pegasus_server_impl.cpp | 3 +- src/server/pegasus_server_write.cpp | 1 + src/server/pegasus_write_service.cpp | 2 -- src/server/pegasus_write_service_impl.h | 7 ++--- src/server/result_writer.cpp | 1 - .../test/capacity_unit_calculator_test.cpp | 5 ++-- src/server/test/hashkey_transform_test.cpp | 1 + src/server/test/hotkey_collector_test.cpp | 1 + src/server/test/pegasus_server_write_test.cpp | 1 + .../test/pegasus_write_service_impl_test.cpp | 1 + .../test/pegasus_write_service_test.cpp | 1 + src/server/test/rocksdb_wrapper_test.cpp | 1 + src/shell/command_helper.h | 4 +++ src/shell/commands/bulk_load.cpp | 2 +- src/shell/commands/data_operations.cpp | 1 - src/shell/commands/detect_hotkey.cpp | 2 -- src/shell/commands/table_management.cpp | 1 - .../base_api_test/test_batch_get.cpp | 1 + src/utils/api_utilities.h | 3 ++ src/utils/error_code.h | 10 +++++-- src/utils/errors.h | 3 ++ src/utils/fail_point_impl.h | 9 ++++-- src/utils/fmt_utils.h | 29 +++++++++++++++++++ src/utils/metrics.cpp | 4 +-- src/utils/string_view.h | 3 ++ src/utils/test/fmt_logging_test.cpp | 2 -- src/utils/test/nth_element_test.cpp | 1 + src/utils/threadpool_code.h | 3 ++ src/zookeeper/lock_struct.h | 2 ++ src/zookeeper/zookeeper_session.h | 3 ++ thirdparty/CMakeLists.txt | 5 ++-- 76 files changed, 210 insertions(+), 72 deletions(-) create mode 100644 src/utils/fmt_utils.h diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 79aba27cb7..bb9e0affc6 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -388,11 +388,9 @@ function(dsn_common_setup) set(BUILD_SHARED_LIBS OFF) - include(CheckCXXCompilerFlag) - CHECK_CXX_COMPILER_FLAG("-std=c++1y" COMPILER_SUPPORTS_CXX1Y) - if(NOT ${COMPILER_SUPPORTS_CXX1Y}) - message(FATAL_ERROR "You need a compiler with C++1y support.") - endif() + set(CMAKE_CXX_STANDARD 14) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) dsn_setup_system_libs() dsn_setup_compiler_flags() diff --git a/src/base/idl_utils.h b/src/base/idl_utils.h index 5b8d93c8a9..bb04018102 100644 --- a/src/base/idl_utils.h +++ b/src/base/idl_utils.h @@ -22,6 +22,7 @@ #include #include "rrdb/rrdb_types.h" +#include "utils/fmt_utils.h" namespace pegasus { @@ -40,3 +41,11 @@ inline bool cas_is_check_operand_needed(dsn::apps::cas_check_type::type type) } } // namespace pegasus + +namespace dsn { +namespace apps { +USER_DEFINED_ENUM_FORMATTER(cas_check_type::type) +USER_DEFINED_ENUM_FORMATTER(filter_type::type) +USER_DEFINED_ENUM_FORMATTER(mutate_operation::type) +} // namespace apps +} // namespace dsn diff --git a/src/base/value_field.h b/src/base/value_field.h index ecb2b766a9..eaa287c632 100644 --- a/src/base/value_field.h +++ b/src/base/value_field.h @@ -19,6 +19,8 @@ #pragma once +#include "utils/fmt_utils.h" + namespace pegasus { enum value_field_type @@ -28,6 +30,7 @@ enum value_field_type USER_DATA, FIELD_COUNT, }; +USER_DEFINED_ENUM_FORMATTER(value_field_type) struct value_field { diff --git a/src/client/replication_ddl_client.cpp b/src/client/replication_ddl_client.cpp index ff0b3269c3..a8a6cf7db6 100644 --- a/src/client/replication_ddl_client.cpp +++ b/src/client/replication_ddl_client.cpp @@ -45,7 +45,7 @@ #include "common/replication.codes.h" #include "common/replication_common.h" #include "common/replication_enums.h" -#include "fmt/ostream.h" +#include "fmt/core.h" #include "meta/meta_rpc_types.h" #include "runtime/api_layer1.h" #include "runtime/rpc/group_address.h" diff --git a/src/client/test/ddl_client_test.cpp b/src/client/test/ddl_client_test.cpp index c489bce373..414c1f029f 100644 --- a/src/client/test/ddl_client_test.cpp +++ b/src/client/test/ddl_client_test.cpp @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. -#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include #include #include #include -#include #include #include diff --git a/src/client_lib/pegasus_client_impl.cpp b/src/client_lib/pegasus_client_impl.cpp index 22f30a3b54..a390e23c32 100644 --- a/src/client_lib/pegasus_client_impl.cpp +++ b/src/client_lib/pegasus_client_impl.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include #include #include #include diff --git a/src/common/duplication_common.h b/src/common/duplication_common.h index c1e77450df..6b953f6e96 100644 --- a/src/common/duplication_common.h +++ b/src/common/duplication_common.h @@ -28,6 +28,7 @@ #include "runtime/rpc/rpc_holder.h" #include "utils/errors.h" #include "utils/flags.h" +#include "utils/fmt_utils.h" namespace dsn { namespace replication { @@ -87,5 +88,7 @@ struct duplication_constants const static std::string kDuplicationEnvMasterMetasKey; }; +USER_DEFINED_ENUM_FORMATTER(duplication_fail_mode::type) +USER_DEFINED_ENUM_FORMATTER(duplication_status::type) } // namespace replication } // namespace dsn diff --git a/src/common/fs_manager.cpp b/src/common/fs_manager.cpp index 6fed98132b..362b29ba87 100644 --- a/src/common/fs_manager.cpp +++ b/src/common/fs_manager.cpp @@ -37,13 +37,13 @@ #include #include #include -#include #include +#include // IWYU pragma: keep + #include "common/gpid.h" #include "common/replication_enums.h" #include "fmt/core.h" -#include "fmt/ostream.h" #include "perf_counter/perf_counter.h" #include "replica_admin_types.h" #include "runtime/api_layer1.h" diff --git a/src/common/gpid.h b/src/common/gpid.h index 1792df3106..ba2f9ee5bc 100644 --- a/src/common/gpid.h +++ b/src/common/gpid.h @@ -29,6 +29,8 @@ #include #include +#include "utils/fmt_utils.h" + namespace dsn { // Group-Partition-ID. @@ -91,6 +93,8 @@ class gpid } // namespace dsn +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::gpid); + namespace std { template <> struct hash<::dsn::gpid> diff --git a/src/common/replication_enums.h b/src/common/replication_enums.h index 5eef1710e6..bd16d81415 100644 --- a/src/common/replication_enums.h +++ b/src/common/replication_enums.h @@ -32,6 +32,7 @@ #include "consensus_types.h" #include "meta_admin_types.h" #include "replica_admin_types.h" +#include "utils/fmt_utils.h" namespace dsn { ENUM_BEGIN2(app_status::type, app_status, app_status::AS_INVALID) @@ -163,4 +164,18 @@ ENUM_REG(replication::manual_compaction_status::QUEUING) ENUM_REG(replication::manual_compaction_status::RUNNING) ENUM_REG(replication::manual_compaction_status::FINISHED) ENUM_END2(replication::manual_compaction_status::type, manual_compaction_status) + +USER_DEFINED_ENUM_FORMATTER(app_status::type) +namespace replication { +USER_DEFINED_ENUM_FORMATTER(bulk_load_status::type) +USER_DEFINED_ENUM_FORMATTER(config_type::type) +USER_DEFINED_ENUM_FORMATTER(detect_action::type) +USER_DEFINED_ENUM_FORMATTER(disk_migration_status::type) +USER_DEFINED_ENUM_FORMATTER(disk_status::type) +USER_DEFINED_ENUM_FORMATTER(learner_status::type) +USER_DEFINED_ENUM_FORMATTER(learn_type::type) +USER_DEFINED_ENUM_FORMATTER(manual_compaction_status::type) +USER_DEFINED_ENUM_FORMATTER(meta_function_level::type) +USER_DEFINED_ENUM_FORMATTER(partition_status::type) +} // namespace replication } // namespace dsn diff --git a/src/http/http_message_parser.cpp b/src/http/http_message_parser.cpp index c6186de30a..ce2e0054ce 100644 --- a/src/http/http_message_parser.cpp +++ b/src/http/http_message_parser.cpp @@ -138,7 +138,9 @@ http_message_parser::http_message_parser() header->hdr_type = http_method::HTTP_METHOD_POST; header->context.u.is_request = 1; } else { - LOG_ERROR("invalid http type {} and method {}", parser->type, parser->method); + // Bit fields don't work with "perfect" forwarding, see + // https://github.com/fmtlib/fmt/issues/1284 + LOG_ERROR("invalid http type {} and method {}", +parser->type, +parser->method); return 1; } return 0; diff --git a/src/http/http_server.cpp b/src/http/http_server.cpp index db66f8e6ce..cbc179c815 100644 --- a/src/http/http_server.cpp +++ b/src/http/http_server.cpp @@ -17,7 +17,6 @@ #include "http_server.h" -#include #include #include #include diff --git a/src/include/pegasus/client.h b/src/include/pegasus/client.h index 8f92b5933c..60e36c55b5 100644 --- a/src/include/pegasus/client.h +++ b/src/include/pegasus/client.h @@ -28,6 +28,8 @@ #include #include +#include "utils/fmt_utils.h" + namespace pegasus { class rrdb_client; @@ -1216,4 +1218,6 @@ class pegasus_client_factory static pegasus_client *get_client(const char *cluster_name, const char *app_name); }; +USER_DEFINED_ENUM_FORMATTER(pegasus_client::filter_type) +USER_DEFINED_ENUM_FORMATTER(pegasus_client::cas_check_type) } // namespace pegasus diff --git a/src/meta/duplication/meta_duplication_service.cpp b/src/meta/duplication/meta_duplication_service.cpp index 663c35d1c9..4d51bac70c 100644 --- a/src/meta/duplication/meta_duplication_service.cpp +++ b/src/meta/duplication/meta_duplication_service.cpp @@ -16,10 +16,8 @@ // under the License. #include -#include #include #include -#include #include #include diff --git a/src/meta/meta_backup_service.h b/src/meta/meta_backup_service.h index e6563dbcc2..2172615ee5 100644 --- a/src/meta/meta_backup_service.h +++ b/src/meta/meta_backup_service.h @@ -41,11 +41,13 @@ #include "runtime/task/task_tracker.h" #include "utils/api_utilities.h" #include "utils/error_code.h" +#include "utils/fmt_utils.h" #include "utils/zlocks.h" namespace dsn { class message_ex; class rpc_address; + namespace dist { namespace block_service { class block_filesystem; @@ -420,3 +422,5 @@ class backup_service }; } // namespace replication } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::backup_start_time); diff --git a/src/meta/meta_options.cpp b/src/meta/meta_options.cpp index 7d9864a354..ebc39ee597 100644 --- a/src/meta/meta_options.cpp +++ b/src/meta/meta_options.cpp @@ -38,6 +38,7 @@ #include #include +#include "common/replication_enums.h" // IWYU pragma: keep #include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/strings.h" diff --git a/src/meta/meta_service.cpp b/src/meta/meta_service.cpp index 4b65d7b299..1aafc61891 100644 --- a/src/meta/meta_service.cpp +++ b/src/meta/meta_service.cpp @@ -26,8 +26,7 @@ // IWYU pragma: no_include #include -// IWYU pragma: no_include -#include +#include #include // for std::remove_if #include #include diff --git a/src/meta/meta_state_service_utils_impl.h b/src/meta/meta_state_service_utils_impl.h index afe4200ce5..f6a1b75918 100644 --- a/src/meta/meta_state_service_utils_impl.h +++ b/src/meta/meta_state_service_utils_impl.h @@ -17,12 +17,12 @@ #pragma once +#include "common/replication.codes.h" +#include "meta/meta_state_service.h" +#include "meta_state_service_utils.h" #include "runtime/pipeline.h" #include "utils/fmt_logging.h" - -#include "meta_state_service_utils.h" -#include "meta/meta_state_service.h" -#include "common/replication.codes.h" +#include "utils/fmt_utils.h" namespace dsn { namespace replication { @@ -61,6 +61,7 @@ struct op_type return op_type_to_string_map[v - 1]; } }; +USER_DEFINED_ENUM_FORMATTER(op_type::type) /// Base class for all operations. struct operation : pipeline::environment diff --git a/src/meta/meta_state_service_zookeeper.cpp b/src/meta/meta_state_service_zookeeper.cpp index 03717f7f6a..a367c1897e 100644 --- a/src/meta/meta_state_service_zookeeper.cpp +++ b/src/meta/meta_state_service_zookeeper.cpp @@ -390,8 +390,9 @@ void meta_state_service_zookeeper::visit_zookeeper_internal(ref_this, { zookeeper_session::zoo_opcontext *op = reinterpret_cast(result); - LOG_DEBUG( - "visit zookeeper internal: ans({}), call type({})", zerror(op->_output.error), op->_optype); + LOG_DEBUG("visit zookeeper internal: ans({}), call type({})", + zerror(op->_output.error), + static_cast(op->_optype)); switch (op->_optype) { case zookeeper_session::ZOO_OPERATION::ZOO_CREATE: diff --git a/src/meta/test/meta_service_test.cpp b/src/meta/test/meta_service_test.cpp index bf993c0dde..161dc7841f 100644 --- a/src/meta/test/meta_service_test.cpp +++ b/src/meta/test/meta_service_test.cpp @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. -#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include diff --git a/src/perf_counter/perf_counter.h b/src/perf_counter/perf_counter.h index dad8009e41..1425bc228c 100644 --- a/src/perf_counter/perf_counter.h +++ b/src/perf_counter/perf_counter.h @@ -33,6 +33,7 @@ #include #include "utils/autoref_ptr.h" +#include "utils/fmt_utils.h" typedef enum dsn_perf_counter_type_t { COUNTER_TYPE_NUMBER, @@ -42,6 +43,7 @@ typedef enum dsn_perf_counter_type_t { COUNTER_TYPE_COUNT, COUNTER_TYPE_INVALID } dsn_perf_counter_type_t; +USER_DEFINED_ENUM_FORMATTER(dsn_perf_counter_type_t) typedef enum dsn_perf_counter_percentile_type_t { COUNTER_PERCENTILE_50, diff --git a/src/replica/backup/cold_backup_context.cpp b/src/replica/backup/cold_backup_context.cpp index f9063a8966..46984c3698 100644 --- a/src/replica/backup/cold_backup_context.cpp +++ b/src/replica/backup/cold_backup_context.cpp @@ -1029,7 +1029,7 @@ void cold_backup_context::on_upload_file_complete(const std::string &local_filen } else { CHECK_GT(total, 0.0); update_progress(static_cast(complete_size / total * 1000)); - LOG_INFO("{}: the progress of upload checkpoint is {}", name, _progress); + LOG_INFO("{}: the progress of upload checkpoint is {}", name, _progress.load()); } if (is_ready_for_upload()) { std::vector upload_files; diff --git a/src/replica/duplication/mutation_batch.cpp b/src/replica/duplication/mutation_batch.cpp index 786c4d61e0..852a8ff127 100644 --- a/src/replica/duplication/mutation_batch.cpp +++ b/src/replica/duplication/mutation_batch.cpp @@ -16,9 +16,7 @@ // under the License. #include -#include #include -#include #include #include #include diff --git a/src/replica/prepare_list.h b/src/replica/prepare_list.h index 96e4d653fb..1882f7b665 100644 --- a/src/replica/prepare_list.h +++ b/src/replica/prepare_list.h @@ -34,6 +34,7 @@ #include "replica/mutation.h" #include "replica/replica_base.h" #include "utils/error_code.h" +#include "utils/fmt_utils.h" namespace dsn { namespace replication { @@ -45,6 +46,7 @@ enum commit_type COMMIT_ALL_READY // commit (last_committed, ... ...] // - only valid when partition_status::PS_SECONDARY or partition_status::PS_PRIMARY }; +USER_DEFINED_ENUM_FORMATTER(commit_type) // prepare_list origins from the concept of `prepared list` in PacificA. // It stores an continuous and ordered list of mutations. diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp index f159d7699e..6184af3b74 100644 --- a/src/replica/replica.cpp +++ b/src/replica/replica.cpp @@ -27,11 +27,9 @@ #include "replica.h" #include -#include #include #include #include -#include #include #include "backup/replica_backup_manager.h" diff --git a/src/replica/replica_learn.cpp b/src/replica/replica_learn.cpp index 75fc9c5894..6f69d16f26 100644 --- a/src/replica/replica_learn.cpp +++ b/src/replica/replica_learn.cpp @@ -46,6 +46,8 @@ #include #include +#include // IWYU pragma: keep + #include "aio/aio_task.h" #include "common/fs_manager.h" #include "common/gpid.h" diff --git a/src/replica/replica_stub.cpp b/src/replica/replica_stub.cpp index e9bbab7a4b..a96b8386fc 100644 --- a/src/replica/replica_stub.cpp +++ b/src/replica/replica_stub.cpp @@ -36,7 +36,6 @@ #include // IWYU pragma: no_include #include -#include #include #include #include diff --git a/src/replica/replica_stub.h b/src/replica/replica_stub.h index b6f54c4850..eddf524ace 100644 --- a/src/replica/replica_stub.h +++ b/src/replica/replica_stub.h @@ -32,7 +32,6 @@ // replica_stub(singleton) --> replica --> replication_app_base // -#include #include #include #include @@ -44,6 +43,8 @@ #include #include +#include + #include "block_service/block_service_manager.h" #include "bulk_load_types.h" #include "common/bulk_load_common.h" @@ -71,12 +72,14 @@ #include "utils/autoref_ptr.h" #include "utils/error_code.h" #include "utils/flags.h" +#include "utils/fmt_utils.h" #include "utils/zlocks.h" namespace dsn { class command_deregister; class message_ex; class nfs_node; + namespace service { class copy_request; class copy_response; @@ -306,6 +309,7 @@ class replica_stub : public serverlet, public ref_counter NS_Connecting, NS_Connected }; + friend USER_DEFINED_ENUM_FORMATTER(replica_stub::replica_node_state); enum replica_life_cycle { diff --git a/src/replica/replication_app_base.h b/src/replica/replication_app_base.h index e48ef8a5dd..aafd97c385 100644 --- a/src/replica/replication_app_base.h +++ b/src/replica/replication_app_base.h @@ -39,6 +39,7 @@ #include "replica/replica_base.h" #include "replica_admin_types.h" #include "utils/error_code.h" +#include "utils/fmt_utils.h" #include "utils/ports.h" namespace dsn { @@ -313,6 +314,6 @@ class replication_app_base : public replica_base explicit replication_app_base(replication::replica *replica); }; - +USER_DEFINED_ENUM_FORMATTER(replication_app_base::chkpt_apply_mode) } // namespace replication } // namespace dsn diff --git a/src/replica/storage/simple_kv/test/case.cpp b/src/replica/storage/simple_kv/test/case.cpp index 79c06503f6..94962e65f7 100644 --- a/src/replica/storage/simple_kv/test/case.cpp +++ b/src/replica/storage/simple_kv/test/case.cpp @@ -42,7 +42,6 @@ #include #include #include -#include #include #include #include diff --git a/src/replica/storage/simple_kv/test/case.h b/src/replica/storage/simple_kv/test/case.h index 0fc0d9fc53..565119808b 100644 --- a/src/replica/storage/simple_kv/test/case.h +++ b/src/replica/storage/simple_kv/test/case.h @@ -46,6 +46,7 @@ #include "meta_admin_types.h" #include "runtime/rpc/rpc_address.h" #include "utils/error_code.h" +#include "utils/fmt_utils.h" #include "utils/singleton.h" #include "utils/zlocks.h" @@ -441,6 +442,7 @@ class client_case_line : public case_line dsn::replication::config_type::type _config_type; rpc_address _config_node; }; +USER_DEFINED_ENUM_FORMATTER(client_case_line::client_type) class test_case : public dsn::utils::singleton { @@ -505,3 +507,5 @@ class test_case : public dsn::utils::singleton } } } + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::test::event); diff --git a/src/replica/storage/simple_kv/test/common.h b/src/replica/storage/simple_kv/test/common.h index 89634acfec..533dadc7e9 100644 --- a/src/replica/storage/simple_kv/test/common.h +++ b/src/replica/storage/simple_kv/test/common.h @@ -47,6 +47,7 @@ #include "common/replication_other_types.h" #include "metadata_types.h" #include "runtime/rpc/rpc_address.h" +#include "utils/fmt_utils.h" namespace dsn { class partition_configuration; @@ -213,3 +214,7 @@ struct parti_config } } } + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::test::parti_config); +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::test::replica_id); +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::test::state_snapshot); diff --git a/src/replica/test/replica_disk_migrate_test.cpp b/src/replica/test/replica_disk_migrate_test.cpp index 024b039cef..e90cbf2fa0 100644 --- a/src/replica/test/replica_disk_migrate_test.cpp +++ b/src/replica/test/replica_disk_migrate_test.cpp @@ -18,11 +18,9 @@ */ #include -#include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include #include diff --git a/src/runtime/rpc/asio_net_provider.cpp b/src/runtime/rpc/asio_net_provider.cpp index 7677d071cb..4f38c0fb5c 100644 --- a/src/runtime/rpc/asio_net_provider.cpp +++ b/src/runtime/rpc/asio_net_provider.cpp @@ -242,7 +242,7 @@ void asio_udp_provider::send_message(message_ex *request) [=](const boost::system::error_code &error, std::size_t bytes_transferred) { if (error) { LOG_WARNING("send udp packet to ep {}:{} failed, message = {}", - ep.address(), + ep.address().to_string(), ep.port(), error.message()); // we do not handle failure here, rpc matcher would handle timeouts diff --git a/src/runtime/rpc/network.h b/src/runtime/rpc/network.h index 72f6108533..1dd15d11d6 100644 --- a/src/runtime/rpc/network.h +++ b/src/runtime/rpc/network.h @@ -39,6 +39,7 @@ #include "runtime/task/task_spec.h" #include "utils/autoref_ptr.h" #include "utils/error_code.h" +#include "utils/fmt_utils.h" #include "utils/join_point.h" #include "utils/link.h" #include "utils/synchronize.h" @@ -282,6 +283,8 @@ class rpc_session : public ref_counter SS_CONNECTED, SS_DISCONNECTED }; + friend USER_DEFINED_ENUM_FORMATTER(rpc_session::session_state); + mutable utils::ex_lock_nr _lock; // [ volatile session_state _connect_state; diff --git a/src/runtime/rpc/rpc_address.h b/src/runtime/rpc/rpc_address.h index 7b8cc68201..b458ca92ad 100644 --- a/src/runtime/rpc/rpc_address.h +++ b/src/runtime/rpc/rpc_address.h @@ -27,13 +27,15 @@ #pragma once #include // IWYU pragma: keep - #include #include +// IWYU pragma: no_include #include #include #include +#include "utils/fmt_utils.h" + namespace apache { namespace thrift { namespace protocol { @@ -47,6 +49,7 @@ typedef enum dsn_host_type_t { HOST_TYPE_IPV4 = 1, HOST_TYPE_GROUP = 2, } dsn_host_type_t; +USER_DEFINED_ENUM_FORMATTER(dsn_host_type_t) namespace dsn { @@ -208,6 +211,8 @@ class rpc_address } // namespace dsn +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::rpc_address); + namespace std { template <> diff --git a/src/runtime/rpc/rpc_host_port.h b/src/runtime/rpc/rpc_host_port.h index bfeba0123e..122ccdd3ef 100644 --- a/src/runtime/rpc/rpc_host_port.h +++ b/src/runtime/rpc/rpc_host_port.h @@ -30,6 +30,7 @@ #include "runtime/rpc/rpc_address.h" #include "utils/errors.h" #include "utils/fmt_logging.h" +#include "utils/fmt_utils.h" namespace apache { namespace thrift { @@ -116,6 +117,8 @@ inline bool operator!=(const host_port &hp1, const host_port &hp2) { return !(hp } // namespace dsn +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::host_port); + namespace std { template <> struct hash<::dsn::host_port> diff --git a/src/runtime/rpc/thrift_message_parser.cpp b/src/runtime/rpc/thrift_message_parser.cpp index e67c2ad0ce..9c8e844c05 100644 --- a/src/runtime/rpc/thrift_message_parser.cpp +++ b/src/runtime/rpc/thrift_message_parser.cpp @@ -46,6 +46,7 @@ #include "utils/crc.h" #include "utils/endians.h" #include "utils/fmt_logging.h" +#include "utils/fmt_utils.h" #include "utils/string_view.h" #include "utils/strings.h" @@ -428,3 +429,5 @@ thrift_message_parser::thrift_message_parser() thrift_message_parser::~thrift_message_parser() = default; } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(apache::thrift::protocol::TMessageType); diff --git a/src/runtime/security/negotiation_manager.cpp b/src/runtime/security/negotiation_manager.cpp index e7c05d6501..d289732dc9 100644 --- a/src/runtime/security/negotiation_manager.cpp +++ b/src/runtime/security/negotiation_manager.cpp @@ -21,7 +21,6 @@ #include "client_negotiation.h" #include "failure_detector/fd.code.definition.h" -#include "fmt/core.h" #include "http/http_server.h" #include "negotiation_utils.h" #include "runtime/rpc/network.h" diff --git a/src/runtime/task/task_code.h b/src/runtime/task/task_code.h index 2dafdc51f3..2dd03bf701 100644 --- a/src/runtime/task/task_code.h +++ b/src/runtime/task/task_code.h @@ -31,6 +31,7 @@ #include #include "utils/enum_helper.h" +#include "utils/fmt_utils.h" #include "utils/ports.h" #include "utils/threadpool_code.h" @@ -53,6 +54,7 @@ typedef enum dsn_task_type_t { TASK_TYPE_COUNT, TASK_TYPE_INVALID } dsn_task_type_t; +USER_DEFINED_ENUM_FORMATTER(dsn_task_type_t) ENUM_BEGIN(dsn_task_type_t, TASK_TYPE_INVALID) ENUM_REG(TASK_TYPE_RPC_REQUEST) @@ -115,9 +117,12 @@ class task_code const char *to_string() const; - constexpr bool operator==(const task_code &r) { return _internal_code == r._internal_code; } + constexpr bool operator==(const task_code &r) const + { + return _internal_code == r._internal_code; + } - constexpr bool operator!=(const task_code &r) { return !(*this == r); } + constexpr bool operator!=(const task_code &r) const { return !(*this == r); } constexpr operator int() const { return _internal_code; } @@ -202,3 +207,5 @@ DEFINE_TASK_CODE(TASK_CODE_INVALID, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) DEFINE_TASK_CODE(TASK_CODE_EXEC_INLINED, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::task_code); diff --git a/src/runtime/task/task_spec.h b/src/runtime/task/task_spec.h index 5de2b56e66..ea3b18ea55 100644 --- a/src/runtime/task/task_spec.h +++ b/src/runtime/task/task_spec.h @@ -48,6 +48,7 @@ #include "utils/enum_helper.h" #include "utils/exp_delay.h" #include "utils/extensible_object.h" +#include "utils/fmt_utils.h" #include "utils/join_point.h" #include "utils/threadpool_code.h" @@ -85,6 +86,7 @@ typedef enum grpc_mode_t { GRPC_COUNT, GRPC_INVALID } grpc_mode_t; +USER_DEFINED_ENUM_FORMATTER(grpc_mode_t) ENUM_BEGIN(grpc_mode_t, GRPC_INVALID) ENUM_REG(GRPC_TO_LEADER) @@ -99,6 +101,7 @@ typedef enum throttling_mode_t { TM_COUNT, TM_INVALID } throttling_mode_t; +USER_DEFINED_ENUM_FORMATTER(throttling_mode_t) ENUM_BEGIN(throttling_mode_t, TM_INVALID) ENUM_REG(TM_NONE) @@ -128,11 +131,13 @@ ENUM_END(dsn_msg_serialize_format) DEFINE_CUSTOMIZED_ID_TYPE(network_header_format) DEFINE_CUSTOMIZED_ID(network_header_format, NET_HDR_INVALID) DEFINE_CUSTOMIZED_ID(network_header_format, NET_HDR_DSN) +USER_DEFINED_ENUM_FORMATTER(network_header_format) // define network channel types for RPC DEFINE_CUSTOMIZED_ID_TYPE(rpc_channel) DEFINE_CUSTOMIZED_ID(rpc_channel, RPC_CHANNEL_TCP) DEFINE_CUSTOMIZED_ID(rpc_channel, RPC_CHANNEL_UDP) +USER_DEFINED_ENUM_FORMATTER(rpc_channel) class aio_task; class message_ex; diff --git a/src/server/available_detector.cpp b/src/server/available_detector.cpp index 73fb81b26c..ca90170fdf 100644 --- a/src/server/available_detector.cpp +++ b/src/server/available_detector.cpp @@ -19,6 +19,7 @@ #include "available_detector.h" +#include // IWYU pragma: no_include #include #include @@ -30,6 +31,8 @@ #include #include +#include // IWYU pragma: keep + #include "base/pegasus_key_schema.h" #include "client/replication_ddl_client.h" #include "common/common.h" diff --git a/src/server/compaction_filter_rule.h b/src/server/compaction_filter_rule.h index 47e780e808..14c157731a 100644 --- a/src/server/compaction_filter_rule.h +++ b/src/server/compaction_filter_rule.h @@ -19,15 +19,17 @@ #pragma once -#include #include #include +#include + #include "base/pegasus_value_schema.h" #include "common/json_helper.h" #include "utils/blob.h" #include "utils/enum_helper.h" #include "utils/factory_store.h" +#include "utils/fmt_utils.h" #include "utils/string_view.h" namespace pegasus { @@ -86,6 +88,7 @@ enum string_match_type SMT_MATCH_POSTFIX, SMT_INVALID, }; +USER_DEFINED_ENUM_FORMATTER(string_match_type) ENUM_BEGIN(string_match_type, SMT_INVALID) ENUM_REG(SMT_MATCH_ANYWHERE) ENUM_REG(SMT_MATCH_PREFIX) diff --git a/src/server/pegasus_event_listener.cpp b/src/server/pegasus_event_listener.cpp index e5414d8b79..e13e2f8635 100644 --- a/src/server/pegasus_event_listener.cpp +++ b/src/server/pegasus_event_listener.cpp @@ -20,10 +20,8 @@ #include "pegasus_event_listener.h" #include -#include #include #include -#include #include #include "common/gpid.h" diff --git a/src/server/pegasus_mutation_duplicator.cpp b/src/server/pegasus_mutation_duplicator.cpp index e9295af356..87c1adbf3e 100644 --- a/src/server/pegasus_mutation_duplicator.cpp +++ b/src/server/pegasus_mutation_duplicator.cpp @@ -20,13 +20,11 @@ #include "pegasus_mutation_duplicator.h" #include -#include #include #include #include #include #include -#include #include #include #include diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 5bc03eafe9..c348952696 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -45,6 +45,7 @@ #include #include +#include "base/idl_utils.h" // IWYU pragma: keep #include "base/pegasus_key_schema.h" #include "base/pegasus_utils.h" #include "base/pegasus_value_schema.h" @@ -3051,7 +3052,7 @@ std::string pegasus_server_impl::compression_type_to_str(rocksdb::CompressionTyp case rocksdb::kZSTD: return "zstd"; default: - LOG_ERROR_PREFIX("Unsupported compression type: {}.", type); + LOG_ERROR_PREFIX("Unsupported compression type: {}.", static_cast(type)); return ""; } } diff --git a/src/server/pegasus_server_write.cpp b/src/server/pegasus_server_write.cpp index 0147050a65..1c6399cc27 100644 --- a/src/server/pegasus_server_write.cpp +++ b/src/server/pegasus_server_write.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include #include #include #include diff --git a/src/server/pegasus_write_service.cpp b/src/server/pegasus_write_service.cpp index 73f6cc8d6f..6ee7109fe0 100644 --- a/src/server/pegasus_write_service.cpp +++ b/src/server/pegasus_write_service.cpp @@ -18,10 +18,8 @@ */ #include -#include #include #include -#include #include #include "base/pegasus_rpc_types.h" diff --git a/src/server/pegasus_write_service_impl.h b/src/server/pegasus_write_service_impl.h index b75b379a3f..ff9f2fe837 100644 --- a/src/server/pegasus_write_service_impl.h +++ b/src/server/pegasus_write_service_impl.h @@ -21,6 +21,7 @@ #include +#include "base/idl_utils.h" #include "base/pegasus_key_schema.h" #include "logging_utils.h" #include "meta_store.h" @@ -270,8 +271,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base if (!is_check_type_supported(update.check_type)) { LOG_ERROR_PREFIX("invalid argument for check_and_set: decree = {}, error = {}", decree, - "check type {} not supported", - update.check_type); + fmt::format("check type {} not supported", update.check_type)); resp.error = rocksdb::Status::kInvalidArgument; // we should write empty record to update rocksdb's last flushed decree return empty_put(decree); @@ -391,8 +391,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base if (!is_check_type_supported(update.check_type)) { LOG_ERROR_PREFIX("invalid argument for check_and_mutate: decree = {}, error = {}", decree, - "check type {} not supported", - update.check_type); + fmt::format("check type {} not supported", update.check_type)); resp.error = rocksdb::Status::kInvalidArgument; // we should write empty record to update rocksdb's last flushed decree return empty_put(decree); diff --git a/src/server/result_writer.cpp b/src/server/result_writer.cpp index 16ec2886bb..d4e9ebeb74 100644 --- a/src/server/result_writer.cpp +++ b/src/server/result_writer.cpp @@ -21,7 +21,6 @@ #include #include -#include #include #include "pegasus/client.h" diff --git a/src/server/test/capacity_unit_calculator_test.cpp b/src/server/test/capacity_unit_calculator_test.cpp index cdfe1bddd4..93db546e14 100644 --- a/src/server/test/capacity_unit_calculator_test.cpp +++ b/src/server/test/capacity_unit_calculator_test.cpp @@ -17,13 +17,14 @@ * under the License. */ -#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include +#include +#include #include #include #include diff --git a/src/server/test/hashkey_transform_test.cpp b/src/server/test/hashkey_transform_test.cpp index 5ece1c7fb6..52f4d5a6be 100644 --- a/src/server/test/hashkey_transform_test.cpp +++ b/src/server/test/hashkey_transform_test.cpp @@ -19,6 +19,7 @@ #include "server/hashkey_transform.h" +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/server/test/hotkey_collector_test.cpp b/src/server/test/hotkey_collector_test.cpp index 7b784a6d2a..be98def07d 100644 --- a/src/server/test/hotkey_collector_test.cpp +++ b/src/server/test/hotkey_collector_test.cpp @@ -17,6 +17,7 @@ #include "server/hotkey_collector.h" +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/server/test/pegasus_server_write_test.cpp b/src/server/test/pegasus_server_write_test.cpp index 0e8c571147..3a9cfc2b1d 100644 --- a/src/server/test/pegasus_server_write_test.cpp +++ b/src/server/test/pegasus_server_write_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/server/test/pegasus_write_service_impl_test.cpp b/src/server/test/pegasus_write_service_impl_test.cpp index f466f9f3ac..0d32540fda 100644 --- a/src/server/test/pegasus_write_service_impl_test.cpp +++ b/src/server/test/pegasus_write_service_impl_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/server/test/pegasus_write_service_test.cpp b/src/server/test/pegasus_write_service_test.cpp index 8971076bb0..f277551bc6 100644 --- a/src/server/test/pegasus_write_service_test.cpp +++ b/src/server/test/pegasus_write_service_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/server/test/rocksdb_wrapper_test.cpp b/src/server/test/rocksdb_wrapper_test.cpp index 24d88e6564..86542c64fe 100644 --- a/src/server/test/rocksdb_wrapper_test.cpp +++ b/src/server/test/rocksdb_wrapper_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index 30d8a22a77..585a64f9c0 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -48,6 +49,7 @@ #include "perf_counter/perf_counter_utils.h" #include "remote_cmd/remote_command.h" #include "tools/mutation_log_tool.h" +#include "utils/fmt_utils.h" #include "utils/string_view.h" #include "utils/strings.h" #include "utils/synchronize.h" @@ -73,6 +75,8 @@ enum scan_data_operator SCAN_GEN_GEO, SCAN_AND_MULTI_SET }; +USER_DEFINED_ENUM_FORMATTER(scan_data_operator) + class top_container { public: diff --git a/src/shell/commands/bulk_load.cpp b/src/shell/commands/bulk_load.cpp index eba70633e8..6f6f16d73b 100644 --- a/src/shell/commands/bulk_load.cpp +++ b/src/shell/commands/bulk_load.cpp @@ -22,10 +22,10 @@ #include // IWYU pragma: no_include #include -#include #include #include #include +#include #include #include #include diff --git a/src/shell/commands/data_operations.cpp b/src/shell/commands/data_operations.cpp index 2a92aea9e7..f13b2fec99 100644 --- a/src/shell/commands/data_operations.cpp +++ b/src/shell/commands/data_operations.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include diff --git a/src/shell/commands/detect_hotkey.cpp b/src/shell/commands/detect_hotkey.cpp index 5eab8b6f9c..b413c63b77 100644 --- a/src/shell/commands/detect_hotkey.cpp +++ b/src/shell/commands/detect_hotkey.cpp @@ -16,10 +16,8 @@ // under the License. #include -#include #include #include -#include #include #include #include diff --git a/src/shell/commands/table_management.cpp b/src/shell/commands/table_management.cpp index b083f9ffbf..818ca8a715 100644 --- a/src/shell/commands/table_management.cpp +++ b/src/shell/commands/table_management.cpp @@ -20,7 +20,6 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include #include diff --git a/src/test/function_test/base_api_test/test_batch_get.cpp b/src/test/function_test/base_api_test/test_batch_get.cpp index 622fc2a425..d0bf4c1e52 100644 --- a/src/test/function_test/base_api_test/test_batch_get.cpp +++ b/src/test/function_test/base_api_test/test_batch_get.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include diff --git a/src/utils/api_utilities.h b/src/utils/api_utilities.h index b32ff52220..4e1d664ed0 100644 --- a/src/utils/api_utilities.h +++ b/src/utils/api_utilities.h @@ -34,6 +34,7 @@ #include #include "ports.h" +#include "utils/fmt_utils.h" /*! @defgroup logging Logging Service @@ -57,6 +58,8 @@ typedef enum dsn_log_level_t { LOG_LEVEL_INVALID } dsn_log_level_t; +USER_DEFINED_ENUM_FORMATTER(dsn_log_level_t) + // logs with level smaller than this start_level will not be logged extern dsn_log_level_t dsn_log_start_level; extern dsn_log_level_t dsn_log_get_start_level(); diff --git a/src/utils/error_code.h b/src/utils/error_code.h index 25feabd48a..45a6e793cc 100644 --- a/src/utils/error_code.h +++ b/src/utils/error_code.h @@ -30,6 +30,7 @@ #include #include +#include "utils/fmt_utils.h" #include "utils/ports.h" namespace apache { @@ -53,9 +54,12 @@ class error_code const char *to_string() const; - constexpr bool operator==(const error_code &r) { return _internal_code == r._internal_code; } + constexpr bool operator==(const error_code &r) const + { + return _internal_code == r._internal_code; + } - constexpr bool operator!=(const error_code &r) { return !(*this == r); } + constexpr bool operator!=(const error_code &r) const { return !(*this == r); } constexpr operator int() const { return _internal_code; } @@ -177,3 +181,5 @@ DEFINE_ERR_CODE(ERR_RDB_CORRUPTION) DEFINE_ERR_CODE(ERR_DISK_IO_ERROR) } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::error_code); diff --git a/src/utils/errors.h b/src/utils/errors.h index 8cc47a74b8..9cbf70e092 100644 --- a/src/utils/errors.h +++ b/src/utils/errors.h @@ -31,6 +31,7 @@ #include "utils/api_utilities.h" #include "utils/error_code.h" #include "utils/fmt_logging.h" +#include "utils/fmt_utils.h" #include "utils/ports.h" #include "utils/smart_pointers.h" #include "utils/string_view.h" @@ -219,6 +220,8 @@ class error_with } // namespace dsn +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::error_s); + #define FMT_ERR(ec, msg, args...) error_s::make(ec, fmt::format(msg, ##args)) #define RETURN_NOT_OK(s) \ diff --git a/src/utils/fail_point_impl.h b/src/utils/fail_point_impl.h index b8b96fd9c2..22212e41d9 100644 --- a/src/utils/fail_point_impl.h +++ b/src/utils/fail_point_impl.h @@ -30,13 +30,15 @@ #pragma once -#include "utils/fail_point.h" -#include "utils/api_utilities.h" -#include "utils/ports.h" #include #include #include +#include "utils/api_utilities.h" +#include "utils/fail_point.h" +#include "utils/fmt_utils.h" +#include "utils/ports.h" + namespace dsn { namespace fail { @@ -97,6 +99,7 @@ struct fail_point int _freq{100}; int _max_cnt{-1}; // TODO(wutao1): not thread-safe }; +USER_DEFINED_ENUM_FORMATTER(fail_point::task_type) struct fail_point_registry { diff --git a/src/utils/fmt_utils.h b/src/utils/fmt_utils.h new file mode 100644 index 0000000000..9624bb881a --- /dev/null +++ b/src/utils/fmt_utils.h @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#define USER_DEFINED_STRUCTURE_FORMATTER(type) \ + template <> \ + struct fmt::formatter : ostream_formatter \ + { \ + } + +#define USER_DEFINED_ENUM_FORMATTER(type) \ + inline auto format_as(type e)->int { return e; } diff --git a/src/utils/metrics.cpp b/src/utils/metrics.cpp index 2a15fb5aa9..40caace7a4 100644 --- a/src/utils/metrics.cpp +++ b/src/utils/metrics.cpp @@ -617,8 +617,8 @@ void metric_timer::on_timer(const boost::system::error_code &ec) } while (0) if (dsn_unlikely(!!ec)) { - CHECK_EQ_MSG(ec, - boost::system::errc::operation_canceled, + CHECK_EQ_MSG(static_cast(boost::system::errc::operation_canceled), + ec.value(), "failed to exec on_timer with an error that cannot be handled: {}", ec.message()); diff --git a/src/utils/string_view.h b/src/utils/string_view.h index ee3664c576..ab867448e2 100644 --- a/src/utils/string_view.h +++ b/src/utils/string_view.h @@ -49,6 +49,7 @@ #include #include "ports.h" +#include "utils/fmt_utils.h" namespace dsn { @@ -428,3 +429,5 @@ inline bool operator!=(string_view x, string_view y) noexcept { return !(x == y) std::ostream &operator<<(std::ostream &o, string_view piece); } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::string_view); diff --git a/src/utils/test/fmt_logging_test.cpp b/src/utils/test/fmt_logging_test.cpp index fc0f18f383..f82dcf0f3e 100644 --- a/src/utils/test/fmt_logging_test.cpp +++ b/src/utils/test/fmt_logging_test.cpp @@ -25,11 +25,9 @@ */ #include -#include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include "common/gpid.h" #include "common/replication.codes.h" diff --git a/src/utils/test/nth_element_test.cpp b/src/utils/test/nth_element_test.cpp index 854cc6fd24..e3771fd538 100644 --- a/src/utils/test/nth_element_test.cpp +++ b/src/utils/test/nth_element_test.cpp @@ -17,6 +17,7 @@ #include "utils/nth_element.h" +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/utils/threadpool_code.h b/src/utils/threadpool_code.h index 2ff31d739f..383c1d5247 100644 --- a/src/utils/threadpool_code.h +++ b/src/utils/threadpool_code.h @@ -29,6 +29,7 @@ #include #include "ports.h" +#include "utils/fmt_utils.h" namespace dsn { class threadpool_code @@ -66,3 +67,5 @@ class threadpool_code DEFINE_THREAD_POOL_CODE(THREAD_POOL_INVALID) DEFINE_THREAD_POOL_CODE(THREAD_POOL_DEFAULT) } + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::threadpool_code); diff --git a/src/zookeeper/lock_struct.h b/src/zookeeper/lock_struct.h index dead1153ea..eca2696ad5 100644 --- a/src/zookeeper/lock_struct.h +++ b/src/zookeeper/lock_struct.h @@ -42,6 +42,7 @@ #include "runtime/task/future_types.h" #include "utils/autoref_ptr.h" #include "utils/distributed_lock_service.h" +#include "utils/fmt_utils.h" #include "utils/thread_access_checker.h" namespace dsn { @@ -57,6 +58,7 @@ enum lock_state unlocking, state_count }; +USER_DEFINED_ENUM_FORMATTER(lock_state) struct zoolock_pair { diff --git a/src/zookeeper/zookeeper_session.h b/src/zookeeper/zookeeper_session.h index e7d3e86258..41323fccf7 100644 --- a/src/zookeeper/zookeeper_session.h +++ b/src/zookeeper/zookeeper_session.h @@ -44,6 +44,7 @@ #include "runtime/service_app.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" +#include "utils/fmt_utils.h" #include "utils/synchronize.h" struct String_vector; @@ -205,3 +206,5 @@ class zookeeper_session }; } } + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::dist::zookeeper_session); diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 577d9bc6ff..4efddc6d79 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -246,9 +246,8 @@ ExternalProject_Add(fds # fmtlib >=6.x requires c++14 support, do not update this library for now ExternalProject_Add(fmt - URL ${OSS_URL_PREFIX}/fmt-5.3.0.tar.gz - https://github.com/fmtlib/fmt/archive/refs/tags/5.3.0.tar.gz - URL_MD5 1015bf3ff2a140dfe03de50ee2469401 + URL https://github.com/fmtlib/fmt/archive/refs/tags/10.1.1.tar.gz + URL_MD5 0d41a16f1b3878d44e6fd7ff1f6cc45a CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DCMAKE_BUILD_TYPE=release -DFMT_TEST=false From 7cb1599942f197f4a820ec38f33ab8475d58e203 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 13 Sep 2023 11:02:08 +0800 Subject: [PATCH 12/42] feat(build): make C++17 as required (#1603) https://github.com/apache/incubator-pegasus/issues/1602 This patch adjust the build to start supporting and requiring C++17 (therefore GCC 7+). `std::random_shuffle` has to be replaced by `std::shuffle`, because the former has been removed since C++17. --- CMakeLists.txt | 6 +++--- cmake_modules/BaseFunctions.cmake | 8 +++----- src/aio/disk_engine.cpp | 1 - src/base/test/utils_test.cpp | 1 - src/common/test/common_test.cpp | 1 + src/geo/lib/geo_client.cpp | 1 - src/geo/lib/latlng_codec.cpp | 1 + src/meta/meta_backup_service.h | 1 + src/meta/meta_service.cpp | 2 +- src/meta/test/meta_service_test.cpp | 1 - src/perf_counter/test/perf_counter_test.cpp | 1 + src/redis_protocol/proxy_ut/redis_proxy_test.cpp | 1 - src/replica/backup/cold_backup_context.cpp | 1 - src/replica/bulk_load/test/replica_bulk_loader_test.cpp | 1 - src/replica/duplication/replica_follower.cpp | 1 - src/replica/mutation_cache.cpp | 2 -- src/replica/replica.cpp | 1 - src/replica/replica_learn.cpp | 1 - src/replica/storage/simple_kv/simple_kv.server.impl.cpp | 1 - src/replica/storage/simple_kv/test/client.cpp | 1 - .../storage/simple_kv/test/simple_kv.server.impl.cpp | 1 - src/replica/test/replica_test.cpp | 1 - src/runtime/providers.common.cpp | 1 - src/runtime/rpc/asio_net_provider.cpp | 1 - src/runtime/rpc/asio_rpc_session.cpp | 1 - src/runtime/rpc/rpc_address.h | 1 + src/runtime/rpc/rpc_host_port.h | 1 + src/runtime/rpc/thrift_message_parser.cpp | 1 - src/runtime/scheduler.cpp | 8 ++++---- src/runtime/service_engine.cpp | 1 - src/runtime/task/task_engine.cpp | 1 - src/runtime/test/client_negotiation_test.cpp | 1 - src/runtime/test/pipeline_test.cpp | 1 + src/runtime/test/server_negotiation_test.cpp | 1 - src/utils/alloc.h | 2 +- src/utils/command_manager.cpp | 1 - src/utils/config_api.cpp | 3 +++ src/utils/filesystem.cpp | 5 ++--- src/utils/flags.h | 1 + src/utils/output_utils.h | 1 + src/utils/simple_logger.cpp | 1 - src/utils/test/autoref_ptr_test.cpp | 1 + src/utils/test/env.cpp | 1 + src/utils/test/fail_point_test.cpp | 1 + src/utils/test/fmt_logging_test.cpp | 1 + src/utils/test/join_point_test.cpp | 1 - src/utils/test/lock.std.cpp | 1 + src/utils/test/main.cpp | 1 + src/utils/test/memutil_test.cpp | 1 + src/utils/test/nth_element_utils.h | 6 +++++- src/utils/test/sema.cpp | 1 + src/utils/test/smart_pointers_test.cpp | 1 + src/utils/utils.h | 2 +- thirdparty/CMakeLists.txt | 7 +++---- 54 files changed, 44 insertions(+), 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bc451ae269..fee1807892 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,9 +19,9 @@ project(pegasus) cmake_minimum_required(VERSION 3.11.0) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - # require at least gcc 5.4.0 - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.4.0) - message(FATAL_ERROR "GCC version must be at least 5.4.0!") + # require at least gcc 7.0.0 + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0.0) + message(FATAL_ERROR "GCC version must be at least 7.0.0!") endif () endif () diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index bb9e0affc6..0147b50298 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -147,6 +147,7 @@ function(dsn_add_project) endif() ms_add_project("${MY_PROJ_TYPE}" "${MY_PROJ_NAME}" "${MY_PROJ_SRC}" "${MY_PROJ_LIBS}" "${MY_BINPLACES}") define_file_basename_for_sources(${MY_PROJ_NAME}) + target_compile_features(${MY_PROJ_NAME} PRIVATE cxx_std_17) endfunction(dsn_add_project) function(dsn_add_static_library) @@ -204,7 +205,7 @@ function(dsn_setup_compiler_flags) # We want access to the PRI* print format macros. add_definitions(-D__STDC_FORMAT_MACROS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -gdwarf-4" CACHE STRING "" FORCE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -gdwarf-4" CACHE STRING "" FORCE) # -Wall: Enable all warnings. add_compile_options(-Wall) @@ -221,9 +222,6 @@ function(dsn_setup_compiler_flags) # use frame pointers to allow simple stack frame walking for backtraces. # This has a small perf hit but worth it for the ability to profile in production add_compile_options( -fno-omit-frame-pointer) - # -Wno-deprecated-register - # kbr5.h uses the legacy 'register' keyword. - add_compile_options(-Wno-deprecated-register) # -Wno-implicit-float-conversion # Poco/Dynamic/VarHolder.h uses 'unsigned long' to 'float' conversion add_compile_options(-Wno-implicit-float-conversion) @@ -388,7 +386,7 @@ function(dsn_common_setup) set(BUILD_SHARED_LIBS OFF) - set(CMAKE_CXX_STANDARD 14) + set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/src/aio/disk_engine.cpp b/src/aio/disk_engine.cpp index 82748de500..0e25bfaac6 100644 --- a/src/aio/disk_engine.cpp +++ b/src/aio/disk_engine.cpp @@ -26,7 +26,6 @@ #include "disk_engine.h" -#include #include // IWYU pragma: no_include #include diff --git a/src/base/test/utils_test.cpp b/src/base/test/utils_test.cpp index d9922c2f0d..ad7dcc337c 100644 --- a/src/base/test/utils_test.cpp +++ b/src/base/test/utils_test.cpp @@ -20,7 +20,6 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include diff --git a/src/common/test/common_test.cpp b/src/common/test/common_test.cpp index dc2bdccaac..c2bf31713a 100644 --- a/src/common/test/common_test.cpp +++ b/src/common/test/common_test.cpp @@ -22,6 +22,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include namespace dsn { TEST(duplication_common, get_current_cluster_name) diff --git a/src/geo/lib/geo_client.cpp b/src/geo/lib/geo_client.cpp index 335afda62e..0d4927695e 100644 --- a/src/geo/lib/geo_client.cpp +++ b/src/geo/lib/geo_client.cpp @@ -32,7 +32,6 @@ #include #include #include -#include #include #include #include diff --git a/src/geo/lib/latlng_codec.cpp b/src/geo/lib/latlng_codec.cpp index 8a74618bc6..171db59c58 100644 --- a/src/geo/lib/latlng_codec.cpp +++ b/src/geo/lib/latlng_codec.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include "utils/error_code.h" #include "utils/errors.h" diff --git a/src/meta/meta_backup_service.h b/src/meta/meta_backup_service.h index 2172615ee5..3c7138cafb 100644 --- a/src/meta/meta_backup_service.h +++ b/src/meta/meta_backup_service.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include #include diff --git a/src/meta/meta_service.cpp b/src/meta/meta_service.cpp index 1aafc61891..5ea615c61d 100644 --- a/src/meta/meta_service.cpp +++ b/src/meta/meta_service.cpp @@ -25,8 +25,8 @@ */ // IWYU pragma: no_include +// IWYU pragma: no_include #include -#include #include // for std::remove_if #include #include diff --git a/src/meta/test/meta_service_test.cpp b/src/meta/test/meta_service_test.cpp index 161dc7841f..2bb7ee2995 100644 --- a/src/meta/test/meta_service_test.cpp +++ b/src/meta/test/meta_service_test.cpp @@ -27,7 +27,6 @@ #include "meta/meta_service.h" #include "meta_admin_types.h" #include "meta_test_base.h" -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_holder.h" diff --git a/src/perf_counter/test/perf_counter_test.cpp b/src/perf_counter/test/perf_counter_test.cpp index f83679bd45..0758ef8071 100644 --- a/src/perf_counter/test/perf_counter_test.cpp +++ b/src/perf_counter/test/perf_counter_test.cpp @@ -38,6 +38,7 @@ // IWYU pragma: no_include #include #include +#include #include #include #include diff --git a/src/redis_protocol/proxy_ut/redis_proxy_test.cpp b/src/redis_protocol/proxy_ut/redis_proxy_test.cpp index 2706ae772f..400c41b4ce 100644 --- a/src/redis_protocol/proxy_ut/redis_proxy_test.cpp +++ b/src/redis_protocol/proxy_ut/redis_proxy_test.cpp @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include diff --git a/src/replica/backup/cold_backup_context.cpp b/src/replica/backup/cold_backup_context.cpp index 46984c3698..845b96fe60 100644 --- a/src/replica/backup/cold_backup_context.cpp +++ b/src/replica/backup/cold_backup_context.cpp @@ -17,7 +17,6 @@ #include "cold_backup_context.h" -#include #include #include #include diff --git a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp index 6b94754e85..df123cc0c7 100644 --- a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp +++ b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp @@ -20,7 +20,6 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include // IWYU pragma: keep #include #include diff --git a/src/replica/duplication/replica_follower.cpp b/src/replica/duplication/replica_follower.cpp index f9c2a996d1..f5a29b94dd 100644 --- a/src/replica/duplication/replica_follower.cpp +++ b/src/replica/duplication/replica_follower.cpp @@ -20,7 +20,6 @@ #include "replica_follower.h" #include -#include #include #include #include diff --git a/src/replica/mutation_cache.cpp b/src/replica/mutation_cache.cpp index c97c9b0451..1a7fa4d338 100644 --- a/src/replica/mutation_cache.cpp +++ b/src/replica/mutation_cache.cpp @@ -30,8 +30,6 @@ // https://github.com/include-what-you-use/include-what-you-use/issues/166 // TODO(yingchun): remove this pragma by using mapping.imp // IWYU pragma: no_include -#include - #include "consensus_types.h" #include "mutation.h" #include "utils/autoref_ptr.h" diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp index 6184af3b74..b99a3c4219 100644 --- a/src/replica/replica.cpp +++ b/src/replica/replica.cpp @@ -28,7 +28,6 @@ #include #include -#include #include #include diff --git a/src/replica/replica_learn.cpp b/src/replica/replica_learn.cpp index 6f69d16f26..3fc23ed73d 100644 --- a/src/replica/replica_learn.cpp +++ b/src/replica/replica_learn.cpp @@ -48,7 +48,6 @@ #include // IWYU pragma: keep -#include "aio/aio_task.h" #include "common/fs_manager.h" #include "common/gpid.h" #include "common/replication.codes.h" diff --git a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp index b6f0a00e3f..a019c0999b 100644 --- a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include diff --git a/src/replica/storage/simple_kv/test/client.cpp b/src/replica/storage/simple_kv/test/client.cpp index b576986080..905cdb080c 100644 --- a/src/replica/storage/simple_kv/test/client.cpp +++ b/src/replica/storage/simple_kv/test/client.cpp @@ -36,7 +36,6 @@ #include "client.h" #include -#include #include #include #include diff --git a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp index 4e2d1e4e78..cf3f418f79 100644 --- a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include diff --git a/src/replica/test/replica_test.cpp b/src/replica/test/replica_test.cpp index f1ff0acc9d..40a93cc95f 100644 --- a/src/replica/test/replica_test.cpp +++ b/src/replica/test/replica_test.cpp @@ -54,7 +54,6 @@ #include "replica/test/mock_utils.h" #include "replica_test_base.h" #include "runtime/api_layer1.h" -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_message.h" diff --git a/src/runtime/providers.common.cpp b/src/runtime/providers.common.cpp index 52fe571f68..b3fc676680 100644 --- a/src/runtime/providers.common.cpp +++ b/src/runtime/providers.common.cpp @@ -33,7 +33,6 @@ * xxxx-xx-xx, author, fix bug about xxx */ -#include #include #include "runtime/env_provider.h" diff --git a/src/runtime/rpc/asio_net_provider.cpp b/src/runtime/rpc/asio_net_provider.cpp index 4f38c0fb5c..95569b2045 100644 --- a/src/runtime/rpc/asio_net_provider.cpp +++ b/src/runtime/rpc/asio_net_provider.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include diff --git a/src/runtime/rpc/asio_rpc_session.cpp b/src/runtime/rpc/asio_rpc_session.cpp index a8518cb261..5006a257f3 100644 --- a/src/runtime/rpc/asio_rpc_session.cpp +++ b/src/runtime/rpc/asio_rpc_session.cpp @@ -28,7 +28,6 @@ #include // IWYU pragma: keep // IWYU pragma: no_include -#include #include #include #include diff --git a/src/runtime/rpc/rpc_address.h b/src/runtime/rpc/rpc_address.h index b458ca92ad..e5afd7270a 100644 --- a/src/runtime/rpc/rpc_address.h +++ b/src/runtime/rpc/rpc_address.h @@ -33,6 +33,7 @@ #include #include #include +#include #include "utils/fmt_utils.h" diff --git a/src/runtime/rpc/rpc_host_port.h b/src/runtime/rpc/rpc_host_port.h index 122ccdd3ef..be825b22c4 100644 --- a/src/runtime/rpc/rpc_host_port.h +++ b/src/runtime/rpc/rpc_host_port.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "runtime/rpc/rpc_address.h" diff --git a/src/runtime/rpc/thrift_message_parser.cpp b/src/runtime/rpc/thrift_message_parser.cpp index 9c8e844c05..e01d342d80 100644 --- a/src/runtime/rpc/thrift_message_parser.cpp +++ b/src/runtime/rpc/thrift_message_parser.cpp @@ -27,7 +27,6 @@ #include "thrift_message_parser.h" #include -#include #include #include #include diff --git a/src/runtime/scheduler.cpp b/src/runtime/scheduler.cpp index 9ccc20afad..458dde7163 100644 --- a/src/runtime/scheduler.cpp +++ b/src/runtime/scheduler.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -276,10 +277,9 @@ void scheduler::schedule() _time_ns = ts; } - // randomize the events, and see - std::random_shuffle( - events->begin(), events->end(), [](int n) { return rand::next_u32(0, n - 1); }); - + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(events->begin(), events->end(), g); for (auto e : *events) { if (e.app_task != nullptr) { task *t = e.app_task; diff --git a/src/runtime/service_engine.cpp b/src/runtime/service_engine.cpp index 90d42a68b7..ee0c742d81 100644 --- a/src/runtime/service_engine.cpp +++ b/src/runtime/service_engine.cpp @@ -27,7 +27,6 @@ #include "service_engine.h" #include -#include #include #include #include diff --git a/src/runtime/task/task_engine.cpp b/src/runtime/task/task_engine.cpp index c729a3b611..11ac1b03fd 100644 --- a/src/runtime/task/task_engine.cpp +++ b/src/runtime/task/task_engine.cpp @@ -28,7 +28,6 @@ // IWYU pragma: no_include #include -#include #include #include diff --git a/src/runtime/test/client_negotiation_test.cpp b/src/runtime/test/client_negotiation_test.cpp index 00eae954be..74dab238ac 100644 --- a/src/runtime/test/client_negotiation_test.cpp +++ b/src/runtime/test/client_negotiation_test.cpp @@ -22,7 +22,6 @@ #include #include -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_holder.h" diff --git a/src/runtime/test/pipeline_test.cpp b/src/runtime/test/pipeline_test.cpp index 22ae223f66..bdc1e5a868 100644 --- a/src/runtime/test/pipeline_test.cpp +++ b/src/runtime/test/pipeline_test.cpp @@ -27,6 +27,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include "common/replication.codes.h" diff --git a/src/runtime/test/server_negotiation_test.cpp b/src/runtime/test/server_negotiation_test.cpp index 907ed0ca8a..0ed7471642 100644 --- a/src/runtime/test/server_negotiation_test.cpp +++ b/src/runtime/test/server_negotiation_test.cpp @@ -23,7 +23,6 @@ #include #include -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_holder.h" diff --git a/src/utils/alloc.h b/src/utils/alloc.h index 195f740f6b..e224eaff6d 100644 --- a/src/utils/alloc.h +++ b/src/utils/alloc.h @@ -24,7 +24,7 @@ #ifdef CACHELINE_SIZE #include -#include +#include // IWYU pragma: keep #include #include #include diff --git a/src/utils/command_manager.cpp b/src/utils/command_manager.cpp index 723f8f3447..38d84054b5 100644 --- a/src/utils/command_manager.cpp +++ b/src/utils/command_manager.cpp @@ -27,7 +27,6 @@ #include "utils/command_manager.h" #include -#include #include #include #include // IWYU pragma: keep diff --git a/src/utils/config_api.cpp b/src/utils/config_api.cpp index 6319f39dab..537ffed659 100644 --- a/src/utils/config_api.cpp +++ b/src/utils/config_api.cpp @@ -25,6 +25,9 @@ */ #include "utils/config_api.h" + +#include + #include "utils/configuration.h" dsn::configuration g_config; diff --git a/src/utils/filesystem.cpp b/src/utils/filesystem.cpp index 8bfe4cfa6f..73aa3a50aa 100644 --- a/src/utils/filesystem.cpp +++ b/src/utils/filesystem.cpp @@ -33,8 +33,6 @@ * xxxx-xx-xx, author, fix bug about xxx */ -// IWYU pragma: no_include -#include // IWYU pragma: keep #include #include #include @@ -46,8 +44,9 @@ #include #include #include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include -#include #include #include "utils/defer.h" diff --git a/src/utils/flags.h b/src/utils/flags.h index 304238b7bb..355fa14ded 100644 --- a/src/utils/flags.h +++ b/src/utils/flags.h @@ -22,6 +22,7 @@ // IWYU pragma: no_include #include #include +#include #include "enum_helper.h" #include "errors.h" diff --git a/src/utils/output_utils.h b/src/utils/output_utils.h index c66d55f734..201cee938b 100644 --- a/src/utils/output_utils.h +++ b/src/utils/output_utils.h @@ -20,6 +20,7 @@ // IWYU pragma: no_include #include #include +#include #include // IWYU pragma: keep #include // IWYU pragma: no_include diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index 98b1693d1a..16c4e95e14 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -29,7 +29,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include diff --git a/src/utils/test/autoref_ptr_test.cpp b/src/utils/test/autoref_ptr_test.cpp index 6a8fe8d9d7..4ff6a15294 100644 --- a/src/utils/test/autoref_ptr_test.cpp +++ b/src/utils/test/autoref_ptr_test.cpp @@ -8,6 +8,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include diff --git a/src/utils/test/env.cpp b/src/utils/test/env.cpp index 5a05f1879b..619e5d7eba 100644 --- a/src/utils/test/env.cpp +++ b/src/utils/test/env.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include "utils/rand.h" diff --git a/src/utils/test/fail_point_test.cpp b/src/utils/test/fail_point_test.cpp index 74d576065a..b7668b03c9 100644 --- a/src/utils/test/fail_point_test.cpp +++ b/src/utils/test/fail_point_test.cpp @@ -31,6 +31,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include "utils/fail_point.h" #include "utils/fail_point_impl.h" diff --git a/src/utils/test/fmt_logging_test.cpp b/src/utils/test/fmt_logging_test.cpp index f82dcf0f3e..a8c5f7d13a 100644 --- a/src/utils/test/fmt_logging_test.cpp +++ b/src/utils/test/fmt_logging_test.cpp @@ -28,6 +28,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include "common/gpid.h" #include "common/replication.codes.h" diff --git a/src/utils/test/join_point_test.cpp b/src/utils/test/join_point_test.cpp index 6751fe94ae..a23740579a 100644 --- a/src/utils/test/join_point_test.cpp +++ b/src/utils/test/join_point_test.cpp @@ -29,7 +29,6 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include namespace dsn { diff --git a/src/utils/test/lock.std.cpp b/src/utils/test/lock.std.cpp index c1530e4bd8..c7027a8861 100644 --- a/src/utils/test/lock.std.cpp +++ b/src/utils/test/lock.std.cpp @@ -27,6 +27,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include "runtime/rpc/rpc_address.h" diff --git a/src/utils/test/main.cpp b/src/utils/test/main.cpp index f611528ef4..2be6302e1a 100644 --- a/src/utils/test/main.cpp +++ b/src/utils/test/main.cpp @@ -16,6 +16,7 @@ // under the License. #include +#include #include "utils/flags.h" #include "utils/logging_provider.h" diff --git a/src/utils/test/memutil_test.cpp b/src/utils/test/memutil_test.cpp index 4ddf62eaa1..87c4da8dc3 100644 --- a/src/utils/test/memutil_test.cpp +++ b/src/utils/test/memutil_test.cpp @@ -19,6 +19,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include TEST(MemUtilTest, memmatch) { diff --git a/src/utils/test/nth_element_utils.h b/src/utils/test/nth_element_utils.h index e8fdc9320f..85e618a318 100644 --- a/src/utils/test/nth_element_utils.h +++ b/src/utils/test/nth_element_utils.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -93,7 +94,10 @@ class nth_element_case_generator auto delta = _rand(_range_size); value += delta; } - std::random_shuffle(array.begin(), array.end()); + + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(array.begin(), array.end(), g); } private: diff --git a/src/utils/test/sema.cpp b/src/utils/test/sema.cpp index 0982ee8837..40b14824b2 100644 --- a/src/utils/test/sema.cpp +++ b/src/utils/test/sema.cpp @@ -39,6 +39,7 @@ // IWYU pragma: no_include #include #include +#include #include TEST(core, Semaphore) diff --git a/src/utils/test/smart_pointers_test.cpp b/src/utils/test/smart_pointers_test.cpp index a2113b4b36..7a6bba05f0 100644 --- a/src/utils/test/smart_pointers_test.cpp +++ b/src/utils/test/smart_pointers_test.cpp @@ -17,6 +17,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include diff --git a/src/utils/utils.h b/src/utils/utils.h index f624ba48f2..fd40cb136f 100644 --- a/src/utils/utils.h +++ b/src/utils/utils.h @@ -28,7 +28,7 @@ #include #include -#include +#include // IWYU pragma: keep #include #include #include diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 4efddc6d79..069f21d299 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -21,9 +21,9 @@ project(pegasus_thirdparties) cmake_minimum_required(VERSION 3.11.0) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - # require at least gcc 5.4.0 - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.4.0) - message(FATAL_ERROR "GCC version must be at least 5.4.0!") + # require at least gcc 7.0.0 + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0.0) + message(FATAL_ERROR "GCC version must be at least 7.0.0!") endif () endif () @@ -244,7 +244,6 @@ ExternalProject_Add(fds BUILD_IN_SOURCE 1 ) -# fmtlib >=6.x requires c++14 support, do not update this library for now ExternalProject_Add(fmt URL https://github.com/fmtlib/fmt/archive/refs/tags/10.1.1.tar.gz URL_MD5 0d41a16f1b3878d44e6fd7ff1f6cc45a From e978e4c78b1436aa0ae06915365f85c601781ba7 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 13 Sep 2023 22:28:01 +0800 Subject: [PATCH 13/42] feat(thirdparty): upgrade rocksdb to 8.5.3 (#1601) https://github.com/apache/incubator-pegasus/issues/1604 This patch upgrade the `rocksdb` to the latest stable version 8.5.3. Since the 2.5 branch has been created, the new version of rocksdb will only take effect on version 2.6 and later. --- .github/workflows/lint_and_test_cpp.yaml | 12 ++++---- .github/workflows/thirdparty-regular-push.yml | 4 +-- docker/thirdparties-bin/Dockerfile | 2 +- run.sh | 4 +-- src/server/pegasus_event_listener.cpp | 1 + src/server/pegasus_server_impl.cpp | 29 ++++++++++--------- src/server/pegasus_server_impl.h | 2 ++ src/server/pegasus_server_impl_init.cpp | 2 ++ .../test/pegasus_compression_options_test.cpp | 1 + src/utils/alloc.h | 2 ++ thirdparty/CMakeLists.txt | 10 ++----- 11 files changed, 38 insertions(+), 31 deletions(-) diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 115f8b124c..e405fa3a47 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -91,7 +91,7 @@ jobs: # Build thirdparties and leave some necessary libraries and source run: | mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* ../scripts/download_hadoop.sh hadoop-bin @@ -151,7 +151,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -296,7 +296,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -438,7 +438,7 @@ jobs: # run: | # rm -f /root/thirdparties-src.zip # mkdir build -# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ +# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ # cmake --build build/ -j $(nproc) # rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* # find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -576,7 +576,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -676,7 +676,7 @@ jobs: run: | export JAVA_HOME="${JAVA_HOME_8_X64}" mkdir -p build - cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} + cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DROCKSDB_PORTABLE=1 cmake --build build/ -j $(sysctl -n hw.physicalcpu) - name: Compilation run: | diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml index 4fb58b59af..48c04e750a 100644 --- a/.github/workflows/thirdparty-regular-push.yml +++ b/.github/workflows/thirdparty-regular-push.yml @@ -167,7 +167,7 @@ jobs: build-args: | GITHUB_BRANCH=${{ github.ref_name }} OS_VERSION=${{ matrix.osversion }} - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 HADOOP_BIN_PATH=hadoop-bin ZOOKEEPER_BIN_PATH=zookeeper-bin - name: Image digest @@ -203,7 +203,7 @@ jobs: build-args: | GITHUB_BRANCH=${{ github.ref_name }} OS_VERSION=${{ matrix.osversion }} - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 USE_JEMALLOC=ON HADOOP_BIN_PATH=hadoop-bin ZOOKEEPER_BIN_PATH=zookeeper-bin diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile index ab91710632..27a0d5146e 100644 --- a/docker/thirdparties-bin/Dockerfile +++ b/docker/thirdparties-bin/Dockerfile @@ -26,7 +26,7 @@ COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip ARG GITHUB_BRANCH=master ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git -ARG ROCKSDB_PORTABLE=OFF +ARG ROCKSDB_PORTABLE=0 ARG USE_JEMALLOC=OFF ARG HADOOP_BIN_PATH ARG ZOOKEEPER_BIN_PATH diff --git a/run.sh b/run.sh index 75b688a8ae..5ac6a19645 100755 --- a/run.sh +++ b/run.sh @@ -118,7 +118,7 @@ function run_build() ENABLE_GPERF=ON SKIP_THIRDPARTY=NO SANITIZER="" - ROCKSDB_PORTABLE=OFF + ROCKSDB_PORTABLE=0 USE_JEMALLOC=OFF BUILD_TEST=OFF IWYU="" @@ -177,7 +177,7 @@ function run_build() SKIP_THIRDPARTY=YES ;; --enable_rocksdb_portable) - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 ;; --use_jemalloc) ENABLE_GPERF=OFF diff --git a/src/server/pegasus_event_listener.cpp b/src/server/pegasus_event_listener.cpp index e13e2f8635..24faf954e7 100644 --- a/src/server/pegasus_event_listener.cpp +++ b/src/server/pegasus_event_listener.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "common/gpid.h" diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index c348952696..8d6ac8c376 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include @@ -1635,12 +1635,12 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) rocksdb::DBOptions loaded_db_opt; std::vector loaded_cf_descs; rocksdb::ColumnFamilyOptions loaded_data_cf_opts; + rocksdb::ConfigOptions config_options; // Set `ignore_unknown_options` true for forward compatibility. - auto status = rocksdb::LoadLatestOptions(rdb_path, - rocksdb::Env::Default(), - &loaded_db_opt, - &loaded_cf_descs, - /*ignore_unknown_options=*/true); + config_options.ignore_unknown_options = true; + config_options.env = rocksdb::Env::Default(); + auto status = + rocksdb::LoadLatestOptions(config_options, rdb_path, &loaded_db_opt, &loaded_cf_descs); if (!status.ok()) { // Here we ignore an invalid argument error related to `pegasus_data_version` and // `pegasus_data` options, which were used in old version rocksdbs (before 2.1.0). @@ -1678,11 +1678,14 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) std::vector column_families( {{DATA_COLUMN_FAMILY_NAME, _table_data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}}); - auto s = rocksdb::CheckOptionsCompatibility(rdb_path, - rocksdb::Env::Default(), - _db_opts, - column_families, - /*ignore_unknown_options=*/true); + rocksdb::ConfigOptions config_options; + config_options.ignore_unknown_options = true; + config_options.ignore_unsupported_options = true; + config_options.sanity_level = + rocksdb::ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible; + config_options.env = rocksdb::Env::Default(); + auto s = + rocksdb::CheckOptionsCompatibility(config_options, rdb_path, _db_opts, column_families); if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) { LOG_ERROR_PREFIX("rocksdb::CheckOptionsCompatibility failed, error = {}", s.ToString()); return dsn::ERR_LOCAL_APP_FAILURE; @@ -2159,7 +2162,7 @@ ::dsn::error_code pegasus_server_impl::copy_checkpoint_to_dir_unsafe(const char {{DATA_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}, {META_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}}); status = rocksdb::DB::OpenForReadOnly( - rocksdb::DBOptions(), checkpoint_dir, column_families, &handles_opened, &snapshot_db); + _db_opts, checkpoint_dir, column_families, &handles_opened, &snapshot_db); if (!status.ok()) { LOG_ERROR_PREFIX( "OpenForReadOnly from {} failed, error = {}", checkpoint_dir, status.ToString()); @@ -3286,7 +3289,7 @@ ::dsn::error_code pegasus_server_impl::check_column_families(const std::string & *missing_meta_cf = true; *missing_data_cf = true; std::vector column_families; - auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path, &column_families); + auto s = rocksdb::DB::ListColumnFamilies(_db_opts, path, &column_families); if (!s.ok()) { LOG_ERROR_PREFIX("rocksdb::DB::ListColumnFamilies failed, error = {}", s.ToString()); if (s.IsCorruption() && diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index c9c5b90ef6..02c6421c7a 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -70,6 +71,7 @@ class WriteBufferManager; namespace dsn { class blob; class message_ex; + namespace replication { class detect_hotkey_request; class detect_hotkey_response; diff --git a/src/server/pegasus_server_impl_init.cpp b/src/server/pegasus_server_impl_init.cpp index c457ebd9fa..528b2bd8e9 100644 --- a/src/server/pegasus_server_impl_init.cpp +++ b/src/server/pegasus_server_impl_init.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -422,6 +423,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) _rng_rd_opts.rocksdb_iteration_threshold_time_ms = FLAGS_rocksdb_iteration_threshold_time_ms; // init rocksdb::DBOptions + _db_opts.env = rocksdb::Env::Default(); _db_opts.create_if_missing = true; // atomic flush data CF and meta CF, aim to keep consistency of 'last flushed decree' in meta CF // and data in data CF. diff --git a/src/server/test/pegasus_compression_options_test.cpp b/src/server/test/pegasus_compression_options_test.cpp index 9fa38daf66..b33eeeac14 100644 --- a/src/server/test/pegasus_compression_options_test.cpp +++ b/src/server/test/pegasus_compression_options_test.cpp @@ -20,6 +20,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include diff --git a/src/utils/alloc.h b/src/utils/alloc.h index e224eaff6d..542f4efb51 100644 --- a/src/utils/alloc.h +++ b/src/utils/alloc.h @@ -17,6 +17,8 @@ #pragma once +#include + #include "utils/ports.h" // The check for the definition of CACHELINE_SIZE has to be put after including "utils/ports.h", diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 069f21d299..7f128c8940 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -358,15 +358,11 @@ ExternalProject_Add(jemalloc BUILD_IN_SOURCE 1 ) -option(ROCKSDB_PORTABLE "build a portable binary" OFF) +option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0) -# The patch name '0879c240' means the patch of rocksdb: -# https://github.com/facebook/rocksdb/commit/0879c240404b00142ba4718f36cd3f2bd537192d ExternalProject_Add(rocksdb - URL ${OSS_URL_PREFIX}/rocksdb-6.6.4.tar.gz - https://github.com/facebook/rocksdb/archive/refs/tags/v6.6.4.tar.gz - URL_MD5 7f7fcca3e96b7d83ef332804c90070c8 - PATCH_COMMAND patch -p1 < ${TP_DIR}/rocksdb_fix_atomic_flush_0879c240.patch + URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.5.3.tar.gz + URL_MD5 f03eac50ec958a21a7cb56183afb8fe4 DEPENDS jemalloc CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DFAIL_ON_WARNINGS=OFF From b15d5771450cb62d0aa2e332c9bbba287cc24fe6 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Thu, 14 Sep 2023 16:03:28 +0800 Subject: [PATCH 14/42] fix(docker): fix build failure of rocksdb when PORTABLE=0 (#1607) https://github.com/apache/incubator-pegasus/issues/1604 Explicit set PORTABLE=native to avoid building failure. The failure looks like: ``` cc1plus: error: bad value ('OFF') for '-march=' switch cc1plus: note: valid arguments to '-march=' switch are: nocona core2 nehalem corei7 westmere sandybridge corei7-avx ivybridge core-avx-i haswell core-avx2 broadwell skylake skylake-avx512 bonnell atom silvermont slm knl x86-64 eden-x2 nano nano-1000 nano-2000 nano-3000 nano-x2 eden-x4 nano-x4 k8 k8-sse3 opteron opteron-sse3 athlon64 athlon64-sse3 athlon-fx amdfam10 barcelona bdver1 bdver2 bdver3 bdver4 znver1 btver1 btver2 ``` --- docker/thirdparties-bin/Dockerfile | 2 +- docker/thirdparties-src/Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile index 27a0d5146e..36f75fe9c5 100644 --- a/docker/thirdparties-bin/Dockerfile +++ b/docker/thirdparties-bin/Dockerfile @@ -26,7 +26,7 @@ COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip ARG GITHUB_BRANCH=master ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git -ARG ROCKSDB_PORTABLE=0 +ARG ROCKSDB_PORTABLE=native ARG USE_JEMALLOC=OFF ARG HADOOP_BIN_PATH ARG ZOOKEEPER_BIN_PATH diff --git a/docker/thirdparties-src/Dockerfile b/docker/thirdparties-src/Dockerfile index 48070659e6..a11538e2e6 100644 --- a/docker/thirdparties-src/Dockerfile +++ b/docker/thirdparties-src/Dockerfile @@ -23,9 +23,10 @@ ARG GITHUB_BRANCH=master ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} +ARG ROCKSDB_PORTABLE=native RUN cd incubator-pegasus/thirdparty \ && mkdir -p build \ - && cmake -DCMAKE_BUILD_TYPE=Release -B build/ . \ + && cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=${ROCKSDB_PORTABLE} -B build/ . \ && cmake --build build/ -j $(($(nproc)/2+1)) RUN cd incubator-pegasus/thirdparty \ From fef5b16c3bb4a87831a3f50c2aca959c46efd4ac Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 15 Sep 2023 19:49:04 +0800 Subject: [PATCH 15/42] chore(CI): unify to use Ubuntu 22.04 for CI (#1609) https://github.com/apache/incubator-pegasus/issues/1604 Prior to this patch, the `thirdparties-bin-test-${OS}-${branch}` docker images are only built for Ubuntu 18.04 for CI purpose, but some CI jobs use Ubuntu 22.04 image, it may cause the CI failed because the Ubuntu 22.04 image is outdated (for example, lack of or outdated of some thirdparties). This patch unify to build and use `thirdparties-bin-test-2204-${branch}` docker image. --- .github/workflows/lint_and_test_cpp.yaml | 8 ++++---- .github/workflows/thirdparty-regular-push.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index e405fa3a47..6c125727c2 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -259,7 +259,7 @@ jobs: needs: cpp_clang_format_linter runs-on: ubuntu-latest container: - image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }} + image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - uses: actions/checkout@v3 - name: Setup cache @@ -370,7 +370,7 @@ jobs: needs: build_ASAN runs-on: ubuntu-latest container: - image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }} + image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} options: --cap-add=SYS_PTRACE steps: - uses: actions/checkout@v3 @@ -401,7 +401,7 @@ jobs: # needs: cpp_clang_format_linter # runs-on: ubuntu-latest # container: -# image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }} +# image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} # steps: # - uses: actions/checkout@v3 # - name: Setup cache @@ -508,7 +508,7 @@ jobs: # needs: build_UBSAN # runs-on: ubuntu-latest # container: -# image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }} +# image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} # options: --cap-add=SYS_PTRACE # steps: # - uses: actions/checkout@v3 diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml index 48c04e750a..f56b3f4c2d 100644 --- a/.github/workflows/thirdparty-regular-push.yml +++ b/.github/workflows/thirdparty-regular-push.yml @@ -144,7 +144,7 @@ jobs: fail-fast: false matrix: osversion: - - ubuntu1804 + - ubuntu2204 steps: - uses: actions/checkout@v3 - name: Set up QEMU From 31ce328583ecbd92682b1a2fb17f9c8e3d089532 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 18 Sep 2023 15:34:01 +0800 Subject: [PATCH 16/42] fix(ASAN): avoid reporting false 'odr-violation' issues (#1611) --- run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run.sh b/run.sh index 5ac6a19645..e6035e957a 100755 --- a/run.sh +++ b/run.sh @@ -26,6 +26,8 @@ export BUILD_LATEST_DIR=${BUILD_ROOT_DIR}/latest export REPORT_DIR="$ROOT/test_report" export THIRDPARTY_ROOT=$ROOT/thirdparty export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:${BUILD_LATEST_DIR}/output/lib:${THIRDPARTY_ROOT}/output/lib:${LD_LIBRARY_PATH} +# Disable AddressSanitizerOneDefinitionRuleViolation, see https://github.com/google/sanitizers/issues/1017 for details. +export ASAN_OPTIONS=detect_odr_violation=0 function usage() { From 9f32017551a8dfc5d200def76200da23d9e00139 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 18 Sep 2023 19:21:51 +0800 Subject: [PATCH 17/42] build(thirdparty): use v8.5.3-pegasus-encrypt branch of pegasus-kv/rocksdb (#1610) https://github.com/apache/incubator-pegasus/issues/1575 This patch changes Pegasus to use the `v8.5.3-pegasus-encrypt` branch of https://github.com/pegasus-kv/rocksdb.git repository. The `v8.5.3-pegasus-encrypt` branch is based on the official `v8.5.3` tag of facebook/rocksdb repository but adds the encryption feature which is implemented by the Pegasus team. There is nothing changed if not enable the encryption feature. --- thirdparty/CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 7f128c8940..a3823e42b5 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -360,9 +360,13 @@ ExternalProject_Add(jemalloc option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0) +# This is the commit "Update the status badge to pegasus-kv/rocksdb's own (#18) " on +# branch v8.5.3-pegasus-encrypt of https://github.com/pegasus-kv/rocksdb.git. +# The v8.5.3-pegasus-encrypt branch is based on the v8.5.3 tag of facebook/rocksdb and add +# the encryption feature. ExternalProject_Add(rocksdb - URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.5.3.tar.gz - URL_MD5 f03eac50ec958a21a7cb56183afb8fe4 + URL https://github.com/pegasus-kv/rocksdb/archive/090c3d9fd1b99c633abc5ab4de2f75d4c6b6528d.zip + URL_MD5 1e556d77ae853b22f62dd2e82e915937 DEPENDS jemalloc CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DFAIL_ON_WARNINGS=OFF From edd1e17c87a7a018dca690cc4a2c569718cd76ea Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 19 Sep 2023 10:37:44 +0800 Subject: [PATCH 18/42] feat(build): build RocksDB with encryption enabled (#1612) https://github.com/apache/incubator-pegasus/issues/1575 Set option -DWITH_OPENSSL=ON to build rocksdb with encryption feature enabled. --- thirdparty/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index a3823e42b5..5784a82289 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -379,6 +379,7 @@ ExternalProject_Add(rocksdb -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF -DUSE_RTTI=ON + -DWITH_OPENSSL=ON # enable encryption -DCMAKE_BUILD_TYPE=Release -DWITH_JEMALLOC=${USE_JEMALLOC} -DJEMALLOC_ROOT_DIR=${TP_OUTPUT} From 22c91816ccd9cce05f2b2449d0ba299ac447493a Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 19 Sep 2023 14:38:27 +0800 Subject: [PATCH 19/42] fix(rocksdb): update rocksdb to fix build failure on lower OpenSSL version (#1614) https://github.com/apache/incubator-pegasus/issues/1575 Fix a build error on lower OpenSSL version, the error looks like: ``` 2023-09-19T02:53:45.4093185Z #11 924.7 /root/incubator-pegasus/thirdparty/build/Source/rocksdb/encryption/encryption.cc: In function 'const EVP_CIPHER* rocksdb::encryption::GetEVPCipher(rocksdb::encryption::EncryptionMethod)': 2023-09-19T02:53:45.4094191Z #11 924.7 /root/incubator-pegasus/thirdparty/build/Source/rocksdb/encryption/encryption.cc:112:44: error: cannot convert 'rocksdb::Status' to 'const EVP_CIPHER* {aka const evp_cipher_st*}' in return 2023-09-19T02:53:45.4094713Z #11 924.7 std::string(OPENSSL_VERSION_TEXT)); 2023-09-19T02:53:45.4094991Z #11 924.7 ^ 2023-09-19T02:53:45.5599505Z #11 924.7 gmake[5]: *** [CMakeFiles/rocksdb.dir/encryption/encryption.cc.o] Error 1 2023-09-19T02:53:45.5599938Z #11 924.7 gmake[4]: *** [CMakeFiles/rocksdb.dir/all] Error 2 2023-09-19T02:53:45.5600266Z #11 924.7 gmake[4]: *** Waiting for unfinished jobs.... ``` --- thirdparty/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 5784a82289..4558aa94f7 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -365,8 +365,8 @@ option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = ba # The v8.5.3-pegasus-encrypt branch is based on the v8.5.3 tag of facebook/rocksdb and add # the encryption feature. ExternalProject_Add(rocksdb - URL https://github.com/pegasus-kv/rocksdb/archive/090c3d9fd1b99c633abc5ab4de2f75d4c6b6528d.zip - URL_MD5 1e556d77ae853b22f62dd2e82e915937 + URL https://github.com/pegasus-kv/rocksdb/archive/e72106597e6b7924485cadc2433b66029f1aaf17.zip + URL_MD5 6f6daef703586ce788643bbb8a984fff DEPENDS jemalloc CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DFAIL_ON_WARNINGS=OFF From 7b2bd09b9298e421b51cac578ca331907d1fb905 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 20 Sep 2023 14:10:28 +0800 Subject: [PATCH 20/42] fix(build): fix a link error on macOS (#1615) https://github.com/apache/incubator-pegasus/issues/1602 Remove the `_test` postfix of function tests to prevent the link errors on macOS look like: ``` [ 50%] Linking CXX executable backup_restore_test ld: can't write output file to 'backup_restore_test' because that path is a directory clang: error: linker command failed with exit code 1 (use -v to see invocation) make[2]: *** [src/test/function_test/backup_restore_test/backup_restore_test] Error 1 make[1]: *** [src/test/function_test/backup_restore_test/CMakeFiles/backup_restore_test.dir/all] Error 2 make[1]: *** Waiting for unfinished jobs.... ``` --- .gitignore | 2 +- run.sh | 6 +++--- src/server/info_collector.h | 3 ++- src/test/function_test/CMakeLists.txt | 16 ++++++++-------- .../CMakeLists.txt | 0 .../main.cpp | 0 .../test_backup_and_restore.cpp | 0 .../{base_api_test => base_api}/CMakeLists.txt | 0 .../integration_test.cpp | 0 .../{base_api_test => base_api}/main.cpp | 0 .../{base_api_test => base_api}/test_basic.cpp | 0 .../test_batch_get.cpp | 0 .../test_check_and_mutate.cpp | 0 .../test_check_and_set.cpp | 0 .../{base_api_test => base_api}/test_copy.cpp | 0 .../{base_api_test => base_api}/test_incr.cpp | 0 .../test_range_read.cpp | 0 .../{base_api_test => base_api}/test_recall.cpp | 0 .../{base_api_test => base_api}/test_scan.cpp | 0 .../{base_api_test => base_api}/test_ttl.cpp | 0 .../{bulk_load_test => bulk_load}/CMakeLists.txt | 0 .../{bulk_load_test => bulk_load}/main.cpp | 0 .../test_bulk_load.cpp | 4 ++-- .../CMakeLists.txt | 0 .../main.cpp | 0 .../test_detect_hotspot.cpp | 0 .../CMakeLists.txt | 0 .../main.cpp | 0 .../test_split.cpp | 0 .../{recovery_test => recovery}/CMakeLists.txt | 0 .../{recovery_test => recovery}/main.cpp | 0 .../test_recovery.cpp | 0 .../{restore_test => restore}/CMakeLists.txt | 0 .../{restore_test => restore}/main.cpp | 0 .../{restore_test => restore}/test_restore.cpp | 0 .../{throttle_test => throttle}/CMakeLists.txt | 0 .../{throttle_test => throttle}/main.cpp | 0 .../test_throttle.cpp | 2 +- src/utils/output_utils.cpp | 1 + 39 files changed, 18 insertions(+), 16 deletions(-) rename src/test/function_test/{backup_restore_test => backup_restore}/CMakeLists.txt (100%) rename src/test/function_test/{backup_restore_test => backup_restore}/main.cpp (100%) rename src/test/function_test/{backup_restore_test => backup_restore}/test_backup_and_restore.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/CMakeLists.txt (100%) rename src/test/function_test/{base_api_test => base_api}/integration_test.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/main.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/test_basic.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/test_batch_get.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/test_check_and_mutate.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/test_check_and_set.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/test_copy.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/test_incr.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/test_range_read.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/test_recall.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/test_scan.cpp (100%) rename src/test/function_test/{base_api_test => base_api}/test_ttl.cpp (100%) rename src/test/function_test/{bulk_load_test => bulk_load}/CMakeLists.txt (100%) rename src/test/function_test/{bulk_load_test => bulk_load}/main.cpp (100%) rename src/test/function_test/{bulk_load_test => bulk_load}/test_bulk_load.cpp (98%) rename src/test/function_test/{detect_hotspot_test => detect_hotspot}/CMakeLists.txt (100%) rename src/test/function_test/{detect_hotspot_test => detect_hotspot}/main.cpp (100%) rename src/test/function_test/{detect_hotspot_test => detect_hotspot}/test_detect_hotspot.cpp (100%) rename src/test/function_test/{partition_split_test => partition_split}/CMakeLists.txt (100%) rename src/test/function_test/{partition_split_test => partition_split}/main.cpp (100%) rename src/test/function_test/{partition_split_test => partition_split}/test_split.cpp (100%) rename src/test/function_test/{recovery_test => recovery}/CMakeLists.txt (100%) rename src/test/function_test/{recovery_test => recovery}/main.cpp (100%) rename src/test/function_test/{recovery_test => recovery}/test_recovery.cpp (100%) rename src/test/function_test/{restore_test => restore}/CMakeLists.txt (100%) rename src/test/function_test/{restore_test => restore}/main.cpp (100%) rename src/test/function_test/{restore_test => restore}/test_restore.cpp (100%) rename src/test/function_test/{throttle_test => throttle}/CMakeLists.txt (100%) rename src/test/function_test/{throttle_test => throttle}/main.cpp (100%) rename src/test/function_test/{throttle_test => throttle}/test_throttle.cpp (99%) diff --git a/.gitignore b/.gitignore index 457dbf3098..2bfd1ed34d 100644 --- a/.gitignore +++ b/.gitignore @@ -115,7 +115,7 @@ scripts/py_utils/*.pyc cmake-build-debug packages -src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files/ +src/test/function_test/bulk_load/pegasus-bulk-load-function-test-files/ config-shell.ini.* *.tar.gz pegasus-server* diff --git a/run.sh b/run.sh index e6035e957a..2672094f7c 100755 --- a/run.sh +++ b/run.sh @@ -437,10 +437,10 @@ function run_test() echo "test_modules=$test_modules" # download bulk load test data - if [[ "$test_modules" =~ "bulk_load_test" && ! -d "$ROOT/src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files" ]]; then + if [[ "$test_modules" =~ "bulk_load_test" && ! -d "$ROOT/src/test/function_test/bulk_load/pegasus-bulk-load-function-test-files" ]]; then echo "Start to download files used for bulk load function test" wget "https://github.com/XiaoMi/pegasus-common/releases/download/deps/pegasus-bulk-load-function-test-files.zip" - unzip "pegasus-bulk-load-function-test-files.zip" -d "$ROOT/src/test/function_test/bulk_load_test" + unzip "pegasus-bulk-load-function-test-files.zip" -d "$ROOT/src/test/function_test/bulk_load" rm "pegasus-bulk-load-function-test-files.zip" echo "Prepare files used for bulk load function test succeed" fi @@ -484,7 +484,7 @@ function run_test() run_stop_zk run_start_zk fi - pushd ${BUILD_LATEST_DIR}/bin/$module + pushd ${BUILD_LATEST_DIR}/bin/${module} REPORT_DIR=${REPORT_DIR} TEST_BIN=${module} TEST_OPTS=${test_opts} ./run.sh if [ $? != 0 ]; then echo "run test \"$module\" in `pwd` failed" diff --git a/src/server/info_collector.h b/src/server/info_collector.h index cf882c1177..258a3d6318 100644 --- a/src/server/info_collector.h +++ b/src/server/info_collector.h @@ -19,10 +19,11 @@ #pragma once -// IWYU pragma: no_include #include #include #include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include #include diff --git a/src/test/function_test/CMakeLists.txt b/src/test/function_test/CMakeLists.txt index d642cfb093..b3453846b3 100644 --- a/src/test/function_test/CMakeLists.txt +++ b/src/test/function_test/CMakeLists.txt @@ -16,11 +16,11 @@ # under the License. add_subdirectory(utils) -add_subdirectory(backup_restore_test) -add_subdirectory(base_api_test) -add_subdirectory(bulk_load_test) -add_subdirectory(detect_hotspot_test) -add_subdirectory(partition_split_test) -add_subdirectory(recovery_test) -add_subdirectory(restore_test) -add_subdirectory(throttle_test) +add_subdirectory(backup_restore) +add_subdirectory(base_api) +add_subdirectory(bulk_load) +add_subdirectory(detect_hotspot) +add_subdirectory(partition_split) +add_subdirectory(recovery) +add_subdirectory(restore) +add_subdirectory(throttle) diff --git a/src/test/function_test/backup_restore_test/CMakeLists.txt b/src/test/function_test/backup_restore/CMakeLists.txt similarity index 100% rename from src/test/function_test/backup_restore_test/CMakeLists.txt rename to src/test/function_test/backup_restore/CMakeLists.txt diff --git a/src/test/function_test/backup_restore_test/main.cpp b/src/test/function_test/backup_restore/main.cpp similarity index 100% rename from src/test/function_test/backup_restore_test/main.cpp rename to src/test/function_test/backup_restore/main.cpp diff --git a/src/test/function_test/backup_restore_test/test_backup_and_restore.cpp b/src/test/function_test/backup_restore/test_backup_and_restore.cpp similarity index 100% rename from src/test/function_test/backup_restore_test/test_backup_and_restore.cpp rename to src/test/function_test/backup_restore/test_backup_and_restore.cpp diff --git a/src/test/function_test/base_api_test/CMakeLists.txt b/src/test/function_test/base_api/CMakeLists.txt similarity index 100% rename from src/test/function_test/base_api_test/CMakeLists.txt rename to src/test/function_test/base_api/CMakeLists.txt diff --git a/src/test/function_test/base_api_test/integration_test.cpp b/src/test/function_test/base_api/integration_test.cpp similarity index 100% rename from src/test/function_test/base_api_test/integration_test.cpp rename to src/test/function_test/base_api/integration_test.cpp diff --git a/src/test/function_test/base_api_test/main.cpp b/src/test/function_test/base_api/main.cpp similarity index 100% rename from src/test/function_test/base_api_test/main.cpp rename to src/test/function_test/base_api/main.cpp diff --git a/src/test/function_test/base_api_test/test_basic.cpp b/src/test/function_test/base_api/test_basic.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_basic.cpp rename to src/test/function_test/base_api/test_basic.cpp diff --git a/src/test/function_test/base_api_test/test_batch_get.cpp b/src/test/function_test/base_api/test_batch_get.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_batch_get.cpp rename to src/test/function_test/base_api/test_batch_get.cpp diff --git a/src/test/function_test/base_api_test/test_check_and_mutate.cpp b/src/test/function_test/base_api/test_check_and_mutate.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_check_and_mutate.cpp rename to src/test/function_test/base_api/test_check_and_mutate.cpp diff --git a/src/test/function_test/base_api_test/test_check_and_set.cpp b/src/test/function_test/base_api/test_check_and_set.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_check_and_set.cpp rename to src/test/function_test/base_api/test_check_and_set.cpp diff --git a/src/test/function_test/base_api_test/test_copy.cpp b/src/test/function_test/base_api/test_copy.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_copy.cpp rename to src/test/function_test/base_api/test_copy.cpp diff --git a/src/test/function_test/base_api_test/test_incr.cpp b/src/test/function_test/base_api/test_incr.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_incr.cpp rename to src/test/function_test/base_api/test_incr.cpp diff --git a/src/test/function_test/base_api_test/test_range_read.cpp b/src/test/function_test/base_api/test_range_read.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_range_read.cpp rename to src/test/function_test/base_api/test_range_read.cpp diff --git a/src/test/function_test/base_api_test/test_recall.cpp b/src/test/function_test/base_api/test_recall.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_recall.cpp rename to src/test/function_test/base_api/test_recall.cpp diff --git a/src/test/function_test/base_api_test/test_scan.cpp b/src/test/function_test/base_api/test_scan.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_scan.cpp rename to src/test/function_test/base_api/test_scan.cpp diff --git a/src/test/function_test/base_api_test/test_ttl.cpp b/src/test/function_test/base_api/test_ttl.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_ttl.cpp rename to src/test/function_test/base_api/test_ttl.cpp diff --git a/src/test/function_test/bulk_load_test/CMakeLists.txt b/src/test/function_test/bulk_load/CMakeLists.txt similarity index 100% rename from src/test/function_test/bulk_load_test/CMakeLists.txt rename to src/test/function_test/bulk_load/CMakeLists.txt diff --git a/src/test/function_test/bulk_load_test/main.cpp b/src/test/function_test/bulk_load/main.cpp similarity index 100% rename from src/test/function_test/bulk_load_test/main.cpp rename to src/test/function_test/bulk_load/main.cpp diff --git a/src/test/function_test/bulk_load_test/test_bulk_load.cpp b/src/test/function_test/bulk_load/test_bulk_load.cpp similarity index 98% rename from src/test/function_test/bulk_load_test/test_bulk_load.cpp rename to src/test/function_test/bulk_load/test_bulk_load.cpp index 83b2dd6ecc..f350c69453 100644 --- a/src/test/function_test/bulk_load_test/test_bulk_load.cpp +++ b/src/test/function_test/bulk_load/test_bulk_load.cpp @@ -90,7 +90,7 @@ class bulk_load_test : public test_util ASSERT_NO_FATAL_FAILURE( run_cmd_from_project_root("mkdir -p onebox/block_service/local_service")); ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root( - "cp -r src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files/" + + "cp -r src/test/function_test/bulk_load/pegasus-bulk-load-function-test-files/" + LOCAL_ROOT + " onebox/block_service/local_service")); string cmd = "echo '{\"app_id\":" + std::to_string(app_id_) + ",\"app_name\":\"temp\",\"partition_count\":8}' > " @@ -114,7 +114,7 @@ class bulk_load_test : public test_util void replace_bulk_load_info() { string cmd = "cp -R " - "src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files/" + "src/test/function_test/bulk_load/pegasus-bulk-load-function-test-files/" "mock_bulk_load_info/. " + bulk_load_local_root_ + "/" + CLUSTER + "/" + app_name_ + "/"; ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root(cmd)); diff --git a/src/test/function_test/detect_hotspot_test/CMakeLists.txt b/src/test/function_test/detect_hotspot/CMakeLists.txt similarity index 100% rename from src/test/function_test/detect_hotspot_test/CMakeLists.txt rename to src/test/function_test/detect_hotspot/CMakeLists.txt diff --git a/src/test/function_test/detect_hotspot_test/main.cpp b/src/test/function_test/detect_hotspot/main.cpp similarity index 100% rename from src/test/function_test/detect_hotspot_test/main.cpp rename to src/test/function_test/detect_hotspot/main.cpp diff --git a/src/test/function_test/detect_hotspot_test/test_detect_hotspot.cpp b/src/test/function_test/detect_hotspot/test_detect_hotspot.cpp similarity index 100% rename from src/test/function_test/detect_hotspot_test/test_detect_hotspot.cpp rename to src/test/function_test/detect_hotspot/test_detect_hotspot.cpp diff --git a/src/test/function_test/partition_split_test/CMakeLists.txt b/src/test/function_test/partition_split/CMakeLists.txt similarity index 100% rename from src/test/function_test/partition_split_test/CMakeLists.txt rename to src/test/function_test/partition_split/CMakeLists.txt diff --git a/src/test/function_test/partition_split_test/main.cpp b/src/test/function_test/partition_split/main.cpp similarity index 100% rename from src/test/function_test/partition_split_test/main.cpp rename to src/test/function_test/partition_split/main.cpp diff --git a/src/test/function_test/partition_split_test/test_split.cpp b/src/test/function_test/partition_split/test_split.cpp similarity index 100% rename from src/test/function_test/partition_split_test/test_split.cpp rename to src/test/function_test/partition_split/test_split.cpp diff --git a/src/test/function_test/recovery_test/CMakeLists.txt b/src/test/function_test/recovery/CMakeLists.txt similarity index 100% rename from src/test/function_test/recovery_test/CMakeLists.txt rename to src/test/function_test/recovery/CMakeLists.txt diff --git a/src/test/function_test/recovery_test/main.cpp b/src/test/function_test/recovery/main.cpp similarity index 100% rename from src/test/function_test/recovery_test/main.cpp rename to src/test/function_test/recovery/main.cpp diff --git a/src/test/function_test/recovery_test/test_recovery.cpp b/src/test/function_test/recovery/test_recovery.cpp similarity index 100% rename from src/test/function_test/recovery_test/test_recovery.cpp rename to src/test/function_test/recovery/test_recovery.cpp diff --git a/src/test/function_test/restore_test/CMakeLists.txt b/src/test/function_test/restore/CMakeLists.txt similarity index 100% rename from src/test/function_test/restore_test/CMakeLists.txt rename to src/test/function_test/restore/CMakeLists.txt diff --git a/src/test/function_test/restore_test/main.cpp b/src/test/function_test/restore/main.cpp similarity index 100% rename from src/test/function_test/restore_test/main.cpp rename to src/test/function_test/restore/main.cpp diff --git a/src/test/function_test/restore_test/test_restore.cpp b/src/test/function_test/restore/test_restore.cpp similarity index 100% rename from src/test/function_test/restore_test/test_restore.cpp rename to src/test/function_test/restore/test_restore.cpp diff --git a/src/test/function_test/throttle_test/CMakeLists.txt b/src/test/function_test/throttle/CMakeLists.txt similarity index 100% rename from src/test/function_test/throttle_test/CMakeLists.txt rename to src/test/function_test/throttle/CMakeLists.txt diff --git a/src/test/function_test/throttle_test/main.cpp b/src/test/function_test/throttle/main.cpp similarity index 100% rename from src/test/function_test/throttle_test/main.cpp rename to src/test/function_test/throttle/main.cpp diff --git a/src/test/function_test/throttle_test/test_throttle.cpp b/src/test/function_test/throttle/test_throttle.cpp similarity index 99% rename from src/test/function_test/throttle_test/test_throttle.cpp rename to src/test/function_test/throttle/test_throttle.cpp index a29c9c5fe1..23583a7bfe 100644 --- a/src/test/function_test/throttle_test/test_throttle.cpp +++ b/src/test/function_test/throttle/test_throttle.cpp @@ -131,7 +131,7 @@ struct throttle_test_recorder // read/write throttle function test // the details of records are saved in -// `./src/builder/test/function_test/throttle_test/throttle_test_result.txt` +// `./src/builder/test/function_test/throttle/throttle_test_result.txt` class throttle_test : public test_util { public: diff --git a/src/utils/output_utils.cpp b/src/utils/output_utils.cpp index 6b30172217..dfaa799633 100644 --- a/src/utils/output_utils.cpp +++ b/src/utils/output_utils.cpp @@ -17,6 +17,7 @@ #include "utils/output_utils.h" +#include // IWYU pragma: no_include #include From 625cebcc90c100bd9ba6d2c6d55ac0d29924ae00 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 20 Sep 2023 16:25:25 +0800 Subject: [PATCH 21/42] feat(encryption): introduce PegasusEnv (#1606) https://github.com/apache/incubator-pegasus/issues/1575 This patch introduces `PegasusEnv()` to obtain the `Env` instance used by RocksDB. Then it's possible to obtain an encrypted Env instance by `PegasusEnv(FileDataType::kSensitive)`, the encrypted Env is used for operating on sensitive files, the writing data to the file will be encrypted and the reading data from the file will be decrypted. Some file operate functions and related unit tests are added as well. --- src/aio/test/CMakeLists.txt | 2 +- src/block_service/test/CMakeLists.txt | 2 +- src/client/test/CMakeLists.txt | 2 +- src/common/test/CMakeLists.txt | 2 +- src/failure_detector/test/CMakeLists.txt | 2 +- src/http/test/CMakeLists.txt | 2 +- src/meta/CMakeLists.txt | 2 +- src/nfs/test/CMakeLists.txt | 2 +- src/perf_counter/test/CMakeLists.txt | 2 +- src/replica/backup/test/CMakeLists.txt | 2 +- src/replica/duplication/test/CMakeLists.txt | 2 +- src/test_util/test_util.h | 11 + src/utils/CMakeLists.txt | 2 +- src/utils/env.cpp | 201 ++++++++++++ src/utils/env.h | 66 ++++ src/utils/filesystem.cpp | 33 +- src/utils/filesystem.h | 10 + src/utils/fmt_logging.h | 11 + src/utils/long_adder_bench/CMakeLists.txt | 2 +- src/utils/test/CMakeLists.txt | 3 +- src/utils/test/env.cpp | 285 +++++++++++++++++- src/utils/test/file_system_test.cpp | 90 +++++- .../test/nth_element_bench/CMakeLists.txt | 2 +- src/zookeeper/test/CMakeLists.txt | 2 +- 24 files changed, 715 insertions(+), 25 deletions(-) create mode 100644 src/utils/env.cpp create mode 100644 src/utils/env.h diff --git a/src/aio/test/CMakeLists.txt b/src/aio/test/CMakeLists.txt index 4228a01481..c1b0a44e46 100644 --- a/src/aio/test/CMakeLists.txt +++ b/src/aio/test/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio) +set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt index a8e47d597e..537778caad 100644 --- a/src/block_service/test/CMakeLists.txt +++ b/src/block_service/test/CMakeLists.txt @@ -36,7 +36,7 @@ set(MY_PROJ_LIBS gtest gtest_main hdfs - ) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/client/test/CMakeLists.txt b/src/client/test/CMakeLists.txt index 90ce5d5e84..bcae3897aa 100644 --- a/src/client/test/CMakeLists.txt +++ b/src/client/test/CMakeLists.txt @@ -27,7 +27,7 @@ set(MY_PROJ_LIBS dsn_runtime dsn_utils gtest -) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/common/test/CMakeLists.txt b/src/common/test/CMakeLists.txt index 78d94000c3..1cdf58407f 100644 --- a/src/common/test/CMakeLists.txt +++ b/src/common/test/CMakeLists.txt @@ -30,7 +30,7 @@ set(MY_PROJ_LIBS dsn_replication_common dsn_runtime gtest - ) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/failure_detector/test/CMakeLists.txt b/src/failure_detector/test/CMakeLists.txt index d529e50ea4..ed4a9703d6 100644 --- a/src/failure_detector/test/CMakeLists.txt +++ b/src/failure_detector/test/CMakeLists.txt @@ -41,7 +41,7 @@ set(MY_PROJ_LIBS dsn.failure_detector gtest hashtable - ) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/http/test/CMakeLists.txt b/src/http/test/CMakeLists.txt index 6a09e42dc5..d4da7e5b3a 100644 --- a/src/http/test/CMakeLists.txt +++ b/src/http/test/CMakeLists.txt @@ -26,7 +26,7 @@ set(MY_PROJ_LIBS dsn_runtime gtest gtest_main - ) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index b38ced7228..def16ec26c 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -54,7 +54,7 @@ set(MY_PROJ_LIBS crypto hashtable hdfs - ) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/nfs/test/CMakeLists.txt b/src/nfs/test/CMakeLists.txt index 735bb29bd6..64c7967ace 100644 --- a/src/nfs/test/CMakeLists.txt +++ b/src/nfs/test/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio) +set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/perf_counter/test/CMakeLists.txt b/src/perf_counter/test/CMakeLists.txt index a02a6923b5..434d97dc11 100644 --- a/src/perf_counter/test/CMakeLists.txt +++ b/src/perf_counter/test/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime) +set(MY_PROJ_LIBS gtest dsn_runtime rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/backup/test/CMakeLists.txt b/src/replica/backup/test/CMakeLists.txt index 21ec10f505..e3dc0ec05b 100644 --- a/src/replica/backup/test/CMakeLists.txt +++ b/src/replica/backup/test/CMakeLists.txt @@ -30,7 +30,7 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_utils hashtable gtest -) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/duplication/test/CMakeLists.txt b/src/replica/duplication/test/CMakeLists.txt index 09619343ec..faab20f7e6 100644 --- a/src/replica/duplication/test/CMakeLists.txt +++ b/src/replica/duplication/test/CMakeLists.txt @@ -30,7 +30,7 @@ set(MY_PROJ_LIBS dsn_meta_server zookeeper hashtable gtest -) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/test_util/test_util.h b/src/test_util/test_util.h index ec7bce6e2e..d33775ff5e 100644 --- a/src/test_util/test_util.h +++ b/src/test_util/test_util.h @@ -21,10 +21,21 @@ #include +#include "gtest/gtest.h" +#include "utils/flags.h" #include "utils/test_macros.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + namespace pegasus { +// A base parameterized test class for testing enable/disable encryption at rest. +class encrypt_data_test_base : public testing::TestWithParam +{ +public: + encrypt_data_test_base() { FLAGS_encrypt_data_at_rest = GetParam(); } +}; + #define ASSERT_EVENTUALLY(expr) \ do { \ AssertEventually(expr); \ diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index e17fef9001..1127a868ef 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -31,7 +31,7 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) -set(MY_PROJ_LIBS dsn_http crypto) +set(MY_PROJ_LIBS dsn_http crypto rocksdb) # Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/utils/env.cpp b/src/utils/env.cpp new file mode 100644 index 0000000000..329406117c --- /dev/null +++ b/src/utils/env.cpp @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "env.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "utils/defer.h" +#include "utils/filesystem.h" +#include "utils/flags.h" +#include "utils/fmt_logging.h" +#include "utils/utils.h" + +DSN_DEFINE_bool(pegasus.server, + encrypt_data_at_rest, + false, + "Whether the sensitive files should be encrypted on the file system."); + +DSN_DEFINE_string(pegasus.server, + server_key_for_testing, + "server_key_for_testing", + "The encrypted server key to use in the filesystem. NOTE: only for testing."); + +DSN_DEFINE_string(pegasus.server, + encryption_method, + "AES128CTR", + "The encryption method to use in the filesystem. Now " + "supports AES128CTR, AES192CTR, AES256CTR and SM4CTR."); + +namespace dsn { +namespace utils { + +rocksdb::Env *NewEncryptedEnv() +{ + // Create an encryption provider. + std::shared_ptr provider; + auto provider_id = + fmt::format("AES:{},{}", FLAGS_server_key_for_testing, FLAGS_encryption_method); + auto s = rocksdb::EncryptionProvider::CreateFromString( + rocksdb::ConfigOptions(), provider_id, &provider); + CHECK(s.ok(), "Failed to create encryption provider: {}", s.ToString()); + + // Create an encrypted env. + return NewEncryptedEnv(rocksdb::Env::Default(), provider); +} + +rocksdb::Env *PegasusEnv(FileDataType type) +{ + // Return an encrypted env only when the file is sensitive and FLAGS_encrypt_data_at_rest + // is enabled at the same time. + if (FLAGS_encrypt_data_at_rest && type == FileDataType::kSensitive) { + static rocksdb::Env *env = NewEncryptedEnv(); + return env; + } + + // Otherwise, return a common non-encrypted env. + static rocksdb::Env *env = rocksdb::Env::Default(); + return env; +} + +namespace { +rocksdb::Status do_copy_file(const std::string &src_fname, + dsn::utils::FileDataType src_type, + const std::string &dst_fname, + dsn::utils::FileDataType dst_type, + int64_t remain_size, + uint64_t *total_size) +{ + rocksdb::EnvOptions src_env_options; + std::unique_ptr src_file; + auto s = + dsn::utils::PegasusEnv(src_type)->NewSequentialFile(src_fname, &src_file, src_env_options); + LOG_AND_RETURN_NOT_RDB_OK(WARNING, s, "failed to open file {} for reading", src_fname); + + // Limit the size of the file to be copied. + int64_t src_file_size; + CHECK_TRUE(dsn::utils::filesystem::file_size(src_fname, src_type, src_file_size)); + if (remain_size == -1) { + // Copy the whole file if 'remain_size' is -1. + remain_size = src_file_size; + } else { + remain_size = std::min(remain_size, src_file_size); + } + + rocksdb::EnvOptions dst_env_options; + std::unique_ptr dst_file; + s = dsn::utils::PegasusEnv(dst_type)->NewWritableFile(dst_fname, &dst_file, dst_env_options); + LOG_AND_RETURN_NOT_RDB_OK(WARNING, s, "failed to open file {} for writing", dst_fname); + + // Read at most 4MB once. + // TODO(yingchun): make it configurable. + const uint64_t kCopyBlockSize = 4 << 20; + auto buffer = dsn::utils::make_shared_array(kCopyBlockSize); + uint64_t offset = 0; + do { + int bytes_per_copy = std::min(remain_size, static_cast(kCopyBlockSize)); + // Reach the EOF. + if (bytes_per_copy <= 0) { + break; + } + + rocksdb::Slice result; + LOG_AND_RETURN_NOT_RDB_OK(WARNING, + src_file->Read(bytes_per_copy, &result, buffer.get()), + "failed to read file {}", + src_fname); + CHECK(!result.empty(), + "read file {} at offset {} with size {} failed", + src_fname, + offset, + bytes_per_copy); + LOG_AND_RETURN_NOT_RDB_OK( + WARNING, dst_file->Append(result), "failed to write file {}", dst_fname); + + offset += result.size(); + remain_size -= result.size(); + + // Reach the EOF. + if (result.size() < bytes_per_copy) { + break; + } + } while (true); + LOG_AND_RETURN_NOT_RDB_OK(WARNING, dst_file->Fsync(), "failed to fsync file {}", dst_fname); + + if (total_size != nullptr) { + *total_size = offset; + } + + LOG_INFO("copy file from {} to {}, total size {}", src_fname, dst_fname, offset); + return rocksdb::Status::OK(); +} +} // anonymous namespace + +rocksdb::Status +copy_file(const std::string &src_fname, const std::string &dst_fname, uint64_t *total_size) +{ + // TODO(yingchun): Consider to use hard link, i.e. rocksdb::Env()::LinkFile(). + return do_copy_file( + src_fname, FileDataType::kSensitive, dst_fname, FileDataType::kSensitive, -1, total_size); +} + +rocksdb::Status +encrypt_file(const std::string &src_fname, const std::string &dst_fname, uint64_t *total_size) +{ + return do_copy_file(src_fname, + FileDataType::kNonSensitive, + dst_fname, + FileDataType::kSensitive, + -1, + total_size); +} + +rocksdb::Status encrypt_file(const std::string &fname, uint64_t *total_size) +{ + // TODO(yingchun): add timestamp to the tmp encrypted file name. + std::string tmp_fname = fname + ".encrypted.tmp"; + auto cleanup = dsn::defer([tmp_fname]() { utils::filesystem::remove_path(tmp_fname); }); + LOG_AND_RETURN_NOT_RDB_OK( + WARNING, encrypt_file(fname, tmp_fname, total_size), "failed to encrypt file {}", fname); + if (!::dsn::utils::filesystem::rename_path(tmp_fname, fname)) { + LOG_WARNING("rename file from {} to {} failed", tmp_fname, fname); + return rocksdb::Status::IOError("rename file failed"); + } + return rocksdb::Status::OK(); +} + +rocksdb::Status +copy_file_by_size(const std::string &src_fname, const std::string &dst_fname, int64_t limit_size) +{ + return do_copy_file(src_fname, + FileDataType::kSensitive, + dst_fname, + FileDataType::kSensitive, + limit_size, + nullptr); +} + +} // namespace utils +} // namespace dsn diff --git a/src/utils/env.h b/src/utils/env.h new file mode 100644 index 0000000000..839bd81ab3 --- /dev/null +++ b/src/utils/env.h @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace dsn { +namespace utils { + +// Indicate whether the file is sensitive or not. +// Only the sensitive file will be encrypted if FLAGS_encrypt_data_at_rest +// is enabled at the same time. +enum class FileDataType +{ + kSensitive = 0, + kNonSensitive = 1 +}; + +static const size_t kEncryptionHeaderkSize = rocksdb::kDefaultPageSize; + +// Get the rocksdb::Env instance for the given file type. +rocksdb::Env *PegasusEnv(FileDataType type); + +// Encrypt the original non-encrypted 'src_fname' to 'dst_fname'. +// The 'total_size' is the total size of the file content, exclude the file encryption header +// (typically 4KB). +rocksdb::Status encrypt_file(const std::string &src_fname, + const std::string &dst_fname, + uint64_t *total_size = nullptr); + +// Similar to the above, but encrypt the file in the same path. +rocksdb::Status encrypt_file(const std::string &fname, uint64_t *total_size = nullptr); + +// Copy the original 'src_fname' to 'dst_fname'. +// Both 'src_fname' and 'dst_fname' are sensitive files. +rocksdb::Status copy_file(const std::string &src_fname, + const std::string &dst_fname, + uint64_t *total_size = nullptr); + +// Similar to the above, but copy the file by a limited size. +// Both 'src_fname' and 'dst_fname' are sensitive files, 'limit_size' is the max size of the +// file to copy, and -1 means no limit. +rocksdb::Status copy_file_by_size(const std::string &src_fname, + const std::string &dst_fname, + int64_t limit_size = -1); +} // namespace utils +} // namespace dsn diff --git a/src/utils/filesystem.cpp b/src/utils/filesystem.cpp index 73aa3a50aa..f2fa59ef87 100644 --- a/src/utils/filesystem.cpp +++ b/src/utils/filesystem.cpp @@ -33,11 +33,8 @@ * xxxx-xx-xx, author, fix bug about xxx */ -#include -#include #include #include -#include #include #include #include @@ -45,11 +42,18 @@ #include #include // IWYU pragma: no_include -#include // IWYU pragma: keep #include -#include + +#include + +#include +#include +#include +#include +#include #include "utils/defer.h" +#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" @@ -388,7 +392,7 @@ bool rename_path(const std::string &path1, const std::string &path2) return ret; } -bool file_size(const std::string &path, int64_t &sz) +bool deprecated_file_size(const std::string &path, int64_t &sz) { struct stat_ st; std::string npath; @@ -417,6 +421,23 @@ bool file_size(const std::string &path, int64_t &sz) return true; } +bool file_size(const std::string &path, int64_t &sz) +{ + return file_size(path, dsn::utils::FileDataType::kNonSensitive, sz); +} + +bool file_size(const std::string &path, FileDataType type, int64_t &sz) +{ + uint64_t file_size = 0; + auto s = dsn::utils::PegasusEnv(type)->GetFileSize(path, &file_size); + if (!s.ok()) { + LOG_ERROR("GetFileSize failed, file '{}', err = {}", path, s.ToString()); + return false; + } + sz = file_size; + return true; +} + static int create_directory_component(const std::string &npath) { int err; diff --git a/src/utils/filesystem.h b/src/utils/filesystem.h index 4229f551d2..9c5d3efea9 100644 --- a/src/utils/filesystem.h +++ b/src/utils/filesystem.h @@ -61,6 +61,8 @@ namespace dsn { namespace utils { +enum class FileDataType; + namespace filesystem { int get_normalized_path(const std::string &path, std::string &npath); @@ -96,7 +98,15 @@ bool remove_path(const std::string &path); // this will always remove target path if exist bool rename_path(const std::string &path1, const std::string &path2); +// Get the file size. The encryption header is considered as part of the file if it is an encrypted +// file. +// TODO(yingchun): refactor to use uint64_t. bool file_size(const std::string &path, int64_t &sz); +// The legacy file_size(), just for testing. +bool deprecated_file_size(const std::string &path, int64_t &sz); +// Get the file size. The encryption header is not considered as part of the file if it is an +// encrypted file and 'type' is specified as FileDataType::kSensitive. +bool file_size(const std::string &path, FileDataType type, int64_t &sz); bool create_directory(const std::string &path); diff --git a/src/utils/fmt_logging.h b/src/utils/fmt_logging.h index f0c74bd293..35c60aaddf 100644 --- a/src/utils/fmt_logging.h +++ b/src/utils/fmt_logging.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include "utils/api_utilities.h" @@ -272,6 +273,16 @@ inline const char *null_str_printer(const char *s) { return s == nullptr ? "(nul LOG_AND_RETURN_NOT_TRUE(level, _err == ::dsn::ERR_OK, _err, __VA_ARGS__); \ } while (0) +// Return the given rocksdb::Status 's' if it is not OK. +#define LOG_AND_RETURN_NOT_RDB_OK(level, s, ...) \ + do { \ + const auto &_s = (s); \ + if (dsn_unlikely(!_s.ok())) { \ + LOG_##level("{}: {}", _s.ToString(), fmt::format(__VA_ARGS__)); \ + return _s; \ + } \ + } while (0) + #ifndef NDEBUG #define DCHECK CHECK #define DCHECK_NOTNULL CHECK_NOTNULL diff --git a/src/utils/long_adder_bench/CMakeLists.txt b/src/utils/long_adder_bench/CMakeLists.txt index f63efc8a96..480b048079 100644 --- a/src/utils/long_adder_bench/CMakeLists.txt +++ b/src/utils/long_adder_bench/CMakeLists.txt @@ -27,7 +27,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime dsn_utils) +set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index fb284b0e86..266b9beeb6 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -33,7 +33,8 @@ set(MY_PROJ_LIBS dsn_http dsn_runtime dsn_utils gtest - ) + rocksdb + test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/utils/test/env.cpp b/src/utils/test/env.cpp index 619e5d7eba..813465de26 100644 --- a/src/utils/test/env.cpp +++ b/src/utils/test/env.cpp @@ -33,18 +33,31 @@ * xxxx-xx-xx, author, fix bug about xxx */ +#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include +#include #include -#include +#include +#include "test_util/test_util.h" +#include "utils/enum_helper.h" +#include "utils/env.h" +#include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/rand.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + using namespace ::dsn; -TEST(core, env) +TEST(env_test, rand) { uint64_t xs[] = {0, std::numeric_limits::max() - 1, 0xdeadbeef}; @@ -56,3 +69,271 @@ TEST(core, env) EXPECT_TRUE(r == x || r == (x + 1)); } } + +TEST(env_test, get_env) +{ + FLAGS_encrypt_data_at_rest = false; + auto *env_no_enc1 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive); + auto *env_no_enc2 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); + ASSERT_EQ(env_no_enc1, env_no_enc2); + + FLAGS_encrypt_data_at_rest = true; + auto *env_no_enc3 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive); + ASSERT_EQ(env_no_enc1, env_no_enc3); + + auto *env_enc1 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); + ASSERT_NE(env_no_enc1, env_enc1); +} + +class env_file_test : public pegasus::encrypt_data_test_base +{ +public: + env_file_test() : pegasus::encrypt_data_test_base() + { + // The size of an actual encrypted file should plus kEncryptionHeaderkSize bytes if consider + // it as kNonSensitive. + if (FLAGS_encrypt_data_at_rest) { + _extra_size = dsn::utils::kEncryptionHeaderkSize; + } + } + uint64_t _extra_size = 0; +}; + +INSTANTIATE_TEST_CASE_P(, env_file_test, ::testing::Values(false, true)); + +TEST_P(env_file_test, encrypt_file_2_files) +{ + const std::string kFileName = "encrypt_file_2_files"; + const std::string kEncryptedFileName = kFileName + ".encrypted"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare a non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + + // Check encrypt_file(src_fname, dst_fname, total_size). + // Loop twice to check overwrite. + for (int i = 0; i < 2; ++i) { + uint64_t encrypt_file_size; + s = dsn::utils::encrypt_file(kFileName, kEncryptedFileName, &encrypt_file_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_EQ(kFileContentSize, encrypt_file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kEncryptedFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kEncryptedFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + kEncryptedFileName, + &data); + ASSERT_EQ(kFileContent, data); + } +} + +TEST_P(env_file_test, encrypt_file_1_file) +{ + const std::string kFileName = "encrypt_file_1_file"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare a non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + + // Check encrypt_file(fname, total_size). + uint64_t encrypt_file_size; + s = dsn::utils::encrypt_file(kFileName, &encrypt_file_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_EQ(kFileContentSize, encrypt_file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), kFileName, &data); + ASSERT_EQ(kFileContent, data); +} + +TEST_P(env_file_test, copy_file) +{ + const std::string kFileName = "copy_file"; + const std::string kCopyFileName = kFileName + ".copy"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare an encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + + // Check copy_file(src_fname, dst_fname, total_size). + // Loop twice to check overwrite. + for (int i = 0; i < 2; ++i) { + uint64_t copy_file_size; + s = dsn::utils::copy_file(kFileName, kCopyFileName, ©_file_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_EQ(kFileContentSize, copy_file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), kCopyFileName, &data); + ASSERT_EQ(kFileContent, data); + } +} + +TEST_P(env_file_test, copy_file_by_size) +{ + const std::string kFileName = "copy_file_by_size"; + std::string kCopyFileName = kFileName + ".copy"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare an encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + + // Check copy_file_by_size(src_fname, dst_fname, limit_size). + struct test_case + { + int64_t limit_size; + int64_t expect_size; + } tests[] = {{-1, kFileContentSize}, + {0, 0}, + {10, 10}, + {kFileContentSize, kFileContentSize}, + {kFileContentSize + 10, kFileContentSize}}; + for (const auto &test : tests) { + s = dsn::utils::copy_file_by_size(kFileName, kCopyFileName, test.limit_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + + int64_t actual_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kSensitive, actual_size)); + ASSERT_EQ(test.expect_size, actual_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(test.expect_size + _extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), kCopyFileName, &data); + ASSERT_EQ(std::string(test.expect_size, 'a'), data); + } +} + +TEST_P(env_file_test, copy_non_encrypt_file) +{ + const std::string kFileName = "copy_non_encrypt_file"; + std::string kCopyFileName = kFileName + ".copy"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare a non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + + // Check copy_file() on non-sensitive file. + s = dsn::utils::copy_file(kFileName, kCopyFileName); + if (FLAGS_encrypt_data_at_rest) { + // copy_file() consider the source file as encrypted, so it will fail. + ASSERT_TRUE(s.IsCorruption()) << s.ToString(); + ASSERT_TRUE(s.ToString().find( + fmt::format("Corruption: Invalid encryption header in {}", kFileName)) == 0) + << s.ToString(); + } else { + // Although copy_file() consider the source file as non-encrypted, but it will succeed if + // FLAGS_encrypt_data_at_rest is disabled. + ASSERT_TRUE(s.ok()) << s.ToString(); + int64_t copy_file_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kNonSensitive, copy_file_size)); + ASSERT_EQ(kFileContentSize, copy_file_size); + } + + // Check copy_file_by_size() on non-sensitive file. + s = dsn::utils::copy_file_by_size(kFileName, kCopyFileName); + if (FLAGS_encrypt_data_at_rest) { + // copy_file_by_size() consider the source file as encrypted, so it will fail. + ASSERT_TRUE(s.IsCorruption()) << s.ToString(); + ASSERT_TRUE(s.ToString().find( + fmt::format("Corruption: Invalid encryption header in {}", kFileName)) == 0) + << s.ToString(); + } else { + // Although copy_file_by_size() consider the source file as non-encrypted, but it will + // succeed if FLAGS_encrypt_data_at_rest is disabled. + ASSERT_TRUE(s.ok()) << s.ToString(); + int64_t copy_file_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kNonSensitive, copy_file_size)); + ASSERT_EQ(kFileContentSize, copy_file_size); + } +} diff --git a/src/utils/test/file_system_test.cpp b/src/utils/test/file_system_test.cpp index f6da3e2b38..ccfc2da2f9 100644 --- a/src/utils/test/file_system_test.cpp +++ b/src/utils/test/file_system_test.cpp @@ -18,16 +18,104 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include +#include #include +#include "utils/env.h" #include "utils/filesystem.h" +#include "utils/flags.h" + +DSN_DECLARE_bool(encrypt_data_at_rest); namespace dsn { namespace utils { namespace filesystem { -TEST(verify_file, verify_file_test) +TEST(filesystem_test, compare_with_legacy_file_size) +{ + const std::string kFileName = "file_size_test"; + std::set test_file_sizes({0, 100}); + for (const auto &test_file_size : test_file_sizes) { + const std::string kFileContent(test_file_size, 'a'); + + // Prepare a non-encrypted test file. + auto s = rocksdb::WriteStringToFile( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // The file size should be the same as the legacy file size. + int64_t actual_file_size; + ASSERT_TRUE(file_size(kFileName, actual_file_size)); + ASSERT_EQ(test_file_size, actual_file_size); + ASSERT_TRUE(deprecated_file_size(kFileName, actual_file_size)); + ASSERT_EQ(test_file_size, actual_file_size); + } +} + +TEST(filesystem_test_p, non_encrypted_file_size) +{ + FLAGS_encrypt_data_at_rest = false; + const std::string kFileName = "non_encrypted_file_size"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare the non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + int64_t actual_file_size; + // Check file_size(path, sz) + ASSERT_TRUE(file_size(kFileName, actual_file_size)); + ASSERT_EQ(kFileContentSize, actual_file_size); + // Check file_size(path, type, sz) + ASSERT_TRUE(file_size(kFileName, FileDataType::kNonSensitive, actual_file_size)); + ASSERT_EQ(kFileContentSize, actual_file_size); + // Check file_size(path, type, sz) with kSensitive type. + // It's able to get the correct file size because FLAGS_encrypt_data_at_rest is disabled. + ASSERT_TRUE(file_size(kFileName, FileDataType::kSensitive, actual_file_size)); + ASSERT_EQ(kFileContentSize, actual_file_size); +} + +TEST(filesystem_test_p, encrypted_file_size) +{ + FLAGS_encrypt_data_at_rest = true; + const std::string kFileName = "encrypted_file_size"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare the non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + int64_t actual_file_size; + // Check file_size(path, sz), the encryption header size is counted. + ASSERT_TRUE(file_size(kFileName, actual_file_size)); + ASSERT_EQ(kFileContentSize + kEncryptionHeaderkSize, actual_file_size); + // Check file_size(path, type, sz) with correct type. + ASSERT_TRUE(file_size(kFileName, FileDataType::kSensitive, actual_file_size)); + ASSERT_EQ(kFileContentSize, actual_file_size); + // Check file_size(path, type, sz) with kNonSensitive type, the encryption header size is + // counted. + ASSERT_TRUE(file_size(kFileName, FileDataType::kNonSensitive, actual_file_size)); + ASSERT_EQ(kFileContentSize + kEncryptionHeaderkSize, actual_file_size); +} + +TEST(filesystem_test, verify_file_test) { const std::string &fname = "test_file"; std::string expected_md5; diff --git a/src/utils/test/nth_element_bench/CMakeLists.txt b/src/utils/test/nth_element_bench/CMakeLists.txt index 217d9c4363..2bd530690c 100644 --- a/src/utils/test/nth_element_bench/CMakeLists.txt +++ b/src/utils/test/nth_element_bench/CMakeLists.txt @@ -27,7 +27,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime dsn_utils) +set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/zookeeper/test/CMakeLists.txt b/src/zookeeper/test/CMakeLists.txt index ed2f742ac7..6179f8d1ec 100644 --- a/src/zookeeper/test/CMakeLists.txt +++ b/src/zookeeper/test/CMakeLists.txt @@ -41,7 +41,7 @@ set(MY_PROJ_LIBS gtest ssl crypto - ) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) From 42136fa4b36288b7597c32c93043541f18b3bba7 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Thu, 21 Sep 2023 10:10:58 +0800 Subject: [PATCH 22/42] refactor(test): refactor bulk load function test (#1616) https://github.com/apache/incubator-pegasus/issues/887 There is no functional changes, but only refactor the bulk_load function test. --- src/block_service/local/local_service.cpp | 12 +- src/block_service/local/local_service.h | 10 + src/block_service/test/local_service_test.cpp | 1 + src/meta/meta_bulk_load_service.h | 4 + .../function_test/bulk_load/CMakeLists.txt | 3 +- .../bulk_load/test_bulk_load.cpp | 341 ++++++++++-------- 6 files changed, 205 insertions(+), 166 deletions(-) diff --git a/src/block_service/local/local_service.cpp b/src/block_service/local/local_service.cpp index 5eb4b3fba4..04cf1cc3d8 100644 --- a/src/block_service/local/local_service.cpp +++ b/src/block_service/local/local_service.cpp @@ -16,17 +16,16 @@ // under the License. #include -#include #include -#include #include +#include #include #include #include #include #include "local_service.h" -#include "nlohmann/detail/macro_scope.hpp" +#include "nlohmann/json.hpp" #include "nlohmann/json_fwd.hpp" #include "runtime/task/async_calls.h" #include "utils/autoref_ptr.h" @@ -52,13 +51,6 @@ namespace block_service { DEFINE_TASK_CODE(LPC_LOCAL_SERVICE_CALL, TASK_PRIORITY_COMMON, THREAD_POOL_BLOCK_SERVICE) -struct file_metadata -{ - uint64_t size; - std::string md5; -}; -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(file_metadata, size, md5) - bool file_metadata_from_json(std::ifstream &fin, file_metadata &fmeta) noexcept { std::string data; diff --git a/src/block_service/local/local_service.h b/src/block_service/local/local_service.h index 9816734cf0..9c944e1d0e 100644 --- a/src/block_service/local/local_service.h +++ b/src/block_service/local/local_service.h @@ -17,6 +17,9 @@ #pragma once +#include +#include // IWYU pragma: keep +#include // IWYU pragma: keep #include #include #include @@ -32,6 +35,13 @@ class task_tracker; namespace dist { namespace block_service { +struct file_metadata +{ + int64_t size = 0; + std::string md5; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(file_metadata, size, md5) + class local_service : public block_filesystem { public: diff --git a/src/block_service/test/local_service_test.cpp b/src/block_service/test/local_service_test.cpp index e355a1b281..72a2012370 100644 --- a/src/block_service/test/local_service_test.cpp +++ b/src/block_service/test/local_service_test.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "block_service/local/local_service.h" diff --git a/src/meta/meta_bulk_load_service.h b/src/meta/meta_bulk_load_service.h index ba151757a8..c411d87f44 100644 --- a/src/meta/meta_bulk_load_service.h +++ b/src/meta/meta_bulk_load_service.h @@ -97,6 +97,10 @@ struct bulk_load_info int32_t app_id; std::string app_name; int32_t partition_count; + bulk_load_info(int32_t id = 0, const std::string &name = "", int32_t pcount = 0) + : app_id(id), app_name(name), partition_count(pcount) + { + } DEFINE_JSON_SERIALIZATION(app_id, app_name, partition_count) }; diff --git a/src/test/function_test/bulk_load/CMakeLists.txt b/src/test/function_test/bulk_load/CMakeLists.txt index 92a677ddf8..a6eac08515 100644 --- a/src/test/function_test/bulk_load/CMakeLists.txt +++ b/src/test/function_test/bulk_load/CMakeLists.txt @@ -37,7 +37,8 @@ set(MY_PROJ_LIBS gssapi_krb5 krb5 function_test_utils - ) + rocksdb + test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/test/function_test/bulk_load/test_bulk_load.cpp b/src/test/function_test/bulk_load/test_bulk_load.cpp index f350c69453..28e52858f3 100644 --- a/src/test/function_test/bulk_load/test_bulk_load.cpp +++ b/src/test/function_test/bulk_load/test_bulk_load.cpp @@ -15,9 +15,15 @@ // specific language governing permissions and limitations // under the License. +#include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include +#include +#include #include #include #include @@ -26,20 +32,24 @@ #include #include #include -#include #include "base/pegasus_const.h" +#include "block_service/local/local_service.h" #include "bulk_load_types.h" +#include "client/partition_resolver.h" #include "client/replication_ddl_client.h" +#include "common/json_helper.h" #include "include/pegasus/client.h" #include "include/pegasus/error.h" +#include "meta/meta_bulk_load_service.h" #include "meta_admin_types.h" -#include "metadata_types.h" #include "test/function_test/utils/test_util.h" +#include "utils/blob.h" +#include "utils/enum_helper.h" #include "utils/error_code.h" #include "utils/errors.h" #include "utils/filesystem.h" -#include "utils/utils.h" +#include "utils/test_macros.h" using namespace ::dsn; using namespace ::dsn::replication; @@ -55,12 +65,12 @@ using std::string; /// - `bulk_load_root` sub-directory stores right data /// - Please do not rename any files or directories under this folder /// -/// The app who is executing bulk load: -/// - app_name is `temp`, app_id is 2, partition_count is 8 +/// The app to test bulk load functionality: +/// - partition count should be 8 /// /// Data: -/// hashkey: hashi sortkey: sorti value: newValue i=[0, 1000] -/// hashkey: hashkeyj sortkey: sortkeyj value: newValue j=[0, 1000] +/// hashkey: hash${i} sortkey: sort${i} value: newValue i=[0, 1000] +/// hashkey: hashkey${j} sortkey: sortkey${j} value: newValue j=[0, 1000] /// class bulk_load_test : public test_util { @@ -68,80 +78,102 @@ class bulk_load_test : public test_util bulk_load_test() : test_util(map({{"rocksdb.allow_ingest_behind", "true"}})) { TRICKY_CODE_TO_AVOID_LINK_ERROR; - bulk_load_local_root_ = - utils::filesystem::path_combine("onebox/block_service/local_service/", LOCAL_ROOT); + bulk_load_local_app_root_ = + fmt::format("{}/{}/{}", kLocalBulkLoadRoot, kCluster, app_name_); } void SetUp() override { test_util::SetUp(); - ASSERT_NO_FATAL_FAILURE(copy_bulk_load_files()); + NO_FATALS(copy_bulk_load_files()); } void TearDown() override { ASSERT_EQ(ERR_OK, ddl_client_->drop_app(app_name_, 0)); - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root("rm -rf onebox/block_service")); + NO_FATALS(run_cmd_from_project_root("rm -rf " + kLocalBulkLoadRoot)); } - void copy_bulk_load_files() + // Generate the 'bulk_load_info' file according to 'bli' to path 'bulk_load_info_path'. + void generate_bulk_load_info(const bulk_load_info &bli, const std::string &bulk_load_info_path) { - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root("mkdir -p onebox/block_service")); - ASSERT_NO_FATAL_FAILURE( - run_cmd_from_project_root("mkdir -p onebox/block_service/local_service")); - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root( - "cp -r src/test/function_test/bulk_load/pegasus-bulk-load-function-test-files/" + - LOCAL_ROOT + " onebox/block_service/local_service")); - string cmd = "echo '{\"app_id\":" + std::to_string(app_id_) + - ",\"app_name\":\"temp\",\"partition_count\":8}' > " - "onebox/block_service/local_service/bulk_load_root/cluster/temp/" - "bulk_load_info"; - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root(cmd)); + auto value = dsn::json::json_forwarder::encode(bli); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(value.data(), value.length()), + bulk_load_info_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); } - error_code start_bulk_load(bool ingest_behind = false) + // Generate the '.bulk_load_info.meta' file according to the 'bulk_load_info' file + // in path 'bulk_load_info_path'. + void generate_bulk_load_info_meta(const std::string &bulk_load_info_path) { - auto err_resp = - ddl_client_->start_bulk_load(app_name_, CLUSTER, PROVIDER, LOCAL_ROOT, ingest_behind); - return err_resp.get_value().err; + dist::block_service::file_metadata fm; + ASSERT_TRUE(utils::filesystem::file_size(bulk_load_info_path, fm.size)); + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(bulk_load_info_path, fm.md5)); + std::string value = nlohmann::json(fm).dump(); + auto bulk_load_info_meta_path = + fmt::format("{}/{}/{}/.bulk_load_info.meta", kLocalBulkLoadRoot, kCluster, app_name_); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(value), + bulk_load_info_meta_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); } - void remove_file(const string &file_path) + void copy_bulk_load_files() + { + // TODO(yingchun): remove the 'mock_bulk_load_info' file, because we can generate it. + // Prepare bulk load files. + // The source data has 8 partitions. + ASSERT_EQ(8, partition_count_); + NO_FATALS(run_cmd_from_project_root("mkdir -p " + kLocalBulkLoadRoot)); + NO_FATALS(run_cmd_from_project_root( + fmt::format("cp -r {}/{} {}", kSourceFilesRoot, kBulkLoad, kLocalServiceRoot))); + + // Generate 'bulk_load_info'. + auto bulk_load_info_path = + fmt::format("{}/{}/{}/bulk_load_info", kLocalBulkLoadRoot, kCluster, app_name_); + NO_FATALS(generate_bulk_load_info(bulk_load_info(app_id_, app_name_, partition_count_), + bulk_load_info_path)); + + // Generate '.bulk_load_info.meta'. + NO_FATALS(generate_bulk_load_info_meta(bulk_load_info_path)); + } + + error_code start_bulk_load(bool ingest_behind = false) { - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root("rm " + file_path)); + return ddl_client_ + ->start_bulk_load(app_name_, kCluster, kProvider, kBulkLoad, ingest_behind) + .get_value() + .err; } - void replace_bulk_load_info() + void remove_file(const string &file_path) { - string cmd = "cp -R " - "src/test/function_test/bulk_load/pegasus-bulk-load-function-test-files/" - "mock_bulk_load_info/. " + - bulk_load_local_root_ + "/" + CLUSTER + "/" + app_name_ + "/"; - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root(cmd)); + NO_FATALS(run_cmd_from_project_root("rm " + file_path)); } void update_allow_ingest_behind(const string &allow_ingest_behind) { - // update app envs - std::vector keys; - keys.emplace_back(ROCKSDB_ALLOW_INGEST_BEHIND); - std::vector values; - values.emplace_back(allow_ingest_behind); - ASSERT_EQ(ERR_OK, ddl_client_->set_app_envs(app_name_, keys, values).get_value().err); + const auto ret = ddl_client_->set_app_envs( + app_name_, {ROCKSDB_ALLOW_INGEST_BEHIND}, {allow_ingest_behind}); + ASSERT_EQ(ERR_OK, ret.get_value().err); std::cout << "sleep 31s to wait app_envs update" << std::endl; std::this_thread::sleep_for(std::chrono::seconds(31)); } - bulk_load_status::type wait_bulk_load_finish(int64_t seconds) + bulk_load_status::type wait_bulk_load_finish(int64_t remain_seconds) { int64_t sleep_time = 5; - error_code err = ERR_OK; + auto err = ERR_OK; - bulk_load_status::type last_status = bulk_load_status::BLS_INVALID; + auto last_status = bulk_load_status::BLS_INVALID; // when bulk load end, err will be ERR_INVALID_STATE - while (seconds > 0 && err == ERR_OK) { - sleep_time = sleep_time > seconds ? seconds : sleep_time; - seconds -= sleep_time; + while (remain_seconds > 0 && err == ERR_OK) { + sleep_time = std::min(sleep_time, remain_seconds); + remain_seconds -= sleep_time; std::cout << "sleep " << sleep_time << "s to query bulk status" << std::endl; std::this_thread::sleep_for(std::chrono::seconds(sleep_time)); @@ -156,52 +188,51 @@ class bulk_load_test : public test_util void verify_bulk_load_data() { - ASSERT_NO_FATAL_FAILURE(verify_data("hashkey", "sortkey")); - ASSERT_NO_FATAL_FAILURE(verify_data(HASHKEY_PREFIX, SORTKEY_PREFIX)); + NO_FATALS(verify_data(kBulkLoadHashKeyPrefix1, kBulkLoadSortKeyPrefix1)); + NO_FATALS(verify_data(kBulkLoadHashKeyPrefix2, kBulkLoadSortKeyPrefix2)); } void verify_data(const string &hashkey_prefix, const string &sortkey_prefix) { - const string &expected_value = VALUE; - for (int i = 0; i < COUNT; ++i) { + for (int i = 0; i < kBulkLoadItemCount; ++i) { string hash_key = hashkey_prefix + std::to_string(i); - for (int j = 0; j < COUNT; ++j) { + for (int j = 0; j < kBulkLoadItemCount; ++j) { string sort_key = sortkey_prefix + std::to_string(j); - string act_value; - ASSERT_EQ(PERR_OK, client_->get(hash_key, sort_key, act_value)) << hash_key << "," - << sort_key; - ASSERT_EQ(expected_value, act_value) << hash_key << "," << sort_key; + string actual_value; + ASSERT_EQ(PERR_OK, client_->get(hash_key, sort_key, actual_value)) + << hash_key << "," << sort_key; + ASSERT_EQ(kBulkLoadValue, actual_value) << hash_key << "," << sort_key; } } } - enum operation + enum class operation { GET, SET, DEL, NO_VALUE }; - void operate_data(bulk_load_test::operation op, const string &value, int count) + void operate_data(operation op, const string &value, int count) { for (int i = 0; i < count; ++i) { - string hash_key = HASHKEY_PREFIX + std::to_string(i); - string sort_key = SORTKEY_PREFIX + std::to_string(i); + auto hash_key = fmt::format("{}{}", kBulkLoadHashKeyPrefix2, i); + auto sort_key = fmt::format("{}{}", kBulkLoadSortKeyPrefix2, i); switch (op) { - case bulk_load_test::operation::GET: { - string act_value; - ASSERT_EQ(PERR_OK, client_->get(hash_key, sort_key, act_value)); - ASSERT_EQ(value, act_value); + case operation::GET: { + string actual_value; + ASSERT_EQ(PERR_OK, client_->get(hash_key, sort_key, actual_value)); + ASSERT_EQ(value, actual_value); } break; - case bulk_load_test::operation::DEL: { + case operation::DEL: { ASSERT_EQ(PERR_OK, client_->del(hash_key, sort_key)); } break; - case bulk_load_test::operation::SET: { + case operation::SET: { ASSERT_EQ(PERR_OK, client_->set(hash_key, sort_key, value)); } break; - case bulk_load_test::operation::NO_VALUE: { - string act_value; - ASSERT_EQ(PERR_NOT_FOUND, client_->get(hash_key, sort_key, act_value)); + case operation::NO_VALUE: { + string actual_value; + ASSERT_EQ(PERR_NOT_FOUND, client_->get(hash_key, sort_key, actual_value)); } break; default: ASSERT_TRUE(false); @@ -210,108 +241,108 @@ class bulk_load_test : public test_util } } -protected: - string bulk_load_local_root_; + void check_bulk_load(bool ingest_behind, + const std::string &value_before_bulk_load, + const std::string &value_after_bulk_load) + { + // Write some data before bulk load. + NO_FATALS(operate_data(operation::SET, value_before_bulk_load, 10)); + NO_FATALS(operate_data(operation::GET, value_before_bulk_load, 10)); + + // Start bulk load and wait until it complete. + ASSERT_EQ(ERR_OK, start_bulk_load(ingest_behind)); + ASSERT_EQ(bulk_load_status::BLS_SUCCEED, wait_bulk_load_finish(300)); + + std::cout << "Start to verify data..." << std::endl; + if (ingest_behind) { + // Values have NOT been overwritten by the bulk load data. + NO_FATALS(operate_data(operation::GET, value_before_bulk_load, 10)); + NO_FATALS(verify_data(kBulkLoadHashKeyPrefix1, kBulkLoadSortKeyPrefix1)); + } else { + // Values have been overwritten by the bulk load data. + NO_FATALS(operate_data(operation::GET, kBulkLoadValue, 10)); + NO_FATALS(verify_bulk_load_data()); + } - const string LOCAL_ROOT = "bulk_load_root"; - const string CLUSTER = "cluster"; - const string PROVIDER = "local_service"; + // Write new data succeed after bulk load. + NO_FATALS(operate_data(operation::SET, value_after_bulk_load, 20)); + NO_FATALS(operate_data(operation::GET, value_after_bulk_load, 20)); - const string HASHKEY_PREFIX = "hash"; - const string SORTKEY_PREFIX = "sort"; - const string VALUE = "newValue"; - const int32_t COUNT = 1000; + // Delete data succeed after bulk load. + NO_FATALS(operate_data(operation::DEL, "", 15)); + NO_FATALS(operate_data(operation::NO_VALUE, "", 15)); + } + +protected: + string bulk_load_local_app_root_; + const string kSourceFilesRoot = + "src/test/function_test/bulk_load/pegasus-bulk-load-function-test-files"; + const string kLocalServiceRoot = "onebox/block_service/local_service"; + const string kLocalBulkLoadRoot = "onebox/block_service/local_service/bulk_load_root"; + const string kBulkLoad = "bulk_load_root"; + const string kCluster = "cluster"; + const string kProvider = "local_service"; + + const int32_t kBulkLoadItemCount = 1000; + const string kBulkLoadHashKeyPrefix1 = "hashkey"; + const string kBulkLoadSortKeyPrefix1 = "sortkey"; + const string kBulkLoadValue = "newValue"; + + // Real time write operations will use this prefix as well. + const string kBulkLoadHashKeyPrefix2 = "hash"; + const string kBulkLoadSortKeyPrefix2 = "sort"; }; -/// -/// case1: lack of `bulk_load_info` file -/// case2: `bulk_load_info` file inconsistent with app_info -/// -TEST_F(bulk_load_test, bulk_load_test_failed) +// Test bulk load failed because the 'bulk_load_info' file is missing +TEST_F(bulk_load_test, missing_bulk_load_info) { - // bulk load failed because `bulk_load_info` file is missing - ASSERT_NO_FATAL_FAILURE( - remove_file(bulk_load_local_root_ + "/" + CLUSTER + "/" + app_name_ + "/bulk_load_info")); + NO_FATALS(remove_file(bulk_load_local_app_root_ + "/bulk_load_info")); ASSERT_EQ(ERR_OBJECT_NOT_FOUND, start_bulk_load()); +} - // bulk load failed because `bulk_load_info` file inconsistent with current app_info - ASSERT_NO_FATAL_FAILURE(replace_bulk_load_info()); - ASSERT_EQ(ERR_INCONSISTENT_STATE, start_bulk_load()); +// Test bulk load failed because the 'bulk_load_info' file is inconsistent with the actual app info. +TEST_F(bulk_load_test, inconsistent_bulk_load_info) +{ + // Only 'app_id' and 'partition_count' will be checked in Pegasus server, so just inject these + // kind of inconsistencies. + bulk_load_info tests[] = {{app_id_ + 1, app_name_, partition_count_}, + {app_id_, app_name_, partition_count_ * 2}}; + for (const auto &test : tests) { + // Generate inconsistent 'bulk_load_info'. + auto bulk_load_info_path = + fmt::format("{}/{}/{}/bulk_load_info", kLocalBulkLoadRoot, kCluster, app_name_); + NO_FATALS(generate_bulk_load_info(test, bulk_load_info_path)); + + // Generate '.bulk_load_info.meta'. + NO_FATALS(generate_bulk_load_info_meta(bulk_load_info_path)); + + ASSERT_EQ(ERR_INCONSISTENT_STATE, start_bulk_load()) << test.app_id << "," << test.app_name + << "," << test.partition_count; + } } -/// -/// case1: lack of `bulk_load_metadata` file -/// case2: bulk load succeed with data verfied -/// case3: bulk load data consistent: -/// - old data will be overrided by bulk load data -/// - get/set/del succeed after bulk load -/// -TEST_F(bulk_load_test, bulk_load_tests) +// Test bulk load failed because partition[0]'s 'bulk_load_metadata' file is missing. +TEST_F(bulk_load_test, missing_p0_bulk_load_metadata) { - // bulk load failed because partition[0] `bulk_load_metadata` file is missing - ASSERT_NO_FATAL_FAILURE(remove_file(bulk_load_local_root_ + "/" + CLUSTER + "/" + app_name_ + - "/0/bulk_load_metadata")); + NO_FATALS(remove_file(bulk_load_local_app_root_ + "/0/bulk_load_metadata")); ASSERT_EQ(ERR_OK, start_bulk_load()); - // bulk load will get FAILED ASSERT_EQ(bulk_load_status::BLS_FAILED, wait_bulk_load_finish(300)); - - // recover complete files - ASSERT_NO_FATAL_FAILURE(copy_bulk_load_files()); - - // write old data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "oldValue", 10)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "oldValue", 10)); - - ASSERT_EQ(ERR_OK, start_bulk_load()); - ASSERT_EQ(bulk_load_status::BLS_SUCCEED, wait_bulk_load_finish(300)); - std::cout << "Start to verify data..." << std::endl; - ASSERT_NO_FATAL_FAILURE(verify_bulk_load_data()); - - // value overide by bulk_loaded_data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, VALUE, 10)); - - // write data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "valueAfterBulkLoad", 20)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "valueAfterBulkLoad", 20)); - - // del data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::DEL, "", 15)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::NO_VALUE, "", 15)); } -/// -/// case1: inconsistent ingest_behind -/// case2: bulk load(ingest_behind) succeed with data verfied -/// case3: bulk load data consistent: -/// - bulk load data will be overrided by old data -/// - get/set/del succeed after bulk load -/// -TEST_F(bulk_load_test, bulk_load_ingest_behind_tests) +// Test bulk load failed because the allow_ingest_behind config is inconsistent. +TEST_F(bulk_load_test, allow_ingest_behind_inconsistent) { - ASSERT_NO_FATAL_FAILURE(update_allow_ingest_behind("false")); - - // app envs allow_ingest_behind = false, request ingest_behind = true + NO_FATALS(update_allow_ingest_behind("false")); ASSERT_EQ(ERR_INCONSISTENT_STATE, start_bulk_load(true)); +} - ASSERT_NO_FATAL_FAILURE(update_allow_ingest_behind("true")); - - // write old data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "oldValue", 10)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "oldValue", 10)); - - ASSERT_EQ(ERR_OK, start_bulk_load(true)); - ASSERT_EQ(bulk_load_status::BLS_SUCCEED, wait_bulk_load_finish(300)); - - std::cout << "Start to verify data..." << std::endl; - // value overide by bulk_loaded_data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "oldValue", 10)); - ASSERT_NO_FATAL_FAILURE(verify_data("hashkey", "sortkey")); - - // write data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "valueAfterBulkLoad", 20)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "valueAfterBulkLoad", 20)); +// Test normal bulk load, old data will be overwritten by bulk load data. +TEST_F(bulk_load_test, normal) { check_bulk_load(false, "oldValue", "valueAfterBulkLoad"); } - // del data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::DEL, "", 15)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::NO_VALUE, "", 15)); +// Test normal bulk load with allow_ingest_behind=true, old data will NOT be overwritten by bulk +// load data. +TEST_F(bulk_load_test, allow_ingest_behind) +{ + NO_FATALS(update_allow_ingest_behind("true")); + check_bulk_load(true, "oldValue", "valueAfterBulkLoad"); } From c8793d22d78da10b01c1a961c81f4b4ad56b04dd Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Thu, 21 Sep 2023 11:31:55 +0800 Subject: [PATCH 23/42] feat(slog): flush and remove all shared logs for garbage collection (#1594) https://github.com/apache/incubator-pegasus/issues/1593 In https://github.com/XiaoMi/rdsn/pull/1019 we've written private logs as WAL instead of shared logs, which also means shared log files would never be appended with new mutations. However, obsolete shared logs that had been applied to rocksdb were not removed since then. There is at least 1 shared log file which is never removed. We should change this policy of garbage collection, to delete all obsolete shared log files. To facilitate the garbage collection of shared log files, we also trigger checkpoints which would flush rocksdb data to disk for each replica that has prevented shared log files from being removed for garbage collection. It is necessary to limit the number of submitted replicas that would be flushed at a time to avoid too much consumption of I/O resources which might affect the processing of read/write requests from clients. --- .../duplication/load_from_private_log.cpp | 14 +- .../duplication/load_from_private_log.h | 3 +- .../duplication/test/duplication_test_base.h | 4 +- src/replica/log_file.cpp | 18 +- src/replica/log_file.h | 19 +- src/replica/mutation_log.cpp | 501 ++++++++---------- src/replica/mutation_log.h | 153 ++++-- src/replica/mutation_log_replay.cpp | 4 +- src/replica/mutation_log_utils.cpp | 2 +- src/replica/mutation_log_utils.h | 9 +- src/replica/replica_stub.cpp | 321 ++++++----- src/replica/replica_stub.h | 39 +- src/replica/replication_app_base.cpp | 1 - src/replica/test/mock_utils.h | 2 +- src/replica/test/mutation_log_test.cpp | 266 +++++++++- src/server/config.ini | 2 +- src/server/pegasus_server_impl.cpp | 3 +- src/server/test/config.ini | 2 +- src/utils/autoref_ptr.h | 2 + src/utils/fmt_logging.h | 3 +- 20 files changed, 872 insertions(+), 496 deletions(-) diff --git a/src/replica/duplication/load_from_private_log.cpp b/src/replica/duplication/load_from_private_log.cpp index aa458b22a4..d3d88a075d 100644 --- a/src/replica/duplication/load_from_private_log.cpp +++ b/src/replica/duplication/load_from_private_log.cpp @@ -16,6 +16,7 @@ // under the License. #include +#include #include #include "common/duplication_common.h" @@ -57,11 +58,11 @@ bool load_from_private_log::will_fail_fast() const // we try to list all files and select a new one to start (find_log_file_to_start). bool load_from_private_log::switch_to_next_log_file() { - auto file_map = _private_log->get_log_file_map(); - auto next_file_it = file_map.find(_current->index() + 1); + const auto &file_map = _private_log->get_log_file_map(); + const auto &next_file_it = file_map.find(_current->index() + 1); if (next_file_it != file_map.end()) { log_file_ptr file; - error_s es = log_utils::open_read(next_file_it->second->path(), file); + const auto &es = log_utils::open_read(next_file_it->second->path(), file); if (!es.is_ok()) { LOG_ERROR_PREFIX("{}", es); _current = nullptr; @@ -123,11 +124,11 @@ void load_from_private_log::run() void load_from_private_log::find_log_file_to_start() { // `file_map` has already excluded the useless log files during replica init. - auto file_map = _private_log->get_log_file_map(); + const auto &file_map = _private_log->get_log_file_map(); // Reopen the files. Because the internal file handle of `file_map` // is cleared once WAL replay finished. They are unable to read. - std::map new_file_map; + mutation_log::log_file_map_by_index new_file_map; for (const auto &pr : file_map) { log_file_ptr file; error_s es = log_utils::open_read(pr.second->path(), file); @@ -141,7 +142,8 @@ void load_from_private_log::find_log_file_to_start() find_log_file_to_start(std::move(new_file_map)); } -void load_from_private_log::find_log_file_to_start(std::map log_file_map) +void load_from_private_log::find_log_file_to_start( + const mutation_log::log_file_map_by_index &log_file_map) { _current = nullptr; if (dsn_unlikely(log_file_map.empty())) { diff --git a/src/replica/duplication/load_from_private_log.h b/src/replica/duplication/load_from_private_log.h index 523002b54a..56651aabfa 100644 --- a/src/replica/duplication/load_from_private_log.h +++ b/src/replica/duplication/load_from_private_log.h @@ -21,7 +21,6 @@ #include #include #include -#include #include "common/replication_other_types.h" #include "mutation_batch.h" @@ -61,7 +60,7 @@ class load_from_private_log final : public replica_base, /// Find the log file that contains `_start_decree`. void find_log_file_to_start(); - void find_log_file_to_start(std::map log_files); + void find_log_file_to_start(const mutation_log::log_file_map_by_index &log_files); void replay_log_block(); diff --git a/src/replica/duplication/test/duplication_test_base.h b/src/replica/duplication/test/duplication_test_base.h index fb0c7ea4f9..4152a9a370 100644 --- a/src/replica/duplication/test/duplication_test_base.h +++ b/src/replica/duplication/test/duplication_test_base.h @@ -68,9 +68,9 @@ class duplication_test_base : public replica_test_base return duplicator; } - std::map open_log_file_map(const std::string &log_dir) + mutation_log::log_file_map_by_index open_log_file_map(const std::string &log_dir) { - std::map log_file_map; + mutation_log::log_file_map_by_index log_file_map; error_s err = log_utils::open_log_file_map(log_dir, log_file_map); EXPECT_EQ(err, error_s::ok()); return log_file_map; diff --git a/src/replica/log_file.cpp b/src/replica/log_file.cpp index 7362aa7c41..e239d4dae1 100644 --- a/src/replica/log_file.cpp +++ b/src/replica/log_file.cpp @@ -166,15 +166,15 @@ log_file::~log_file() { close(); } log_file::log_file( const char *path, disk_file *handle, int index, int64_t start_offset, bool is_read) - : _is_read(is_read) + : _crc32(0), + _start_offset(start_offset), + _end_offset(start_offset), + _handle(handle), + _is_read(is_read), + _path(path), + _index(index), + _last_write_time(0) { - _start_offset = start_offset; - _end_offset = start_offset; - _handle = handle; - _path = path; - _index = index; - _crc32 = 0; - _last_write_time = 0; memset(&_header, 0, sizeof(_header)); if (is_read) { @@ -357,7 +357,7 @@ void log_file::reset_stream(size_t offset /*default = 0*/) } } -decree log_file::previous_log_max_decree(const dsn::gpid &pid) +decree log_file::previous_log_max_decree(const dsn::gpid &pid) const { auto it = _previous_log_max_decrees.find(pid); return it == _previous_log_max_decrees.end() ? 0 : it->second.max_decree; diff --git a/src/replica/log_file.h b/src/replica/log_file.h index fb1fe3b6d4..fcaff2187a 100644 --- a/src/replica/log_file.h +++ b/src/replica/log_file.h @@ -65,9 +65,9 @@ struct log_file_header // a structure to record replica's log info struct replica_log_info { - int64_t max_decree; + decree max_decree; int64_t valid_start_offset; // valid start offset in global space - replica_log_info(int64_t d, int64_t o) + replica_log_info(decree d, int64_t o) { max_decree = d; valid_start_offset = o; @@ -184,11 +184,14 @@ class log_file : public ref_counter // file path const std::string &path() const { return _path; } // previous decrees - const replica_log_info_map &previous_log_max_decrees() { return _previous_log_max_decrees; } + const replica_log_info_map &previous_log_max_decrees() const + { + return _previous_log_max_decrees; + } // previous decree for speicified gpid - decree previous_log_max_decree(const gpid &pid); + decree previous_log_max_decree(const gpid &pid) const; // file header - log_file_header &header() { return _header; } + const log_file_header &header() const { return _header; } // read file header from reader, return byte count consumed int read_file_header(binary_reader &reader); @@ -213,7 +216,7 @@ class log_file : public ref_counter friend class mock_log_file; uint32_t _crc32; - int64_t _start_offset; // start offset in the global space + const int64_t _start_offset; // start offset in the global space std::atomic _end_offset; // end offset in the global space: end_offset = start_offset + file_size class file_streamer; @@ -221,8 +224,8 @@ class log_file : public ref_counter std::unique_ptr _stream; disk_file *_handle; // file handle const bool _is_read; // if opened for read or write - std::string _path; // file path - int _index; // file index + const std::string _path; // file path + const int _index; // file index log_file_header _header; // file header uint64_t _last_write_time; // seconds from epoch time diff --git a/src/replica/mutation_log.cpp b/src/replica/mutation_log.cpp index c618c7cd47..ad745670b2 100644 --- a/src/replica/mutation_log.cpp +++ b/src/replica/mutation_log.cpp @@ -527,8 +527,8 @@ error_code mutation_log::open(replay_callback read_callback, io_failure_callback write_error_callback, const std::map &replay_condition) { - CHECK(!_is_opened, "cannot open a opened mutation_log"); - CHECK(nullptr == _current_log_file, ""); + CHECK(!_is_opened, "cannot open an opened mutation_log"); + CHECK_NULL(_current_log_file, ""); // create dir if necessary if (!dsn::utils::filesystem::path_exists(_dir)) { @@ -562,9 +562,8 @@ error_code mutation_log::open(replay_callback read_callback, err == ERR_INVALID_PARAMETERS) { LOG_WARNING("skip file {} during log init, err = {}", fpath, err); continue; - } else { - return err; } + return err; } if (_is_private) { @@ -592,8 +591,8 @@ error_code mutation_log::open(replay_callback read_callback, file_list.clear(); // filter useless log - std::map::iterator replay_begin = _log_files.begin(); - std::map::iterator replay_end = _log_files.end(); + log_file_map_by_index::iterator replay_begin = _log_files.begin(); + log_file_map_by_index::iterator replay_end = _log_files.end(); if (!replay_condition.empty()) { if (_is_private) { auto find = replay_condition.find(_private_gpid); @@ -609,7 +608,7 @@ error_code mutation_log::open(replay_callback read_callback, } else { // find the largest file which can be ignored. // after iterate, the 'mark_it' will point to the largest file which can be ignored. - std::map::reverse_iterator mark_it; + log_file_map_by_index::reverse_iterator mark_it; std::set kickout_replicas; replica_log_info_map max_decrees; // max_decrees for log file at mark_it. for (mark_it = _log_files.rbegin(); mark_it != _log_files.rend(); ++mark_it) { @@ -666,7 +665,7 @@ error_code mutation_log::open(replay_callback read_callback, } // replay with the found files - std::map replay_logs(replay_begin, replay_end); + log_file_map_by_index replay_logs(replay_begin, replay_end); int64_t end_offset = 0; err = replay( replay_logs, @@ -863,11 +862,8 @@ decree mutation_log::max_decree(gpid gpid) const CHECK_EQ(gpid, _private_gpid); return _private_log_info.max_decree; } else { - auto it = _shared_log_info_map.find(gpid); - if (it != _shared_log_info_map.end()) - return it->second.max_decree; - else - return 0; + const auto &it = _shared_log_info_map.find(gpid); + return it != _shared_log_info_map.end() ? it->second.max_decree : 0; } } @@ -923,7 +919,7 @@ int64_t mutation_log::total_size() const int64_t mutation_log::total_size_no_lock() const { - return _log_files.size() > 0 ? _global_end_offset - _global_start_offset : 0; + return _log_files.empty() ? 0 : _global_end_offset - _global_start_offset; } error_code mutation_log::reset_from(const std::string &dir, @@ -1095,7 +1091,7 @@ bool mutation_log::get_learn_state(gpid gpid, decree start, /*out*/ learn_state state.meta = temp_writer.get_buffer(); } - std::map files; + log_file_map_by_index files; { zauto_lock l(_lock); @@ -1202,13 +1198,13 @@ void mutation_log::get_parent_mutations_and_logs(gpid pid, // no memory data and no disk data return; } - std::map file_map = get_log_file_map(); + const auto &file_map = get_log_file_map(); bool skip_next = false; std::list learn_files; decree last_max_decree = 0; for (auto itr = file_map.rbegin(); itr != file_map.rend(); ++itr) { - log_file_ptr &log = itr->second; + const log_file_ptr &log = itr->second; if (log->end_offset() <= _private_log_info.valid_start_offset) break; @@ -1287,7 +1283,7 @@ int mutation_log::garbage_collection(gpid gpid, { CHECK(_is_private, "this method is only valid for private log"); - std::map files; + log_file_map_by_index files; decree max_decree = invalid_decree; int current_file_index = -1; @@ -1295,23 +1291,24 @@ int mutation_log::garbage_collection(gpid gpid, zauto_lock l(_lock); files = _log_files; max_decree = _private_log_info.max_decree; - if (_current_log_file != nullptr) + if (_current_log_file != nullptr) { current_file_index = _current_log_file->index(); + } } if (files.size() <= 1) { // nothing to do return 0; - } else { - // the last one should be the current log file - CHECK(current_file_index == -1 || files.rbegin()->first == current_file_index, - "invalid current_file_index, index = {}", - current_file_index); } + // the last one should be the current log file + CHECK(current_file_index == -1 || files.rbegin()->first == current_file_index, + "invalid current_file_index, index = {}", + current_file_index); + // find the largest file which can be deleted. // after iterate, the 'mark_it' will point to the largest file which can be deleted. - std::map::reverse_iterator mark_it; + log_file_map_by_index::reverse_iterator mark_it; int64_t already_reserved_size = 0; for (mark_it = files.rbegin(); mark_it != files.rend(); ++mark_it) { log_file_ptr log = mark_it->second; @@ -1403,315 +1400,253 @@ int mutation_log::garbage_collection(gpid gpid, return deleted; } -int mutation_log::garbage_collection(const replica_log_info_map &gc_condition, - int file_count_limit, - std::set &prevent_gc_replicas) +struct gc_summary_info { - CHECK(!_is_private, "this method is only valid for shared log"); - - std::map files; - replica_log_info_map max_decrees; - int current_log_index = -1; - int64_t total_log_size = 0; + dsn::gpid pid; + int min_file_index = 0; + dsn::replication::decree max_decree_gap = 0; + dsn::replication::decree garbage_max_decree = 0; + dsn::replication::decree slog_max_decree = 0; + std::string to_string() const { - zauto_lock l(_lock); - files = _log_files; - max_decrees = _shared_log_info_map; - if (_current_log_file != nullptr) - current_log_index = _current_log_file->index(); - total_log_size = total_size_no_lock(); + return fmt::format("gc_summary_info = [pid = {}, min_file_index = {}, max_decree_gap = {}, " + "garbage_max_decree = {}, slog_max_decree = {}]", + pid, + min_file_index, + max_decree_gap, + garbage_max_decree, + slog_max_decree); } - if (files.size() <= 1) { - // nothing to do - LOG_INFO("gc_shared: too few files to delete, file_count_limit = {}, reserved_log_count " - "= {}, reserved_log_size = {}, current_log_index = {}", - file_count_limit, - files.size(), - total_log_size, - current_log_index); - return (int)files.size(); - } else { - // the last one should be the current log file - CHECK(-1 == current_log_index || files.rbegin()->first == current_log_index, - "invalid current_log_index, index = {}", - current_log_index); + friend std::ostream &operator<<(std::ostream &os, const gc_summary_info &gc_summary) + { + return os << gc_summary.to_string(); } +}; - int reserved_log_count = files.size(); - int64_t reserved_log_size = total_log_size; - int reserved_smallest_log = files.begin()->first; - int reserved_largest_log = current_log_index; - - // find the largest file which can be deleted. - // after iterate, the 'mark_it' will point to the largest file which can be deleted. - std::map::reverse_iterator mark_it; - std::set kickout_replicas; - gpid stop_gc_replica; - int stop_gc_log_index = 0; - decree stop_gc_decree_gap = 0; - decree stop_gc_garbage_max_decree = 0; - decree stop_gc_log_max_decree = 0; - int file_count = 0; - for (mark_it = files.rbegin(); mark_it != files.rend(); ++mark_it) { - log_file_ptr log = mark_it->second; - CHECK_EQ(mark_it->first, log->index()); - file_count++; - - bool delete_ok = true; - - // skip current file - if (current_log_index == log->index()) { - delete_ok = false; - } +namespace { - if (delete_ok) { - std::set prevent_gc_replicas_for_this_log; +bool can_gc_replica_slog(const dsn::replication::replica_log_info_map &slog_max_decrees, + const dsn::replication::log_file_ptr &file, + const dsn::gpid &pid, + const dsn::replication::replica_log_info &replica_durable_info, + dsn::replication::gc_summary_info &gc_summary) +{ + const auto &garbage_max_decree = replica_durable_info.max_decree; + const auto &valid_start_offset = replica_durable_info.valid_start_offset; + + const auto &it = slog_max_decrees.find(pid); + if (it == slog_max_decrees.end()) { + // There's no log found in this file for this replica, thus all decrees of + // this replica in this file could deleted. + // + // `valid_start_offset` might be reset to 0 if initialize_on_load() returned + // `ERR_INCOMPLETE_DATA`, thus it's possible that `valid_start_offset == 0`. + CHECK(valid_start_offset == 0 || file->end_offset() <= valid_start_offset, + "valid start offset must be 0 or greater than the end of this log file"); + LOG_DEBUG("gc @ {}: max_decree for {} is missing vs {} as garbage max decree, it's " + "safe to delete this and all older logs for this replica", + pid, + file->path(), + garbage_max_decree); + return true; + } else if (file->end_offset() <= valid_start_offset) { + // This file has been invalid for this replica, since `valid_start_offset` was reset + // to a file with larger index than this file. Thus all decrees of this replica in + // this file could be deleted. + LOG_DEBUG("gc @ {}: log is invalid for {}, as valid start offset vs log end offset = " + "{} vs {}, it is therefore safe to delete this and all older logs for this " + "replica", + pid, + file->path(), + valid_start_offset, + file->end_offset()); + return true; + } else if (it->second.max_decree <= garbage_max_decree) { + // All decrees are no more than the garbage max decree. Since all decrees less than + // garbage max decree would be deleted, all decrees of this replica in this file + // could be deleted. + LOG_DEBUG("gc @ {}: max_decree for {} is {} vs {} as garbage max decree, it is " + "therefore safe to delete this and all older logs for this replica", + pid, + file->path(), + it->second.max_decree, + garbage_max_decree); + return true; + } - for (auto &kv : gc_condition) { - if (kickout_replicas.find(kv.first) != kickout_replicas.end()) { - // no need to consider this replica - continue; - } + // it->second.max_decree > garbage_max_decree + // + // Some decrees are more than garbage max decree, thus this file should not be deleted + // for now. + LOG_DEBUG("gc @ {}: max_decree for {} is {} vs {} as garbage max decree, it " + "is therefore not allowed to delete this and all older logs", + pid, + file->path(), + it->second.max_decree, + garbage_max_decree); + + auto gap = it->second.max_decree - garbage_max_decree; + if (file->index() < gc_summary.min_file_index || gap > gc_summary.max_decree_gap) { + // Find the oldest file of this round of iteration for gc of slog files, with the + // max decree gap between the garbage max decree and the oldest slog file. + gc_summary.pid = pid; + gc_summary.min_file_index = file->index(); + gc_summary.max_decree_gap = gap; + gc_summary.garbage_max_decree = garbage_max_decree; + gc_summary.slog_max_decree = it->second.max_decree; + } - gpid gpid = kv.first; - decree garbage_max_decree = kv.second.max_decree; - int64_t valid_start_offset = kv.second.valid_start_offset; + return false; +} - bool delete_ok_for_this_replica = false; - bool kickout_this_replica = false; - auto it3 = max_decrees.find(gpid); +} // anonymous namespace - // log not found for this replica, ok to delete - if (it3 == max_decrees.end()) { - // valid_start_offset may be reset to 0 if initialize_on_load() returns - // ERR_INCOMPLETE_DATA - CHECK(valid_start_offset == 0 || valid_start_offset >= log->end_offset(), - "valid start offset must be 0 or greater than the end of this log file"); +void mutation_log::garbage_collection(const replica_log_info_map &replica_durable_decrees, + std::set &prevent_gc_replicas) +{ + CHECK(!_is_private, "this method is only valid for shared log"); - LOG_DEBUG( - "gc @ {}: max_decree for {} is missing vs {} as garbage max decree, it's " - "safe to delete this and all older logs for this replica", - gpid, - log->path(), - garbage_max_decree); - delete_ok_for_this_replica = true; - kickout_this_replica = true; - } + // Fetch the snapshot of the latest states of the slog, such as the max decree it maintains + // for each partition. + log_file_map_by_index files; + replica_log_info_map slog_max_decrees; + int64_t total_log_size = 0; + { + zauto_lock l(_lock); + total_log_size = total_size_no_lock(); + if (_log_files.empty()) { + CHECK_EQ(total_log_size, 0); + LOG_INFO("gc_shared: slog file not found"); + return; + } - // log is invalid for this replica, ok to delete - else if (log->end_offset() <= valid_start_offset) { - LOG_DEBUG( - "gc @ {}: log is invalid for {}, as valid start offset vs log end offset = " - "{} vs {}, it is therefore safe to delete this and all older logs for this " - "replica", - gpid, - log->path(), - valid_start_offset, - log->end_offset()); - delete_ok_for_this_replica = true; - kickout_this_replica = true; - } + CHECK_NULL(_current_log_file, + "shared logs have been deprecated, thus could not be created"); + files = _log_files; + slog_max_decrees = _shared_log_info_map; + } - // all decrees are no more than garbage max decree, ok to delete - else if (it3->second.max_decree <= garbage_max_decree) { - LOG_DEBUG("gc @ {}: max_decree for {} is {} vs {} as garbage max decree, it is " - "therefore safe to delete this and all older logs for this replica", - gpid, - log->path(), - it3->second.max_decree, - garbage_max_decree); - delete_ok_for_this_replica = true; - kickout_this_replica = true; - } + reserved_slog_info reserved_slog = { + files.size(), total_log_size, files.begin()->first, files.rbegin()->first}; - else // it3->second.max_decree > garbage_max_decree - { - // should not delete this file - LOG_DEBUG("gc @ {}: max_decree for {} is {} vs {} as garbage max decree, it " - "is therefore not allowed to delete this and all older logs", - gpid, - log->path(), - it3->second.max_decree, - garbage_max_decree); - prevent_gc_replicas_for_this_log.insert(gpid); - decree gap = it3->second.max_decree - garbage_max_decree; - if (log->index() < stop_gc_log_index || gap > stop_gc_decree_gap) { - // record the max gap replica for the smallest log - stop_gc_replica = gpid; - stop_gc_log_index = log->index(); - stop_gc_decree_gap = gap; - stop_gc_garbage_max_decree = garbage_max_decree; - stop_gc_log_max_decree = it3->second.max_decree; - } - } + // Iterate over the slog files from the newest to the oldest in descending order(i.e. + // file index in descending order), to find the newest file that could be deleted(after + // iterating, `mark_it` would point to the newest file that could be deleted). + log_file_map_by_index::reverse_iterator mark_it; + std::set kickout_replicas; + gc_summary_info gc_summary; + for (mark_it = files.rbegin(); mark_it != files.rend(); ++mark_it) { + const auto &file = mark_it->second; + CHECK_EQ(mark_it->first, file->index()); - if (kickout_this_replica) { - // files before this file is useless for this replica, - // so from now on, this replica will not be considered anymore - kickout_replicas.insert(gpid); - } + bool can_gc_all_replicas_slog = true; - if (!delete_ok_for_this_replica) { - // can not delete this file, mark it, and continue to check other replicas - delete_ok = false; - } + for (const auto &replica_durable_info : replica_durable_decrees) { + if (kickout_replicas.find(replica_durable_info.first) != kickout_replicas.end()) { + // There's no need to consider this replica. + continue; } - // update prevent_gc_replicas - if (file_count > file_count_limit && !prevent_gc_replicas_for_this_log.empty()) { - prevent_gc_replicas.insert(prevent_gc_replicas_for_this_log.begin(), - prevent_gc_replicas_for_this_log.end()); + if (can_gc_replica_slog(slog_max_decrees, + file, + replica_durable_info.first, + replica_durable_info.second, + gc_summary)) { + // Log files before this file is useless for this replica, + // so from now on, this replica would not be considered any more. + kickout_replicas.insert(replica_durable_info.first); + continue; } + + // For now, this file could not be deleted. + can_gc_all_replicas_slog = false; + prevent_gc_replicas.insert(replica_durable_info.first); } - if (delete_ok) { - // found the largest file which can be deleted + if (can_gc_all_replicas_slog) { + // The newest file that could be deleted has been found. break; } - // update max_decrees for the next log file - max_decrees = log->previous_log_max_decrees(); + // Fetch max decrees of the next slog file. + slog_max_decrees = file->previous_log_max_decrees(); } if (mark_it == files.rend()) { - // no file to delete - if (stop_gc_decree_gap > 0) { - LOG_INFO("gc_shared: no file can be deleted, file_count_limit = {}, " - "reserved_log_count = {}, reserved_log_size = {}, " - "reserved_smallest_log = {}, reserved_largest_log = {}, " - "stop_gc_log_index = {}, stop_gc_replica_count = {}, " - "stop_gc_replica = {}, stop_gc_decree_gap = {}, " - "stop_gc_garbage_max_decree = {}, stop_gc_log_max_decree = {}", - file_count_limit, - reserved_log_count, - reserved_log_size, - reserved_smallest_log, - reserved_largest_log, - stop_gc_log_index, - prevent_gc_replicas.size(), - stop_gc_replica, - stop_gc_decree_gap, - stop_gc_garbage_max_decree, - stop_gc_log_max_decree); - } else { - LOG_INFO("gc_shared: no file can be deleted, file_count_limit = {}, " - "reserved_log_count = {}, reserved_log_size = {}, " - "reserved_smallest_log = {}, reserved_largest_log = {}", - file_count_limit, - reserved_log_count, - reserved_log_size, - reserved_smallest_log, - reserved_largest_log); - } - - return reserved_log_count; + // There's no file that could be deleted. + LOG_INFO("gc_shared: no file can be deleted: {}, {}, prevent_gc_replicas = {}", + reserved_slog, + gc_summary, + prevent_gc_replicas.size()); + return; } - // ok, let's delete files in increasing order of file index - // to avoid making a hole in the file list - int largest_log_to_delete = mark_it->second->index(); - int to_delete_log_count = 0; - int64_t to_delete_log_size = 0; - int deleted_log_count = 0; - int64_t deleted_log_size = 0; - int deleted_smallest_log = 0; - int deleted_largest_log = 0; - for (auto it = files.begin(); it != files.end() && it->second->index() <= largest_log_to_delete; + slog_deletion_info slog_deletion; + + // Delete files in ascending order of file index. Otherwise, deleting files in descending + // order would lead to a hole in the file list once a file failed to be deleted. + remove_obsolete_slog_files(mark_it->second->index(), files, reserved_slog, slog_deletion); + LOG_INFO("gc_shared: deleted some files: {}, {}, {}, prevent_gc_replicas = {}", + reserved_slog, + slog_deletion, + gc_summary, + prevent_gc_replicas.size()); +} + +void mutation_log::remove_obsolete_slog_files(const int max_file_index_to_delete, + log_file_map_by_index &files, + reserved_slog_info &reserved_slog, + slog_deletion_info &slog_deletion) +{ + for (auto it = files.begin(); + it != files.end() && it->second->index() <= max_file_index_to_delete; ++it) { - log_file_ptr log = it->second; - CHECK_EQ(it->first, log->index()); - to_delete_log_count++; - to_delete_log_size += log->end_offset() - log->start_offset(); + auto &file = it->second; + CHECK_EQ(it->first, file->index()); + slog_deletion.to_delete_file_count++; + slog_deletion.to_delete_log_size += file->end_offset() - file->start_offset(); - // close first - log->close(); + // Firstly close the log file. + file->close(); - // delete file - auto &fpath = log->path(); + // Delete the log file. + const auto &fpath = file->path(); if (!dsn::utils::filesystem::remove_path(fpath)) { LOG_ERROR("gc_shared: fail to remove {}, stop current gc cycle ...", fpath); break; } - // delete succeed + // The log file is deleted successfully. LOG_INFO("gc_shared: log file {} is removed", fpath); - deleted_log_count++; - deleted_log_size += log->end_offset() - log->start_offset(); - if (deleted_smallest_log == 0) - deleted_smallest_log = log->index(); - deleted_largest_log = log->index(); + slog_deletion.deleted_file_count++; + slog_deletion.deleted_log_size += file->end_offset() - file->start_offset(); + if (slog_deletion.deleted_min_file_index == 0) { + slog_deletion.deleted_min_file_index = file->index(); + } + slog_deletion.deleted_max_file_index = file->index(); - // erase from _log_files + // Remove the log file from _log_files. { zauto_lock l(_lock); _log_files.erase(it->first); _global_start_offset = _log_files.size() > 0 ? _log_files.begin()->second->start_offset() : 0; - reserved_log_count = _log_files.size(); - reserved_log_size = total_size_no_lock(); - if (reserved_log_count > 0) { - reserved_smallest_log = _log_files.begin()->first; - reserved_largest_log = _log_files.rbegin()->first; + reserved_slog.file_count = _log_files.size(); + reserved_slog.log_size = total_size_no_lock(); + if (reserved_slog.file_count > 0) { + reserved_slog.min_file_index = _log_files.begin()->first; + reserved_slog.max_file_index = _log_files.rbegin()->first; } else { - reserved_smallest_log = -1; - reserved_largest_log = -1; + reserved_slog.min_file_index = -1; + reserved_slog.max_file_index = -1; } } } - - if (stop_gc_decree_gap > 0) { - LOG_INFO("gc_shared: deleted some files, file_count_limit = {}, " - "reserved_log_count = {}, reserved_log_size = {}, " - "reserved_smallest_log = {}, reserved_largest_log = {}, " - "to_delete_log_count = {}, to_delete_log_size = {}, " - "deleted_log_count = {}, deleted_log_size = {}, " - "deleted_smallest_log = {}, deleted_largest_log = {}, " - "stop_gc_log_index = {}, stop_gc_replica_count = {}, " - "stop_gc_replica = {}, stop_gc_decree_gap = {}, " - "stop_gc_garbage_max_decree = {}, stop_gc_log_max_decree = {}", - file_count_limit, - reserved_log_count, - reserved_log_size, - reserved_smallest_log, - reserved_largest_log, - to_delete_log_count, - to_delete_log_size, - deleted_log_count, - deleted_log_size, - deleted_smallest_log, - deleted_largest_log, - stop_gc_log_index, - prevent_gc_replicas.size(), - stop_gc_replica, - stop_gc_decree_gap, - stop_gc_garbage_max_decree, - stop_gc_log_max_decree); - } else { - LOG_INFO("gc_shared: deleted some files, file_count_limit = {}, " - "reserved_log_count = {}, reserved_log_size = {}, " - "reserved_smallest_log = {}, reserved_largest_log = {}, " - "to_delete_log_count = {}, to_delete_log_size = {}, " - "deleted_log_count = {}, deleted_log_size = {}, " - "deleted_smallest_log = {}, deleted_largest_log = {}", - file_count_limit, - reserved_log_count, - reserved_log_size, - reserved_smallest_log, - reserved_largest_log, - to_delete_log_count, - to_delete_log_size, - deleted_log_count, - deleted_log_size, - deleted_smallest_log, - deleted_largest_log); - } - - return reserved_log_count; } -std::map mutation_log::get_log_file_map() const +mutation_log::log_file_map_by_index mutation_log::get_log_file_map() const { zauto_lock l(_lock); return _log_files; @@ -1719,3 +1654,5 @@ std::map mutation_log::get_log_file_map() const } // namespace replication } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(dsn::replication::gc_summary_info); diff --git a/src/replica/mutation_log.h b/src/replica/mutation_log.h index 6636f808af..b6223e847c 100644 --- a/src/replica/mutation_log.h +++ b/src/replica/mutation_log.h @@ -26,11 +26,13 @@ #pragma once +#include #include #include #include #include #include +#include #include #include #include @@ -50,6 +52,7 @@ #include "utils/autoref_ptr.h" #include "utils/error_code.h" #include "utils/errors.h" +#include "utils/fmt_utils.h" #include "utils/zlocks.h" namespace dsn { @@ -221,19 +224,17 @@ class mutation_log : public ref_counter int64_t reserve_max_size, int64_t reserve_max_time); - // garbage collection for shared log, returns reserved file count. - // `prevent_gc_replicas' will store replicas which prevent log files out of `file_count_limit' - // to be deleted. - // remove log files if satisfy: - // - for each replica "r": - // r is not in file.max_decree - // || file.max_decree[r] <= gc_condition[r].max_decree - // || file.end_offset[r] <= gc_condition[r].valid_start_offset - // - the current log file should not be removed - // thread safe - int garbage_collection(const replica_log_info_map &gc_condition, - int file_count_limit, - std::set &prevent_gc_replicas); + // Garbage collection for shared log. + // `prevent_gc_replicas' will store replicas which prevent log files from being deleted + // for gc. + // + // Since slog had been deprecated, no new slog files would be created. Therefore, our + // target is to remove all of the existing slog files according to the progressive durable + // decree for each replica. + // + // Thread safe. + void garbage_collection(const replica_log_info_map &replica_durable_decrees, + std::set &prevent_gc_replicas); // // when this is a private log, log files are learned by remote replicas @@ -285,8 +286,10 @@ class mutation_log : public ref_counter decree max_gced_decree(gpid gpid) const; decree max_gced_decree_no_lock(gpid gpid) const; + using log_file_map_by_index = std::map; + // thread-safe - std::map get_log_file_map() const; + log_file_map_by_index get_log_file_map() const; // check the consistence of valid_start_offset // thread safe @@ -300,6 +303,58 @@ class mutation_log : public ref_counter task_tracker *tracker() { return &_tracker; } + struct reserved_slog_info + { + size_t file_count = 0; + int64_t log_size = 0; + int min_file_index = 0; + int max_file_index = 0; + + std::string to_string() const + { + return fmt::format("reserved_slog_info = [file_count = {}, log_size = {}, " + "min_file_index = {}, max_file_index = {}]", + file_count, + log_size, + min_file_index, + max_file_index); + } + + friend std::ostream &operator<<(std::ostream &os, const reserved_slog_info &reserved_log) + { + return os << reserved_log.to_string(); + } + }; + + struct slog_deletion_info + { + int to_delete_file_count = 0; + int64_t to_delete_log_size = 0; + int deleted_file_count = 0; + int64_t deleted_log_size = 0; + int deleted_min_file_index = 0; + int deleted_max_file_index = 0; + + std::string to_string() const + { + return fmt::format("slog_deletion_info = [to_delete_file_count = {}, " + "to_delete_log_size = {}, deleted_file_count = {}, " + "deleted_log_size = {}, deleted_min_file_index = {}, " + "deleted_max_file_index = {}]", + to_delete_file_count, + to_delete_log_size, + deleted_file_count, + deleted_log_size, + deleted_min_file_index, + deleted_max_file_index); + } + + friend std::ostream &operator<<(std::ostream &os, const slog_deletion_info &log_deletion) + { + return os << log_deletion.to_string(); + } + }; + protected: // thread-safe // 'size' is data size to write; the '_global_end_offset' will be updated by 'size'. @@ -325,7 +380,7 @@ class mutation_log : public ref_counter replay_callback callback, /*out*/ int64_t &end_offset); - static error_code replay(std::map &log_files, + static error_code replay(log_file_map_by_index &log_files, replay_callback callback, /*out*/ int64_t &end_offset); @@ -345,6 +400,13 @@ class mutation_log : public ref_counter // get total size ithout lock. int64_t total_size_no_lock() const; + // Closing and remove all of slog files whose indexes are less than (i.e. older) or equal to + // `max_file_index_to_delete`. + void remove_obsolete_slog_files(const int max_file_index_to_delete, + log_file_map_by_index &files, + reserved_slog_info &reserved_log, + slog_deletion_info &log_deletion); + protected: std::string _dir; bool _is_private; @@ -373,12 +435,12 @@ class mutation_log : public ref_counter bool _switch_file_demand; // logs - int _last_file_index; // new log file index = _last_file_index + 1 - std::map _log_files; // index -> log_file_ptr - log_file_ptr _current_log_file; // current log file - int64_t _global_start_offset; // global start offset of all files. - // invalid if _log_files.size() == 0. - int64_t _global_end_offset; // global end offset currently + int _last_file_index; // new log file index = _last_file_index + 1 + log_file_map_by_index _log_files; // index -> log_file_ptr + log_file_ptr _current_log_file; // current log file + int64_t _global_start_offset; // global start offset of all files. + // invalid if _log_files.size() == 0. + int64_t _global_end_offset; // global end offset currently // replica log info // - log_info.max_decree: the max decree of mutations up to now @@ -410,21 +472,21 @@ class mutation_log_shared : public mutation_log { } - virtual ~mutation_log_shared() override + ~mutation_log_shared() override { close(); _tracker.cancel_outstanding_tasks(); } - virtual ::dsn::task_ptr append(mutation_ptr &mu, - dsn::task_code callback_code, - dsn::task_tracker *tracker, - aio_handler &&callback, - int hash = 0, - int64_t *pending_size = nullptr) override; + ::dsn::task_ptr append(mutation_ptr &mu, + dsn::task_code callback_code, + dsn::task_tracker *tracker, + aio_handler &&callback, + int hash = 0, + int64_t *pending_size = nullptr) override; - virtual void flush() override; - virtual void flush_once() override; + void flush() override; + void flush_once() override; private: // async write pending mutations into log file @@ -467,24 +529,22 @@ class mutation_log_private : public mutation_log, private replica_base _tracker.cancel_outstanding_tasks(); } - virtual ::dsn::task_ptr append(mutation_ptr &mu, - dsn::task_code callback_code, - dsn::task_tracker *tracker, - aio_handler &&callback, - int hash = 0, - int64_t *pending_size = nullptr) override; + ::dsn::task_ptr append(mutation_ptr &mu, + dsn::task_code callback_code, + dsn::task_tracker *tracker, + aio_handler &&callback, + int hash = 0, + int64_t *pending_size = nullptr) override; - virtual bool get_learn_state_in_memory(decree start_decree, - binary_writer &writer) const override; + bool get_learn_state_in_memory(decree start_decree, binary_writer &writer) const override; // get in-memory mutations, including pending and writing mutations - virtual void - get_in_memory_mutations(decree start_decree, - ballot start_ballot, - /*out*/ std::vector &mutation_list) const override; + void get_in_memory_mutations(decree start_decree, + ballot start_ballot, + /*out*/ std::vector &mutation_list) const override; - virtual void flush() override; - virtual void flush_once() override; + void flush() override; + void flush_once() override; private: // async write pending mutations into log file @@ -499,7 +559,7 @@ class mutation_log_private : public mutation_log, private replica_base std::shared_ptr &pending, decree max_commit); - virtual void init_states() override; + void init_states() override; // flush at most count times // if count <= 0, means flush until all data is on disk @@ -521,3 +581,6 @@ class mutation_log_private : public mutation_log, private replica_base } // namespace replication } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::mutation_log::reserved_slog_info); +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::mutation_log::slog_deletion_info); diff --git a/src/replica/mutation_log_replay.cpp b/src/replica/mutation_log_replay.cpp index 261ff4706a..316c12396e 100644 --- a/src/replica/mutation_log_replay.cpp +++ b/src/replica/mutation_log_replay.cpp @@ -133,7 +133,7 @@ namespace replication { replay_callback callback, /*out*/ int64_t &end_offset) { - std::map logs; + log_file_map_by_index logs; for (auto &fpath : log_files) { error_code err; log_file_ptr log = log_file::open_read(fpath.c_str(), err); @@ -154,7 +154,7 @@ namespace replication { return replay(logs, callback, end_offset); } -/*static*/ error_code mutation_log::replay(std::map &logs, +/*static*/ error_code mutation_log::replay(log_file_map_by_index &logs, replay_callback callback, /*out*/ int64_t &end_offset) { diff --git a/src/replica/mutation_log_utils.cpp b/src/replica/mutation_log_utils.cpp index 5083897494..46c32b4df9 100644 --- a/src/replica/mutation_log_utils.cpp +++ b/src/replica/mutation_log_utils.cpp @@ -64,7 +64,7 @@ namespace log_utils { } /*extern*/ -error_s check_log_files_continuity(const std::map &logs) +error_s check_log_files_continuity(const mutation_log::log_file_map_by_index &logs) { if (logs.empty()) { return error_s::ok(); diff --git a/src/replica/mutation_log_utils.h b/src/replica/mutation_log_utils.h index 4b61846696..9673546f82 100644 --- a/src/replica/mutation_log_utils.h +++ b/src/replica/mutation_log_utils.h @@ -31,6 +31,7 @@ #include #include "replica/log_file.h" +#include "replica/mutation_log.h" #include "utils/autoref_ptr.h" #include "utils/errors.h" #include "utils/string_view.h" @@ -44,7 +45,7 @@ extern error_s open_read(string_view path, /*out*/ log_file_ptr &file); extern error_s list_all_files(const std::string &dir, /*out*/ std::vector &files); inline error_s open_log_file_map(const std::vector &log_files, - /*out*/ std::map &log_file_map) + /*out*/ mutation_log::log_file_map_by_index &log_file_map) { for (const std::string &fname : log_files) { log_file_ptr lf; @@ -58,7 +59,7 @@ inline error_s open_log_file_map(const std::vector &log_files, } inline error_s open_log_file_map(const std::string &dir, - /*out*/ std::map &log_file_map) + /*out*/ mutation_log::log_file_map_by_index &log_file_map) { std::vector log_files; error_s es = list_all_files(dir, log_files); @@ -68,11 +69,11 @@ inline error_s open_log_file_map(const std::string &dir, return open_log_file_map(log_files, log_file_map) << "open_log_file_map(dir)"; } -extern error_s check_log_files_continuity(const std::map &logs); +extern error_s check_log_files_continuity(const mutation_log::log_file_map_by_index &logs); inline error_s check_log_files_continuity(const std::string &dir) { - std::map log_file_map; + mutation_log::log_file_map_by_index log_file_map; error_s es = open_log_file_map(dir, log_file_map); if (!es.is_ok()) { return es << "check_log_files_continuity(dir)"; diff --git a/src/replica/replica_stub.cpp b/src/replica/replica_stub.cpp index a96b8386fc..c5bd5000a9 100644 --- a/src/replica/replica_stub.cpp +++ b/src/replica/replica_stub.cpp @@ -36,6 +36,7 @@ #include // IWYU pragma: no_include #include +#include #include #include #include @@ -43,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -167,7 +169,13 @@ DSN_DEFINE_int32(replication, 32, "shared log maximum segment file size (MB)"); -DSN_DEFINE_int32(replication, log_shared_file_count_limit, 100, "shared log maximum file count"); +DSN_DEFINE_uint64( + replication, + log_shared_gc_flush_replicas_limit, + 64, + "The number of submitted replicas that are flushed for gc shared logs; 0 means no limit"); +DSN_TAG_VARIABLE(log_shared_gc_flush_replicas_limit, FT_MUTABLE); + DSN_DEFINE_int32( replication, mem_release_check_interval_ms, @@ -192,6 +200,9 @@ bool replica_stub::s_not_exit_on_log_failure = false; replica_stub::replica_stub(replica_state_subscriber subscriber /*= nullptr*/, bool is_long_subscriber /* = true*/) : serverlet("replica_stub"), + _last_prevent_gc_replica_count(0), + _real_log_shared_gc_flush_replicas_limit(0), + _mock_flush_replicas_for_test(0), _deny_client(false), _verbose_client_log(false), _verbose_commit_log(false), @@ -578,13 +589,7 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f LOG_INFO("primary_address = {}", _primary_address_str); set_options(opts); - std::ostringstream oss; - for (int i = 0; i < _options.meta_servers.size(); ++i) { - if (i != 0) - oss << ","; - oss << _options.meta_servers[i].to_string(); - } - LOG_INFO("meta_servers = {}", oss.str()); + LOG_INFO("meta_servers = {}", fmt::join(_options.meta_servers, ", ")); _deny_client = FLAGS_deny_client_on_start; _verbose_client_log = FLAGS_verbose_client_log_on_start; @@ -1767,138 +1772,202 @@ void replica_stub::on_gc_replica(replica_stub_ptr this_, gpid id) } } -void replica_stub::on_gc() +void replica_stub::gc_slog(const replica_gc_info_map &replica_gc_map) { - uint64_t start = dsn_now_ns(); + if (_log == nullptr) { + return; + } - struct gc_info - { - replica_ptr rep; - partition_status::type status; - mutation_log_ptr plog; - decree last_durable_decree; - int64_t init_offset_in_shared_log; - }; - - std::unordered_map rs; - { - zauto_read_lock l(_replicas_lock); - // collect info in lock to prevent the case that the replica is closed in replica::close() - for (auto &kv : _replicas) { - const replica_ptr &rep = kv.second; - gc_info &info = rs[kv.first]; - info.rep = rep; - info.status = rep->status(); - info.plog = rep->private_log(); - info.last_durable_decree = rep->last_durable_decree(); - info.init_offset_in_shared_log = rep->get_app()->init_info().init_offset_in_shared_log; + replica_log_info_map replica_durable_decrees; + for (auto &replica_gc : replica_gc_map) { + replica_log_info replica_log; + auto &rep = replica_gc.second.rep; + auto &plog = replica_gc.second.plog; + if (plog) { + // Flush private log to update `plog_max_commit_on_disk`, and just flush once + // to avoid flushing infinitely. + plog->flush_once(); + auto plog_max_commit_on_disk = plog->max_commit_on_disk(); + + replica_log.max_decree = + std::min(replica_gc.second.last_durable_decree, plog_max_commit_on_disk); + LOG_INFO("gc_shared: gc condition for {}, status = {}, garbage_max_decree = {}, " + "last_durable_decree= {}, plog_max_commit_on_disk = {}", + rep->name(), + enum_to_string(replica_gc.second.status), + replica_log.max_decree, + replica_gc.second.last_durable_decree, + plog_max_commit_on_disk); + } else { + replica_log.max_decree = replica_gc.second.last_durable_decree; + LOG_INFO("gc_shared: gc condition for {}, status = {}, garbage_max_decree = {}, " + "last_durable_decree = {}", + rep->name(), + enum_to_string(replica_gc.second.status), + replica_log.max_decree, + replica_gc.second.last_durable_decree); } + replica_log.valid_start_offset = replica_gc.second.init_offset_in_shared_log; + replica_durable_decrees[replica_gc.first] = replica_log; } - LOG_INFO("start to garbage collection, replica_count = {}", rs.size()); + // Garbage collection for shared log files. + std::set prevent_gc_replicas; + _log->garbage_collection(replica_durable_decrees, prevent_gc_replicas); + + // Trigger checkpoint to flush memtables once some replicas were found that prevent + // slog files from being removed for gc. + flush_replicas_for_slog_gc(replica_gc_map, prevent_gc_replicas); - // gc shared prepare log + auto total_size = _log->total_size(); + _counter_shared_log_size->set(total_size / (1024 * 1024)); + + // TODO(wangdan): currently we could not yet call _log.reset() as below to close slog and + // reset it to nullptr even if it was found that slog had become empty (which means there + // had not been any file for slog). + // if (total_size == 0) { + // _log.reset(); + // } // - // Now that checkpoint is very important for gc, we must be able to trigger checkpoint when - // necessary. - // that is, we should be able to trigger memtable flush when necessary. + // The reason for this point is that on_gc() is scheduled by timer to run asynchronously + // during the initialization of replica_stub. It might happen before slog.on_partition_reset() + // (building slog._shared_log_info_map), which means slog would be closed mistakenly before + // it was initialized completely. // - // How to trigger memtable flush? - // we add a parameter `is_emergency' in dsn_app_async_checkpoint() function, when set true, - // the undering storage system should flush memtable as soon as possiable. + // All of slog files would removed on v2.5; thus it is safe to remove all of slog code (which + // means even slog object would not be created) on the next version (namely 2.6), and this + // problem would also be resolved. +} + +void replica_stub::limit_flush_replicas_for_slog_gc(size_t prevent_gc_replica_count) +{ + const size_t log_shared_gc_flush_replicas_limit = FLAGS_log_shared_gc_flush_replicas_limit; + if (log_shared_gc_flush_replicas_limit == 0) { + // 0 for log_shared_gc_flush_replicas_limit means no limit. + _real_log_shared_gc_flush_replicas_limit = std::numeric_limits::max(); + return; + } + + if (_last_prevent_gc_replica_count == 0) { + // Initialize it for the 1st time. + _real_log_shared_gc_flush_replicas_limit = log_shared_gc_flush_replicas_limit; + return; + } + + CHECK_GE(_last_prevent_gc_replica_count, prevent_gc_replica_count); + size_t flushed_replicas = _last_prevent_gc_replica_count - prevent_gc_replica_count; + if (flushed_replicas == 0) { + // It's too busy to process more flush tasks. + _real_log_shared_gc_flush_replicas_limit = + std::min(2UL, log_shared_gc_flush_replicas_limit); + return; + } + + if (_real_log_shared_gc_flush_replicas_limit == 0 || + _real_log_shared_gc_flush_replicas_limit == std::numeric_limits::max()) { + // Once it was previously set with some special values, it should be reset. + _real_log_shared_gc_flush_replicas_limit = log_shared_gc_flush_replicas_limit; + return; + } + + if (flushed_replicas < _real_log_shared_gc_flush_replicas_limit) { + // Keep it unchanged. + return; + } + + // Increase it to process more flush tasks. + _real_log_shared_gc_flush_replicas_limit = + std::min(log_shared_gc_flush_replicas_limit, _real_log_shared_gc_flush_replicas_limit << 1); +} + +void replica_stub::flush_replicas_for_slog_gc(const replica_gc_info_map &replica_gc_map, + const std::set &prevent_gc_replicas) +{ + // Trigger checkpoints to flush memtables once some replicas were found that prevent slog files + // from being removed for gc. // - // When to trigger memtable flush? - // 1. Using `[replication].checkpoint_max_interval_hours' option, we can set max interval time - // of two adjacent checkpoints; If the time interval is arrived, then emergency checkpoint - // will be triggered. - // 2. Using `[replication].log_shared_file_count_limit' option, we can set max file count of - // shared log; If the limit is exceeded, then emergency checkpoint will be triggered; Instead - // of triggering all replicas to do checkpoint, we will only trigger a few of necessary - // replicas which block garbage collection of the oldest log file. + // How to trigger memtable flush ? + // A parameter `is_emergency' was added for `replica::background_async_checkpoint()` function; + // once it's set true, underlying storage engine would flush memtable as soon as possiable. // - if (_log != nullptr) { - replica_log_info_map gc_condition; - for (auto &kv : rs) { - replica_log_info ri; - replica_ptr &rep = kv.second.rep; - mutation_log_ptr &plog = kv.second.plog; - if (plog) { - // flush private log to update plog_max_commit_on_disk, - // and just flush once to avoid flushing infinitely - plog->flush_once(); - - decree plog_max_commit_on_disk = plog->max_commit_on_disk(); - ri.max_decree = std::min(kv.second.last_durable_decree, plog_max_commit_on_disk); - LOG_INFO("gc_shared: gc condition for {}, status = {}, garbage_max_decree = {}, " - "last_durable_decree= {}, plog_max_commit_on_disk = {}", - rep->name(), - enum_to_string(kv.second.status), - ri.max_decree, - kv.second.last_durable_decree, - plog_max_commit_on_disk); - } else { - ri.max_decree = kv.second.last_durable_decree; - LOG_INFO("gc_shared: gc condition for {}, status = {}, garbage_max_decree = {}, " - "last_durable_decree = {}", - rep->name(), - enum_to_string(kv.second.status), - ri.max_decree, - kv.second.last_durable_decree); - } - ri.valid_start_offset = kv.second.init_offset_in_shared_log; - gc_condition[kv.first] = ri; + // When memtable flush is triggered ? + // 1. After a fixed interval (specified by `[replication].gc_interval_ms` option), try to find + // if there are some replicas preventing slog files from being removed for gc; if any, all of + // them would be deleted "gradually" ("gradually" means the number of the replicas whose + // memtables are submitted to storage engine to be flushed would be limited). + // 2. `[replication].checkpoint_max_interval_hours' option specified the max interval between + // the two adjacent checkpoints. + + if (prevent_gc_replicas.empty()) { + return; + } + + limit_flush_replicas_for_slog_gc(prevent_gc_replicas.size()); + _last_prevent_gc_replica_count = prevent_gc_replicas.size(); + + LOG_INFO("gc_shared: trigger emergency checkpoints to flush replicas for gc shared logs: " + "log_shared_gc_flush_replicas_limit = {}/{}, prevent_gc_replicas({}) = {}", + _real_log_shared_gc_flush_replicas_limit, + FLAGS_log_shared_gc_flush_replicas_limit, + prevent_gc_replicas.size(), + fmt::join(prevent_gc_replicas, ", ")); + + size_t i = 0; + for (const auto &pid : prevent_gc_replicas) { + const auto &replica_gc = replica_gc_map.find(pid); + if (replica_gc == replica_gc_map.end()) { + continue; } - std::set prevent_gc_replicas; - int reserved_log_count = _log->garbage_collection( - gc_condition, FLAGS_log_shared_file_count_limit, prevent_gc_replicas); - if (reserved_log_count > FLAGS_log_shared_file_count_limit * 2) { - LOG_INFO( - "gc_shared: trigger emergency checkpoint by FLAGS_log_shared_file_count_limit, " - "file_count_limit = {}, reserved_log_count = {}, trigger all replicas to do " - "checkpoint", - FLAGS_log_shared_file_count_limit, - reserved_log_count); - for (auto &kv : rs) { - tasking::enqueue( - LPC_PER_REPLICA_CHECKPOINT_TIMER, - kv.second.rep->tracker(), - std::bind(&replica_stub::trigger_checkpoint, this, kv.second.rep, true), - kv.first.thread_hash(), - std::chrono::milliseconds(rand::next_u32(0, FLAGS_gc_interval_ms / 2))); - } - } else if (reserved_log_count > FLAGS_log_shared_file_count_limit) { - std::ostringstream oss; - int c = 0; - for (auto &i : prevent_gc_replicas) { - if (c != 0) - oss << ", "; - oss << i.to_string(); - c++; - } - LOG_INFO( - "gc_shared: trigger emergency checkpoint by FLAGS_log_shared_file_count_limit, " - "file_count_limit = {}, reserved_log_count = {}, prevent_gc_replica_count = " - "{}, trigger them to do checkpoint: {}", - FLAGS_log_shared_file_count_limit, - reserved_log_count, - prevent_gc_replicas.size(), - oss.str()); - for (auto &id : prevent_gc_replicas) { - auto find = rs.find(id); - if (find != rs.end()) { - tasking::enqueue( - LPC_PER_REPLICA_CHECKPOINT_TIMER, - find->second.rep->tracker(), - std::bind(&replica_stub::trigger_checkpoint, this, find->second.rep, true), - id.thread_hash(), - std::chrono::milliseconds(rand::next_u32(0, FLAGS_gc_interval_ms / 2))); - } - } + if (++i > _real_log_shared_gc_flush_replicas_limit) { + break; } - _counter_shared_log_size->set(_log->total_size() / (1024 * 1024)); + bool mock_flush = false; + FAIL_POINT_INJECT_NOT_RETURN_F( + "mock_flush_replicas_for_slog_gc", [&mock_flush, this, i](dsn::string_view str) { + CHECK(buf2bool(str, mock_flush), + "invalid mock_flush_replicas_for_slog_gc toggle, should be true or false: {}", + str); + _mock_flush_replicas_for_test = i; + }); + if (dsn_unlikely(mock_flush)) { + continue; + } + + tasking::enqueue( + LPC_PER_REPLICA_CHECKPOINT_TIMER, + replica_gc->second.rep->tracker(), + std::bind(&replica_stub::trigger_checkpoint, this, replica_gc->second.rep, true), + pid.thread_hash(), + std::chrono::milliseconds(rand::next_u32(0, FLAGS_gc_interval_ms / 2))); } +} + +void replica_stub::on_gc() +{ + uint64_t start = dsn_now_ns(); + + replica_gc_info_map replica_gc_map; + { + zauto_read_lock l(_replicas_lock); + // A replica was removed from _replicas before it would be closed by replica::close(). + // Thus it's safe to use the replica after fetching its ref pointer from _replicas. + for (const auto &rep_pair : _replicas) { + const replica_ptr &rep = rep_pair.second; + + auto &replica_gc = replica_gc_map[rep_pair.first]; + replica_gc.rep = rep; + replica_gc.status = rep->status(); + replica_gc.plog = rep->private_log(); + replica_gc.last_durable_decree = rep->last_durable_decree(); + replica_gc.init_offset_in_shared_log = + rep->get_app()->init_info().init_offset_in_shared_log; + } + } + + LOG_INFO("start to garbage collection, replica_count = {}", replica_gc_map.size()); + gc_slog(replica_gc_map); // statistic learning info uint64_t learning_count = 0; @@ -1914,7 +1983,7 @@ void replica_stub::on_gc() uint64_t splitting_max_duration_time_ms = 0; uint64_t splitting_max_async_learn_time_ms = 0; uint64_t splitting_max_copy_file_size = 0; - for (auto &kv : rs) { + for (auto &kv : replica_gc_map) { replica_ptr &rep = kv.second.rep; if (rep->status() == partition_status::PS_POTENTIAL_SECONDARY) { learning_count++; diff --git a/src/replica/replica_stub.h b/src/replica/replica_stub.h index eddf524ace..e71e8f864e 100644 --- a/src/replica/replica_stub.h +++ b/src/replica/replica_stub.h @@ -32,19 +32,20 @@ // replica_stub(singleton) --> replica --> replication_app_base // +#include +#include #include #include #include #include #include +#include #include #include #include #include #include -#include - #include "block_service/block_service_manager.h" #include "bulk_load_types.h" #include "common/bulk_load_common.h" @@ -361,6 +362,30 @@ class replica_stub : public serverlet, public ref_counter replica_life_cycle get_replica_life_cycle(gpid id); void on_gc_replica(replica_stub_ptr this_, gpid id); + struct replica_gc_info + { + replica_ptr rep; + partition_status::type status; + mutation_log_ptr plog; + decree last_durable_decree; + int64_t init_offset_in_shared_log; + }; + using replica_gc_info_map = std::unordered_map; + + // Try to remove obsolete files of shared log for garbage collection according to the provided + // states of all replicas. The purpose is to remove all of the files of shared log, since it + // has been deprecated, and would not be appended any more. + void gc_slog(const replica_gc_info_map &replica_gc_map); + + // The number of flushed replicas for the garbage collection of shared log at a time should be + // limited. + void limit_flush_replicas_for_slog_gc(size_t prevent_gc_replica_count); + + // Flush rocksdb data to sst files for replicas to facilitate garbage collection of more files + // of shared log. + void flush_replicas_for_slog_gc(const replica_gc_info_map &replica_gc_map, + const std::set &prevent_gc_replicas); + void response_client(gpid id, bool is_read, dsn::message_ex *request, @@ -423,6 +448,7 @@ class replica_stub : public serverlet, public ref_counter FRIEND_TEST(open_replica_test, open_replica_add_decree_and_ballot_check); FRIEND_TEST(replica_error_test, test_auto_trash_of_corruption); FRIEND_TEST(replica_test, test_clear_on_failure); + FRIEND_TEST(GcSlogFlushFeplicasTest, FlushReplicas); typedef std::unordered_map opening_replicas; typedef std::unordered_map> @@ -436,6 +462,15 @@ class replica_stub : public serverlet, public ref_counter closing_replicas _closing_replicas; closed_replicas _closed_replicas; + // The number of replicas that prevent slog files from being removed for gc at the last round. + size_t _last_prevent_gc_replica_count; + + // The real limit of flushed replicas for the garbage collection of shared log. + size_t _real_log_shared_gc_flush_replicas_limit; + + // The number of flushed replicas, mocked only for test. + size_t _mock_flush_replicas_for_test; + mutation_log_ptr _log; ::dsn::rpc_address _primary_address; char _primary_address_str[64]; diff --git a/src/replica/replication_app_base.cpp b/src/replica/replication_app_base.cpp index db92bdf17e..4aa7a8e145 100644 --- a/src/replica/replication_app_base.cpp +++ b/src/replica/replication_app_base.cpp @@ -51,7 +51,6 @@ #include "runtime/task/task_code.h" #include "runtime/task/task_spec.h" #include "runtime/task/task_tracker.h" -#include "utils/autoref_ptr.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" #include "utils/blob.h" diff --git a/src/replica/test/mock_utils.h b/src/replica/test/mock_utils.h index 104bd2c072..48072f546a 100644 --- a/src/replica/test/mock_utils.h +++ b/src/replica/test/mock_utils.h @@ -445,7 +445,7 @@ class mock_mutation_log_shared : public mutation_log_shared dsn::task_tracker *tracker, aio_handler &&callback, int hash = 0, - int64_t *pending_size = nullptr) + int64_t *pending_size = nullptr) override { _mu_list.push_back(mu); return nullptr; diff --git a/src/replica/test/mutation_log_test.cpp b/src/replica/test/mutation_log_test.cpp index 6bca13336a..527c1e8d98 100644 --- a/src/replica/test/mutation_log_test.cpp +++ b/src/replica/test/mutation_log_test.cpp @@ -26,11 +26,16 @@ #include "replica/mutation_log.h" +// IWYU pragma: no_include // IWYU pragma: no_include +// IWYU pragma: no_include // IWYU pragma: no_include #include #include #include +#include +#include +#include #include #include "aio/aio_task.h" @@ -40,12 +45,17 @@ #include "replica/log_block.h" #include "replica/log_file.h" #include "replica/mutation.h" +#include "replica/replica_stub.h" #include "replica/test/mock_utils.h" #include "replica_test_base.h" +#include "rrdb/rrdb.code.definition.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" #include "utils/blob.h" +#include "utils/defer.h" +#include "utils/fail_point.h" #include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/ports.h" @@ -121,7 +131,7 @@ TEST(replication, log_file) lf->write_file_header(temp_writer, mdecrees); writer->add(temp_writer.get_buffer()); ASSERT_EQ(mdecrees, lf->previous_log_max_decrees()); - log_file_header &h = lf->header(); + const auto &h = lf->header(); ASSERT_EQ(100, h.start_global_offset); } @@ -450,6 +460,83 @@ class mutation_log_test : public replica_test_base ASSERT_GE(log_files.size(), 1); } } + + mutation_ptr generate_slog_mutation(const gpid &pid, const decree d, const std::string &data) + { + mutation_ptr mu(new mutation()); + mu->data.header.ballot = 1; + mu->data.header.decree = d; + mu->data.header.pid = pid; + mu->data.header.last_committed_decree = d - 1; + mu->data.header.log_offset = 0; + mu->data.header.timestamp = d; + + mu->data.updates.push_back(mutation_update()); + mu->data.updates.back().code = dsn::apps::RPC_RRDB_RRDB_PUT; + mu->data.updates.back().data = blob::create_from_bytes(std::string(data)); + + mu->client_requests.push_back(nullptr); + + return mu; + } + + void generate_slog_file(const std::vector> &replica_mutations, + mutation_log_ptr &mlog, + decree &d, + std::unordered_map &valid_start_offsets, + std::pair &slog_file_start_offset) + { + for (size_t i = 0; i < replica_mutations.size(); ++i) { + const auto &pid = replica_mutations[i].first; + + for (size_t j = 0; j < replica_mutations[i].second; ++j) { + if (i == 0) { + // Record the start offset of each slog file. + slog_file_start_offset.first = pid; + slog_file_start_offset.second = mlog->get_global_offset(); + } + + const auto &it = valid_start_offsets.find(pid); + if (it == valid_start_offsets.end()) { + // Add new partition with its start offset in slog. + valid_start_offsets.emplace(pid, mlog->get_global_offset()); + mlog->set_valid_start_offset_on_open(pid, mlog->get_global_offset()); + } + + // Append a mutation. + auto mu = generate_slog_mutation(pid, d++, "test data"); + mlog->append(mu, LPC_AIO_IMMEDIATE_CALLBACK, mlog->tracker(), nullptr, 0); + } + } + + // Wait until all mutations are written into this file. + mlog->tracker()->wait_outstanding_tasks(); + } + + void generate_slog_files(const std::vector>> &files, + mutation_log_ptr &mlog, + std::unordered_map &valid_start_offsets, + std::vector> &slog_file_start_offsets) + { + valid_start_offsets.clear(); + slog_file_start_offsets.resize(files.size()); + + decree d = 1; + for (size_t i = 0; i < files.size(); ++i) { + generate_slog_file(files[i], mlog, d, valid_start_offsets, slog_file_start_offsets[i]); + if (i + 1 < files.size()) { + // Do not create a new slog file after the last file is generated. + mlog->create_new_log_file(); + // Wait until file header is written. + mlog->tracker()->wait_outstanding_tasks(); + } + } + + // Close and reset `_current_log_file` since slog has been deprecated and would not be + // used again. + mlog->_current_log_file->close(); + mlog->_current_log_file = nullptr; + } }; TEST_F(mutation_log_test, replay_single_file_1000) { test_replay_single_file(1000); } @@ -606,5 +693,182 @@ TEST_F(mutation_log_test, reset_from_while_writing) mlog->flush(); ASSERT_EQ(actual.size(), expected.size()); } + +TEST_F(mutation_log_test, gc_slog) +{ + // Remove the slog dir and create a new one. + const std::string slog_dir("./slog_test"); + ASSERT_TRUE(dsn::utils::filesystem::remove_path(slog_dir)); + ASSERT_TRUE(dsn::utils::filesystem::create_directory(slog_dir)); + + // Create and open slog object, which would be closed at the end of the scope. + mutation_log_ptr mlog = new mutation_log_shared(slog_dir, 1, false); + auto cleanup = dsn::defer([mlog]() { mlog->close(); }); + ASSERT_EQ(ERR_OK, mlog->open(nullptr, nullptr)); + + // Each line describes a sequence of mutations written to specified replicas by + // specified numbers. + // + // From these sequences the decrees for each partition could be concluded as below: + // {1, 1}: 9 ~ 15 + // {1, 2}: 16 ~ 22 + // {2, 5}: 1 ~ 8, 23 ~ 38 + // {2, 7}: 39 ~ 46 + // {5, 6}: 47 ~ 73 + const std::vector>> files = { + {{{2, 5}, 8}, {{1, 1}, 7}, {{1, 2}, 2}}, + {{{1, 2}, 5}}, + {{{2, 5}, 16}, {{2, 7}, 8}, {{5, 6}, 27}}}; + + // Each line describes a progress of durable decrees for all of replicas: decrees are + // continuously being applied and becoming durable. + const std::vector> durable_decrees = { + {{{1, 1}, 10}, {{1, 2}, 17}, {{2, 5}, 6}, {{2, 7}, 39}, {{5, 6}, 47}}, + {{{1, 1}, 15}, {{1, 2}, 18}, {{2, 5}, 7}, {{2, 7}, 40}, {{5, 6}, 57}}, + {{{1, 1}, 15}, {{1, 2}, 20}, {{2, 5}, 8}, {{2, 7}, 42}, {{5, 6}, 61}}, + {{{1, 1}, 15}, {{1, 2}, 22}, {{2, 5}, 23}, {{2, 7}, 44}, {{5, 6}, 65}}, + {{{1, 1}, 15}, {{1, 2}, 22}, {{2, 5}, 27}, {{2, 7}, 46}, {{5, 6}, 66}}, + {{{1, 1}, 15}, {{1, 2}, 22}, {{2, 5}, 32}, {{2, 7}, 46}, {{5, 6}, 67}}, + {{{1, 1}, 15}, {{1, 2}, 22}, {{2, 5}, 38}, {{2, 7}, 46}, {{5, 6}, 72}}, + {{{1, 1}, 15}, {{1, 2}, 22}, {{2, 5}, 38}, {{2, 7}, 46}, {{5, 6}, 73}}, + }; + const std::vector remaining_slog_files = {3, 3, 2, 1, 1, 1, 1, 0}; + const std::vector> expected_prevent_gc_replicas = { + {{1, 1}, {1, 2}, {2, 5}, {2, 7}, {5, 6}}, + {{1, 2}, {2, 5}, {2, 7}, {5, 6}}, + {{1, 2}, {2, 5}, {2, 7}, {5, 6}}, + {{2, 5}, {2, 7}, {5, 6}}, + {{2, 5}, {5, 6}}, + {{2, 5}, {5, 6}}, + {{5, 6}}, + {}, + }; + + // Each line describes an action, that during a round (related to the index of + // `durable_decrees`), which replica should be reset to the start offset of an + // slog file (related to the index of `files` and `slog_file_start_offsets`). + const std::unordered_map set_to_slog_file_start_offsets = { + {2, 1}, + }; + + // Create slog files and write some data into them according to test cases. + std::unordered_map valid_start_offsets; + std::vector> slog_file_start_offsets; + generate_slog_files(files, mlog, valid_start_offsets, slog_file_start_offsets); + + for (size_t i = 0; i < durable_decrees.size(); ++i) { + std::cout << "Update No." << i << " group of durable decrees" << std::endl; + + // Update the progress of durable_decrees for each partition. + replica_log_info_map replica_durable_decrees; + for (const auto &d : durable_decrees[i]) { + replica_durable_decrees.emplace( + d.first, replica_log_info(d.second, valid_start_offsets[d.first])); + } + + // Test condition for `valid_start_offset`, see `can_gc_replica_slog`. + const auto &set_to_start = set_to_slog_file_start_offsets.find(i); + if (set_to_start != set_to_slog_file_start_offsets.end()) { + const auto &start_offset = slog_file_start_offsets[set_to_start->second]; + replica_durable_decrees[start_offset.first].valid_start_offset = start_offset.second; + } + + // Run garbage collection for a round. + std::set actual_prevent_gc_replicas; + mlog->garbage_collection(replica_durable_decrees, actual_prevent_gc_replicas); + + // Check if the number of remaining slog files after garbage collection is desired. + std::vector file_list; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(slog_dir, file_list, false)); + ASSERT_EQ(remaining_slog_files[i], file_list.size()); + + // Check if the replicas that prevent garbage collection (i.e. cannot be removed by + // garbage collection) is expected. + ASSERT_EQ(expected_prevent_gc_replicas[i], actual_prevent_gc_replicas); + } +} + +using gc_slog_flush_replicas_case = std::tuple, uint64_t, size_t, size_t, size_t>; + +class GcSlogFlushFeplicasTest : public testing::TestWithParam +{ +}; + +DSN_DECLARE_uint64(log_shared_gc_flush_replicas_limit); + +TEST_P(GcSlogFlushFeplicasTest, FlushReplicas) +{ + std::set prevent_gc_replicas; + size_t last_prevent_gc_replica_count; + uint64_t limit; + size_t last_limit; + size_t expected_flush_replicas; + std::tie(prevent_gc_replicas, + last_prevent_gc_replica_count, + limit, + last_limit, + expected_flush_replicas) = GetParam(); + + replica_stub::replica_gc_info_map replica_gc_map; + for (const auto &r : prevent_gc_replicas) { + replica_gc_map.emplace(r, replica_stub::replica_gc_info()); + } + + const auto reserved_log_shared_gc_flush_replicas_limit = + FLAGS_log_shared_gc_flush_replicas_limit; + FLAGS_log_shared_gc_flush_replicas_limit = limit; + + dsn::fail::setup(); + dsn::fail::cfg("mock_flush_replicas_for_slog_gc", "void(true)"); + + replica_stub stub; + stub._last_prevent_gc_replica_count = last_prevent_gc_replica_count; + stub._real_log_shared_gc_flush_replicas_limit = last_limit; + + stub.flush_replicas_for_slog_gc(replica_gc_map, prevent_gc_replicas); + EXPECT_EQ(expected_flush_replicas, stub._mock_flush_replicas_for_test); + + dsn::fail::teardown(); + + FLAGS_log_shared_gc_flush_replicas_limit = reserved_log_shared_gc_flush_replicas_limit; +} + +const std::vector gc_slog_flush_replicas_tests = { + // Initially, there is no limit on flushed replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 0, 0, 0, 6}, + // Initially, there is no limit on flushed replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 1, 0, 5, 6}, + // Initially, limit is less than the number of replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 0, 1, 0, 1}, + // Initially, limit is less than the number of replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 0, 2, 0, 2}, + // Initially, limit is just equal to the number of replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 0, 6, 0, 6}, + // Initially, limit is more than the number of replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 0, 7, 0, 6}, + // No replica has been flushed during previous round. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 6, 6, 6, 2}, + // No replica has been flushed during previous round. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 6, 1, 2, 1}, + // The previous limit is 0. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 7, 5, 0, 5}, + // The previous limit is infinite. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 7, 5, std::numeric_limits::max(), 5}, + // The number of previously flushed replicas is less than the previous limit. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 7, 5, 0, 5}, + // The number of previously flushed replicas reaches the previous limit. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 8, 6, 2, 4}, + // The number of previously flushed replicas reaches the previous limit. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 12, 6, 6, 6}, + // The number of previously flushed replicas is more than the previous limit. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 9, 3, 2, 3}, + // The number of previously flushed replicas is more than the previous limit. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 9, 5, 2, 4}, +}; + +INSTANTIATE_TEST_CASE_P(MutationLogTest, + GcSlogFlushFeplicasTest, + testing::ValuesIn(gc_slog_flush_replicas_tests)); + } // namespace replication } // namespace dsn diff --git a/src/server/config.ini b/src/server/config.ini index 2d2b7d19d6..543fa316f6 100644 --- a/src/server/config.ini +++ b/src/server/config.ini @@ -278,7 +278,7 @@ stateful = true plog_force_flush = false log_shared_file_size_mb = 128 - log_shared_file_count_limit = 100 + log_shared_gc_flush_replicas_limit = 64 log_shared_batch_buffer_kb = 0 log_shared_force_flush = false log_shared_pending_size_throttling_threshold_kb = 0 diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 8d6ac8c376..9bc325ba9d 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -2043,7 +2043,8 @@ ::dsn::error_code pegasus_server_impl::async_checkpoint(bool flush_memtable) } int64_t checkpoint_decree = 0; - ::dsn::error_code err = copy_checkpoint_to_dir_unsafe(tmp_dir.c_str(), &checkpoint_decree); + ::dsn::error_code err = + copy_checkpoint_to_dir_unsafe(tmp_dir.c_str(), &checkpoint_decree, flush_memtable); if (err != ::dsn::ERR_OK) { LOG_ERROR_PREFIX("copy_checkpoint_to_dir_unsafe failed with err = {}", err.to_string()); return ::dsn::ERR_LOCAL_APP_FAILURE; diff --git a/src/server/test/config.ini b/src/server/test/config.ini index 018d5b28d7..1ec547264a 100644 --- a/src/server/test/config.ini +++ b/src/server/test/config.ini @@ -187,7 +187,7 @@ log_private_reserve_max_size_mb = 0 log_private_reserve_max_time_seconds = 0 log_shared_file_size_mb = 32 -log_shared_file_count_limit = 32 +log_shared_gc_flush_replicas_limit = 64 log_shared_batch_buffer_kb = 0 log_shared_force_flush = false diff --git a/src/utils/autoref_ptr.h b/src/utils/autoref_ptr.h index a08ebc6ee4..a3a7dfd2b2 100644 --- a/src/utils/autoref_ptr.h +++ b/src/utils/autoref_ptr.h @@ -159,6 +159,8 @@ class ref_ptr void swap(ref_ptr &r) noexcept { std::swap(_obj, r._obj); } + void reset(T *obj = nullptr) { *this = obj; } + T *get() const { return _obj; } operator T *() const { return _obj; } diff --git a/src/utils/fmt_logging.h b/src/utils/fmt_logging.h index 35c60aaddf..0a6ac64d99 100644 --- a/src/utils/fmt_logging.h +++ b/src/utils/fmt_logging.h @@ -65,7 +65,8 @@ } while (false) #define CHECK(x, ...) CHECK_EXPRESSION(x, x, __VA_ARGS__) -#define CHECK_NOTNULL(p, ...) CHECK(p != nullptr, __VA_ARGS__) +#define CHECK_NOTNULL(p, ...) CHECK((p) != nullptr, __VA_ARGS__) +#define CHECK_NULL(p, ...) CHECK((p) == nullptr, __VA_ARGS__) // Macros for writing log message prefixed by log_prefix(). #define LOG_DEBUG_PREFIX(...) LOG_DEBUG("[{}] {}", log_prefix(), fmt::format(__VA_ARGS__)) From f1fee10dd56f3cc6a485a9ff7878b0a31cc6f2e8 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 25 Sep 2023 11:28:39 +0800 Subject: [PATCH 24/42] refactor(aio_test): refactor aio unit test (#1617) https://github.com/apache/incubator-pegasus/issues/887 There is no functional changes, but only refactor the aio unit test. The `aio_pwrite_incomplete` fail point injection has been removed, it's useless in the following aio refactor. --- src/aio/native_linux_aio_provider.cpp | 9 - src/aio/test/aio.cpp | 413 +++++++++++++++++--------- 2 files changed, 273 insertions(+), 149 deletions(-) diff --git a/src/aio/native_linux_aio_provider.cpp b/src/aio/native_linux_aio_provider.cpp index 470f3d63ac..56a0efa1e1 100644 --- a/src/aio/native_linux_aio_provider.cpp +++ b/src/aio/native_linux_aio_provider.cpp @@ -35,12 +35,10 @@ #include "aio/disk_engine.h" #include "runtime/service_engine.h" #include "runtime/task/async_calls.h" -#include "utils/fail_point.h" #include "utils/fmt_logging.h" #include "utils/latency_tracer.h" #include "utils/ports.h" #include "utils/safe_strerror_posix.h" -#include "utils/string_view.h" namespace dsn { @@ -99,13 +97,6 @@ error_code native_linux_aio_provider::write(const aio_context &aio_ctx, return resp; } - // mock the `ret` to reproduce the `write incomplete` case in the first write - FAIL_POINT_INJECT_NOT_RETURN_F("aio_pwrite_incomplete", [&](string_view s) -> void { - if (dsn_unlikely(buffer_offset == 0)) { - --ret; - } - }); - buffer_offset += ret; if (dsn_unlikely(buffer_offset != aio_ctx.buffer_size)) { LOG_WARNING( diff --git a/src/aio/test/aio.cpp b/src/aio/test/aio.cpp index 7037996526..a95cbeb8f0 100644 --- a/src/aio/test/aio.cpp +++ b/src/aio/test/aio.cpp @@ -24,156 +24,264 @@ * THE SOFTWARE. */ -#include #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include +#include #include +#include +#include #include #include +#include #include +#include #include "aio/aio_task.h" #include "aio/file_io.h" #include "runtime/task/task_code.h" #include "runtime/tool_api.h" +#include "test_util/test_util.h" #include "utils/autoref_ptr.h" +#include "utils/env.h" #include "utils/error_code.h" -#include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/ports.h" -#include "utils/strings.h" +#include "utils/test_macros.h" #include "utils/threadpool_code.h" -namespace dsn { -class disk_file; -} // namespace dsn - using namespace ::dsn; DEFINE_THREAD_POOL_CODE(THREAD_POOL_TEST_SERVER) DEFINE_TASK_CODE_AIO(LPC_AIO_TEST, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER); -TEST(core, aio) +class aio_test : public pegasus::encrypt_data_test_base { - fail::setup(); - fail::cfg("aio_pwrite_incomplete", "void()"); - const char *buffer = "hello, world"; - int len = (int)strlen(buffer); +public: + void SetUp() override { utils::filesystem::remove_path(kTestFileName); } - // write - auto fp = file::open("tmp", O_RDWR | O_CREAT | O_BINARY, 0666); + const std::string kTestFileName = "aio_test.txt"; +}; - std::list tasks; - uint64_t offset = 0; +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, aio_test, ::testing::Values(false)); - // new write - for (int i = 0; i < 100; i++) { - auto t = ::dsn::file::write(fp, buffer, len, offset, LPC_AIO_TEST, nullptr, nullptr); - tasks.push_back(t); - offset += len; - } +TEST_P(aio_test, basic) +{ + const char *kUnitBuffer = "hello, world"; + const size_t kUnitBufferLength = strlen(kUnitBuffer); + const int kTotalBufferCount = 100; + const int kBufferCountPerBatch = 10; + const int64_t kFileSize = kUnitBufferLength * kTotalBufferCount; + ASSERT_EQ(0, kTotalBufferCount % kBufferCountPerBatch); + + auto check_callback = [kUnitBufferLength](::dsn::error_code err, size_t n) { + // Use CHECK_* instead of ASSERT_* to exit the tests immediately when error occurs. + CHECK_EQ(ERR_OK, err); + CHECK_EQ(kUnitBufferLength, n); + }; + auto verify_data = [=]() { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + kTestFileName.c_str(), dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(kFileSize, file_size); + + // Create a read file handler. + auto rfile = file::open(kTestFileName.c_str(), O_RDONLY | O_BINARY, 0); + ASSERT_NE(rfile, nullptr); + + // 1. Check sequential read. + { + uint64_t offset = 0; + std::list tasks; + for (int i = 0; i < kTotalBufferCount; i++) { + char read_buffer[kUnitBufferLength + 1]; + read_buffer[kUnitBufferLength] = 0; + auto t = ::dsn::file::read(rfile, + read_buffer, + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + ASSERT_STREQ(kUnitBuffer, read_buffer); + } + } - for (auto &t : tasks) { - t->wait(); - } + // 2. Check concurrent read. + { + uint64_t offset = 0; + std::list tasks; + char read_buffers[kTotalBufferCount][kUnitBufferLength + 1]; + for (int i = 0; i < kTotalBufferCount; i++) { + read_buffers[i][kUnitBufferLength] = 0; + auto t = ::dsn::file::read(rfile, + read_buffers[i], + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + for (int i = 0; i < kTotalBufferCount; i++) { + ASSERT_STREQ(kUnitBuffer, read_buffers[i]); + } + } + ASSERT_EQ(ERR_OK, file::close(rfile)); + }; - // overwrite - offset = 0; - tasks.clear(); - for (int i = 0; i < 100; i++) { - auto t = ::dsn::file::write(fp, buffer, len, offset, LPC_AIO_TEST, nullptr, nullptr); - tasks.push_back(t); - offset += len; + // 1. Sequential write. + { + auto wfile = file::open(kTestFileName.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + ASSERT_NE(wfile, nullptr); + + uint64_t offset = 0; + std::list tasks; + for (int i = 0; i < kTotalBufferCount; i++) { + auto t = ::dsn::file::write(wfile, + kUnitBuffer, + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); } + NO_FATALS(verify_data()); - for (auto &t : tasks) { - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == (size_t)len); - } + // 2. Un-sequential write. + { + auto wfile = file::open(kTestFileName.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + ASSERT_NE(wfile, nullptr); - // vector write - tasks.clear(); - std::unique_ptr buffers(new dsn_file_buffer_t[100]); - for (int i = 0; i < 10; i++) { - buffers[i].buffer = static_cast(const_cast(buffer)); - buffers[i].size = len; - } - for (int i = 0; i < 10; i++) { - tasks.push_back(::dsn::file::write_vector( - fp, buffers.get(), 10, offset, LPC_AIO_TEST, nullptr, nullptr)); - offset += 10 * len; - } - for (auto &t : tasks) { - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == 10 * len); - } - auto err = file::close(fp); - EXPECT_TRUE(err == ERR_OK); - - // read - char *buffer2 = (char *)alloca((size_t)len); - fp = file::open("tmp", O_RDONLY | O_BINARY, 0); - - // concurrent read - offset = 0; - tasks.clear(); - for (int i = 0; i < 100; i++) { - auto t = ::dsn::file::read(fp, buffer2, len, offset, LPC_AIO_TEST, nullptr, nullptr); - tasks.push_back(t); - offset += len; - } + std::vector offsets; + offsets.reserve(kTotalBufferCount); + for (int i = 0; i < kTotalBufferCount; i++) { + offsets.push_back(i * kUnitBufferLength); + } - for (auto &t : tasks) { - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == (size_t)len); - } + std::random_device rd; + std::mt19937 gen(rd()); + std::shuffle(offsets.begin(), offsets.end(), gen); - // sequential read - offset = 0; - tasks.clear(); - for (int i = 0; i < 200; i++) { - buffer2[0] = 'x'; - auto t = ::dsn::file::read(fp, buffer2, len, offset, LPC_AIO_TEST, nullptr, nullptr); - offset += len; - - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == (size_t)len); - EXPECT_TRUE(dsn::utils::mequals(buffer, buffer2, len)); + std::list tasks; + for (const auto &offset : offsets) { + auto t = ::dsn::file::write(wfile, + kUnitBuffer, + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); } - - err = file::close(fp); - fail::teardown(); - EXPECT_TRUE(err == ERR_OK); - - utils::filesystem::remove_path("tmp"); + NO_FATALS(verify_data()); + + // 3. Overwrite. + { + auto wfile = file::open(kTestFileName.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + ASSERT_NE(wfile, nullptr); + + uint64_t offset = 0; + std::list tasks; + for (int i = 0; i < kTotalBufferCount; i++) { + auto t = ::dsn::file::write(wfile, + kUnitBuffer, + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); + } + NO_FATALS(verify_data()); + + // 4. Vector write. + { + auto wfile = file::open(kTestFileName.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + ASSERT_NE(wfile, nullptr); + + uint64_t offset = 0; + std::list tasks; + std::unique_ptr buffers(new dsn_file_buffer_t[kBufferCountPerBatch]); + for (int i = 0; i < kBufferCountPerBatch; i++) { + buffers[i].buffer = static_cast(const_cast(kUnitBuffer)); + buffers[i].size = kUnitBufferLength; + } + for (int i = 0; i < kTotalBufferCount / kBufferCountPerBatch; i++) { + tasks.push_back( + ::dsn::file::write_vector(wfile, + buffers.get(), + kBufferCountPerBatch, + offset, + LPC_AIO_TEST, + nullptr, + [=](::dsn::error_code err, size_t n) { + CHECK_EQ(ERR_OK, err); + CHECK_EQ(kBufferCountPerBatch * kUnitBufferLength, n); + })); + offset += kBufferCountPerBatch * kUnitBufferLength; + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kBufferCountPerBatch * kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); + } + NO_FATALS(verify_data()); } -TEST(core, aio_share) +TEST_P(aio_test, aio_share) { - auto fp = file::open("tmp", O_WRONLY | O_CREAT | O_BINARY, 0666); - EXPECT_TRUE(fp != nullptr); + auto wfile = file::open(kTestFileName.c_str(), O_WRONLY | O_CREAT | O_BINARY, 0666); + ASSERT_NE(wfile, nullptr); - auto fp2 = file::open("tmp", O_RDONLY | O_BINARY, 0); - EXPECT_TRUE(fp2 != nullptr); + auto rfile = file::open(kTestFileName.c_str(), O_RDONLY | O_BINARY, 0); + ASSERT_NE(rfile, nullptr); - file::close(fp); - file::close(fp2); - - utils::filesystem::remove_path("tmp"); + ASSERT_EQ(ERR_OK, file::close(wfile)); + ASSERT_EQ(ERR_OK, file::close(rfile)); } -TEST(core, operation_failed) +TEST_P(aio_test, operation_failed) { - fail::setup(); - fail::cfg("aio_pwrite_incomplete", "void()"); - - auto fp = file::open("tmp_test_file", O_WRONLY, 0600); - EXPECT_TRUE(fp == nullptr); - auto err = std::make_unique(); auto count = std::make_unique(); auto io_callback = [&err, &count](::dsn::error_code e, size_t n) { @@ -181,39 +289,42 @@ TEST(core, operation_failed) *count = n; }; - fp = file::open("tmp_test_file", O_WRONLY | O_CREAT | O_BINARY, 0666); - EXPECT_TRUE(fp != nullptr); - char buffer[512]; - const char *str = "hello file"; - auto t = ::dsn::file::write(fp, str, strlen(str), 0, LPC_AIO_TEST, nullptr, io_callback, 0); + auto wfile = file::open(kTestFileName.c_str(), O_WRONLY | O_CREAT | O_BINARY, 0666); + ASSERT_NE(wfile, nullptr); + + char buff[512] = {0}; + const char *kUnitBuffer = "hello file"; + const size_t kUnitBufferLength = strlen(kUnitBuffer); + auto t = ::dsn::file::write( + wfile, kUnitBuffer, kUnitBufferLength, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_OK && *count == strlen(str)); + ASSERT_EQ(ERR_OK, *err); + ASSERT_EQ(kUnitBufferLength, *count); - t = ::dsn::file::read(fp, buffer, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(wfile, buff, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_FILE_OPERATION_FAILED); + ASSERT_EQ(ERR_FILE_OPERATION_FAILED, *err); - auto fp2 = file::open("tmp_test_file", O_RDONLY | O_BINARY, 0); - EXPECT_TRUE(fp2 != nullptr); + auto rfile = file::open(kTestFileName.c_str(), O_RDONLY | O_BINARY, 0); + ASSERT_NE(nullptr, rfile); - t = ::dsn::file::read(fp2, buffer, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(rfile, buff, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_OK && *count == strlen(str)); - EXPECT_TRUE(dsn::utils::equals(buffer, str, 10)); + ASSERT_EQ(ERR_OK, *err); + ASSERT_EQ(kUnitBufferLength, *count); + ASSERT_STREQ(kUnitBuffer, buff); - t = ::dsn::file::read(fp2, buffer, 5, 0, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(rfile, buff, 5, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_OK && *count == 5); - EXPECT_TRUE(dsn::utils::equals(buffer, str, 5)); + ASSERT_EQ(ERR_OK, *err); + ASSERT_EQ(5, *count); + ASSERT_STREQ(kUnitBuffer, buff); - t = ::dsn::file::read(fp2, buffer, 512, 100, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(rfile, buff, 512, 100, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - LOG_INFO("error code: {}", *err); - file::close(fp); - file::close(fp2); - fail::teardown(); - - EXPECT_TRUE(utils::filesystem::remove_path("tmp_test_file")); + ASSERT_EQ(ERR_HANDLE_EOF, *err); + ASSERT_EQ(ERR_OK, file::close(wfile)); + ASSERT_EQ(ERR_OK, file::close(rfile)); } DEFINE_TASK_CODE_AIO(LPC_AIO_TEST_READ, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) @@ -223,22 +334,39 @@ struct aio_result dsn::error_code err; size_t sz; }; -TEST(core, dsn_file) + +TEST_P(aio_test, dsn_file) { - int64_t fin_size, fout_size; - ASSERT_TRUE(utils::filesystem::file_size("copy_source.txt", fin_size)); - ASSERT_LT(0, fin_size); + std::string src_file = "copy_source.txt"; + std::string dst_file = "copy_dest.txt"; + if (FLAGS_encrypt_data_at_rest) { + auto s = dsn::utils::encrypt_file(src_file, src_file + ".encrypted"); + ASSERT_TRUE(s.ok()) << s.ToString(); + src_file += ".encrypted"; + + s = dsn::utils::encrypt_file(dst_file, dst_file + ".encrypted"); + ASSERT_TRUE(s.ok()) << s.ToString(); + dst_file += ".encrypted"; + } + + int64_t src_file_size; + ASSERT_TRUE(utils::filesystem::file_size( + src_file, dsn::utils::FileDataType::kSensitive, src_file_size)); + ASSERT_LT(0, src_file_size); + std::string src_file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(src_file, src_file_md5)); + ASSERT_FALSE(src_file_md5.empty()); - dsn::disk_file *fin = file::open("copy_source.txt", O_RDONLY, 0); + auto fin = file::open(src_file.c_str(), O_RDONLY | O_BINARY, 0); ASSERT_NE(nullptr, fin); - dsn::disk_file *fout = file::open("copy_dest.txt", O_RDWR | O_CREAT | O_TRUNC, 0666); + auto fout = file::open(dst_file.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0666); ASSERT_NE(nullptr, fout); - char buffer[1024]; + char kUnitBuffer[1024]; uint64_t offset = 0; while (true) { aio_result rin; aio_task_ptr tin = file::read(fin, - buffer, + kUnitBuffer, 1024, offset, LPC_AIO_TEST_READ, @@ -270,7 +398,7 @@ TEST(core, dsn_file) aio_result rout; aio_task_ptr tout = file::write(fout, - buffer, + kUnitBuffer, rin.sz, offset, LPC_AIO_TEST_WRITE, @@ -296,10 +424,15 @@ TEST(core, dsn_file) offset += rin.sz; } - ASSERT_EQ((uint64_t)fin_size, offset); + ASSERT_EQ(static_cast(src_file_size), offset); ASSERT_EQ(ERR_OK, file::close(fout)); ASSERT_EQ(ERR_OK, file::close(fin)); - ASSERT_TRUE(utils::filesystem::file_size("copy_dest.txt", fout_size)); - ASSERT_EQ(fin_size, fout_size); + int64_t dst_file_size; + ASSERT_TRUE(utils::filesystem::file_size( + dst_file, dsn::utils::FileDataType::kSensitive, dst_file_size)); + ASSERT_EQ(src_file_size, dst_file_size); + std::string dst_file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(src_file, dst_file_md5)); + ASSERT_EQ(src_file_md5, dst_file_md5); } From 73d53dbbb923fc15a7b3b5f4def8186c2fc97c73 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 25 Sep 2023 16:45:15 +0800 Subject: [PATCH 25/42] refactor(hdfs): use rocksdb API to read/write file (#1618) https://github.com/apache/incubator-pegasus/issues/887 There is no functional changes, but only refactor the HDFS service and related unit test. The directio_writable_file.* files have been removed because it has been implemented by the rocksdb API options. --- src/block_service/directio_writable_file.cpp | 169 --------- src/block_service/directio_writable_file.h | 57 --- src/block_service/hdfs/CMakeLists.txt | 8 +- src/block_service/hdfs/hdfs_service.cpp | 181 ++++++---- src/block_service/test/config-test.ini | 4 +- src/block_service/test/hdfs_service_test.cpp | 358 +++++++++++-------- src/replica/CMakeLists.txt | 2 +- src/utils/env.cpp | 8 + src/utils/env.h | 4 +- 9 files changed, 333 insertions(+), 458 deletions(-) delete mode 100644 src/block_service/directio_writable_file.cpp delete mode 100644 src/block_service/directio_writable_file.h diff --git a/src/block_service/directio_writable_file.cpp b/src/block_service/directio_writable_file.cpp deleted file mode 100644 index 2c74ae05d2..0000000000 --- a/src/block_service/directio_writable_file.cpp +++ /dev/null @@ -1,169 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "block_service/directio_writable_file.h" - -#include -#include -#include // posix_memalign -#include -#include // getpagesize -#include -#include -#include - -#include "utils/flags.h" -#include "utils/fmt_logging.h" -#include "utils/safe_strerror_posix.h" - -namespace dsn { -namespace dist { -namespace block_service { - -DSN_DEFINE_uint32(replication, - direct_io_buffer_pages, - 64, - "Number of pages we need to set to direct io buffer"); -DSN_TAG_VARIABLE(direct_io_buffer_pages, FT_MUTABLE); - -DSN_DEFINE_bool(replication, - enable_direct_io, - false, - "Whether to enable direct I/O when download files"); -DSN_TAG_VARIABLE(enable_direct_io, FT_MUTABLE); - -const uint32_t g_page_size = getpagesize(); - -direct_io_writable_file::direct_io_writable_file(const std::string &file_path) - : _file_path(file_path), - _fd(-1), - _file_size(0), - _buffer(nullptr), - _buffer_size(FLAGS_direct_io_buffer_pages * g_page_size), - _offset(0) -{ -} - -direct_io_writable_file::~direct_io_writable_file() -{ - if (!_buffer || _fd < 0) { - return; - } - // Here is an ensurance, users shuold call finalize manually - CHECK_EQ_MSG(_offset, 0, "finalize() should be called before destructor"); - - ::free(_buffer); - CHECK_EQ_MSG( - 0, ::close(_fd), "Failed to close {}, err = {}", _file_path, utils::safe_strerror(errno)); -} - -bool direct_io_writable_file::initialize() -{ - if (posix_memalign(&_buffer, g_page_size, _buffer_size) != 0) { - LOG_ERROR("Allocate memaligned buffer failed, err = {}", utils::safe_strerror(errno)); - return false; - } - - int flag = O_WRONLY | O_TRUNC | O_CREAT; -#if !defined(__APPLE__) - flag |= O_DIRECT; -#endif - // TODO(yingchun): there maybe serious error of the disk driver when these system call failed, - // maybe just terminate the process or mark the disk as failed would be better - _fd = ::open(_file_path.c_str(), flag, S_IRUSR | S_IWUSR | S_IRGRP); - if (_fd < 0) { - LOG_ERROR("Failed to open {} with flag {}, err = {}", - _file_path, - flag, - utils::safe_strerror(errno)); - ::free(_buffer); - _buffer = nullptr; - return false; - } - return true; -} - -bool direct_io_writable_file::finalize() -{ - CHECK(_buffer && _fd >= 0, "Initialize the instance first"); - - if (_offset > 0) { - ssize_t written_bytes = ::write(_fd, _buffer, _buffer_size); - if (dsn_unlikely(written_bytes < 0)) { - LOG_ERROR("Failed to write the last chunk, file_path = {}, err = {}", - _file_path, - utils::safe_strerror(errno)); - return false; - } - // TODO(yingchun): would better to retry - if (dsn_unlikely(written_bytes != _buffer_size)) { - LOG_ERROR("Failed to write the last chunk, file_path = {}, data bytes = {}, written " - "bytes = {}", - _file_path, - _buffer_size, - written_bytes); - return false; - } - _offset = 0; - if (::ftruncate(_fd, _file_size) < 0) { - LOG_ERROR("Failed to truncate {}, err = {}", _file_path, utils::safe_strerror(errno)); - return false; - } - } - return true; -} - -bool direct_io_writable_file::write(const char *s, size_t n) -{ - CHECK(_buffer && _fd >= 0, "Initialize the instance first"); - - uint32_t remaining = n; - while (remaining > 0) { - uint32_t bytes = std::min((_buffer_size - _offset), remaining); - memcpy((char *)_buffer + _offset, s, bytes); - _offset += bytes; - remaining -= bytes; - s += bytes; - // buffer is full, flush to file - if (_offset == _buffer_size) { - ssize_t written_bytes = ::write(_fd, _buffer, _buffer_size); - if (dsn_unlikely(written_bytes < 0)) { - LOG_ERROR("Failed to write chunk, file_path = {}, err = {}", - _file_path, - utils::safe_strerror(errno)); - return false; - } - // TODO(yingchun): would better to retry - if (dsn_unlikely(written_bytes != _buffer_size)) { - LOG_ERROR( - "Failed to write chunk, file_path = {}, data bytes = {}, written bytes = {}", - _file_path, - _buffer_size, - written_bytes); - return false; - } - // reset offset - _offset = 0; - } - } - _file_size += n; - return true; -} - -} // namespace block_service -} // namespace dist -} // namespace dsn diff --git a/src/block_service/directio_writable_file.h b/src/block_service/directio_writable_file.h deleted file mode 100644 index d4f99949ff..0000000000 --- a/src/block_service/directio_writable_file.h +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "utils/ports.h" - -namespace dsn { -namespace dist { -namespace block_service { - -class direct_io_writable_file -{ -public: - explicit direct_io_writable_file(const std::string &file_path); - ~direct_io_writable_file(); - - bool initialize(); - bool write(const char *s, size_t n); - bool finalize(); - -private: - DISALLOW_COPY_AND_ASSIGN(direct_io_writable_file); - - std::string _file_path; - int _fd; - uint32_t _file_size; - - // page size aligned buffer - void *_buffer; - // buffer size - uint32_t _buffer_size; - // buffer offset - uint32_t _offset; -}; - -} // namespace block_service -} // namespace dist -} // namespace dsn diff --git a/src/block_service/hdfs/CMakeLists.txt b/src/block_service/hdfs/CMakeLists.txt index 803e85bec3..2bba8b96bb 100644 --- a/src/block_service/hdfs/CMakeLists.txt +++ b/src/block_service/hdfs/CMakeLists.txt @@ -17,20 +17,16 @@ set(MY_PROJ_NAME dsn.block_service.hdfs) -set(DIRECTIO_SRC - ../directio_writable_file.cpp - ) - #Source files under CURRENT project directory will be automatically included. #You can manually set MY_PROJ_SRC to include source files under other directories. -set(MY_PROJ_SRC "${DIRECTIO_SRC}") +set(MY_PROJ_SRC "") #Search mode for source files under CURRENT project directory ? #"GLOB_RECURSE" for recursive search #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS hdfs) +set(MY_PROJ_LIBS hdfs rocksdb) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/hdfs/hdfs_service.cpp b/src/block_service/hdfs/hdfs_service.cpp index 459108e111..5909f40de0 100644 --- a/src/block_service/hdfs/hdfs_service.cpp +++ b/src/block_service/hdfs/hdfs_service.cpp @@ -17,19 +17,21 @@ #include #include +#include #include -#include #include #include -#include "block_service/directio_writable_file.h" #include "hdfs/hdfs.h" #include "hdfs_service.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/task/async_calls.h" #include "runtime/task/task.h" #include "utils/TokenBucket.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -37,6 +39,8 @@ #include "utils/safe_strerror_posix.h" #include "utils/strings.h" +DSN_DECLARE_bool(enable_direct_io); + struct hdfsBuilder; namespace dsn { @@ -65,8 +69,6 @@ DSN_DEFINE_uint64(replication, "hdfs write batch size, the default value is 64MB"); DSN_TAG_VARIABLE(hdfs_write_batch_size_bytes, FT_MUTABLE); -DSN_DECLARE_bool(enable_direct_io); - hdfs_service::hdfs_service() { _read_token_bucket.reset(new folly::DynamicTokenBucket()); @@ -108,12 +110,12 @@ error_code hdfs_service::create_fs() hdfsBuilderSetNameNode(builder, _hdfs_name_node.c_str()); _fs = hdfsBuilderConnect(builder); if (!_fs) { - LOG_ERROR("Fail to connect hdfs name node {}, error: {}.", + LOG_ERROR("Fail to connect HDFS name node {}, error: {}.", _hdfs_name_node, utils::safe_strerror(errno)); return ERR_FS_INTERNAL; } - LOG_INFO("Succeed to connect hdfs name node {}.", _hdfs_name_node); + LOG_INFO("Succeed to connect HDFS name node {}.", _hdfs_name_node); return ERR_OK; } @@ -122,10 +124,10 @@ void hdfs_service::close() // This method should be carefully called. // Calls to hdfsDisconnect() by individual threads would terminate // all other connections handed out via hdfsConnect() to the same URI. - LOG_INFO("Try to disconnect hdfs."); + LOG_INFO("Try to disconnect HDFS."); int result = hdfsDisconnect(_fs); if (result == -1) { - LOG_ERROR("Fail to disconnect from the hdfs file system, error: {}.", + LOG_ERROR("Fail to disconnect from the HDFS file system, error: {}.", utils::safe_strerror(errno)); } // Even if there is an error, the resources associated with the hdfsFS will be freed. @@ -134,7 +136,7 @@ void hdfs_service::close() std::string hdfs_service::get_hdfs_entry_name(const std::string &hdfs_path) { - // get exact file name from an hdfs path. + // get exact file name from an HDFS path. int pos = hdfs_path.find_last_of("/"); return hdfs_path.substr(pos + 1); } @@ -305,7 +307,7 @@ error_code hdfs_file_object::write_data_in_batches(const char *data, hdfsFile write_file = hdfsOpenFile(_service->get_fs(), file_name().c_str(), O_WRONLY | O_CREAT, 0, 0, 0); if (!write_file) { - LOG_ERROR("Failed to open hdfs file {} for writting, error: {}.", + LOG_ERROR("Failed to open HDFS file {} for writting, error: {}.", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; @@ -323,7 +325,7 @@ error_code hdfs_file_object::write_data_in_batches(const char *data, (void *)(data + cur_pos), static_cast(write_len)); if (num_written_bytes == -1) { - LOG_ERROR("Failed to write hdfs file {}, error: {}.", + LOG_ERROR("Failed to write HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); hdfsCloseFile(_service->get_fs(), write_file); @@ -333,18 +335,18 @@ error_code hdfs_file_object::write_data_in_batches(const char *data, } if (hdfsHFlush(_service->get_fs(), write_file) != 0) { LOG_ERROR( - "Failed to flush hdfs file {}, error: {}.", file_name(), utils::safe_strerror(errno)); + "Failed to flush HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); hdfsCloseFile(_service->get_fs(), write_file); return ERR_FS_INTERNAL; } written_size = cur_pos; if (hdfsCloseFile(_service->get_fs(), write_file) != 0) { LOG_ERROR( - "Failed to close hdfs file {}, error: {}", file_name(), utils::safe_strerror(errno)); + "Failed to close HDFS file {}, error: {}", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; } - LOG_INFO("start to synchronize meta data after successfully wrote data to hdfs"); + LOG_INFO("start to synchronize meta data after successfully wrote data to HDFS"); return get_file_meta(); } @@ -376,23 +378,51 @@ dsn::task_ptr hdfs_file_object::upload(const upload_request &req, add_ref(); auto upload_background = [this, req, t]() { + LOG_INFO("start to upload from '{}' to '{}'", req.input_local_name, file_name()); + upload_response resp; - resp.uploaded_size = 0; - std::ifstream is(req.input_local_name, std::ios::binary | std::ios::in); - if (is.is_open()) { - int64_t file_sz = 0; - dsn::utils::filesystem::file_size(req.input_local_name, file_sz); - std::unique_ptr buffer(new char[file_sz]); - is.read(buffer.get(), file_sz); - is.close(); - resp.err = write_data_in_batches(buffer.get(), file_sz, resp.uploaded_size); - } else { - LOG_ERROR("HDFS upload failed: open local file {} failed when upload to {}, error: {}", - req.input_local_name, - file_name(), - utils::safe_strerror(errno)); - resp.err = dsn::ERR_FILE_OPERATION_FAILED; - } + do { + rocksdb::EnvOptions env_options; + env_options.use_direct_reads = FLAGS_enable_direct_io; + std::unique_ptr rfile; + auto s = rocksdb::Env::Default()->NewSequentialFile( + req.input_local_name, &rfile, env_options); + if (!s.ok()) { + LOG_ERROR( + "open local file '{}' failed, err = {}", req.input_local_name, s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + int64_t file_size; + if (!dsn::utils::filesystem::file_size( + req.input_local_name, dsn::utils::FileDataType::kSensitive, file_size)) { + LOG_ERROR("get size of local file '{}' failed", req.input_local_name); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + rocksdb::Slice result; + char scratch[file_size]; + s = rfile->Read(file_size, &result, scratch); + if (!s.ok()) { + LOG_ERROR( + "read local file '{}' failed, err = {}", req.input_local_name, s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + resp.err = write_data_in_batches(result.data(), result.size(), resp.uploaded_size); + if (resp.err != ERR_OK) { + LOG_ERROR("write data to remote '{}' failed, err = {}", file_name(), resp.err); + break; + } + + LOG_INFO("finish to upload from '{}' to '{}', size = {}", + req.input_local_name, + file_name(), + resp.uploaded_size); + } while (false); t->enqueue_with(resp); release_ref(); }; @@ -417,7 +447,7 @@ error_code hdfs_file_object::read_data_in_batches(uint64_t start_pos, hdfsFile read_file = hdfsOpenFile(_service->get_fs(), file_name().c_str(), O_RDONLY, 0, 0, 0); if (!read_file) { - LOG_ERROR("Failed to open hdfs file {} for reading, error: {}.", + LOG_ERROR("Failed to open HDFS file {} for reading, error: {}.", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; @@ -446,7 +476,7 @@ error_code hdfs_file_object::read_data_in_batches(uint64_t start_pos, cur_pos += num_read_bytes; dst_buf += num_read_bytes; } else if (num_read_bytes == -1) { - LOG_ERROR("Failed to read hdfs file {}, error: {}.", + LOG_ERROR("Failed to read HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); read_success = false; @@ -455,7 +485,7 @@ error_code hdfs_file_object::read_data_in_batches(uint64_t start_pos, } if (hdfsCloseFile(_service->get_fs(), read_file) != 0) { LOG_ERROR( - "Failed to close hdfs file {}, error: {}.", file_name(), utils::safe_strerror(errno)); + "Failed to close HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; } if (read_success) { @@ -504,48 +534,53 @@ dsn::task_ptr hdfs_file_object::download(const download_request &req, auto download_background = [this, req, t]() { download_response resp; resp.downloaded_size = 0; - std::string read_buffer; - size_t read_length = 0; - resp.err = - read_data_in_batches(req.remote_pos, req.remote_length, read_buffer, read_length); - if (resp.err == ERR_OK) { - bool write_succ = false; - if (FLAGS_enable_direct_io) { - auto dio_file = std::make_unique(req.output_local_name); - do { - if (!dio_file->initialize()) { - break; - } - bool wr_ret = dio_file->write(read_buffer.c_str(), read_length); - if (!wr_ret) { - break; - } - if (dio_file->finalize()) { - resp.downloaded_size = read_length; - resp.file_md5 = utils::string_md5(read_buffer.c_str(), read_length); - write_succ = true; - } - } while (0); - } else { - std::ofstream out(req.output_local_name, - std::ios::binary | std::ios::out | std::ios::trunc); - if (out.is_open()) { - out.write(read_buffer.c_str(), read_length); - out.close(); - resp.downloaded_size = read_length; - resp.file_md5 = utils::string_md5(read_buffer.c_str(), read_length); - write_succ = true; - } + resp.err = ERR_OK; + bool write_succ = false; + std::string target_file = req.output_local_name; + do { + LOG_INFO("start to download from '{}' to '{}'", file_name(), target_file); + + std::string read_buffer; + size_t read_length = 0; + resp.err = + read_data_in_batches(req.remote_pos, req.remote_length, read_buffer, read_length); + if (resp.err != ERR_OK) { + LOG_ERROR("read data from remote '{}' failed, err = {}", file_name(), resp.err); + break; } - if (!write_succ) { - LOG_ERROR("HDFS download failed: fail to open localfile {} when download {}, " - "error: {}", - req.output_local_name, - file_name(), - utils::safe_strerror(errno)); - resp.err = ERR_FILE_OPERATION_FAILED; - resp.downloaded_size = 0; + + rocksdb::EnvOptions env_options; + env_options.use_direct_writes = FLAGS_enable_direct_io; + std::unique_ptr wfile; + auto s = rocksdb::Env::Default()->NewWritableFile(target_file, &wfile, env_options); + if (!s.ok()) { + LOG_ERROR("create local file '{}' failed, err = {}", target_file, s.ToString()); + break; + } + + s = wfile->Append(rocksdb::Slice(read_buffer.data(), read_length)); + if (!s.ok()) { + LOG_ERROR("append local file '{}' failed, err = {}", target_file, s.ToString()); + break; + } + + s = wfile->Fsync(); + if (!s.ok()) { + LOG_ERROR("fsync local file '{}' failed, err = {}", target_file, s.ToString()); + break; } + + resp.downloaded_size = read_length; + resp.file_md5 = utils::string_md5(read_buffer.c_str(), read_length); + write_succ = true; + } while (false); + + if (!write_succ) { + LOG_ERROR("HDFS download failed: fail to write local file {} when download {}", + target_file, + file_name()); + resp.err = ERR_FILE_OPERATION_FAILED; + resp.downloaded_size = 0; } t->enqueue_with(resp); release_ref(); diff --git a/src/block_service/test/config-test.ini b/src/block_service/test/config-test.ini index c1996d5518..2acb86e2bb 100644 --- a/src/block_service/test/config-test.ini +++ b/src/block_service/test/config-test.ini @@ -53,7 +53,7 @@ max_size = 150 worker_count = 8 [hdfs_test] -test_name_node = -test_backup_path = +test_name_node = +test_backup_path = num_test_file_lines = 4096 num_total_files_for_hdfs_concurrent_test = 64 diff --git a/src/block_service/test/hdfs_service_test.cpp b/src/block_service/test/hdfs_service_test.cpp index 6cfcfb379c..d7b33a8daf 100644 --- a/src/block_service/test/hdfs_service_test.cpp +++ b/src/block_service/test/hdfs_service_test.cpp @@ -15,15 +15,20 @@ // specific language governing permissions and limitations // under the License. +#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include #include #include #include #include -#include +#include #include #include #include @@ -35,27 +40,23 @@ #include "runtime/task/task.h" #include "runtime/task/task_code.h" #include "runtime/task/task_tracker.h" +#include "test_util/test_util.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" #include "utils/enum_helper.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" -#include "utils/strings.h" +#include "utils/fmt_logging.h" +#include "utils/test_macros.h" #include "utils/threadpool_code.h" using namespace dsn; using namespace dsn::dist::block_service; -static std::string example_name_node = ""; -static std::string example_backup_path = ""; -// Please modify following paras in 'config-test.ini' to enable hdfs_service_test, -// or hdfs_service_test will be skipped and return true. -DSN_DEFINE_string(hdfs_test, test_name_node, "", "hdfs name node"); -DSN_DEFINE_string(hdfs_test, - test_backup_path, - "", - "path for uploading and downloading test files"); +DSN_DEFINE_string(hdfs_test, test_name_node, "", "hdfs name node"); +DSN_DEFINE_string(hdfs_test, test_backup_path, "", "path for uploading and downloading test files"); DSN_DEFINE_uint32(hdfs_test, num_test_file_lines, 4096, "number of lines in test file"); DSN_DEFINE_uint32(hdfs_test, @@ -65,206 +66,257 @@ DSN_DEFINE_uint32(hdfs_test, DEFINE_TASK_CODE(LPC_TEST_HDFS, TASK_PRIORITY_HIGH, dsn::THREAD_POOL_DEFAULT) -class HDFSClientTest : public testing::Test +class HDFSClientTest : public pegasus::encrypt_data_test_base { protected: - virtual void SetUp() override; - virtual void TearDown() override; - void generate_test_file(const char *filename); - void write_test_files_async(task_tracker *tracker); - std::string name_node; - std::string backup_path; - std::string local_test_dir; - std::string test_data_str; + void generate_test_file(const std::string &filename); + void write_test_files_async(const std::string &local_test_path, + int total_files, + int *success_count, + task_tracker *tracker); }; -void HDFSClientTest::SetUp() +void HDFSClientTest::generate_test_file(const std::string &filename) { - name_node = FLAGS_test_name_node; - backup_path = FLAGS_test_backup_path; - local_test_dir = "test_dir"; - test_data_str = ""; - for (int i = 0; i < FLAGS_num_test_file_lines; ++i) { - test_data_str += "test"; + int lines = FLAGS_num_test_file_lines; + std::unique_ptr wfile; + auto s = rocksdb::Env::Default()->NewWritableFile(filename, &wfile, rocksdb::EnvOptions()); + ASSERT_TRUE(s.ok()) << s.ToString(); + for (int i = 0; i < lines; ++i) { + rocksdb::Slice data(fmt::format("{:04}d_this_is_a_simple_test_file\n", i)); + s = wfile->Append(data); + ASSERT_TRUE(s.ok()) << s.ToString(); } + s = wfile->Fsync(); + ASSERT_TRUE(s.ok()) << s.ToString(); } -void HDFSClientTest::TearDown() {} - -void HDFSClientTest::generate_test_file(const char *filename) +void HDFSClientTest::write_test_files_async(const std::string &local_test_path, + int total_files, + int *success_count, + task_tracker *tracker) { - // generate a local test file. - int lines = FLAGS_num_test_file_lines; - FILE *fp = fopen(filename, "wb"); - for (int i = 0; i < lines; ++i) { - fprintf(fp, "%04d_this_is_a_simple_test_file\n", i); + dsn::utils::filesystem::create_directory(local_test_path); + std::string local_test_data; + for (int i = 0; i < FLAGS_num_test_file_lines; ++i) { + local_test_data += "test"; + } + for (int i = 0; i < total_files; ++i) { + tasking::enqueue( + LPC_TEST_HDFS, tracker, [&local_test_path, &local_test_data, i, success_count]() { + // mock the writing process in hdfs_file_object::download(). + std::string test_file_name = local_test_path + "/test_file_" + std::to_string(i); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(local_test_data), + test_file_name, + /* should_sync */ true); + if (s.ok()) { + ++(*success_count); + } else { + CHECK(s.IsIOError(), "{}", s.ToString()); + auto pos1 = s.ToString().find( + "IO error: No such file or directory: While open a file for appending: "); + auto pos2 = s.ToString().find("IO error: While appending to file: "); + CHECK(pos1 == 0 || pos2 == 0, "{}", s.ToString()); + } + }); } - fclose(fp); } -void HDFSClientTest::write_test_files_async(task_tracker *tracker) +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, HDFSClientTest, ::testing::Values(false)); + +TEST_P(HDFSClientTest, test_hdfs_read_write) { - dsn::utils::filesystem::create_directory(local_test_dir); - for (int i = 0; i < 100; ++i) { - tasking::enqueue(LPC_TEST_HDFS, tracker, [this, i]() { - // mock the writing process in hdfs_file_object::download(). - std::string test_file_name = local_test_dir + "/test_file_" + std::to_string(i); - std::ofstream out(test_file_name, std::ios::binary | std::ios::out | std::ios::trunc); - if (out.is_open()) { - out.write(test_data_str.c_str(), test_data_str.length()); - } - out.close(); - }); + if (strlen(FLAGS_test_name_node) == 0 || strlen(FLAGS_test_backup_path) == 0) { + // TODO(yingchun): use GTEST_SKIP after upgrading gtest. + std::cout << "Set hdfs_test.* configs in config-test.ini to enable hdfs_service_test." + << std::endl; + return; } + + auto s = std::make_shared(); + ASSERT_EQ(dsn::ERR_OK, s->initialize({FLAGS_test_name_node, FLAGS_test_backup_path})); + + const std::string kRemoteTestPath = "hdfs_client_test"; + const std::string kRemoteTestRWFile = kRemoteTestPath + "/test_write_file"; + const std::string kTestBuffer = "write_hello_world_for_test"; + const int kTestBufferLength = kTestBuffer.size(); + + // 1. clean up all old file in remote test directory. + printf("clean up all old files.\n"); + remove_path_response rem_resp; + s->remove_path(remove_path_request{kRemoteTestPath, true}, + LPC_TEST_HDFS, + [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, + nullptr) + ->wait(); + ASSERT_TRUE(dsn::ERR_OK == rem_resp.err || dsn::ERR_OBJECT_NOT_FOUND == rem_resp.err); + + // 2. create file. + printf("test write operation.\n"); + create_file_response cf_resp; + s->create_file(create_file_request{kRemoteTestRWFile, false}, + LPC_TEST_HDFS, + [&cf_resp](const create_file_response &r) { cf_resp = r; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, cf_resp.err); + + // 3. write file. + dsn::blob bb(kTestBuffer.c_str(), 0, kTestBufferLength); + write_response w_resp; + cf_resp.file_handle + ->write(write_request{bb}, + LPC_TEST_HDFS, + [&w_resp](const write_response &w) { w_resp = w; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, w_resp.err); + ASSERT_EQ(kTestBufferLength, w_resp.written_size); + ASSERT_EQ(kTestBufferLength, cf_resp.file_handle->get_size()); + + // 4. read file. + printf("test read just written contents.\n"); + read_response r_resp; + cf_resp.file_handle + ->read(read_request{0, -1}, + LPC_TEST_HDFS, + [&r_resp](const read_response &r) { r_resp = r; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, r_resp.err); + ASSERT_EQ(kTestBufferLength, r_resp.buffer.length()); + ASSERT_EQ(kTestBuffer, r_resp.buffer.to_string()); + + // 5. partial read. + const uint64_t kOffset = 5; + const int64_t kSize = 10; + cf_resp.file_handle + ->read(read_request{kOffset, kSize}, + LPC_TEST_HDFS, + [&r_resp](const read_response &r) { r_resp = r; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, r_resp.err); + ASSERT_EQ(kSize, r_resp.buffer.length()); + ASSERT_EQ(kTestBuffer.substr(kOffset, kSize), r_resp.buffer.to_string()); } -TEST_F(HDFSClientTest, test_basic_operation) +TEST_P(HDFSClientTest, test_upload_and_download) { - if (name_node == example_name_node || backup_path == example_backup_path) { + if (strlen(FLAGS_test_name_node) == 0 || strlen(FLAGS_test_backup_path) == 0) { + // TODO(yingchun): use GTEST_SKIP after upgrading gtest. + std::cout << "Set hdfs_test.* configs in config-test.ini to enable hdfs_service_test." + << std::endl; return; } - std::vector args = {name_node, backup_path}; - std::shared_ptr s = std::make_shared(); - ASSERT_EQ(dsn::ERR_OK, s->initialize(args)); + auto s = std::make_shared(); + ASSERT_EQ(dsn::ERR_OK, s->initialize({FLAGS_test_name_node, FLAGS_test_backup_path})); - std::string local_test_file = "test_file"; - std::string remote_test_file = "hdfs_client_test/test_file"; - int64_t test_file_size = 0; + const std::string kLocalFile = "test_file"; + const std::string kRemoteTestPath = "hdfs_client_test"; + const std::string kRemoteTestFile = kRemoteTestPath + "/test_file"; - generate_test_file(local_test_file.c_str()); - dsn::utils::filesystem::file_size(local_test_file, test_file_size); + // 0. generate test file. + NO_FATALS(generate_test_file(kLocalFile)); + int64_t local_file_size = 0; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kLocalFile, dsn::utils::FileDataType::kSensitive, local_file_size)); + std::string local_file_md5sum; + dsn::utils::filesystem::md5sum(kLocalFile, local_file_md5sum); - // fisrt clean up all old file in test directory. + // 1. clean up all old file in remote test directory. printf("clean up all old files.\n"); remove_path_response rem_resp; - s->remove_path(remove_path_request{"hdfs_client_test", true}, + s->remove_path(remove_path_request{kRemoteTestPath, true}, LPC_TEST_HDFS, [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, nullptr) ->wait(); ASSERT_TRUE(dsn::ERR_OK == rem_resp.err || dsn::ERR_OBJECT_NOT_FOUND == rem_resp.err); - // test upload file. - printf("create and upload: %s.\n", remote_test_file.c_str()); + // 2. create file. + printf("create and upload: %s.\n", kRemoteTestFile.c_str()); create_file_response cf_resp; - s->create_file(create_file_request{remote_test_file, true}, + s->create_file(create_file_request{kRemoteTestFile, true}, LPC_TEST_HDFS, [&cf_resp](const create_file_response &r) { cf_resp = r; }, nullptr) ->wait(); - ASSERT_EQ(cf_resp.err, dsn::ERR_OK); + ASSERT_EQ(dsn::ERR_OK, cf_resp.err); + + // 3. upload file. upload_response u_resp; cf_resp.file_handle - ->upload(upload_request{local_test_file}, + ->upload(upload_request{kLocalFile}, LPC_TEST_HDFS, [&u_resp](const upload_response &r) { u_resp = r; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, u_resp.err); - ASSERT_EQ(test_file_size, cf_resp.file_handle->get_size()); + ASSERT_EQ(local_file_size, cf_resp.file_handle->get_size()); - // test list directory. + // 4. list directory. ls_response l_resp; - s->list_dir(ls_request{"hdfs_client_test"}, + s->list_dir(ls_request{kRemoteTestPath}, LPC_TEST_HDFS, [&l_resp](const ls_response &resp) { l_resp = resp; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, l_resp.err); ASSERT_EQ(1, l_resp.entries->size()); - ASSERT_EQ("test_file", l_resp.entries->at(0).entry_name); + ASSERT_EQ(kLocalFile, l_resp.entries->at(0).entry_name); ASSERT_EQ(false, l_resp.entries->at(0).is_directory); - // test download file. + // 5. download file. download_response d_resp; - printf("test download %s.\n", remote_test_file.c_str()); - s->create_file(create_file_request{remote_test_file, false}, + printf("test download %s.\n", kRemoteTestFile.c_str()); + s->create_file(create_file_request{kRemoteTestFile, false}, LPC_TEST_HDFS, [&cf_resp](const create_file_response &resp) { cf_resp = resp; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_EQ(test_file_size, cf_resp.file_handle->get_size()); - std::string local_file_for_download = "test_file_d"; + ASSERT_EQ(local_file_size, cf_resp.file_handle->get_size()); + std::string kLocalDownloadFile = "test_file_d"; cf_resp.file_handle - ->download(download_request{local_file_for_download, 0, -1}, + ->download(download_request{kLocalDownloadFile, 0, -1}, LPC_TEST_HDFS, [&d_resp](const download_response &resp) { d_resp = resp; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, d_resp.err); - ASSERT_EQ(test_file_size, d_resp.downloaded_size); - - // compare local_test_file and local_file_for_download. - int64_t file_size = 0; - dsn::utils::filesystem::file_size(local_file_for_download, file_size); - ASSERT_EQ(test_file_size, file_size); - std::string test_file_md5sum; - dsn::utils::filesystem::md5sum(local_test_file, test_file_md5sum); - std::string downloaded_file_md5sum; - dsn::utils::filesystem::md5sum(local_file_for_download, downloaded_file_md5sum); - ASSERT_EQ(test_file_md5sum, downloaded_file_md5sum); - - // test read and write. - printf("test read write operation.\n"); - std::string test_write_file = "hdfs_client_test/test_write_file"; - s->create_file(create_file_request{test_write_file, false}, - LPC_TEST_HDFS, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - const char *test_buffer = "write_hello_world_for_test"; - int length = strlen(test_buffer); - dsn::blob bb(test_buffer, 0, length); - write_response w_resp; - cf_resp.file_handle - ->write(write_request{bb}, - LPC_TEST_HDFS, - [&w_resp](const write_response &w) { w_resp = w; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, w_resp.err); - ASSERT_EQ(length, w_resp.written_size); - ASSERT_EQ(length, cf_resp.file_handle->get_size()); - printf("test read just written contents.\n"); - read_response r_resp; - cf_resp.file_handle - ->read(read_request{0, -1}, - LPC_TEST_HDFS, - [&r_resp](const read_response &r) { r_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, r_resp.err); - ASSERT_EQ(length, r_resp.buffer.length()); - ASSERT_TRUE(dsn::utils::mequals(r_resp.buffer.data(), test_buffer, length)); - - // test partitial read. - cf_resp.file_handle - ->read(read_request{5, 10}, - LPC_TEST_HDFS, - [&r_resp](const read_response &r) { r_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, r_resp.err); - ASSERT_EQ(10, r_resp.buffer.length()); - ASSERT_TRUE(dsn::utils::mequals(r_resp.buffer.data(), test_buffer + 5, 10)); - - // clean up local test files. - utils::filesystem::remove_path(local_test_file); - utils::filesystem::remove_path(local_file_for_download); + ASSERT_EQ(local_file_size, d_resp.downloaded_size); + + // 6. compare kLocalFile and kLocalDownloadFile. + // 6.1 check file size. + int64_t local_download_file_size = 0; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kLocalDownloadFile, dsn::utils::FileDataType::kSensitive, local_download_file_size)); + ASSERT_EQ(local_file_size, local_download_file_size); + // 6.2 check file md5sum. + std::string local_download_file_md5sum; + dsn::utils::filesystem::md5sum(kLocalDownloadFile, local_download_file_md5sum); + ASSERT_EQ(local_file_md5sum, local_download_file_md5sum); + + // 7. clean up local test files. + utils::filesystem::remove_path(kLocalFile); + utils::filesystem::remove_path(kLocalDownloadFile); } -TEST_F(HDFSClientTest, test_concurrent_upload_download) +TEST_P(HDFSClientTest, test_concurrent_upload_download) { - if (name_node == example_name_node || backup_path == example_backup_path) { + if (strlen(FLAGS_test_name_node) == 0 || strlen(FLAGS_test_backup_path) == 0) { + // TODO(yingchun): use GTEST_SKIP after upgrading gtest. + std::cout << "Set hdfs_test.* configs in config-test.ini to enable hdfs_service_test." + << std::endl; return; } - std::vector args = {name_node, backup_path}; - std::shared_ptr s = std::make_shared(); - ASSERT_EQ(dsn::ERR_OK, s->initialize(args)); + auto s = std::make_shared(); + ASSERT_EQ(dsn::ERR_OK, s->initialize({FLAGS_test_name_node, FLAGS_test_backup_path})); int total_files = FLAGS_num_total_files_for_hdfs_concurrent_test; std::vector local_file_names; @@ -282,11 +334,12 @@ TEST_F(HDFSClientTest, test_concurrent_upload_download) // generate test files. for (int i = 0; i < total_files; ++i) { std::string file_name = "randomfile" + std::to_string(i); - generate_test_file(file_name.c_str()); + NO_FATALS(generate_test_file(file_name)); int64_t file_size = 0; - dsn::utils::filesystem::file_size(file_name, file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + file_name, dsn::utils::FileDataType::kSensitive, file_size)); std::string md5sum; - dsn::utils::filesystem::md5sum(file_name, md5sum); + ASSERT_EQ(ERR_OK, dsn::utils::filesystem::md5sum(file_name, md5sum)); local_file_names.emplace_back(file_name); remote_file_names.emplace_back("hdfs_concurrent_test/" + file_name); @@ -385,13 +438,22 @@ TEST_F(HDFSClientTest, test_concurrent_upload_download) } } -TEST_F(HDFSClientTest, test_rename_path_while_writing) +TEST_P(HDFSClientTest, test_rename_path_while_writing) { + std::string kLocalTestPath = "test_dir"; + const int kTotalFiles = 100; + task_tracker tracker; - write_test_files_async(&tracker); + int success_count = 0; + write_test_files_async(kLocalTestPath, kTotalFiles, &success_count, &tracker); usleep(100); - std::string rename_dir = "rename_dir." + std::to_string(dsn_now_ms()); - // rename succeed but writing failed. - ASSERT_TRUE(dsn::utils::filesystem::rename_path(local_test_dir, rename_dir)); + + std::string kLocalRenamedTestPath = "rename_dir." + std::to_string(dsn_now_ms()); + // Rename succeed but some files write failed. + ASSERT_TRUE(dsn::utils::filesystem::rename_path(kLocalTestPath, kLocalRenamedTestPath)); tracker.cancel_outstanding_tasks(); + // Generally, we can assume partial files are written failed. + // It maybe flaky, please retry if it failed. + ASSERT_GT(success_count, 0) << success_count; + ASSERT_LT(success_count, kTotalFiles) << success_count; } diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt index 7e27ef9b77..d4609d9e58 100644 --- a/src/replica/CMakeLists.txt +++ b/src/replica/CMakeLists.txt @@ -75,7 +75,7 @@ set(MY_PROJ_LIBS PocoFoundation PocoNetSSL PocoJSON - ) + rocksdb) set(MY_BOOST_LIBS Boost::filesystem Boost::regex) diff --git a/src/utils/env.cpp b/src/utils/env.cpp index 329406117c..7e49d999a5 100644 --- a/src/utils/env.cpp +++ b/src/utils/env.cpp @@ -49,6 +49,12 @@ DSN_DEFINE_string(pegasus.server, "The encryption method to use in the filesystem. Now " "supports AES128CTR, AES192CTR, AES256CTR and SM4CTR."); +DSN_DEFINE_bool(replication, + enable_direct_io, + false, + "Whether to enable direct I/O when download files"); +DSN_TAG_VARIABLE(enable_direct_io, FT_MUTABLE); + namespace dsn { namespace utils { @@ -89,6 +95,7 @@ rocksdb::Status do_copy_file(const std::string &src_fname, uint64_t *total_size) { rocksdb::EnvOptions src_env_options; + src_env_options.use_direct_reads = FLAGS_enable_direct_io; std::unique_ptr src_file; auto s = dsn::utils::PegasusEnv(src_type)->NewSequentialFile(src_fname, &src_file, src_env_options); @@ -105,6 +112,7 @@ rocksdb::Status do_copy_file(const std::string &src_fname, } rocksdb::EnvOptions dst_env_options; + dst_env_options.use_direct_writes = FLAGS_enable_direct_io; std::unique_ptr dst_file; s = dsn::utils::PegasusEnv(dst_type)->NewWritableFile(dst_fname, &dst_file, dst_env_options); LOG_AND_RETURN_NOT_RDB_OK(WARNING, s, "failed to open file {} for writing", dst_fname); diff --git a/src/utils/env.h b/src/utils/env.h index 839bd81ab3..cdaf16aaa5 100644 --- a/src/utils/env.h +++ b/src/utils/env.h @@ -19,8 +19,8 @@ #include #include -#include -#include +#include +#include #include namespace dsn { From 64abf905595d8f2ac2520c66b0554c121b01b6bb Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 26 Sep 2023 04:08:43 -0500 Subject: [PATCH 26/42] refactor(local service): use rocksdb API to read/write file (#1619) https://github.com/apache/incubator-pegasus/issues/887 There is no functional changes, but only refactor the local service and related unit test. And now FLAGS_enable_direct_io takes effect on local block service. --- src/block_service/local/CMakeLists.txt | 2 +- src/block_service/local/local_service.cpp | 339 ++++++++++-------- src/block_service/test/local_service_test.cpp | 64 ++-- 3 files changed, 227 insertions(+), 178 deletions(-) diff --git a/src/block_service/local/CMakeLists.txt b/src/block_service/local/CMakeLists.txt index 0886bece9a..9d830f7825 100644 --- a/src/block_service/local/CMakeLists.txt +++ b/src/block_service/local/CMakeLists.txt @@ -26,7 +26,7 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS "") +set(MY_PROJ_LIBS rocksdb) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/local/local_service.cpp b/src/block_service/local/local_service.cpp index 04cf1cc3d8..9175961340 100644 --- a/src/block_service/local/local_service.cpp +++ b/src/block_service/local/local_service.cpp @@ -15,10 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include -#include +#include #include -#include #include #include #include @@ -27,18 +25,22 @@ #include "local_service.h" #include "nlohmann/json.hpp" #include "nlohmann/json_fwd.hpp" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/task/async_calls.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" -#include "utils/defer.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/fail_point.h" #include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" -#include "utils/safe_strerror_posix.h" #include "utils/string_view.h" #include "utils/strings.h" +DSN_DECLARE_bool(enable_direct_io); + namespace dsn { class task_tracker; } // namespace dsn @@ -51,15 +53,13 @@ namespace block_service { DEFINE_TASK_CODE(LPC_LOCAL_SERVICE_CALL, TASK_PRIORITY_COMMON, THREAD_POOL_BLOCK_SERVICE) -bool file_metadata_from_json(std::ifstream &fin, file_metadata &fmeta) noexcept +bool file_metadata_from_json(const std::string &data, file_metadata &fmeta) noexcept { - std::string data; - fin >> data; try { nlohmann::json::parse(data).get_to(fmeta); return true; } catch (nlohmann::json::exception &exp) { - LOG_WARNING("decode meta data from json failed: {} [{}]", exp.what(), data); + LOG_WARNING("decode metadata from json failed: {} [{}]", exp.what(), data); return false; } } @@ -191,7 +191,7 @@ dsn::task_ptr local_service::create_file(const create_file_request &req, if (utils::filesystem::file_exists(file_path) && utils::filesystem::file_exists(meta_file_path)) { - LOG_DEBUG("file({}) already exist", file_path); + LOG_INFO("file({}) already exist", file_path); resp.err = f->load_metadata(); } @@ -268,17 +268,17 @@ error_code local_file_object::load_metadata() return ERR_OK; std::string metadata_path = local_service::get_metafile(file_name()); - std::ifstream is(metadata_path, std::ios::in); - if (!is.is_open()) { - LOG_WARNING( - "load meta data from {} failed, err = {}", metadata_path, utils::safe_strerror(errno)); + std::string data; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), metadata_path, &data); + if (!s.ok()) { + LOG_WARNING("read file '{}' failed, err = {}", metadata_path, s.ToString()); return ERR_FS_INTERNAL; } - auto cleanup = dsn::defer([&is]() { is.close(); }); file_metadata meta; - bool ans = file_metadata_from_json(is, meta); + bool ans = file_metadata_from_json(data, meta); if (!ans) { + LOG_WARNING("decode metadata '{}' file content failed", metadata_path); return ERR_FS_INTERNAL; } _size = meta.size; @@ -292,16 +292,16 @@ error_code local_file_object::store_metadata() file_metadata meta; meta.md5 = _md5_value; meta.size = _size; - + std::string data = nlohmann::json(meta).dump(); std::string metadata_path = local_service::get_metafile(file_name()); - std::ofstream os(metadata_path, std::ios::out | std::ios::trunc); - if (!os.is_open()) { - LOG_WARNING( - "store to metadata file {} failed, err={}", metadata_path, utils::safe_strerror(errno)); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(data), + metadata_path, + /* should_sync */ true); + if (!s.ok()) { + LOG_WARNING("store to metadata file {} failed, err={}", metadata_path, s.ToString()); return ERR_FS_INTERNAL; } - auto cleanup = dsn::defer([&os]() { os.close(); }); - os << nlohmann::json(meta); return ERR_OK; } @@ -337,24 +337,36 @@ dsn::task_ptr local_file_object::write(const write_request &req, } if (resp.err == ERR_OK) { - LOG_DEBUG("start write file, file = {}", file_name()); + LOG_INFO("start write file, file = {}", file_name()); + + do { + auto s = rocksdb::WriteStringToFile( + rocksdb::Env::Default(), + rocksdb::Slice(req.buffer.data(), req.buffer.length()), + file_name(), + /* should_sync */ true); + if (!s.ok()) { + LOG_WARNING("write file '{}' failed, err = {}", file_name(), s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; + } - std::ofstream fout(file_name(), std::ifstream::out | std::ifstream::trunc); - if (!fout.is_open()) { - resp.err = ERR_FS_INTERNAL; - } else { - fout.write(req.buffer.data(), req.buffer.length()); resp.written_size = req.buffer.length(); - fout.close(); // Currently we calc the meta data from source data, which save the io bandwidth // a lot, but it is somewhat not correct. _size = resp.written_size; _md5_value = utils::string_md5(req.buffer.data(), req.buffer.length()); + // TODO(yingchun): make store_metadata as a local function, do not depend on the + // member variables (i.e. _size and _md5_value). + auto err = store_metadata(); + if (err != ERR_OK) { + LOG_WARNING("store_metadata failed"); + resp.err = ERR_FS_INTERNAL; + break; + } _has_meta_synced = true; - - store_metadata(); - } + } while (false); } tsk->enqueue_with(resp); release_ref(); @@ -375,38 +387,65 @@ dsn::task_ptr local_file_object::read(const read_request &req, auto read_func = [this, req, tsk]() { read_response resp; - resp.err = ERR_OK; - if (!utils::filesystem::file_exists(file_name()) || - !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { - resp.err = ERR_OBJECT_NOT_FOUND; - } else { - if ((resp.err = load_metadata()) != ERR_OK) { - LOG_WARNING("load meta data of {} failed", file_name()); + do { + if (!utils::filesystem::file_exists(file_name()) || + !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { + LOG_WARNING("data file '{}' or metadata file '{}' not exist", + file_name(), + local_service::get_metafile(file_name())); + resp.err = ERR_OBJECT_NOT_FOUND; + break; + } + + resp.err = load_metadata(); + if (resp.err != ERR_OK) { + LOG_WARNING("load metadata of {} failed", file_name()); + break; + } + + int64_t file_sz = _size; + int64_t total_sz = 0; + if (req.remote_length == -1 || req.remote_length + req.remote_pos > file_sz) { + total_sz = file_sz - req.remote_pos; } else { - int64_t file_sz = _size; - int64_t total_sz = 0; - if (req.remote_length == -1 || req.remote_length + req.remote_pos > file_sz) { - total_sz = file_sz - req.remote_pos; - } else { - total_sz = req.remote_length; - } + total_sz = req.remote_length; + } - LOG_DEBUG("read file({}), size = {}", file_name(), total_sz); - std::string buf; - buf.resize(total_sz + 1); - std::ifstream fin(file_name(), std::ifstream::in); - if (!fin.is_open()) { - resp.err = ERR_FS_INTERNAL; - } else { - fin.seekg(static_cast(req.remote_pos), fin.beg); - fin.read((char *)buf.c_str(), total_sz); - buf[fin.gcount()] = '\0'; - resp.buffer = blob::create_from_bytes(std::move(buf)); - } - fin.close(); + LOG_INFO("start to read file '{}', offset = {}, size = {}", + file_name(), + req.remote_pos, + total_sz); + rocksdb::EnvOptions env_options; + env_options.use_direct_reads = FLAGS_enable_direct_io; + std::unique_ptr sfile; + auto s = rocksdb::Env::Default()->NewSequentialFile(file_name(), &sfile, env_options); + if (!s.ok()) { + LOG_WARNING("open file '{}' failed, err = {}", file_name(), s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; } - } + s = sfile->Skip(req.remote_pos); + if (!s.ok()) { + LOG_WARNING( + "skip '{}' for {} failed, err = {}", file_name(), req.remote_pos, s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; + } + + rocksdb::Slice result; + std::string buf; + buf.resize(total_sz + 1); + s = sfile->Read(total_sz, &result, buf.data()); + if (!s.ok()) { + LOG_WARNING("read file '{}' failed, err = {}", file_name(), s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; + } + + buf[result.size()] = 0; + resp.buffer = blob::create_from_bytes(std::move(buf)); + } while (false); tsk->enqueue_with(resp); release_ref(); }; @@ -424,58 +463,50 @@ dsn::task_ptr local_file_object::upload(const upload_request &req, upload_future_ptr tsk(new upload_future(code, cb, 0)); tsk->set_tracker(tracker); auto upload_file_func = [this, req, tsk]() { - upload_response resp; - resp.err = ERR_OK; - std::ifstream fin(req.input_local_name, std::ios_base::in); - if (!fin.is_open()) { - LOG_WARNING("open source file {} for read failed, err({})", - req.input_local_name, - utils::safe_strerror(errno)); - resp.err = ERR_FILE_OPERATION_FAILED; - } + LOG_INFO("start to upload from '{}' to '{}'", req.input_local_name, file_name()); - utils::filesystem::create_file(file_name()); - std::ofstream fout(file_name(), std::ios_base::out | std::ios_base::trunc); - if (!fout.is_open()) { - LOG_WARNING("open target file {} for write failed, err({})", - file_name(), - utils::safe_strerror(errno)); - resp.err = ERR_FS_INTERNAL; - } - - if (resp.err == ERR_OK) { - LOG_DEBUG("start to transfer from src_file({}) to dst_file({})", - req.input_local_name, - file_name()); - int64_t total_sz = 0; - char buf[max_length] = {'\0'}; - while (!fin.eof()) { - fin.read(buf, max_length); - total_sz += fin.gcount(); - fout.write(buf, fin.gcount()); + upload_response resp; + do { + // Create the directory. + std::string path = dsn::utils::filesystem::remove_file_name(file_name()); + if (!dsn::utils::filesystem::create_directory(path)) { + LOG_WARNING("create directory '{}' failed", path); + resp.err = ERR_FILE_OPERATION_FAILED; + break; } - LOG_DEBUG("finish upload file, file = {}, total_size = {}", file_name(), total_sz); - fout.close(); - fin.close(); - - resp.uploaded_size = static_cast(total_sz); - // calc the md5sum by source file for simplicity - _size = total_sz; - error_code res = utils::filesystem::md5sum(req.input_local_name, _md5_value); - if (res == dsn::ERR_OK) { - _has_meta_synced = true; - store_metadata(); - } else { + uint64_t file_size; + auto s = dsn::utils::copy_file(req.input_local_name, file_name(), &file_size); + if (!s.ok()) { + LOG_WARNING("upload from '{}' to '{}' failed, err = {}", + req.input_local_name, + file_name(), + s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + LOG_INFO("finish to upload from '{}' to '{}', size = {}", + req.input_local_name, + file_name(), + file_size); + + resp.uploaded_size = file_size; + _size = file_size; + auto res = utils::filesystem::md5sum(file_name(), _md5_value); + if (res != dsn::ERR_OK) { + LOG_WARNING("calculate md5sum for '{}' failed", file_name()); resp.err = ERR_FS_INTERNAL; + break; } - } else { - if (fin.is_open()) - fin.close(); - if (fout.is_open()) - fout.close(); - } + auto err = store_metadata(); + if (err != ERR_OK) { + LOG_ERROR("store_metadata failed"); + resp.err = ERR_FS_INTERNAL; + break; + } + _has_meta_synced = true; + } while (false); tsk->enqueue_with(resp); release_ref(); }; @@ -497,65 +528,61 @@ dsn::task_ptr local_file_object::download(const download_request &req, download_response resp; resp.err = ERR_OK; std::string target_file = req.output_local_name; - if (target_file.empty()) { - LOG_ERROR( - "download {} failed, because target name({}) is invalid", file_name(), target_file); - resp.err = ERR_INVALID_PARAMETERS; - } - if (resp.err == ERR_OK && !_has_meta_synced) { - if (!utils::filesystem::file_exists(file_name()) || - !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { - resp.err = ERR_OBJECT_NOT_FOUND; + do { + if (target_file.empty()) { + LOG_WARNING("download {} failed, because target name({}) is invalid", + file_name(), + target_file); + resp.err = ERR_INVALID_PARAMETERS; + break; } - } - if (resp.err == ERR_OK) { - std::ifstream fin(file_name(), std::ifstream::in); - if (!fin.is_open()) { - LOG_ERROR("open block file({}) failed, err({})", - file_name(), - utils::safe_strerror(errno)); - resp.err = ERR_FS_INTERNAL; + if (!_has_meta_synced) { + if (!utils::filesystem::file_exists(file_name()) || + !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { + LOG_WARNING("file '{}' or metadata file '{}' not found", + file_name(), + local_service::get_metafile(file_name())); + resp.err = ERR_OBJECT_NOT_FOUND; + break; + } } - std::ofstream fout(target_file, std::ios_base::out | std::ios_base::trunc); - if (!fout.is_open()) { - if (fin.is_open()) - fin.close(); - LOG_ERROR("open target file({}) failed, err({})", - target_file, - utils::safe_strerror(errno)); + LOG_INFO("start to download from '{}' to '{}'", file_name(), target_file); + + // Create the directory. + std::string path = dsn::utils::filesystem::remove_file_name(file_name()); + if (!dsn::utils::filesystem::create_directory(path)) { + LOG_WARNING("create directory '{}' failed", path); resp.err = ERR_FILE_OPERATION_FAILED; + break; } - if (resp.err == ERR_OK) { - LOG_DEBUG( - "start to transfer, src_file({}), dst_file({})", file_name(), target_file); - int64_t total_sz = 0; - char buf[max_length] = {'\0'}; - while (!fin.eof()) { - fin.read(buf, max_length); - total_sz += fin.gcount(); - fout.write(buf, fin.gcount()); - } - LOG_DEBUG("finish download file({}), total_size = {}", target_file, total_sz); - fout.close(); - fin.close(); - resp.downloaded_size = static_cast(total_sz); - - _size = total_sz; - if ((resp.err = utils::filesystem::md5sum(target_file, _md5_value)) != ERR_OK) { - LOG_WARNING("download {} failed when calculate the md5sum of {}", - file_name(), - target_file); - } else { - _has_meta_synced = true; - resp.file_md5 = _md5_value; - } + uint64_t file_size; + auto s = dsn::utils::copy_file(file_name(), target_file, &file_size); + if (!s.ok()) { + LOG_WARNING("download from '{}' to '{}' failed, err = {}", + file_name(), + target_file, + s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + auto res = utils::filesystem::md5sum(target_file, _md5_value); + if (res != dsn::ERR_OK) { + LOG_WARNING("calculate md5sum for {} failed", target_file); + resp.err = ERR_FILE_OPERATION_FAILED; + break; } - } + LOG_INFO("finish download file '{}', size = {}", target_file, file_size); + resp.downloaded_size = file_size; + resp.file_md5 = _md5_value; + _size = file_size; + _has_meta_synced = true; + } while (false); tsk->enqueue_with(resp); release_ref(); }; diff --git a/src/block_service/test/local_service_test.cpp b/src/block_service/test/local_service_test.cpp index 72a2012370..e4e6aeb2ee 100644 --- a/src/block_service/test/local_service_test.cpp +++ b/src/block_service/test/local_service_test.cpp @@ -19,17 +19,23 @@ #include // IWYU pragma: no_include +// IWYU pragma: no_include // IWYU pragma: no_include #include #include #include #include -#include +#include +#include +#include #include +#include #include +#include #include #include "block_service/local/local_service.h" +#include "test_util/test_util.h" #include "utils/error_code.h" namespace dsn { @@ -37,53 +43,69 @@ namespace dist { namespace block_service { // Simple tests for nlohmann::json serialization, via NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE. +class local_service_test : public pegasus::encrypt_data_test_base +{ +}; + +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, local_service_test, ::testing::Values(false)); -TEST(local_service, store_metadata) +TEST_P(local_service_test, store_metadata) { local_file_object file("a.txt"); error_code ec = file.store_metadata(); - ASSERT_EQ(ec, ERR_OK); + ASSERT_EQ(ERR_OK, ec); auto meta_file_path = local_service::get_metafile(file.file_name()); ASSERT_TRUE(boost::filesystem::exists(meta_file_path)); - std::ifstream ifs(meta_file_path); - nlohmann::json j; - ifs >> j; - ASSERT_EQ(j["md5"], ""); - ASSERT_EQ(j["size"], 0); + std::string data; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), meta_file_path, &data); + ASSERT_TRUE(s.ok()) << s.ToString(); + + nlohmann::json j = nlohmann::json::parse(data); + ASSERT_EQ("", j["md5"]); + ASSERT_EQ(0, j["size"]); } -TEST(local_service, load_metadata) +TEST_P(local_service_test, load_metadata) { local_file_object file("a.txt"); auto meta_file_path = local_service::get_metafile(file.file_name()); { - std::ofstream ofs(meta_file_path); nlohmann::json j({{"md5", "abcde"}, {"size", 5}}); - ofs << j; - ofs.close(); + std::string data = j.dump(); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(data), + meta_file_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); - ASSERT_EQ(file.load_metadata(), ERR_OK); - ASSERT_EQ(file.get_md5sum(), "abcde"); - ASSERT_EQ(file.get_size(), 5); + ASSERT_EQ(ERR_OK, file.load_metadata()); + ASSERT_EQ("abcde", file.get_md5sum()); + ASSERT_EQ(5, file.get_size()); } { - std::ofstream ofs(meta_file_path); - ofs << "invalid json string"; - ofs.close(); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice("invalid json string"), + meta_file_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); local_file_object file2("a.txt"); ASSERT_EQ(file2.load_metadata(), ERR_FS_INTERNAL); } { - std::ofstream ofs(meta_file_path); nlohmann::json j({{"md5", "abcde"}, {"no such key", "illegal"}}); - ofs << j; - ofs.close(); + std::string data = j.dump(); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(data), + meta_file_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); local_file_object file2("a.txt"); ASSERT_EQ(file2.load_metadata(), ERR_FS_INTERNAL); From a2f2ebd3d2b5f2075497ea3f3bd18bf8fe411578 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 26 Sep 2023 04:25:22 -0500 Subject: [PATCH 27/42] refactor(simple_meta_state): use rocksdb API to read/write file (#1620) https://github.com/apache/incubator-pegasus/issues/887 There is no functional changes, but only refactor the meta_state_service_simple and related unit test. --- src/meta/distributed_lock_service_simple.h | 1 + src/meta/dump_file.h | 2 + src/meta/meta_state_service_simple.cpp | 123 +++++++++++------- src/meta/meta_state_service_simple.h | 1 + .../test/meta_state/meta_state_service.cpp | 88 ++++++++----- 5 files changed, 131 insertions(+), 84 deletions(-) diff --git a/src/meta/distributed_lock_service_simple.h b/src/meta/distributed_lock_service_simple.h index a2da463b49..59b2bde5e6 100644 --- a/src/meta/distributed_lock_service_simple.h +++ b/src/meta/distributed_lock_service_simple.h @@ -53,6 +53,7 @@ namespace dsn { namespace dist { +// Only for test purpose. class distributed_lock_service_simple : public distributed_lock_service { public: diff --git a/src/meta/dump_file.h b/src/meta/dump_file.h index 4e8018ca58..062ffd1fcc 100644 --- a/src/meta/dump_file.h +++ b/src/meta/dump_file.h @@ -63,6 +63,8 @@ struct block_header uint32_t crc32; }; +// TODO(yingchun): use rocksdb APIs to unify the file operations. +// A tool to dump app_states of meta server to local file, used by remote command "meta.dump". class dump_file { public: diff --git a/src/meta/meta_state_service_simple.cpp b/src/meta/meta_state_service_simple.cpp index f2db0da0a7..6ba00176ea 100644 --- a/src/meta/meta_state_service_simple.cpp +++ b/src/meta/meta_state_service_simple.cpp @@ -27,7 +27,6 @@ #include "meta_state_service_simple.h" #include -#include #include #include #include @@ -35,6 +34,9 @@ #include #include "aio/file_io.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/service_app.h" #include "runtime/task/async_calls.h" #include "runtime/task/task.h" @@ -228,7 +230,7 @@ error_code meta_state_service_simple::apply_transaction( default: CHECK(false, "unsupported operation"); } - CHECK_EQ_MSG(ec, ERR_OK, "unexpected error when applying"); + CHECK_EQ_MSG(ERR_OK, ec, "unexpected error when applying"); } return ERR_OK; @@ -242,53 +244,73 @@ error_code meta_state_service_simple::initialize(const std::vector _offset = 0; std::string log_path = dsn::utils::filesystem::path_combine(work_dir, "meta_state_service.log"); if (utils::filesystem::file_exists(log_path)) { - if (FILE *fd = fopen(log_path.c_str(), "rb")) { - for (;;) { - log_header header; - if (fread(&header, sizeof(log_header), 1, fd) != 1) { - break; - } - if (header.magic != log_header::default_magic) { - break; - } - std::shared_ptr buffer(dsn::utils::make_shared_array(header.size)); - if (fread(buffer.get(), header.size, 1, fd) != 1) { - break; - } - _offset += sizeof(header) + header.size; - binary_reader reader(blob(buffer, (int)header.size)); - int op_type = 0; - reader.read(op_type); - - switch (static_cast(op_type)) { - case operation_type::create_node: { - std::string node; - blob data; - create_node_log::parse(reader, node, data); - create_node_internal(node, data); - break; - } - case operation_type::delete_node: { - std::string node; - bool recursively_delete; - delete_node_log::parse(reader, node, recursively_delete); - delete_node_internal(node, recursively_delete); - break; - } - case operation_type::set_data: { - std::string node; - blob data; - set_data_log::parse(reader, node, data); - set_data_internal(node, data); - break; - } - default: - // The log is complete but its content is modified by cosmic ray. This is - // unacceptable - CHECK(false, "meta state server log corrupted"); - } + std::unique_ptr log_file; + auto s = + rocksdb::Env::Default()->NewSequentialFile(log_path, &log_file, rocksdb::EnvOptions()); + CHECK(s.ok(), "open log file '{}' failed, err = {}", log_path, s.ToString()); + + while (true) { + static const int kLogHeaderSize = sizeof(log_header); + static const int kDefaultMagic = 0xdeadbeef; + rocksdb::Slice result; + + // Read header. + char scratch[kLogHeaderSize] = {0}; + s = log_file->PositionedRead(_offset, kLogHeaderSize, &result, scratch); + CHECK(s.ok(), "read log file '{}' header failed, err = {}", log_path, s.ToString()); + if (result.empty()) { + LOG_INFO("read EOF of log file '{}'", log_path); + break; + } + log_header *header = reinterpret_cast(scratch); + if (header->magic != kDefaultMagic) { + LOG_WARNING("read log file '{}' header with bad magic {}, skip the left!", + log_path, + header->magic); + break; + } + _offset += kLogHeaderSize; + + // Read body. + std::shared_ptr buffer(dsn::utils::make_shared_array(header->size)); + s = log_file->PositionedRead(_offset, header->size, &result, buffer.get()); + CHECK(s.ok(), + "read log file '{}' header with bad body, err = {}", + log_path, + s.ToString()); + _offset += header->size; + + binary_reader reader(blob(buffer, header->size)); + int op_type = 0; + CHECK_EQ(sizeof(op_type), reader.read(op_type)); + + switch (static_cast(op_type)) { + case operation_type::create_node: { + std::string node; + blob data; + create_node_log::parse(reader, node, data); + create_node_internal(node, data); + break; + } + case operation_type::delete_node: { + std::string node; + bool recursively_delete; + delete_node_log::parse(reader, node, recursively_delete); + delete_node_internal(node, recursively_delete); + break; + } + case operation_type::set_data: { + std::string node; + blob data; + set_data_log::parse(reader, node, data); + set_data_internal(node, data); + break; + } + default: + // The log is complete but its content is modified by cosmic ray. This is + // unacceptable + CHECK(false, "meta state server log corrupted"); } - fclose(fd); } } @@ -507,8 +529,9 @@ task_ptr meta_state_service_simple::get_children(const std::string &node, meta_state_service_simple::~meta_state_service_simple() { - _tracker.cancel_outstanding_tasks(); - file::close(_log); + _tracker.wait_outstanding_tasks(); + CHECK_EQ(ERR_OK, file::flush(_log)); + CHECK_EQ(ERR_OK, file::close(_log)); for (const auto &kv : _quick_map) { if ("/" != kv.first) { diff --git a/src/meta/meta_state_service_simple.h b/src/meta/meta_state_service_simple.h index f26e33e174..697b5b159a 100644 --- a/src/meta/meta_state_service_simple.h +++ b/src/meta/meta_state_service_simple.h @@ -66,6 +66,7 @@ DEFINE_TASK_CODE_AIO(LPC_META_STATE_SERVICE_SIMPLE_INTERNAL, TASK_PRIORITY_HIGH, THREAD_POOL_DEFAULT); +// Only for test purpose. class meta_state_service_simple : public meta_state_service { public: diff --git a/src/meta/test/meta_state/meta_state_service.cpp b/src/meta/test/meta_state/meta_state_service.cpp index a286c5067a..4959670b29 100644 --- a/src/meta/test/meta_state/meta_state_service.cpp +++ b/src/meta/test/meta_state/meta_state_service.cpp @@ -27,6 +27,7 @@ #include "meta/meta_state_service.h" #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include @@ -36,12 +37,18 @@ #include "meta/meta_state_service_simple.h" #include "meta/meta_state_service_zookeeper.h" +#include "runtime/service_app.h" #include "runtime/task/task_tracker.h" +#include "test_util/test_util.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" +#include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/threadpool_code.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + using namespace dsn; using namespace dsn::dist; @@ -50,8 +57,8 @@ DEFINE_TASK_CODE(META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, TASK_PRIORITY_HIGH, TH typedef std::function service_creator_func; typedef std::function service_deleter_func; -#define expect_ok [](error_code ec) { EXPECT_TRUE(ec == ERR_OK); } -#define expect_err [](error_code ec) { EXPECT_FALSE(ec == ERR_OK); } +#define expect_ok [](error_code ec) { CHECK_EQ(ERR_OK, ec); } +#define expect_err [](error_code ec) { CHECK_NE(ERR_OK, ec); } void provider_basic_test(const service_creator_func &service_creator, const service_deleter_func &service_deleter) @@ -70,9 +77,9 @@ void provider_basic_test(const service_creator_func &service_creator, service->get_children("/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const std::vector &children) { - CHECK(ec == ERR_OK && children.size() == 1 && - *children.begin() == "1", - "unexpected child"); + CHECK_EQ(ERR_OK, ec); + CHECK_EQ(1, children.size()); + CHECK_EQ("1", *children.begin()); }); service->node_exist("/1/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok)->wait(); service->delete_node("/1", false, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_err) @@ -107,11 +114,11 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_data("/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const dsn::blob &value) { - expect_ok(ec); + CHECK_EQ(ERR_OK, ec); dsn::binary_reader reader(value); int read_value = 0; reader.read(read_value); - CHECK_EQ(read_value, 0xdeadbeef); + CHECK_EQ(0xdeadbeef, read_value); }) ->wait(); writer = dsn::binary_writer(); @@ -124,27 +131,26 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_data("/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const dsn::blob &value) { - expect_ok(ec); + CHECK_EQ(ERR_OK, ec); dsn::binary_reader reader(value); int read_value = 0; reader.read(read_value); - CHECK_EQ(read_value, 0xbeefdead); + CHECK_EQ(0xbeefdead, read_value); }) ->wait(); } - // clean the node created in previos code-block, to support test in next round + // clean the node created in previous code-block, to support test in next round { service->delete_node("/1", false, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok) ->wait(); } - typedef dsn::dist::meta_state_service::transaction_entries TEntries; // transaction op { // basic dsn::binary_writer writer; writer.write(0xdeadbeef); - std::shared_ptr entries = service->new_transaction_entries(5); + auto entries = service->new_transaction_entries(5); entries->create_node("/2"); entries->create_node("/2/2"); entries->create_node("/2/3"); @@ -155,11 +161,11 @@ void provider_basic_test(const service_creator_func &service_creator, entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok); tsk->wait(); for (unsigned int i = 0; i < 5; ++i) { - EXPECT_TRUE(entries->get_result(i) == ERR_OK); + ASSERT_EQ(ERR_OK, entries->get_result(i)); } // an invalid operation will stop whole transaction - entries = service->new_transaction_entries(5); + entries = service->new_transaction_entries(4); entries->create_node("/3"); entries->create_node("/4"); entries->delete_node("/2"); // delete a non empty dir @@ -168,11 +174,12 @@ void provider_basic_test(const service_creator_func &service_creator, service->submit_transaction(entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_err) ->wait(); error_code err[4] = {ERR_OK, ERR_OK, ERR_INVALID_PARAMETERS, ERR_INCONSISTENT_STATE}; - for (unsigned int i = 0; i < 4; ++i) - EXPECT_EQ(err[i], entries->get_result(i)); + for (unsigned int i = 0; i < 4; ++i) { + ASSERT_EQ(err[i], entries->get_result(i)); + } // another invalid transaction - entries = service->new_transaction_entries(5); + entries = service->new_transaction_entries(4); entries->create_node("/3"); entries->create_node("/4"); entries->delete_node("/5"); // delete a non exist dir @@ -182,8 +189,9 @@ void provider_basic_test(const service_creator_func &service_creator, err[2] = ERR_OBJECT_NOT_FOUND; service->submit_transaction(entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_err) ->wait(); - for (unsigned int i = 0; i < 4; ++i) - EXPECT_EQ(err[i], entries->get_result(i)); + for (unsigned int i = 0; i < 4; ++i) { + ASSERT_EQ(err[i], entries->get_result(i)); + } } // check replay with transaction @@ -195,7 +203,9 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_children("/2", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const std::vector &children) { - ASSERT_TRUE(children.size() == 1 && children[0] == "2"); + CHECK_EQ(ERR_OK, ec); + CHECK_EQ(1, children.size()); + CHECK_EQ("2", children[0]); }) ->wait(); @@ -203,27 +213,27 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_data("/2", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const blob &value) { - ASSERT_TRUE(ec == ERR_OK); + CHECK_EQ(ERR_OK, ec); binary_reader reader(value); int content_value; reader.read(content_value); - ASSERT_TRUE(content_value == 0xdeadbeef); + CHECK_EQ(0xdeadbeef, content_value); }) ->wait(); } // delete the nodes created just now, using transaction delete { - std::shared_ptr entries = service->new_transaction_entries(2); + auto entries = service->new_transaction_entries(2); entries->delete_node("/2/2"); entries->delete_node("/2"); service->submit_transaction(entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok) ->wait(); - error_code err[2] = {ERR_OK, ERR_OK}; - for (unsigned int i = 0; i < 2; ++i) - EXPECT_EQ(err[i], entries->get_result(i)); + for (unsigned int i = 0; i < 2; ++i) { + ASSERT_EQ(ERR_OK, entries->get_result(i)); + } } service_deleter(service); @@ -235,7 +245,7 @@ void recursively_create_node_callback(meta_state_service *service, int current_layer, error_code ec) { - ASSERT_TRUE(ec == ERR_OK); + ASSERT_EQ(ERR_OK, ec); if (current_layer <= 0) return; @@ -279,31 +289,41 @@ void provider_recursively_create_delete_test(const service_creator_func &creator deleter(service); } -#undef expect_ok -#undef expect_err +class meta_state_service_test : public pegasus::encrypt_data_test_base +{ +}; + +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, meta_state_service_test, ::testing::Values(false)); -TEST(meta_state_service, simple) +TEST_P(meta_state_service_test, simple) { auto simple_service_creator = [] { meta_state_service_simple *svc = new meta_state_service_simple(); - svc->initialize({}); + auto err = svc->initialize({}); + CHECK_EQ(ERR_OK, err); return svc; }; auto simple_service_deleter = [](meta_state_service *simple_svc) { delete simple_svc; }; provider_basic_test(simple_service_creator, simple_service_deleter); provider_recursively_create_delete_test(simple_service_creator, simple_service_deleter); + + std::string log_path = dsn::utils::filesystem::path_combine( + service_app::current_service_app_info().data_dir, "meta_state_service.log"); + ASSERT_TRUE(dsn::utils::filesystem::remove_path(log_path)); } -TEST(meta_state_service, zookeeper) +TEST_P(meta_state_service_test, zookeeper) { auto zookeeper_service_creator = [] { meta_state_service_zookeeper *svc = new meta_state_service_zookeeper(); - svc->initialize({}); + auto err = svc->initialize({}); + CHECK_EQ(ERR_OK, err); return svc; }; auto zookeeper_service_deleter = [](meta_state_service *zookeeper_svc) { - ASSERT_EQ(zookeeper_svc->finalize(), ERR_OK); + ASSERT_EQ(ERR_OK, zookeeper_svc->finalize()); }; provider_basic_test(zookeeper_service_creator, zookeeper_service_deleter); From c4470fa4b5631fda4ed9a12ef63449d5fd6de9bc Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 26 Sep 2023 04:43:11 -0500 Subject: [PATCH 28/42] refactor(nfs_test): use rocksdb API to read/write file (#1621) https://github.com/apache/incubator-pegasus/issues/887 There is no functional changes, but only refactor the nfs unit test. --- src/nfs/test/main.cpp | 159 +++++++++++++++++++++++++++--------------- 1 file changed, 103 insertions(+), 56 deletions(-) diff --git a/src/nfs/test/main.cpp b/src/nfs/test/main.cpp index 445c4732fe..0788129928 100644 --- a/src/nfs/test/main.cpp +++ b/src/nfs/test/main.cpp @@ -24,13 +24,16 @@ * THE SOFTWARE. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include #include -#include +#include +#include #include #include +#include #include #include "aio/aio_task.h" @@ -40,7 +43,9 @@ #include "runtime/rpc/rpc_address.h" #include "runtime/task/task_code.h" #include "runtime/tool_api.h" +#include "test_util/test_util.h" #include "utils/autoref_ptr.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/threadpool_code.h" @@ -54,36 +59,55 @@ struct aio_result size_t sz; }; -TEST(nfs, basic) +class nfs_test : public pegasus::encrypt_data_test_base { - std::unique_ptr nfs(dsn::nfs_node::create()); +}; + +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, nfs_test, ::testing::Values(false)); + +TEST_P(nfs_test, basic) +{ + auto nfs = dsn::nfs_node::create(); nfs->start(); nfs->register_async_rpc_handler_for_test(); dsn::gpid fake_pid = gpid(1, 0); - utils::filesystem::remove_path("nfs_test_dir"); - utils::filesystem::remove_path("nfs_test_dir_copy"); - - ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir")); - ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir_copy")); - - ASSERT_TRUE(utils::filesystem::create_directory("nfs_test_dir")); - ASSERT_TRUE(utils::filesystem::directory_exists("nfs_test_dir")); + // Prepare the destination directory. + const std::string kDstDir = "nfs_test_dir"; + ASSERT_TRUE(utils::filesystem::remove_path(kDstDir)); + ASSERT_FALSE(utils::filesystem::directory_exists(kDstDir)); + ASSERT_TRUE(utils::filesystem::create_directory(kDstDir)); + ASSERT_TRUE(utils::filesystem::directory_exists(kDstDir)); + + // Prepare the source files information. + std::vector kSrcFilenames({"nfs_test_file1", "nfs_test_file2"}); + std::vector src_file_sizes; + std::vector src_file_md5s; + for (const auto &src_filename : kSrcFilenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + src_filename, dsn::utils::FileDataType::kSensitive, file_size)); + src_file_sizes.push_back(file_size); + std::string src_file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(src_filename, src_file_md5)); + src_file_md5s.emplace_back(std::move(src_file_md5)); + } + // copy files to the destination directory. { - // copy nfs_test_file1 nfs_test_file2 nfs_test_dir - ASSERT_FALSE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1")); - ASSERT_FALSE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2")); - - std::vector files{"nfs_test_file1", "nfs_test_file2"}; + // The destination directory is empty before copying. + std::vector dst_filenames; + ASSERT_TRUE(utils::filesystem::get_subfiles(kDstDir, dst_filenames, true)); + ASSERT_TRUE(dst_filenames.empty()); aio_result r; dsn::aio_task_ptr t = nfs->copy_remote_files(dsn::rpc_address("localhost", 20101), "default", ".", - files, + kSrcFilenames, "default", - "nfs_test_dir", + kDstDir, fake_pid, false, false, @@ -100,32 +124,32 @@ TEST(nfs, basic) ASSERT_EQ(ERR_OK, r.err); ASSERT_EQ(r.sz, t->get_transferred_size()); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2")); - - int64_t sz1, sz2; - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_file1", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file1", sz2)); - ASSERT_EQ(sz1, sz2); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_file2", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file2", sz2)); - ASSERT_EQ(sz1, sz2); + // The destination files equal to the source files after copying. + ASSERT_TRUE(utils::filesystem::get_subfiles(kDstDir, dst_filenames, true)); + std::sort(dst_filenames.begin(), dst_filenames.end()); + ASSERT_EQ(kSrcFilenames.size(), dst_filenames.size()); + int i = 0; + for (const auto &dst_filename : dst_filenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + dst_filename, dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(src_file_sizes[i], file_size); + std::string file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(dst_filename, file_md5)); + ASSERT_EQ(src_file_md5s[i], file_md5); + i++; + } } + // copy files to the destination directory, files will be overwritten. { - // copy files again, overwrite - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2")); - - std::vector files{"nfs_test_file1", "nfs_test_file2"}; - aio_result r; dsn::aio_task_ptr t = nfs->copy_remote_files(dsn::rpc_address("localhost", 20101), "default", ".", - files, + kSrcFilenames, "default", - "nfs_test_dir", + kDstDir, fake_pid, true, false, @@ -141,22 +165,42 @@ TEST(nfs, basic) ASSERT_EQ(r.err, t->error()); ASSERT_EQ(ERR_OK, r.err); ASSERT_EQ(r.sz, t->get_transferred_size()); + // this is only true for simulator if (dsn::tools::get_current_tool()->name() == "simulator") { ASSERT_EQ(1, t->get_count()); } + + // The destination files equal to the source files after overwrite copying. + std::vector dst_filenames; + ASSERT_TRUE(utils::filesystem::get_subfiles(kDstDir, dst_filenames, true)); + std::sort(dst_filenames.begin(), dst_filenames.end()); + ASSERT_EQ(kSrcFilenames.size(), dst_filenames.size()); + int i = 0; + for (const auto &dst_filename : dst_filenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + dst_filename, dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(src_file_sizes[i], file_size); + std::string file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(dst_filename, file_md5)); + ASSERT_EQ(src_file_md5s[i], file_md5); + i++; + } } + // copy files from kDstDir to kNewDstDir. { - // copy nfs_test_dir nfs_test_dir_copy - ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir_copy")); + const std::string kNewDstDir = "nfs_test_dir_copy"; + ASSERT_TRUE(utils::filesystem::remove_path(kNewDstDir)); + ASSERT_FALSE(utils::filesystem::directory_exists(kNewDstDir)); aio_result r; dsn::aio_task_ptr t = nfs->copy_remote_directory(dsn::rpc_address("localhost", 20101), "default", - "nfs_test_dir", + kDstDir, "default", - "nfs_test_dir_copy", + kNewDstDir, fake_pid, false, false, @@ -173,22 +217,25 @@ TEST(nfs, basic) ASSERT_EQ(ERR_OK, r.err); ASSERT_EQ(r.sz, t->get_transferred_size()); - ASSERT_TRUE(utils::filesystem::directory_exists("nfs_test_dir_copy")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir_copy/nfs_test_file1")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir_copy/nfs_test_file2")); - - std::vector sub1, sub2; - ASSERT_TRUE(utils::filesystem::get_subfiles("nfs_test_dir", sub1, true)); - ASSERT_TRUE(utils::filesystem::get_subfiles("nfs_test_dir_copy", sub2, true)); - ASSERT_EQ(sub1.size(), sub2.size()); - - int64_t sz1, sz2; - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file1", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir_copy/nfs_test_file1", sz2)); - ASSERT_EQ(sz1, sz2); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file2", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir_copy/nfs_test_file2", sz2)); - ASSERT_EQ(sz1, sz2); + // The kNewDstDir will be created automatically. + ASSERT_TRUE(utils::filesystem::directory_exists(kNewDstDir)); + + std::vector new_dst_filenames; + ASSERT_TRUE(utils::filesystem::get_subfiles(kNewDstDir, new_dst_filenames, true)); + std::sort(new_dst_filenames.begin(), new_dst_filenames.end()); + ASSERT_EQ(kSrcFilenames.size(), new_dst_filenames.size()); + + int i = 0; + for (const auto &new_dst_filename : new_dst_filenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + new_dst_filename, dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(src_file_sizes[i], file_size); + std::string file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(new_dst_filename, file_md5)); + ASSERT_EQ(src_file_md5s[i], file_md5); + i++; + } } nfs->stop(); From 916c7741455fde3c8f1ec8a12a50bd1245bdfe45 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 26 Sep 2023 05:19:54 -0500 Subject: [PATCH 29/42] refactor(simple_kv): use rocksdb API to read/write file (#1622) https://github.com/apache/incubator-pegasus/issues/887 There is no functional changes, but only refactor the simple_kv module. --- src/replica/storage/simple_kv/CMakeLists.txt | 2 +- .../simple_kv/simple_kv.server.impl.cpp | 127 +++++++++++------ .../storage/simple_kv/test/CMakeLists.txt | 3 +- .../storage/simple_kv/test/case-000.ini | 3 + .../storage/simple_kv/test/case-001.ini | 3 + .../storage/simple_kv/test/case-002.ini | 3 + .../storage/simple_kv/test/case-003.ini | 3 + .../storage/simple_kv/test/case-004.ini | 3 + .../storage/simple_kv/test/case-005.ini | 3 + .../storage/simple_kv/test/case-006.ini | 3 + .../storage/simple_kv/test/case-100.ini | 3 + .../storage/simple_kv/test/case-101.ini | 3 + .../storage/simple_kv/test/case-102.ini | 3 + .../storage/simple_kv/test/case-103.ini | 3 + .../storage/simple_kv/test/case-104.ini | 3 + .../storage/simple_kv/test/case-105.ini | 3 + .../storage/simple_kv/test/case-106.ini | 3 + .../storage/simple_kv/test/case-107.ini | 3 + .../storage/simple_kv/test/case-108.ini | 3 + .../storage/simple_kv/test/case-109.ini | 3 + .../storage/simple_kv/test/case-200.ini | 3 + .../storage/simple_kv/test/case-201.ini | 3 + .../storage/simple_kv/test/case-202-0.ini | 3 + .../storage/simple_kv/test/case-202-1.ini | 3 + .../storage/simple_kv/test/case-203-0.ini | 3 + .../storage/simple_kv/test/case-204.ini | 3 + .../storage/simple_kv/test/case-205.ini | 3 + .../storage/simple_kv/test/case-206.ini | 3 + .../storage/simple_kv/test/case-207.ini | 3 + .../storage/simple_kv/test/case-208.ini | 3 + .../storage/simple_kv/test/case-209.ini | 3 + .../storage/simple_kv/test/case-210.ini | 3 + .../storage/simple_kv/test/case-211.ini | 3 + .../storage/simple_kv/test/case-212.ini | 3 + .../storage/simple_kv/test/case-213.ini | 3 + .../storage/simple_kv/test/case-214.ini | 3 + .../storage/simple_kv/test/case-215.ini | 3 + .../storage/simple_kv/test/case-216.ini | 3 + .../storage/simple_kv/test/case-300-0.ini | 3 + .../storage/simple_kv/test/case-300-1.ini | 3 + .../storage/simple_kv/test/case-300-2.ini | 3 + .../storage/simple_kv/test/case-301.ini | 3 + .../storage/simple_kv/test/case-302.ini | 3 + .../storage/simple_kv/test/case-303.ini | 3 + .../storage/simple_kv/test/case-304.ini | 3 + .../storage/simple_kv/test/case-305.ini | 3 + .../storage/simple_kv/test/case-306.ini | 3 + .../storage/simple_kv/test/case-307.ini | 3 + .../storage/simple_kv/test/case-400.ini | 3 + .../storage/simple_kv/test/case-401.ini | 3 + .../storage/simple_kv/test/case-402.ini | 3 + .../storage/simple_kv/test/case-600.ini | 3 + .../storage/simple_kv/test/case-601.ini | 3 + .../storage/simple_kv/test/case-602.ini | 3 + .../storage/simple_kv/test/case-603.ini | 3 + src/replica/storage/simple_kv/test/run.sh | 8 ++ .../simple_kv/test/simple_kv.server.impl.cpp | 129 ++++++++++++------ 57 files changed, 340 insertions(+), 85 deletions(-) diff --git a/src/replica/storage/simple_kv/CMakeLists.txt b/src/replica/storage/simple_kv/CMakeLists.txt index cb7d328fc1..667b4ab862 100644 --- a/src/replica/storage/simple_kv/CMakeLists.txt +++ b/src/replica/storage/simple_kv/CMakeLists.txt @@ -37,7 +37,7 @@ set(MY_PROJ_SRC ${SIMPLE_KV_THRIFT_SRCS}) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime hashtable) +set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime hashtable rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp index a019c0999b..81a718a45f 100644 --- a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp @@ -35,24 +35,36 @@ #include "simple_kv.server.impl.h" +#include +#include #include +#include #include #include #include #include -#include +#include #include #include +#include "aio/aio_task.h" +#include "aio/file_io.h" +#include "common/replication.codes.h" #include "consensus_types.h" #include "replica/storage/simple_kv/simple_kv.server.h" +#include "rocksdb/env.h" +#include "rocksdb/status.h" #include "runtime/serverlet.h" #include "simple_kv_types.h" +#include "utils/autoref_ptr.h" +#include "utils/binary_reader.h" +#include "utils/blob.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" +#include "utils/ports.h" +#include "utils/utils.h" namespace dsn { -class blob; namespace replication { class replica; @@ -175,75 +187,104 @@ void simple_kv_service_impl::recover(const std::string &name, int64_t version) { zauto_lock l(_lock); - std::ifstream is(name.c_str(), std::ios::binary); - if (!is.is_open()) - return; + std::unique_ptr rfile; + auto s = rocksdb::Env::Default()->NewSequentialFile(name, &rfile, rocksdb::EnvOptions()); + CHECK(s.ok(), "open log file '{}' failed, err = {}", name, s.ToString()); _store.clear(); - uint64_t count; - int magic; - - is.read((char *)&count, sizeof(count)); - is.read((char *)&magic, sizeof(magic)); + // Read header. + uint64_t count = 0; + int magic = 0; + rocksdb::Slice result; + static const uint64_t kHeaderSize = sizeof(count) + sizeof(magic); + char buff[kHeaderSize] = {0}; + s = rfile->Read(kHeaderSize, &result, buff); + CHECK(s.ok(), "read header failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + binary_reader reader(blob(buff, 0, kHeaderSize)); + CHECK_EQ(sizeof(count), reader.read(count)); + CHECK_EQ(sizeof(magic), reader.read(magic)); CHECK_EQ_MSG(magic, 0xdeadbeef, "invalid checkpoint"); + // Read kv pairs. for (uint64_t i = 0; i < count; i++) { - std::string key; - std::string value; - - uint32_t sz; - is.read((char *)&sz, (uint32_t)sizeof(sz)); - key.resize(sz); - - is.read((char *)&key[0], sz); - - is.read((char *)&sz, (uint32_t)sizeof(sz)); - value.resize(sz); - - is.read((char *)&value[0], sz); - + // Read key. + uint32_t sz = 0; + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read key size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr key_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, key_buffer.get()); + CHECK(s.ok(), "read key failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string key = result.ToString(); + + // Read value. + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read value size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr value_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, value_buffer.get()); + CHECK(s.ok(), "read value failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string value = result.ToString(); + + // Store the kv pair. _store[key] = value; } - is.close(); } ::dsn::error_code simple_kv_service_impl::sync_checkpoint() { - char name[256]; int64_t last_commit = _last_committed_decree.load(); - sprintf(name, "%s/checkpoint.%" PRId64, _dir_data.c_str(), last_commit); + std::string fname = fmt::format("{}/checkpoint.{}", data_dir(), last_commit); zauto_lock l(_lock); - if (last_commit == last_durable_decree()) { - CHECK(utils::filesystem::file_exists(name), "checkpoint file {} is missing!", name); + CHECK(utils::filesystem::file_exists(fname), "checkpoint file {} is missing!", fname); return ERR_OK; } - std::ofstream os(name, std::ios::binary); + auto wfile = file::open(fname.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + CHECK_NOTNULL(wfile, ""); +#define WRITE_DATA_SIZE(data, size) \ + do { \ + auto tsk = ::dsn::file::write( \ + wfile, (char *)&data, size, offset, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr); \ + tsk->wait(); \ + offset += size; \ + } while (false) + +#define WRITE_DATA(data) WRITE_DATA_SIZE(data, sizeof(data)) + + uint64_t offset = 0; uint64_t count = (uint64_t)_store.size(); - int magic = 0xdeadbeef; + WRITE_DATA(count); - os.write((const char *)&count, (uint32_t)sizeof(count)); - os.write((const char *)&magic, (uint32_t)sizeof(magic)); + int magic = 0xdeadbeef; + WRITE_DATA(magic); - for (auto it = _store.begin(); it != _store.end(); ++it) { - const std::string &k = it->first; + for (const auto &kv : _store) { + const std::string &k = kv.first; uint32_t sz = (uint32_t)k.length(); + WRITE_DATA(sz); + WRITE_DATA_SIZE(k[0], sz); - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&k[0], sz); - - const std::string &v = it->second; + const std::string &v = kv.second; sz = (uint32_t)v.length(); - - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&v[0], sz); + WRITE_DATA(sz); + WRITE_DATA_SIZE(v[0], sz); } +#undef WRITE_DATA +#undef WRITE_DATA_SIZE - os.close(); + CHECK_EQ(ERR_OK, file::flush(wfile)); + CHECK_EQ(ERR_OK, file::close(wfile)); // TODO: gc checkpoints set_last_durable_decree(last_commit); diff --git a/src/replica/storage/simple_kv/test/CMakeLists.txt b/src/replica/storage/simple_kv/test/CMakeLists.txt index 1d64070fb6..de86358d09 100644 --- a/src/replica/storage/simple_kv/test/CMakeLists.txt +++ b/src/replica/storage/simple_kv/test/CMakeLists.txt @@ -39,7 +39,8 @@ set(MY_PROJ_LIBS dsn_replica_server zookeeper hashtable gtest - ) + dsn_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/storage/simple_kv/test/case-000.ini b/src/replica/storage/simple_kv/test/case-000.ini index a05a95e53f..4cb2b55450 100644 --- a/src/replica/storage/simple_kv/test/case-000.ini +++ b/src/replica/storage/simple_kv/test/case-000.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-001.ini b/src/replica/storage/simple_kv/test/case-001.ini index 4f214ef80b..08c42f699a 100644 --- a/src/replica/storage/simple_kv/test/case-001.ini +++ b/src/replica/storage/simple_kv/test/case-001.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-002.ini b/src/replica/storage/simple_kv/test/case-002.ini index d75bbc1013..e883afcc03 100644 --- a/src/replica/storage/simple_kv/test/case-002.ini +++ b/src/replica/storage/simple_kv/test/case-002.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-003.ini b/src/replica/storage/simple_kv/test/case-003.ini index 8d60d50d66..5ee053ad4d 100644 --- a/src/replica/storage/simple_kv/test/case-003.ini +++ b/src/replica/storage/simple_kv/test/case-003.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-004.ini b/src/replica/storage/simple_kv/test/case-004.ini index 5c74f7e281..a2eebf4ee6 100644 --- a/src/replica/storage/simple_kv/test/case-004.ini +++ b/src/replica/storage/simple_kv/test/case-004.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-005.ini b/src/replica/storage/simple_kv/test/case-005.ini index 4c8c84b324..1b75197419 100644 --- a/src/replica/storage/simple_kv/test/case-005.ini +++ b/src/replica/storage/simple_kv/test/case-005.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-006.ini b/src/replica/storage/simple_kv/test/case-006.ini index 18e782b779..4a8d09e289 100644 --- a/src/replica/storage/simple_kv/test/case-006.ini +++ b/src/replica/storage/simple_kv/test/case-006.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-100.ini b/src/replica/storage/simple_kv/test/case-100.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-100.ini +++ b/src/replica/storage/simple_kv/test/case-100.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-101.ini b/src/replica/storage/simple_kv/test/case-101.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-101.ini +++ b/src/replica/storage/simple_kv/test/case-101.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-102.ini b/src/replica/storage/simple_kv/test/case-102.ini index 8c35047783..4ac4bcc5f8 100644 --- a/src/replica/storage/simple_kv/test/case-102.ini +++ b/src/replica/storage/simple_kv/test/case-102.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-103.ini b/src/replica/storage/simple_kv/test/case-103.ini index 64a643dd0a..3d70ea9c5e 100644 --- a/src/replica/storage/simple_kv/test/case-103.ini +++ b/src/replica/storage/simple_kv/test/case-103.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-104.ini b/src/replica/storage/simple_kv/test/case-104.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-104.ini +++ b/src/replica/storage/simple_kv/test/case-104.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-105.ini b/src/replica/storage/simple_kv/test/case-105.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-105.ini +++ b/src/replica/storage/simple_kv/test/case-105.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-106.ini b/src/replica/storage/simple_kv/test/case-106.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-106.ini +++ b/src/replica/storage/simple_kv/test/case-106.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-107.ini b/src/replica/storage/simple_kv/test/case-107.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-107.ini +++ b/src/replica/storage/simple_kv/test/case-107.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-108.ini b/src/replica/storage/simple_kv/test/case-108.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-108.ini +++ b/src/replica/storage/simple_kv/test/case-108.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-109.ini b/src/replica/storage/simple_kv/test/case-109.ini index d95415e373..9a1d88c697 100644 --- a/src/replica/storage/simple_kv/test/case-109.ini +++ b/src/replica/storage/simple_kv/test/case-109.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-200.ini b/src/replica/storage/simple_kv/test/case-200.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-200.ini +++ b/src/replica/storage/simple_kv/test/case-200.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-201.ini b/src/replica/storage/simple_kv/test/case-201.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-201.ini +++ b/src/replica/storage/simple_kv/test/case-201.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-202-0.ini b/src/replica/storage/simple_kv/test/case-202-0.ini index a25eb9145e..dabdf01544 100644 --- a/src/replica/storage/simple_kv/test/case-202-0.ini +++ b/src/replica/storage/simple_kv/test/case-202-0.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-202-1.ini b/src/replica/storage/simple_kv/test/case-202-1.ini index a25eb9145e..dabdf01544 100644 --- a/src/replica/storage/simple_kv/test/case-202-1.ini +++ b/src/replica/storage/simple_kv/test/case-202-1.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-203-0.ini b/src/replica/storage/simple_kv/test/case-203-0.ini index 82d693aa9b..42c9c006c9 100644 --- a/src/replica/storage/simple_kv/test/case-203-0.ini +++ b/src/replica/storage/simple_kv/test/case-203-0.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-204.ini b/src/replica/storage/simple_kv/test/case-204.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-204.ini +++ b/src/replica/storage/simple_kv/test/case-204.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-205.ini b/src/replica/storage/simple_kv/test/case-205.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-205.ini +++ b/src/replica/storage/simple_kv/test/case-205.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-206.ini b/src/replica/storage/simple_kv/test/case-206.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-206.ini +++ b/src/replica/storage/simple_kv/test/case-206.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-207.ini b/src/replica/storage/simple_kv/test/case-207.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-207.ini +++ b/src/replica/storage/simple_kv/test/case-207.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-208.ini b/src/replica/storage/simple_kv/test/case-208.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-208.ini +++ b/src/replica/storage/simple_kv/test/case-208.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-209.ini b/src/replica/storage/simple_kv/test/case-209.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-209.ini +++ b/src/replica/storage/simple_kv/test/case-209.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-210.ini b/src/replica/storage/simple_kv/test/case-210.ini index 23784c15f0..a9dd9383cc 100644 --- a/src/replica/storage/simple_kv/test/case-210.ini +++ b/src/replica/storage/simple_kv/test/case-210.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-211.ini b/src/replica/storage/simple_kv/test/case-211.ini index 23784c15f0..a9dd9383cc 100644 --- a/src/replica/storage/simple_kv/test/case-211.ini +++ b/src/replica/storage/simple_kv/test/case-211.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-212.ini b/src/replica/storage/simple_kv/test/case-212.ini index 4582c07878..a61a428f21 100644 --- a/src/replica/storage/simple_kv/test/case-212.ini +++ b/src/replica/storage/simple_kv/test/case-212.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-213.ini b/src/replica/storage/simple_kv/test/case-213.ini index 103a599a63..660afcfff9 100644 --- a/src/replica/storage/simple_kv/test/case-213.ini +++ b/src/replica/storage/simple_kv/test/case-213.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-214.ini b/src/replica/storage/simple_kv/test/case-214.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-214.ini +++ b/src/replica/storage/simple_kv/test/case-214.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-215.ini b/src/replica/storage/simple_kv/test/case-215.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-215.ini +++ b/src/replica/storage/simple_kv/test/case-215.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-216.ini b/src/replica/storage/simple_kv/test/case-216.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-216.ini +++ b/src/replica/storage/simple_kv/test/case-216.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-300-0.ini b/src/replica/storage/simple_kv/test/case-300-0.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-300-0.ini +++ b/src/replica/storage/simple_kv/test/case-300-0.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-300-1.ini b/src/replica/storage/simple_kv/test/case-300-1.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-300-1.ini +++ b/src/replica/storage/simple_kv/test/case-300-1.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-300-2.ini b/src/replica/storage/simple_kv/test/case-300-2.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-300-2.ini +++ b/src/replica/storage/simple_kv/test/case-300-2.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-301.ini b/src/replica/storage/simple_kv/test/case-301.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-301.ini +++ b/src/replica/storage/simple_kv/test/case-301.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-302.ini b/src/replica/storage/simple_kv/test/case-302.ini index 38d3a831f1..fdfb947769 100644 --- a/src/replica/storage/simple_kv/test/case-302.ini +++ b/src/replica/storage/simple_kv/test/case-302.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-303.ini b/src/replica/storage/simple_kv/test/case-303.ini index 04932bc943..db032f0a27 100644 --- a/src/replica/storage/simple_kv/test/case-303.ini +++ b/src/replica/storage/simple_kv/test/case-303.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-304.ini b/src/replica/storage/simple_kv/test/case-304.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-304.ini +++ b/src/replica/storage/simple_kv/test/case-304.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-305.ini b/src/replica/storage/simple_kv/test/case-305.ini index 04932bc943..db032f0a27 100644 --- a/src/replica/storage/simple_kv/test/case-305.ini +++ b/src/replica/storage/simple_kv/test/case-305.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-306.ini b/src/replica/storage/simple_kv/test/case-306.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-306.ini +++ b/src/replica/storage/simple_kv/test/case-306.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-307.ini b/src/replica/storage/simple_kv/test/case-307.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-307.ini +++ b/src/replica/storage/simple_kv/test/case-307.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-400.ini b/src/replica/storage/simple_kv/test/case-400.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-400.ini +++ b/src/replica/storage/simple_kv/test/case-400.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-401.ini b/src/replica/storage/simple_kv/test/case-401.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-401.ini +++ b/src/replica/storage/simple_kv/test/case-401.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-402.ini b/src/replica/storage/simple_kv/test/case-402.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-402.ini +++ b/src/replica/storage/simple_kv/test/case-402.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-600.ini b/src/replica/storage/simple_kv/test/case-600.ini index 18e782b779..4a8d09e289 100644 --- a/src/replica/storage/simple_kv/test/case-600.ini +++ b/src/replica/storage/simple_kv/test/case-600.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-601.ini b/src/replica/storage/simple_kv/test/case-601.ini index 9a25f26cc8..20ee3c8bf0 100644 --- a/src/replica/storage/simple_kv/test/case-601.ini +++ b/src/replica/storage/simple_kv/test/case-601.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-602.ini b/src/replica/storage/simple_kv/test/case-602.ini index 18e782b779..4a8d09e289 100644 --- a/src/replica/storage/simple_kv/test/case-602.ini +++ b/src/replica/storage/simple_kv/test/case-602.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-603.ini b/src/replica/storage/simple_kv/test/case-603.ini index 1531c037fe..563b86adf8 100644 --- a/src/replica/storage/simple_kv/test/case-603.ini +++ b/src/replica/storage/simple_kv/test/case-603.ini @@ -151,6 +151,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/run.sh b/src/replica/storage/simple_kv/test/run.sh index b3daa127d1..e10210fa41 100755 --- a/src/replica/storage/simple_kv/test/run.sh +++ b/src/replica/storage/simple_kv/test/run.sh @@ -112,6 +112,14 @@ else fi if [ ! -z "${cases}" ]; then + OLD_TEST_OPTS=${TEST_OPTS} + TEST_OPTS=${OLD_TEST_OPTS},encrypt_data_at_rest=false + for id in ${cases}; do + run_case ${id} + echo + done + # TODO(yingchun): ENCRYPTION: add enable encryption test. + # TEST_OPTS=${OLD_TEST_OPTS},encrypt_data_at_rest=true for id in ${cases}; do run_case ${id} echo diff --git a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp index cf3f418f79..ae04a7f4e6 100644 --- a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp @@ -25,22 +25,37 @@ */ #include "simple_kv.server.impl.h" +#include +#include #include #include #include #include -#include +#include #include #include +#include "aio/aio_task.h" +#include "aio/file_io.h" #include "consensus_types.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/serverlet.h" +#include "runtime/task/task_code.h" #include "simple_kv_types.h" +#include "utils/autoref_ptr.h" +#include "utils/binary_reader.h" +#include "utils/blob.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" +#include "utils/ports.h" +#include "utils/threadpool_code.h" +#include "utils/utils.h" +// TODO(yingchun): most of the code are the same as +// src/replica/storage/simple_kv/simple_kv.server.impl.cpp, unify the code! namespace dsn { -class blob; namespace replication { class replica; @@ -53,6 +68,8 @@ namespace dsn { namespace replication { namespace test { +DEFINE_TASK_CODE(LPC_AIO_IMMEDIATE_CALLBACK, TASK_PRIORITY_COMMON, dsn::THREAD_POOL_DEFAULT) + bool simple_kv_service_impl::s_simple_kv_open_fail = false; bool simple_kv_service_impl::s_simple_kv_close_fail = false; bool simple_kv_service_impl::s_simple_kv_get_checkpoint_fail = false; @@ -173,34 +190,53 @@ void simple_kv_service_impl::recover(const std::string &name, int64_t version) { dsn::zauto_lock l(_lock); - std::ifstream is(name.c_str(), std::ios::binary); - if (!is.is_open()) - return; + std::unique_ptr rfile; + auto s = rocksdb::Env::Default()->NewSequentialFile(name, &rfile, rocksdb::EnvOptions()); + CHECK(s.ok(), "open log file '{}' failed, err = {}", name, s.ToString()); _store.clear(); - uint64_t count; - int magic; - - is.read((char *)&count, sizeof(count)); - is.read((char *)&magic, sizeof(magic)); + // Read header. + uint64_t count = 0; + int magic = 0; + rocksdb::Slice result; + static const uint64_t kHeaderSize = sizeof(count) + sizeof(magic); + char buff[kHeaderSize] = {0}; + s = rfile->Read(kHeaderSize, &result, buff); + CHECK(s.ok(), "read header failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + binary_reader reader(blob(buff, 0, kHeaderSize)); + CHECK_EQ(sizeof(count), reader.read(count)); + CHECK_EQ(sizeof(magic), reader.read(magic)); CHECK_EQ_MSG(magic, 0xdeadbeef, "invalid checkpoint"); + // Read kv pairs. for (uint64_t i = 0; i < count; i++) { - std::string key; - std::string value; - - uint32_t sz; - is.read((char *)&sz, (uint32_t)sizeof(sz)); - key.resize(sz); - - is.read((char *)&key[0], sz); - - is.read((char *)&sz, (uint32_t)sizeof(sz)); - value.resize(sz); - - is.read((char *)&value[0], sz); - + // Read key. + uint32_t sz = 0; + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read key size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr key_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, key_buffer.get()); + CHECK(s.ok(), "read key failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string key = result.ToString(); + + // Read value. + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read value size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr value_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, value_buffer.get()); + CHECK(s.ok(), "read value failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string value = result.ToString(); + + // Store the kv pair. _store[key] = value; } } @@ -217,30 +253,43 @@ ::dsn::error_code simple_kv_service_impl::sync_checkpoint() return ERR_OK; } - // TODO: should use async write instead - char name[256]; - sprintf(name, "%s/checkpoint.%" PRId64, data_dir().c_str(), last_commit); - std::ofstream os(name, std::ios::binary); + std::string fname = fmt::format("{}/checkpoint.{}", data_dir(), last_commit); + auto wfile = file::open(fname.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + CHECK_NOTNULL(wfile, ""); +#define WRITE_DATA_SIZE(data, size) \ + do { \ + auto tsk = ::dsn::file::write( \ + wfile, (char *)&data, size, offset, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr); \ + tsk->wait(); \ + offset += size; \ + } while (false) + +#define WRITE_DATA(data) WRITE_DATA_SIZE(data, sizeof(data)) + + uint64_t offset = 0; uint64_t count = (uint64_t)_store.size(); - int magic = 0xdeadbeef; + WRITE_DATA(count); - os.write((const char *)&count, (uint32_t)sizeof(count)); - os.write((const char *)&magic, (uint32_t)sizeof(magic)); + int magic = 0xdeadbeef; + WRITE_DATA(magic); - for (auto it = _store.begin(); it != _store.end(); ++it) { - const std::string &k = it->first; + for (const auto &kv : _store) { + const std::string &k = kv.first; uint32_t sz = (uint32_t)k.length(); + WRITE_DATA(sz); + WRITE_DATA_SIZE(k[0], sz); - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&k[0], sz); - - const std::string &v = it->second; + const std::string &v = kv.second; sz = (uint32_t)v.length(); - - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&v[0], sz); + WRITE_DATA(sz); + WRITE_DATA_SIZE(v[0], sz); } +#undef WRITE_DATA +#undef WRITE_DATA_SIZE + + CHECK_EQ(ERR_OK, file::flush(wfile)); + CHECK_EQ(ERR_OK, file::close(wfile)); set_last_durable_decree(last_commit); LOG_INFO("simple_kv_service_impl create checkpoint succeed, " From a1c463d48abe1b0c3f742b700fd7d52c5f456764 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 26 Sep 2023 06:56:19 -0500 Subject: [PATCH 30/42] refactor(replica): use rocksdb API to read/write file (#1623) https://github.com/apache/incubator-pegasus/issues/887 There is no functional changes, but only refactor the replica metadata files read/write module. --- src/replica/replica_restore.cpp | 41 +++------ src/replica/replication_app_base.cpp | 122 +++++++-------------------- src/replica/replication_app_base.h | 8 +- 3 files changed, 49 insertions(+), 122 deletions(-) diff --git a/src/replica/replica_restore.cpp b/src/replica/replica_restore.cpp index 1a37479777..7236f31a15 100644 --- a/src/replica/replica_restore.cpp +++ b/src/replica/replica_restore.cpp @@ -17,6 +17,8 @@ #include #include +#include +#include #include #include #include @@ -52,7 +54,6 @@ #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" -#include "utils/utils.h" using namespace dsn::dist::block_service; @@ -93,41 +94,25 @@ bool replica::remove_useless_file_under_chkpt(const std::string &chkpt_dir, return true; } -bool replica::read_cold_backup_metadata(const std::string &file, +bool replica::read_cold_backup_metadata(const std::string &fname, cold_backup_metadata &backup_metadata) { - if (!::dsn::utils::filesystem::file_exists(file)) { + if (!::dsn::utils::filesystem::file_exists(fname)) { LOG_ERROR_PREFIX( - "checkpoint on remote storage media is damaged, coz file({}) doesn't exist", file); + "checkpoint on remote storage media is damaged, coz file({}) doesn't exist", fname); return false; } - int64_t file_sz = 0; - if (!::dsn::utils::filesystem::file_size(file, file_sz)) { - LOG_ERROR_PREFIX("get file({}) size failed", file); - return false; - } - std::shared_ptr buf = utils::make_shared_array(file_sz + 1); - std::ifstream fin(file, std::ifstream::in); - if (!fin.is_open()) { - LOG_ERROR_PREFIX("open file({}) failed", file); + std::string data; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &data); + if (!s.ok()) { + LOG_ERROR_PREFIX("read file '{}' failed, err = {}", fname, s.ToString()); return false; } - fin.read(buf.get(), file_sz); - CHECK_EQ_MSG(file_sz, - fin.gcount(), - "{}: read file({}) failed, need {}, but read {}", - name(), - file, - file_sz, - fin.gcount()); - fin.close(); - - buf.get()[fin.gcount()] = '\0'; - blob bb; - bb.assign(std::move(buf), 0, file_sz); - if (!::dsn::json::json_forwarder::decode(bb, backup_metadata)) { - LOG_ERROR_PREFIX("file({}) under checkpoint is damaged", file); + + if (!::dsn::json::json_forwarder::decode( + blob::create_from_bytes(std::move(data)), backup_metadata)) { + LOG_ERROR_PREFIX("file({}) under checkpoint is damaged", fname); return false; } return true; diff --git a/src/replica/replication_app_base.cpp b/src/replica/replication_app_base.cpp index 4aa7a8e145..bc84c0c0a6 100644 --- a/src/replica/replication_app_base.cpp +++ b/src/replica/replication_app_base.cpp @@ -25,16 +25,14 @@ */ #include -#include +#include +#include #include -#include #include #include #include #include -#include "aio/aio_task.h" -#include "aio/file_io.h" #include "common/bulk_load_common.h" #include "common/duplication_common.h" #include "common/replica_envs.h" @@ -50,7 +48,6 @@ #include "runtime/rpc/serialization.h" #include "runtime/task/task_code.h" #include "runtime/task/task_spec.h" -#include "runtime/task/task_tracker.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" #include "utils/blob.h" @@ -60,57 +57,31 @@ #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/latency_tracer.h" -#include "utils/ports.h" #include "utils/string_view.h" -#include "utils/threadpool_code.h" -#include "utils/utils.h" namespace dsn { -class disk_file; namespace replication { const std::string replica_init_info::kInitInfo = ".init-info"; -DEFINE_TASK_CODE_AIO(LPC_AIO_INFO_WRITE, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) - namespace { -error_code write_blob_to_file(const std::string &file, const blob &data) +error_code write_blob_to_file(const std::string &fname, const blob &data) { - std::string tmp_file = file + ".tmp"; - disk_file *hfile = file::open(tmp_file.c_str(), O_WRONLY | O_CREAT | O_BINARY | O_TRUNC, 0666); + std::string tmp_fname = fname + ".tmp"; + auto cleanup = defer([tmp_fname]() { utils::filesystem::remove_path(tmp_fname); }); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(data.data(), data.length()), + tmp_fname, + /* should_sync */ true); LOG_AND_RETURN_NOT_TRUE( - ERROR, hfile, ERR_FILE_OPERATION_FAILED, "open file {} failed", tmp_file); - auto cleanup = defer([tmp_file]() { utils::filesystem::remove_path(tmp_file); }); - - error_code err; - size_t sz = 0; - task_tracker tracker; - aio_task_ptr tsk = file::write(hfile, - data.data(), - data.length(), - 0, - LPC_AIO_INFO_WRITE, - &tracker, - [&err, &sz](error_code e, size_t s) { - err = e; - sz = s; - }, - 0); - CHECK_NOTNULL(tsk, "create file::write task failed"); - tracker.wait_outstanding_tasks(); - file::flush(hfile); - file::close(hfile); - LOG_AND_RETURN_NOT_OK(ERROR, err, "write file {} failed", tmp_file); - CHECK_EQ(data.length(), sz); - // TODO(yingchun): need fsync too？ + ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "write file {} failed", tmp_fname); LOG_AND_RETURN_NOT_TRUE(ERROR, - utils::filesystem::rename_path(tmp_file, file), + utils::filesystem::rename_path(tmp_fname, fname), ERR_FILE_OPERATION_FAILED, "move file from {} to {} failed", - tmp_file, - file); - + tmp_fname, + fname); return ERR_OK; } } // namespace @@ -145,38 +116,23 @@ error_code replica_init_info::store(const std::string &dir) return ERR_OK; } -error_code replica_init_info::load_json(const std::string &file) +error_code replica_init_info::load_json(const std::string &fname) { - std::ifstream is(file, std::ios::binary); - LOG_AND_RETURN_NOT_TRUE( - ERROR, is.is_open(), ERR_FILE_OPERATION_FAILED, "open file {} failed", file); - - int64_t sz = 0; + std::string data; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &data); + LOG_AND_RETURN_NOT_TRUE(ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "read file {} failed", fname); LOG_AND_RETURN_NOT_TRUE(ERROR, - utils::filesystem::file_size(std::string(file), sz), + json::json_forwarder::decode( + blob::create_from_bytes(std::move(data)), *this), ERR_FILE_OPERATION_FAILED, - "get file size of {} failed", - file); - - std::shared_ptr buffer(utils::make_shared_array(sz)); - is.read((char *)buffer.get(), sz); - LOG_AND_RETURN_NOT_TRUE( - ERROR, !is.bad(), ERR_FILE_OPERATION_FAILED, "read file {} failed", file); - is.close(); - - LOG_AND_RETURN_NOT_TRUE( - ERROR, - json::json_forwarder::decode(blob(buffer, sz), *this), - ERR_FILE_OPERATION_FAILED, - "decode json from file {} failed", - file); - + "decode json from file {} failed", + fname); return ERR_OK; } -error_code replica_init_info::store_json(const std::string &file) +error_code replica_init_info::store_json(const std::string &fname) { - return write_blob_to_file(file, json::json_forwarder::encode(*this)); + return write_blob_to_file(fname, json::json_forwarder::encode(*this)); } std::string replica_init_info::to_string() @@ -189,35 +145,21 @@ std::string replica_init_info::to_string() return oss.str(); } -error_code replica_app_info::load(const std::string &file) +error_code replica_app_info::load(const std::string &fname) { - std::ifstream is(file, std::ios::binary); - LOG_AND_RETURN_NOT_TRUE( - ERROR, is.is_open(), ERR_FILE_OPERATION_FAILED, "open file {} failed", file); - - int64_t sz = 0; - LOG_AND_RETURN_NOT_TRUE(ERROR, - utils::filesystem::file_size(std::string(file), sz), - ERR_FILE_OPERATION_FAILED, - "get file size of {} failed", - file); - - std::shared_ptr buffer(utils::make_shared_array(sz)); - is.read((char *)buffer.get(), sz); - is.close(); - - binary_reader reader(blob(buffer, sz)); - int magic; + std::string data; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &data); + LOG_AND_RETURN_NOT_TRUE(ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "read file {} failed", fname); + binary_reader reader(blob::create_from_bytes(std::move(data))); + int magic = 0; unmarshall(reader, magic, DSF_THRIFT_BINARY); - LOG_AND_RETURN_NOT_TRUE( - ERROR, magic == 0xdeadbeef, ERR_INVALID_DATA, "data in file {} is invalid (magic)", file); - + ERROR, magic == 0xdeadbeef, ERR_INVALID_DATA, "data in file {} is invalid (magic)", fname); unmarshall(reader, *_app, DSF_THRIFT_JSON); return ERR_OK; } -error_code replica_app_info::store(const std::string &file) +error_code replica_app_info::store(const std::string &fname) { binary_writer writer; int magic = 0xdeadbeef; @@ -237,7 +179,7 @@ error_code replica_app_info::store(const std::string &file) marshall(writer, tmp, DSF_THRIFT_JSON); } - return write_blob_to_file(file, writer.get_buffer()); + return write_blob_to_file(fname, writer.get_buffer()); } /*static*/ diff --git a/src/replica/replication_app_base.h b/src/replica/replication_app_base.h index aafd97c385..5ae162a1bb 100644 --- a/src/replica/replication_app_base.h +++ b/src/replica/replication_app_base.h @@ -76,8 +76,8 @@ class replica_init_info std::string to_string(); private: - error_code load_json(const std::string &file); - error_code store_json(const std::string &file); + error_code load_json(const std::string &fname); + error_code store_json(const std::string &fname); }; class replica_app_info @@ -87,8 +87,8 @@ class replica_app_info public: replica_app_info(app_info *app) { _app = app; } - error_code load(const std::string &file); - error_code store(const std::string &file); + error_code load(const std::string &fname); + error_code store(const std::string &fname); }; /// The store engine interface of Pegasus. From c65c39af34071a4de28a0507098b5306b212afa1 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 27 Sep 2023 08:01:22 -0500 Subject: [PATCH 31/42] refactor(misc): use rocksdb API to read/write file (#1625) https://github.com/apache/incubator-pegasus/issues/887 There is no functional changes, but only refactor the files read/write method of some unit tests. --- src/block_service/test/CMakeLists.txt | 1 + .../test/block_service_manager_test.cpp | 68 +++++++++---------- src/replica/bulk_load/test/CMakeLists.txt | 3 +- .../test/replica_bulk_loader_test.cpp | 59 ++++++---------- .../test/load_from_private_log_test.cpp | 64 +++++++++++------ src/replica/test/CMakeLists.txt | 3 +- src/replica/test/mutation_log_test.cpp | 66 +++++++----------- .../function_test/bulk_load/CMakeLists.txt | 4 +- src/test_util/CMakeLists.txt | 2 +- src/test_util/test_util.cpp | 21 ++++++ src/test_util/test_util.h | 11 ++- src/utils/test/CMakeLists.txt | 4 +- 12 files changed, 160 insertions(+), 146 deletions(-) diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt index 537778caad..19dd06808b 100644 --- a/src/block_service/test/CMakeLists.txt +++ b/src/block_service/test/CMakeLists.txt @@ -36,6 +36,7 @@ set(MY_PROJ_LIBS gtest gtest_main hdfs + test_utils rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/block_service/test/block_service_manager_test.cpp b/src/block_service/test/block_service_manager_test.cpp index 8ead04fd2a..ef93ed1962 100644 --- a/src/block_service/test/block_service_manager_test.cpp +++ b/src/block_service/test/block_service_manager_test.cpp @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include #include -#include #include #include #include @@ -29,14 +29,16 @@ #include "block_service/local/local_service.h" #include "block_service_mock.h" #include "metadata_types.h" +#include "test_util/test_util.h" #include "utils/error_code.h" #include "utils/filesystem.h" +#include "utils/test_macros.h" namespace dsn { namespace dist { namespace block_service { -class block_service_manager_test : public ::testing::Test +class block_service_manager_test : public pegasus::encrypt_data_test_base { public: block_service_manager_test() @@ -54,20 +56,6 @@ class block_service_manager_test : public ::testing::Test PROVIDER, LOCAL_DIR, FILE_NAME, _fs.get(), download_size); } - void create_local_file(const std::string &file_name) - { - std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, file_name); - utils::filesystem::create_file(whole_name); - std::ofstream test_file; - test_file.open(whole_name); - test_file << "write some data.\n"; - test_file.close(); - - _file_meta.name = whole_name; - utils::filesystem::md5sum(whole_name, _file_meta.md5); - utils::filesystem::file_size(whole_name, _file_meta.size); - } - void create_remote_file(const std::string &file_name, int64_t size, const std::string &md5) { std::string whole_file_name = utils::filesystem::path_combine(PROVIDER, file_name); @@ -84,8 +72,10 @@ class block_service_manager_test : public ::testing::Test std::string FILE_NAME = "test_file"; }; -// download_file unit tests -TEST_F(block_service_manager_test, do_download_remote_file_not_exist) +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, block_service_manager_test, ::testing::Values(false)); + +TEST_P(block_service_manager_test, remote_file_not_exist) { utils::filesystem::remove_path(LOCAL_DIR); auto fs = std::make_unique(); @@ -93,31 +83,35 @@ TEST_F(block_service_manager_test, do_download_remote_file_not_exist) uint64_t download_size = 0; error_code err = _block_service_manager.download_file( PROVIDER, LOCAL_DIR, FILE_NAME, fs.get(), download_size); - ASSERT_EQ(err, ERR_CORRUPTION); // file does not exist + ASSERT_EQ(ERR_CORRUPTION, err); } -TEST_F(block_service_manager_test, do_download_same_name_file) +TEST_P(block_service_manager_test, local_file_exist) { - // local file exists, but md5 not matched with remote file - create_local_file(FILE_NAME); - create_remote_file(FILE_NAME, 2333, "md5_not_match"); - uint64_t download_size = 0; - ASSERT_EQ(test_download_file(download_size), ERR_PATH_ALREADY_EXIST); - ASSERT_EQ(download_size, 0); -} - -TEST_F(block_service_manager_test, do_download_file_exist) -{ - create_local_file(FILE_NAME); - create_remote_file(FILE_NAME, _file_meta.size, _file_meta.md5); - uint64_t download_size = 0; - ASSERT_EQ(test_download_file(download_size), ERR_PATH_ALREADY_EXIST); - ASSERT_EQ(download_size, 0); + NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME), + &_file_meta)); + struct remote_file_info + { + int64_t size; + std::string md5; + } tests[]{ + {2333, "bad_md5"}, // wrong size and md5 + {2333, _file_meta.md5}, // wrong size + {_file_meta.size, "bad_md5"} // wrong md5 + }; + for (const auto &test : tests) { + // The remote file will be overwritten when repeatedly created. + create_remote_file(FILE_NAME, test.size, test.md5); + uint64_t download_size = 0; + ASSERT_EQ(ERR_PATH_ALREADY_EXIST, test_download_file(download_size)); + ASSERT_EQ(0, download_size); + } } -TEST_F(block_service_manager_test, do_download_succeed) +TEST_P(block_service_manager_test, do_download_succeed) { - create_local_file(FILE_NAME); + NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME), + &_file_meta)); create_remote_file(FILE_NAME, _file_meta.size, _file_meta.md5); // remove local file to mock condition that file not existed std::string file_name = utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME); diff --git a/src/replica/bulk_load/test/CMakeLists.txt b/src/replica/bulk_load/test/CMakeLists.txt index 77081196c5..a15861f61c 100644 --- a/src/replica/bulk_load/test/CMakeLists.txt +++ b/src/replica/bulk_load/test/CMakeLists.txt @@ -27,7 +27,8 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_runtime hashtable gtest -) + test_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp index df123cc0c7..9eb7322a61 100644 --- a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp +++ b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp @@ -17,9 +17,14 @@ #include "replica/bulk_load/replica_bulk_loader.h" +#include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include // IWYU pragma: keep #include #include @@ -32,10 +37,11 @@ #include "replica/test/replica_test_base.h" #include "runtime/rpc/rpc_address.h" #include "runtime/task/task_tracker.h" +#include "test_util/test_util.h" #include "utils/blob.h" #include "utils/fail_point.h" #include "utils/filesystem.h" -#include "utils/fmt_logging.h" +#include "utils/test_macros.h" namespace dsn { namespace replication { @@ -247,44 +253,21 @@ class replica_bulk_loader_test : public replica_test_base _replica->set_primary_partition_configuration(config); } - void create_local_file(const std::string &file_name) + void create_local_metadata_file() { - std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, file_name); - utils::filesystem::create_file(whole_name); - std::ofstream test_file; - test_file.open(whole_name); - test_file << "write some data.\n"; - test_file.close(); + NO_FATALS(pegasus::create_local_test_file( + utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME), &_file_meta)); - _file_meta.name = whole_name; - utils::filesystem::md5sum(whole_name, _file_meta.md5); - utils::filesystem::file_size(whole_name, _file_meta.size); - } - - error_code create_local_metadata_file() - { - create_local_file(FILE_NAME); _metadata.files.emplace_back(_file_meta); _metadata.file_total_size = _file_meta.size; - std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); - utils::filesystem::create_file(whole_name); - std::ofstream os(whole_name.c_str(), - (std::ofstream::out | std::ios::binary | std::ofstream::trunc)); - if (!os.is_open()) { - LOG_ERROR("open file {} failed", whole_name); - return ERR_FILE_OPERATION_FAILED; - } - blob bb = json::json_forwarder::encode(_metadata); - os.write((const char *)bb.data(), (std::streamsize)bb.length()); - if (os.bad()) { - LOG_ERROR("write file {} failed", whole_name); - return ERR_FILE_OPERATION_FAILED; - } - os.close(); - - return ERR_OK; + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(bb.data(), bb.length()), + whole_name, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << fmt::format( + "write file {} failed, err = {}", whole_name, s.ToString()); } bool validate_metadata() @@ -549,21 +532,21 @@ TEST_F(replica_bulk_loader_test, bulk_load_metadata_corrupt) { // create file can not parse as bulk_load_metadata structure utils::filesystem::create_directory(LOCAL_DIR); - create_local_file(METADATA); + NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, METADATA), + &_file_meta)); std::string metadata_file_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); error_code ec = test_parse_bulk_load_metadata(metadata_file_name); - ASSERT_EQ(ec, ERR_CORRUPTION); + ASSERT_EQ(ERR_CORRUPTION, ec); utils::filesystem::remove_path(LOCAL_DIR); } TEST_F(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) { utils::filesystem::create_directory(LOCAL_DIR); - error_code ec = create_local_metadata_file(); - ASSERT_EQ(ec, ERR_OK); + NO_FATALS(create_local_metadata_file()); std::string metadata_file_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); - ec = test_parse_bulk_load_metadata(metadata_file_name); + auto ec = test_parse_bulk_load_metadata(metadata_file_name); ASSERT_EQ(ec, ERR_OK); ASSERT_TRUE(validate_metadata()); utils::filesystem::remove_path(LOCAL_DIR); diff --git a/src/replica/duplication/test/load_from_private_log_test.cpp b/src/replica/duplication/test/load_from_private_log_test.cpp index 342669f63c..2782f55f11 100644 --- a/src/replica/duplication/test/load_from_private_log_test.cpp +++ b/src/replica/duplication/test/load_from_private_log_test.cpp @@ -15,16 +15,19 @@ // specific language governing permissions and limitations // under the License. -#include -#include +// IWYU pragma: no_include #include #include // IWYU pragma: no_include +// IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include #include -#include +#include "aio/aio_task.h" +#include "aio/file_io.h" #include "common/gpid.h" #include "common/replication.codes.h" #include "common/replication_other_types.h" @@ -44,12 +47,14 @@ #include "runtime/task/task_tracker.h" #include "utils/autoref_ptr.h" #include "utils/chrono_literals.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/errors.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" +#include "utils/ports.h" #define BOOST_NO_CXX11_SCOPED_ENUMS #include @@ -58,7 +63,6 @@ #include #include #include -#include #include #include #include @@ -303,42 +307,43 @@ TEST_F(load_from_private_log_test, start_duplication_100000_4MB) // Ensure replica_duplicator can correctly handle real-world log file TEST_F(load_from_private_log_test, handle_real_private_log) { + std::vector log_files({"log.1.0.handle_real_private_log", + "log.1.0.handle_real_private_log2", + "log.1.0.all_loaded_are_write_empties"}); + struct test_data { - std::string fname; int puts; int total; gpid id; } tests[] = { // PUT, PUT, PUT, EMPTY, PUT, EMPTY, EMPTY - {"log.1.0.handle_real_private_log", 4, 6, gpid(1, 4)}, + {4, 6, gpid(1, 4)}, // EMPTY, PUT, EMPTY - {"log.1.0.handle_real_private_log2", 1, 2, gpid(1, 4)}, + {1, 2, gpid(1, 4)}, // EMPTY, EMPTY, EMPTY - {"log.1.0.all_loaded_are_write_empties", 0, 2, gpid(1, 5)}, + {0, 2, gpid(1, 5)}, }; - for (auto tt : tests) { + ASSERT_EQ(log_files.size(), sizeof(tests) / sizeof(test_data)); + for (int i = 0; i < log_files.size(); i++) { // reset replica to specified gpid duplicator.reset(nullptr); - _replica = create_mock_replica(stub.get(), tt.id.get_app_id(), tt.id.get_partition_index()); + _replica = create_mock_replica( + stub.get(), tests[i].id.get_app_id(), tests[i].id.get_partition_index()); // Update '_log_dir' to the corresponding replica created above. _log_dir = _replica->dir(); ASSERT_TRUE(utils::filesystem::path_exists(_log_dir)) << _log_dir; // Copy the log file to '_log_dir' - boost::filesystem::path file(tt.fname); - ASSERT_TRUE(dsn::utils::filesystem::file_exists(tt.fname)) << tt.fname; - boost::system::error_code ec; - boost::filesystem::copy_file( - file, _log_dir + "/log.1.0", boost::filesystem::copy_option::overwrite_if_exists, ec); - ASSERT_TRUE(!ec) << ec.value() << ", " << ec.category().name() << ", " << ec.message(); + auto s = dsn::utils::copy_file(log_files[i], _log_dir + "/log.1.0"); + ASSERT_TRUE(s.ok()) << s.ToString(); // Start to verify. - load_and_wait_all_entries_loaded(tt.puts, tt.total, tt.id, 1, 0); + load_and_wait_all_entries_loaded(tests[i].puts, tests[i].total, tests[i].id, 1, 0); } } @@ -451,12 +456,27 @@ TEST_F(load_fail_mode_test, fail_skip_real_corrupted_file) { { // inject some bad data in the middle of the first file std::string log_path = _log_dir + "/log.1.0"; - auto file_size = boost::filesystem::file_size(log_path); - int fd = open(log_path.c_str(), O_WRONLY); + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + log_path, dsn::utils::FileDataType::kSensitive, file_size)); + auto wfile = file::open(log_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + ASSERT_NE(wfile, nullptr); + const char buf[] = "xxxxxx"; - auto written_size = pwrite(fd, buf, sizeof(buf), file_size / 2); - ASSERT_EQ(written_size, sizeof(buf)); - close(fd); + auto buff_len = sizeof(buf); + auto t = ::dsn::file::write(wfile, + buf, + buff_len, + file_size / 2, + LPC_AIO_IMMEDIATE_CALLBACK, + nullptr, + [=](::dsn::error_code err, size_t n) { + CHECK_EQ(ERR_OK, err); + CHECK_EQ(buff_len, n); + }); + t->wait(); + ASSERT_EQ(ERR_OK, ::dsn::file::flush(wfile)); + ASSERT_EQ(ERR_OK, ::dsn::file::close(wfile)); } duplicator->update_fail_mode(duplication_fail_mode::FAIL_SKIP); diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt index 3b82d28e12..587826baad 100644 --- a/src/replica/test/CMakeLists.txt +++ b/src/replica/test/CMakeLists.txt @@ -47,7 +47,8 @@ set(MY_PROJ_LIBS dsn_meta_server zookeeper hashtable gtest - test_utils) + test_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/test/mutation_log_test.cpp b/src/replica/test/mutation_log_test.cpp index 527c1e8d98..980a538bf1 100644 --- a/src/replica/test/mutation_log_test.cpp +++ b/src/replica/test/mutation_log_test.cpp @@ -26,12 +26,12 @@ #include "replica/mutation_log.h" +#include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include #include @@ -39,6 +39,7 @@ #include #include "aio/aio_task.h" +#include "aio/file_io.h" #include "backup_types.h" #include "common/replication.codes.h" #include "consensus_types.h" @@ -53,6 +54,7 @@ #include "utils/binary_writer.h" #include "utils/blob.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -66,43 +68,26 @@ class message_ex; using namespace ::dsn; using namespace ::dsn::replication; -static void copy_file(const char *from_file, const char *to_file, int64_t to_size = -1) -{ - int64_t from_size; - ASSERT_TRUE(dsn::utils::filesystem::file_size(from_file, from_size)); - ASSERT_LE(to_size, from_size); - FILE *from = fopen(from_file, "rb"); - ASSERT_TRUE(from != nullptr); - FILE *to = fopen(to_file, "wb"); - ASSERT_TRUE(to != nullptr); - if (to_size == -1) - to_size = from_size; - if (to_size > 0) { - std::unique_ptr buf(new char[to_size]); - auto n = fread(buf.get(), 1, to_size, from); - ASSERT_EQ(to_size, n); - n = fwrite(buf.get(), 1, to_size, to); - ASSERT_EQ(to_size, n); - } - int r = fclose(from); - ASSERT_EQ(0, r); - r = fclose(to); - ASSERT_EQ(0, r); -} - static void overwrite_file(const char *file, int offset, const void *buf, int size) { - FILE *f = fopen(file, "r+b"); - ASSERT_TRUE(f != nullptr); - int r = fseek(f, offset, SEEK_SET); - ASSERT_EQ(0, r); - size_t n = fwrite(buf, 1, size, f); - ASSERT_EQ(size, n); - r = fclose(f); - ASSERT_EQ(0, r); + auto wfile = file::open(file, O_RDWR | O_CREAT | O_BINARY, 0666); + ASSERT_NE(wfile, nullptr); + auto t = ::dsn::file::write(wfile, + (const char *)buf, + size, + offset, + LPC_AIO_IMMEDIATE_CALLBACK, + nullptr, + [=](::dsn::error_code err, size_t n) { + CHECK_EQ(ERR_OK, err); + CHECK_EQ(size, n); + }); + t->wait(); + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); } -TEST(replication, log_file) +TEST(replication_test, log_file) { replica_log_info_map mdecrees; gpid gpid(1, 0); @@ -198,7 +183,7 @@ TEST(replication, log_file) // bad file data: empty file ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.0")); - copy_file(fpath.c_str(), "log.1.0", 0); + dsn::utils::copy_file_by_size(fpath, "log.1.0", 0); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.0")); lf = log_file::open_read("log.1.0", err); ASSERT_TRUE(lf == nullptr); @@ -208,7 +193,7 @@ TEST(replication, log_file) // bad file data: incomplete log_block_header ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.1")); - copy_file(fpath.c_str(), "log.1.1", sizeof(log_block_header) - 1); + dsn::utils::copy_file_by_size(fpath, "log.1.1", sizeof(log_block_header) - 1); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.1")); lf = log_file::open_read("log.1.1", err); ASSERT_TRUE(lf == nullptr); @@ -218,7 +203,7 @@ TEST(replication, log_file) // bad file data: bad log_block_header (magic = 0xfeadbeef) ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.2")); - copy_file(fpath.c_str(), "log.1.2"); + dsn::utils::copy_file_by_size(fpath, "log.1.2"); int32_t bad_magic = 0xfeadbeef; overwrite_file("log.1.2", FIELD_OFFSET(log_block_header, magic), &bad_magic, sizeof(bad_magic)); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.2")); @@ -230,7 +215,7 @@ TEST(replication, log_file) // bad file data: bad log_block_header (crc check failed) ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.3")); - copy_file(fpath.c_str(), "log.1.3"); + dsn::utils::copy_file_by_size(fpath, "log.1.3"); int32_t bad_crc = 0; overwrite_file("log.1.3", FIELD_OFFSET(log_block_header, body_crc), &bad_crc, sizeof(bad_crc)); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.3")); @@ -242,14 +227,13 @@ TEST(replication, log_file) // bad file data: incomplete block body ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.4")); - copy_file(fpath.c_str(), "log.1.4", sizeof(log_block_header) + 1); + dsn::utils::copy_file_by_size(fpath, "log.1.4", sizeof(log_block_header) + 1); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.4")); lf = log_file::open_read("log.1.4", err); ASSERT_TRUE(lf == nullptr); ASSERT_EQ(ERR_INCOMPLETE_DATA, err); ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.4")); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.4.removed")); - ASSERT_TRUE(dsn::utils::filesystem::rename_path("log.1.4.removed", "log.1.4")); // read the file for test offset = 100; @@ -259,7 +243,7 @@ TEST(replication, log_file) ASSERT_EQ(1, lf->index()); ASSERT_EQ(100, lf->start_offset()); int64_t sz; - ASSERT_TRUE(dsn::utils::filesystem::file_size(fpath, sz)); + ASSERT_TRUE(dsn::utils::filesystem::file_size(fpath, dsn::utils::FileDataType::kSensitive, sz)); ASSERT_EQ(lf->start_offset() + sz, lf->end_offset()); // read data diff --git a/src/test/function_test/bulk_load/CMakeLists.txt b/src/test/function_test/bulk_load/CMakeLists.txt index a6eac08515..ea27eed37b 100644 --- a/src/test/function_test/bulk_load/CMakeLists.txt +++ b/src/test/function_test/bulk_load/CMakeLists.txt @@ -37,8 +37,8 @@ set(MY_PROJ_LIBS gssapi_krb5 krb5 function_test_utils - rocksdb - test_utils) + test_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/test_util/CMakeLists.txt b/src/test_util/CMakeLists.txt index 267c825e40..b1e7ac2dff 100644 --- a/src/test_util/CMakeLists.txt +++ b/src/test_util/CMakeLists.txt @@ -22,6 +22,6 @@ set(MY_PROJ_NAME test_utils) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest) +set(MY_PROJ_LIBS gtest rocksdb) dsn_add_static_library() diff --git a/src/test_util/test_util.cpp b/src/test_util/test_util.cpp index 5ab185503d..0789c4678b 100644 --- a/src/test_util/test_util.cpp +++ b/src/test_util/test_util.cpp @@ -26,12 +26,33 @@ #include "gtest/gtest-message.h" #include "gtest/gtest-test-part.h" #include "gtest/gtest.h" +#include "metadata_types.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/api_layer1.h" #include "utils/defer.h" +#include "utils/env.h" +#include "utils/error_code.h" +#include "utils/filesystem.h" #include "utils/fmt_logging.h" namespace pegasus { +void create_local_test_file(const std::string &full_name, dsn::replication::file_meta *fm) +{ + ASSERT_NE(fm, nullptr); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice("write some data."), + full_name, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + fm->name = full_name; + ASSERT_EQ(dsn::ERR_OK, dsn::utils::filesystem::md5sum(full_name, fm->md5)); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + full_name, dsn::utils::FileDataType::kSensitive, fm->size)); +} + void AssertEventually(const std::function &f, int timeout_sec, WaitBackoff backoff) { // TODO(yingchun): should use mono time diff --git a/src/test_util/test_util.h b/src/test_util/test_util.h index d33775ff5e..762d154367 100644 --- a/src/test_util/test_util.h +++ b/src/test_util/test_util.h @@ -19,12 +19,19 @@ #pragma once +#include #include +#include -#include "gtest/gtest.h" #include "utils/flags.h" #include "utils/test_macros.h" +namespace dsn { +namespace replication { +class file_meta; +} // namespace replication +} // namespace dsn + DSN_DECLARE_bool(encrypt_data_at_rest); namespace pegasus { @@ -36,6 +43,8 @@ class encrypt_data_test_base : public testing::TestWithParam encrypt_data_test_base() { FLAGS_encrypt_data_at_rest = GetParam(); } }; +void create_local_test_file(const std::string &full_name, dsn::replication::file_meta *fm); + #define ASSERT_EVENTUALLY(expr) \ do { \ AssertEventually(expr); \ diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index 266b9beeb6..d77464c360 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -33,8 +33,8 @@ set(MY_PROJ_LIBS dsn_http dsn_runtime dsn_utils gtest - rocksdb - test_utils) + test_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) From c2d9affe8f997d6d95c08cb449b7008005fc2cca Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 27 Sep 2023 08:10:06 -0500 Subject: [PATCH 32/42] refactor(configuration): use rocksdb API to read/write file (#1626) https://github.com/apache/incubator-pegasus/issues/887 There is no functional changes, but only refactor the files read/write method of configuration module. --- src/utils/configuration.cpp | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/src/utils/configuration.cpp b/src/utils/configuration.cpp index 9aea53448e..c69a556d2c 100644 --- a/src/utils/configuration.cpp +++ b/src/utils/configuration.cpp @@ -33,14 +33,16 @@ * xxxx-xx-xx, author, fix bug about xxx */ -#include +#include +#include +#include #include #include #include #include #include "utils/configuration.h" -#include "utils/filesystem.h" +#include "utils/env.h" #include "utils/strings.h" namespace dsn { @@ -67,33 +69,17 @@ bool configuration::load(const char *file_name, const char *arguments) { _file_name = std::string(file_name); - FILE *fd = ::fopen(file_name, "rb"); - if (fd == nullptr) { - std::string cdir; - dsn::utils::filesystem::get_current_directory(cdir); - printf("ERROR: cannot open file %s in %s, err = %s\n", - file_name, - cdir.c_str(), - strerror(errno)); - return false; - } - ::fseek(fd, 0, SEEK_END); - int len = ftell(fd); - if (len == -1 || len == 0) { - printf("ERROR: cannot get length of %s, err = %s\n", file_name, strerror(errno)); - ::fclose(fd); + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), _file_name, &_file_data); + if (!s.ok()) { + fmt::print(stderr, "ERROR: read file '{}' failed, err = {}\n", _file_name, s.ToString()); return false; } - _file_data.resize(len + 1); - ::fseek(fd, 0, SEEK_SET); - auto sz = ::fread((char *)_file_data.c_str(), len, 1, fd); - ::fclose(fd); - if (sz != 1) { - printf("ERROR: cannot read correct data of %s, err = %s\n", file_name, strerror(errno)); + if (_file_data.empty()) { + fmt::print(stderr, "ERROR: file '{}' is empty\n", _file_name); return false; } - _file_data[len] = '\n'; // replace data with arguments if (arguments != nullptr) { From 4a605825def1b3f102dbd7d8196f3e9b80a0348a Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 27 Sep 2023 08:11:46 -0500 Subject: [PATCH 33/42] fix(http_server): add deregister_http_call back (#1627) https://github.com/apache/incubator-pegasus/issues/887 `deregister_http_call()` is defined but not declared in .h files, this patch adds the declaration back and calls it in the desctructor of class `replica_http_service`. --- src/http/http_server.h | 2 ++ src/replica/replica_http_service.h | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/http/http_server.h b/src/http/http_server.h index b182947e41..c84055261c 100644 --- a/src/http/http_server.h +++ b/src/http/http_server.h @@ -148,6 +148,8 @@ class http_server_base : public http_service // ``` extern http_call ®ister_http_call(std::string full_path); +extern void deregister_http_call(const std::string &full_path); + // Starts serving HTTP requests. // The internal HTTP server will reuse the rDSN server port. extern void start_http_server(); diff --git a/src/replica/replica_http_service.h b/src/replica/replica_http_service.h index d02707441d..35b2352ac1 100644 --- a/src/replica/replica_http_service.h +++ b/src/replica/replica_http_service.h @@ -48,7 +48,14 @@ class replica_http_service : public http_server_base this, std::placeholders::_1, std::placeholders::_2), - "ip:port/replica/maual_compaction?app_id="); + "ip:port/replica/manual_compaction?app_id="); + } + + ~replica_http_service() + { + deregister_http_call("replica/duplication"); + deregister_http_call("replica/data_version"); + deregister_http_call("replica/manual_compaction"); } std::string path() const override { return "replica"; } From 58ac765758bc616b17568f7ae02988c37224126b Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 27 Sep 2023 12:03:47 -0500 Subject: [PATCH 34/42] refactor(file_system): use rocksdb API to read/write file (#1624) https://github.com/apache/incubator-pegasus/issues/887 There is no functional changes, but only refactor the files read/write method of filesystem module. --- src/replica/bulk_load/replica_bulk_loader.cpp | 7 +- src/replica/replica_restore.cpp | 4 +- src/server/pegasus_write_service_impl.h | 4 +- src/utils/filesystem.cpp | 154 ++++++++++-------- src/utils/filesystem.h | 13 +- src/utils/test/file_system_test.cpp | 55 ++++++- src/utils/test/file_utils.cpp | 9 +- 7 files changed, 156 insertions(+), 90 deletions(-) diff --git a/src/replica/bulk_load/replica_bulk_loader.cpp b/src/replica/bulk_load/replica_bulk_loader.cpp index 60f3e5a043..5261e4183c 100644 --- a/src/replica/bulk_load/replica_bulk_loader.cpp +++ b/src/replica/bulk_load/replica_bulk_loader.cpp @@ -44,6 +44,7 @@ #include "utils/autoref_ptr.h" #include "utils/blob.h" #include "utils/chrono_literals.h" +#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" @@ -497,7 +498,8 @@ void replica_bulk_loader::download_sst_file(const std::string &remote_dir, // We are not sure if the file was cached by system. And we couldn't // afford the io overhead which is cased by reading file in verify_file(), // so if file exist we just verify file size - if (utils::filesystem::verify_file_size(file_name, f_meta.size)) { + if (utils::filesystem::verify_file_size( + file_name, utils::FileDataType::kSensitive, f_meta.size)) { // local file exist and is verified ec = ERR_OK; f_size = f_meta.size; @@ -520,7 +522,8 @@ void replica_bulk_loader::download_sst_file(const std::string &remote_dir, if (ec == ERR_OK && !verified) { if (!f_meta.md5.empty() && f_md5 != f_meta.md5) { ec = ERR_CORRUPTION; - } else if (!utils::filesystem::verify_file_size(file_name, f_meta.size)) { + } else if (!utils::filesystem::verify_file_size( + file_name, utils::FileDataType::kSensitive, f_meta.size)) { ec = ERR_CORRUPTION; } } diff --git a/src/replica/replica_restore.cpp b/src/replica/replica_restore.cpp index 7236f31a15..f2494482ff 100644 --- a/src/replica/replica_restore.cpp +++ b/src/replica/replica_restore.cpp @@ -51,6 +51,7 @@ #include "runtime/task/task_tracker.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" @@ -145,7 +146,8 @@ error_code replica::download_checkpoint(const configuration_restore_request &req const std::string file_name = utils::filesystem::path_combine(local_chkpt_dir, f_meta.name); if (download_err == ERR_OK || download_err == ERR_PATH_ALREADY_EXIST) { - if (!utils::filesystem::verify_file(file_name, f_meta.md5, f_meta.size)) { + if (!utils::filesystem::verify_file( + file_name, utils::FileDataType::kSensitive, f_meta.md5, f_meta.size)) { download_err = ERR_CORRUPTION; } else if (download_err == ERR_PATH_ALREADY_EXIST) { download_err = ERR_OK; diff --git a/src/server/pegasus_write_service_impl.h b/src/server/pegasus_write_service_impl.h index ff9f2fe837..69b71c6ba4 100644 --- a/src/server/pegasus_write_service_impl.h +++ b/src/server/pegasus_write_service_impl.h @@ -29,6 +29,7 @@ #include "pegasus_write_service.h" #include "rocksdb_wrapper.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/string_conv.h" #include "utils/strings.h" @@ -76,7 +77,8 @@ inline dsn::error_code get_external_files_path(const std::string &bulk_load_dir, for (const auto &f_meta : metadata.files) { const auto &file_name = dsn::utils::filesystem::path_combine(bulk_load_dir, f_meta.name); if (verify_before_ingest && - !dsn::utils::filesystem::verify_file(file_name, f_meta.md5, f_meta.size)) { + !dsn::utils::filesystem::verify_file( + file_name, dsn::utils::FileDataType::kSensitive, f_meta.md5, f_meta.size)) { break; } files_path.emplace_back(file_name); diff --git a/src/utils/filesystem.cpp b/src/utils/filesystem.cpp index f2fa59ef87..2f80a51d50 100644 --- a/src/utils/filesystem.cpp +++ b/src/utils/filesystem.cpp @@ -33,24 +33,23 @@ * xxxx-xx-xx, author, fix bug about xxx */ +#include +#include #include -#include +#include #include #include #include +#include +#include +#include #include #include #include // IWYU pragma: no_include #include - #include - -#include -#include -#include -#include -#include +#include #include "utils/defer.h" #include "utils/env.h" @@ -60,7 +59,6 @@ #include "utils/ports.h" #include "utils/safe_strerror_posix.h" #include "utils/string_view.h" -#include "utils/strings.h" #define getcwd_ getcwd #define rmdir_ rmdir @@ -417,7 +415,6 @@ bool deprecated_file_size(const std::string &path, int64_t &sz) } sz = st.st_size; - return true; } @@ -517,53 +514,28 @@ bool create_directory(const std::string &path) bool create_file(const std::string &path) { - size_t pos; std::string npath; - int fd; - int mode; - int err; - - if (path.empty()) { - return false; - } - - if (_FS_ISSEP(path.back())) { - return false; - } - - err = get_normalized_path(path, npath); + int err = get_normalized_path(path, npath); if (err != 0) { return false; } - if (dsn::utils::filesystem::path_exists_internal(npath, FTW_F)) { - return true; - } - - if (dsn::utils::filesystem::path_exists_internal(npath, FTW_D)) { - return false; - } - - pos = npath.find_last_of("\\/"); + auto pos = npath.find_last_of("\\/"); if ((pos != std::string::npos) && (pos > 0)) { auto ppath = npath.substr(0, pos); if (!dsn::utils::filesystem::create_directory(ppath)) { + LOG_WARNING("fail to create directory {}", ppath); return false; } } - mode = 0775; - fd = ::creat(npath.c_str(), mode); - if (fd == -1) { - err = errno; - LOG_WARNING("create_file {} failed, err = {}", path, safe_strerror(err)); + std::unique_ptr wfile; + auto s = rocksdb::Env::Default()->ReopenWritableFile(path, &wfile, rocksdb::EnvOptions()); + if (dsn_unlikely(!s.ok())) { + LOG_WARNING("fail to create file {}, err={}", path, s.ToString()); return false; } - if (::close_(fd) != 0) { - LOG_WARNING("create_file {}, failed to close the file handle.", path); - } - return true; } @@ -745,6 +717,54 @@ bool link_file(const std::string &src, const std::string &target) } error_code md5sum(const std::string &file_path, /*out*/ std::string &result) +{ + result.clear(); + if (!::dsn::utils::filesystem::file_exists(file_path)) { + LOG_ERROR("md5sum error: file {} not exist", file_path); + return ERR_OBJECT_NOT_FOUND; + } + + std::unique_ptr sfile; + auto s = rocksdb::Env::Default()->NewSequentialFile(file_path, &sfile, rocksdb::EnvOptions()); + if (!sfile) { + LOG_ERROR("md5sum error: open file {} failed, err={}", file_path, s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } + + const int64_t kBufferSize = 4096; + char buf[kBufferSize]; + unsigned char out[MD5_DIGEST_LENGTH] = {0}; + MD5_CTX c; + CHECK_EQ(1, MD5_Init(&c)); + while (true) { + rocksdb::Slice res; + s = sfile->Read(kBufferSize, &res, buf); + if (!s.ok()) { + MD5_Final(out, &c); + LOG_ERROR("md5sum error: read file {} failed, err={}", file_path, s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } + if (res.empty()) { + break; + } + CHECK_EQ(1, MD5_Update(&c, buf, res.size())); + if (res.size() < kBufferSize) { + break; + } + } + CHECK_EQ(1, MD5_Final(out, &c)); + + char str[MD5_DIGEST_LENGTH * 2 + 1]; + str[MD5_DIGEST_LENGTH * 2] = 0; + for (int n = 0; n < MD5_DIGEST_LENGTH; n++) { + sprintf(str + n + n, "%02x", out[n]); + } + result.assign(str); + + return ERR_OK; +} + +error_code deprecated_md5sum(const std::string &file_path, /*out*/ std::string &result) { result.clear(); // if file not exist, we return ERR_OBJECT_NOT_FOUND @@ -841,6 +861,7 @@ error_code read_file(const std::string &fname, std::string &buf) } bool verify_file(const std::string &fname, + FileDataType type, const std::string &expected_md5, const int64_t &expected_fsize) { @@ -849,7 +870,7 @@ bool verify_file(const std::string &fname, return false; } int64_t f_size = 0; - if (!file_size(fname, f_size)) { + if (!file_size(fname, type, f_size)) { LOG_ERROR("verify file({}) failed, becaused failed to get file size", fname); return false; } @@ -870,14 +891,14 @@ bool verify_file(const std::string &fname, return true; } -bool verify_file_size(const std::string &fname, const int64_t &expected_fsize) +bool verify_file_size(const std::string &fname, FileDataType type, const int64_t &expected_fsize) { if (!file_exists(fname)) { LOG_ERROR("file({}) is not existed", fname); return false; } int64_t f_size = 0; - if (!file_size(fname, f_size)) { + if (!file_size(fname, type, f_size)) { LOG_ERROR("verify file({}) size failed, becaused failed to get file size", fname); return false; } @@ -891,22 +912,6 @@ bool verify_file_size(const std::string &fname, const int64_t &expected_fsize) return true; } -bool verify_data_md5(const std::string &fname, - const char *data, - const size_t data_size, - const std::string &expected_md5) -{ - std::string md5 = string_md5(data, data_size); - if (md5 != expected_md5) { - LOG_ERROR("verify data({}) failed, because data damaged, size: md5: {} VS {}", - fname, - md5, - expected_md5); - return false; - } - return true; -} - bool create_directory(const std::string &path, std::string &absolute_path, std::string &err_msg) { FAIL_POINT_INJECT_F("filesystem_create_directory", [path](string_view str) { @@ -952,23 +957,28 @@ bool check_dir_rw(const std::string &path, std::string &err_msg) path.find(broken_disk_dir) == std::string::npos; }); - std::string fname = "read_write_test_file"; - std::string fpath = path_combine(path, fname); - if (!create_file(fpath)) { - err_msg = fmt::format("Fail to create test file {}.", fpath); + static const std::string kTestValue = "test_value"; + static const std::string kFname = "read_write_test_file"; + std::string fpath = path_combine(path, kFname); + auto cleanup = defer([&fpath]() { remove_path(fpath); }); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(kTestValue), + fpath, + /* should_sync */ true); + if (dsn_unlikely(!s.ok())) { + err_msg = fmt::format("fail to write file {}, err={}", fpath, s.ToString()); return false; } - auto cleanup = defer([&fpath]() { remove_path(fpath); }); - std::string value = "test_value"; - if (!write_file(fpath, value)) { - err_msg = fmt::format("Fail to write file {}.", fpath); + std::string read_data; + s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fpath, &read_data); + if (dsn_unlikely(!s.ok())) { + err_msg = fmt::format("fail to read file {}, err={}", fpath, s.ToString()); return false; } - std::string buf; - if (read_file(fpath, buf) != ERR_OK || buf != value) { - err_msg = fmt::format("Fail to read file {} or get wrong value({}).", fpath, buf); + if (dsn_unlikely(read_data != kTestValue)) { + err_msg = fmt::format("get wrong value '{}' from file {}", read_data, fpath); return false; } diff --git a/src/utils/filesystem.h b/src/utils/filesystem.h index 9c5d3efea9..440128a97d 100644 --- a/src/utils/filesystem.h +++ b/src/utils/filesystem.h @@ -65,6 +65,8 @@ enum class FileDataType; namespace filesystem { +// TODO(yingchun): Consider using rocksdb APIs to rewrite the following functions. + int get_normalized_path(const std::string &path, std::string &npath); bool get_absolute_path(const std::string &path1, std::string &path2); @@ -136,6 +138,7 @@ bool get_disk_space_info(const std::string &path, disk_space_info &info); bool link_file(const std::string &src, const std::string &target); error_code md5sum(const std::string &file_path, /*out*/ std::string &result); +error_code deprecated_md5sum(const std::string &file_path, /*out*/ std::string &result); // return value: // - : @@ -143,25 +146,23 @@ error_code md5sum(const std::string &file_path, /*out*/ std::string &result); // B is represent wheter the directory is empty, true means empty, otherwise false std::pair is_directory_empty(const std::string &dirname); +// TODO(yingchun): remove it! error_code read_file(const std::string &fname, /*out*/ std::string &buf); // compare file metadata calculated by fname with expected md5 and file_size bool verify_file(const std::string &fname, + FileDataType type, const std::string &expected_md5, const int64_t &expected_fsize); -bool verify_file_size(const std::string &fname, const int64_t &expected_fsize); - -bool verify_data_md5(const std::string &fname, - const char *data, - const size_t data_size, - const std::string &expected_md5); +bool verify_file_size(const std::string &fname, FileDataType type, const int64_t &expected_fsize); // create driectory and get absolute path bool create_directory(const std::string &path, /*out*/ std::string &absolute_path, /*out*/ std::string &err_msg); +// TODO(yingchun): remove it! bool write_file(const std::string &fname, std::string &buf); // check if directory is readable and writable diff --git a/src/utils/test/file_system_test.cpp b/src/utils/test/file_system_test.cpp index ccfc2da2f9..f497a908b3 100644 --- a/src/utils/test/file_system_test.cpp +++ b/src/utils/test/file_system_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -26,6 +27,7 @@ #include #include "utils/env.h" +#include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -115,18 +117,63 @@ TEST(filesystem_test_p, encrypted_file_size) ASSERT_EQ(kFileContentSize + kEncryptionHeaderkSize, actual_file_size); } +// The old filesystem API doesn't support sensitive files, so skip testing +// FLAGS_encrypt_data_at_rest=true. +TEST(filesystem_test, check_new_md5sum) +{ + FLAGS_encrypt_data_at_rest = false; + + struct file_info + { + int64_t size; + } tests[]{{4095}, {4096}, {4097}}; + + for (const auto &test : tests) { + std::string fname = "test_file"; + // deprecated_md5sum doesn't support kSensitive files, so use kNonSensitive here. + auto s = rocksdb::WriteStringToFile( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(std::string(test.size, 'a')), + fname, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + // Check the file size. + int64_t file_fsize; + ASSERT_TRUE(file_size(fname, FileDataType::kNonSensitive, file_fsize)); + ASSERT_EQ(test.size, file_fsize); + + // Get the md5sum. + std::string md5sum1; + ASSERT_EQ(ERR_OK, md5sum(fname, md5sum1)); + ASSERT_FALSE(md5sum1.empty()); + + // Check the md5sum is repeatable. + std::string md5sum2; + ASSERT_EQ(ERR_OK, md5sum(fname, md5sum2)); + ASSERT_EQ(md5sum1, md5sum2); + + // Check the md5sum is the same to deprecated_md5sum. + ASSERT_EQ(ERR_OK, deprecated_md5sum(fname, md5sum2)); + ASSERT_EQ(md5sum1, md5sum2); + + utils::filesystem::remove_path(fname); + } +} + TEST(filesystem_test, verify_file_test) { + FLAGS_encrypt_data_at_rest = false; + const std::string &fname = "test_file"; std::string expected_md5; int64_t expected_fsize; create_file(fname); md5sum(fname, expected_md5); - file_size(fname, expected_fsize); + ASSERT_TRUE(file_size(fname, FileDataType::kNonSensitive, expected_fsize)); - ASSERT_TRUE(verify_file(fname, expected_md5, expected_fsize)); - ASSERT_FALSE(verify_file(fname, "wrong_md5", 10086)); - ASSERT_FALSE(verify_file("file_not_exists", "wrong_md5", 10086)); + ASSERT_TRUE(verify_file(fname, FileDataType::kNonSensitive, expected_md5, expected_fsize)); + ASSERT_FALSE(verify_file(fname, FileDataType::kNonSensitive, "wrong_md5", 10086)); + ASSERT_FALSE(verify_file("file_not_exists", FileDataType::kNonSensitive, "wrong_md5", 10086)); remove_path(fname); } diff --git a/src/utils/test/file_utils.cpp b/src/utils/test/file_utils.cpp index 004e228b4b..0cc2173e8e 100644 --- a/src/utils/test/file_utils.cpp +++ b/src/utils/test/file_utils.cpp @@ -33,6 +33,7 @@ #include #include +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" @@ -683,12 +684,12 @@ static void file_utils_test_file_size() bool ret; path = "./file_utils_temp.txt"; - ret = dsn::utils::filesystem::file_size(path, sz); - EXPECT_TRUE(ret); - EXPECT_TRUE(sz == 12); + ret = dsn::utils::filesystem::file_size(path, dsn::utils::FileDataType::kNonSensitive, sz); + ASSERT_TRUE(ret); + ASSERT_EQ(12, sz); path = "./file_utils_temp2.txt"; - ret = dsn::utils::filesystem::file_size(path, sz); + ret = dsn::utils::filesystem::file_size(path, dsn::utils::FileDataType::kNonSensitive, sz); EXPECT_FALSE(ret); } From 0db1ff90d75de72aa0864892794017cedb5e4840 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Thu, 28 Sep 2023 15:25:20 +0800 Subject: [PATCH 35/42] feat(http): implement http client based on libcurl (#1583) https://github.com/apache/incubator-pegasus/issues/1582 Http client is implemented based on libcurl. Both GET and POST methods are supported. --- src/http/http_call_registry.h | 17 +- src/http/http_client.cpp | 324 +++++++++++++++++++++++++++++ src/http/http_client.h | 154 ++++++++++++++ src/http/http_message_parser.cpp | 7 +- src/http/http_method.h | 36 ++++ src/http/http_server.cpp | 5 +- src/http/http_server.h | 7 +- src/http/pprof_http_service.cpp | 3 +- src/http/test/CMakeLists.txt | 6 +- src/http/test/config-test.ini | 75 +++++++ src/http/test/http_client_test.cpp | 213 +++++++++++++++++++ src/http/test/http_server_test.cpp | 21 +- src/http/test/main.cpp | 120 +++++++++++ src/utils/error_code.h | 2 + src/utils/errors.h | 4 +- src/utils/metrics.cpp | 3 +- src/utils/test/metrics_test.cpp | 2 +- thirdparty/CMakeLists.txt | 19 +- 18 files changed, 986 insertions(+), 32 deletions(-) create mode 100644 src/http/http_client.cpp create mode 100644 src/http/http_client.h create mode 100644 src/http/http_method.h create mode 100644 src/http/test/config-test.ini create mode 100644 src/http/test/http_client_test.cpp create mode 100644 src/http/test/main.cpp diff --git a/src/http/http_call_registry.h b/src/http/http_call_registry.h index fef3b0c467..2e944463ce 100644 --- a/src/http/http_call_registry.h +++ b/src/http/http_call_registry.h @@ -35,11 +35,11 @@ class http_call_registry : public utils::singleton std::shared_ptr find(const std::string &path) const { std::lock_guard guard(_mu); - auto it = _call_map.find(path); - if (it == _call_map.end()) { + const auto &iter = _call_map.find(path); + if (iter == _call_map.end()) { return nullptr; } - return it->second; + return iter->second; } void remove(const std::string &path) @@ -48,14 +48,19 @@ class http_call_registry : public utils::singleton _call_map.erase(path); } - void add(std::unique_ptr call_uptr) + void add(const std::shared_ptr &call) { - auto call = std::shared_ptr(call_uptr.release()); std::lock_guard guard(_mu); - CHECK_EQ_MSG(_call_map.count(call->path), 0, call->path); + CHECK_EQ_MSG(_call_map.count(call->path), 0, "{} has been added", call->path); _call_map[call->path] = call; } + void add(std::unique_ptr call_uptr) + { + auto call = std::shared_ptr(call_uptr.release()); + add(call); + } + std::vector> list_all_calls() const { std::lock_guard guard(_mu); diff --git a/src/http/http_client.cpp b/src/http/http_client.cpp new file mode 100644 index 0000000000..32ae5eaea1 --- /dev/null +++ b/src/http/http_client.cpp @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "http/http_client.h" + +#include +#include +#include + +#include "curl/curl.h" +#include "utils/error_code.h" +#include "utils/flags.h" +#include "utils/fmt_logging.h" + +namespace dsn { + +DSN_DEFINE_uint32(http, + curl_timeout_ms, + 10000, + "The maximum time in milliseconds that you allow the libcurl transfer operation " + "to complete"); + +http_client::http_client() + : _curl(nullptr), + _method(http_method::GET), + _recv_callback(nullptr), + _header_changed(true), + _header_list(nullptr) +{ + // Since `kErrorBufferBytes` is private, `static_assert` have to be put in constructor. + static_assert(http_client::kErrorBufferBytes >= CURL_ERROR_SIZE, + "The error buffer used by libcurl must be at least CURL_ERROR_SIZE bytes big"); + + clear_error_buf(); +} + +http_client::~http_client() +{ + if (_curl != nullptr) { + curl_easy_cleanup(_curl); + _curl = nullptr; + } + + free_header_list(); +} + +namespace { + +inline dsn::error_code to_error_code(CURLcode code) +{ + switch (code) { + case CURLE_OK: + return dsn::ERR_OK; + case CURLE_OPERATION_TIMEDOUT: + return dsn::ERR_TIMEOUT; + default: + return dsn::ERR_CURL_FAILED; + } +} + +} // anonymous namespace + +#define RETURN_IF_CURL_NOT_OK(expr, ...) \ + do { \ + const auto code = (expr); \ + if (dsn_unlikely(code != CURLE_OK)) { \ + std::string msg(fmt::format("{}: {}", fmt::format(__VA_ARGS__), to_error_msg(code))); \ + return dsn::error_s::make(to_error_code(code), msg); \ + } \ + } while (0) + +#define RETURN_IF_SETOPT_NOT_OK(opt, input) \ + RETURN_IF_CURL_NOT_OK(curl_easy_setopt(_curl, opt, input), \ + "failed to set " #opt " to" \ + " " #input) + +#define RETURN_IF_GETINFO_NOT_OK(info, output) \ + RETURN_IF_CURL_NOT_OK(curl_easy_getinfo(_curl, info, output), "failed to get from " #info) + +#define RETURN_IF_EXEC_METHOD_NOT_OK() \ + RETURN_IF_CURL_NOT_OK(curl_easy_perform(_curl), \ + "failed to perform http request(method={}, url={})", \ + enum_to_string(_method), \ + _url) + +dsn::error_s http_client::init() +{ + if (_curl == nullptr) { + _curl = curl_easy_init(); + if (_curl == nullptr) { + return dsn::error_s::make(dsn::ERR_CURL_FAILED, "fail to initialize curl"); + } + } else { + curl_easy_reset(_curl); + } + + clear_header_fields(); + free_header_list(); + + // Additional messages for errors are needed. + clear_error_buf(); + RETURN_IF_SETOPT_NOT_OK(CURLOPT_ERRORBUFFER, _error_buf); + + // Set with NOSIGNAL since we are multi-threaded. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_NOSIGNAL, 1L); + + // Redirects are supported. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_FOLLOWLOCATION, 1L); + + // Before 8.3.0, CURLOPT_MAXREDIRS was unlimited. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_MAXREDIRS, 20); + + // Set common timeout for transfer operation. Users could also change it with their + // custom values by `set_timeout`. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_TIMEOUT_MS, static_cast(FLAGS_curl_timeout_ms)); + + // A lambda can only be converted to a function pointer if it does not capture: + // https://stackoverflow.com/questions/28746744/passing-capturing-lambda-as-function-pointer + // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2012/n3337.pdf + curl_write_callback callback = [](char *buffer, size_t size, size_t nmemb, void *param) { + http_client *client = reinterpret_cast(param); + return client->on_response_data(buffer, size * nmemb); + }; + RETURN_IF_SETOPT_NOT_OK(CURLOPT_WRITEFUNCTION, callback); + + // This http_client object itself is passed to the callback function. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_WRITEDATA, reinterpret_cast(this)); + + return dsn::error_s::ok(); +} + +void http_client::clear_error_buf() { _error_buf[0] = 0; } + +bool http_client::is_error_buf_empty() const { return _error_buf[0] == 0; } + +std::string http_client::to_error_msg(CURLcode code) const +{ + std::string err_msg = + fmt::format("code={}, desc=\"{}\"", static_cast(code), curl_easy_strerror(code)); + if (is_error_buf_empty()) { + return err_msg; + } + + err_msg += fmt::format(", msg=\"{}\"", _error_buf); + return err_msg; +} + +// `data` passed to this function is NOT null-terminated. +// `length` might be zero. +size_t http_client::on_response_data(const void *data, size_t length) +{ + if (_recv_callback == nullptr) { + return length; + } + + if (!(*_recv_callback)) { + // callback function is empty. + return length; + } + + // According to libcurl, callback should return the number of bytes actually taken care of. + // If that amount differs from the amount passed to callback function, it would signals an + // error condition. This causes the transfer to get aborted and the libcurl function used + // returns CURLE_WRITE_ERROR. Therefore, here we just return the max limit of size_t for + // failure. + // + // See https://curl.se/libcurl/c/CURLOPT_WRITEFUNCTION.html for details. + return (*_recv_callback)(data, length) ? length : std::numeric_limits::max(); +} + +dsn::error_s http_client::set_url(const std::string &url) +{ + RETURN_IF_SETOPT_NOT_OK(CURLOPT_URL, url.c_str()); + + _url = url; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::with_post_method(const std::string &data) +{ + // No need to enable CURLOPT_POST by `RETURN_IF_SETOPT_NOT_OK(CURLOPT_POST, 1L)`, since using + // either of CURLOPT_POSTFIELDS or CURLOPT_COPYPOSTFIELDS implies setting CURLOPT_POST to 1. + // See https://curl.se/libcurl/c/CURLOPT_POSTFIELDS.html for details. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_POSTFIELDSIZE, static_cast(data.size())); + RETURN_IF_SETOPT_NOT_OK(CURLOPT_COPYPOSTFIELDS, data.data()); + _method = http_method::POST; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::with_get_method() { return set_method(http_method::GET); } + +dsn::error_s http_client::set_method(http_method method) +{ + // No need to process the case of http_method::POST, since it should be enabled by + // `with_post_method`. + switch (method) { + case http_method::GET: + RETURN_IF_SETOPT_NOT_OK(CURLOPT_HTTPGET, 1L); + break; + default: + LOG_FATAL("Unsupported http_method"); + } + + _method = method; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::set_timeout(long timeout_ms) +{ + RETURN_IF_SETOPT_NOT_OK(CURLOPT_TIMEOUT_MS, timeout_ms); + return dsn::error_s::ok(); +} + +void http_client::clear_header_fields() +{ + _header_fields.clear(); + + _header_changed = true; +} + +void http_client::free_header_list() +{ + if (_header_list == nullptr) { + return; + } + + curl_slist_free_all(_header_list); + _header_list = nullptr; +} + +void http_client::set_header_field(dsn::string_view key, dsn::string_view val) +{ + _header_fields[std::string(key)] = std::string(val); + _header_changed = true; +} + +void http_client::set_accept(dsn::string_view val) { set_header_field("Accept", val); } + +void http_client::set_content_type(dsn::string_view val) { set_header_field("Content-Type", val); } + +dsn::error_s http_client::process_header() +{ + if (!_header_changed) { + return dsn::error_s::ok(); + } + + free_header_list(); + + for (const auto &field : _header_fields) { + auto str = fmt::format("{}: {}", field.first, field.second); + + // A null pointer is returned if anything went wrong, otherwise the new list pointer is + // returned. To avoid overwriting an existing non-empty list on failure, the new list + // should be returned to a temporary variable which can be tested for NULL before updating + // the original list pointer. (https://curl.se/libcurl/c/curl_slist_append.html) + struct curl_slist *temp = curl_slist_append(_header_list, str.c_str()); + if (temp == nullptr) { + free_header_list(); + return dsn::error_s::make(dsn::ERR_CURL_FAILED, "curl_slist_append failed"); + } + _header_list = temp; + } + + // This would work well even if `_header_list` is NULL pointer. Pass a NULL to this option + // to reset back to no custom headers. (https://curl.se/libcurl/c/CURLOPT_HTTPHEADER.html) + RETURN_IF_SETOPT_NOT_OK(CURLOPT_HTTPHEADER, _header_list); + + // New header has been built successfully, thus mark it unchanged. + _header_changed = false; + + return dsn::error_s::ok(); +} + +dsn::error_s http_client::exec_method(const http_client::recv_callback &callback) +{ + // `curl_easy_perform` would run synchronously, thus it is safe to use the pointer to + // `callback`. + _recv_callback = &callback; + + RETURN_NOT_OK(process_header()); + + RETURN_IF_EXEC_METHOD_NOT_OK(); + return dsn::error_s::ok(); +} + +dsn::error_s http_client::exec_method(std::string *response) +{ + if (response == nullptr) { + return exec_method(); + } + + auto callback = [response](const void *data, size_t length) { + response->append(reinterpret_cast(data), length); + return true; + }; + + return exec_method(callback); +} + +dsn::error_s http_client::get_http_status(long &http_status) const +{ + RETURN_IF_GETINFO_NOT_OK(CURLINFO_RESPONSE_CODE, &http_status); + return dsn::error_s::ok(); +} + +#undef RETURN_IF_EXEC_METHOD_NOT_OK +#undef RETURN_IF_SETOPT_NOT_OK +#undef RETURN_IF_CURL_NOT_OK + +} // namespace dsn diff --git a/src/http/http_client.h b/src/http/http_client.h new file mode 100644 index 0000000000..190af11f2b --- /dev/null +++ b/src/http/http_client.h @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "http/http_method.h" +#include "utils/errors.h" +#include "utils/ports.h" +#include "utils/string_view.h" + +namespace dsn { + +// A library for http client that provides convenient APIs to access http services, implemented +// based on libcurl (https://curl.se/libcurl/c/). +// +// This class is not thread-safe. Thus maintain one instance for each thread. +// +// Example of submitting GET request to remote http service +// -------------------------------------------------------- +// Create an instance of http_client: +// http_client client; +// +// It's necessary to initialize the new instance before coming into use: +// auto err = client.init(); +// +// Specify the target url that you would request for: +// err = client.set_url(method); +// +// If you would use GET method, call `with_get_method`: +// err = client.with_get_method(); +// +// If you would use POST method, call `with_post_method` with post data: +// err = client.with_post_method(post_data); +// +// Submit the request to remote http service: +// err = client.exec_method(); +// +// If response data should be processed, use callback function: +// auto callback = [...](const void *data, size_t length) { +// ...... +// return true; +// }; +// err = client.exec_method(callback); +// +// Or just provide a string pointer: +// std::string response; +// err = client.exec_method(&response); +// +// Get the http status code after requesting: +// long http_status; +// err = client.get_http_status(http_status); +class http_client +{ +public: + using recv_callback = std::function; + + http_client(); + ~http_client(); + + // Before coming into use, init() must be called to initialize http client. It could also be + // called to reset the http clients that have been initialized previously. + dsn::error_s init(); + + // Specify the target url that the request would be sent for. + dsn::error_s set_url(const std::string &url); + + // Using post method, with `data` as the payload for post body. + dsn::error_s with_post_method(const std::string &data); + + // Using get method. + dsn::error_s with_get_method(); + + // Specify the maximum time in milliseconds that a request is allowed to complete. + dsn::error_s set_timeout(long timeout_ms); + + // Operations for the header fields. + void clear_header_fields(); + void set_accept(dsn::string_view val); + void set_content_type(dsn::string_view val); + + // Submit request to remote http service, with response processed by callback function. + // + // `callback` function gets called by libcurl as soon as there is data received that needs + // to be saved. For most transfers, this callback gets called many times and each invoke + // delivers another chunk of data. + // + // This function would run synchronously, which means it would wait until the response was + // returned and processed appropriately. + dsn::error_s exec_method(const recv_callback &callback = {}); + + // Submit request to remote http service, with response data returned in a string. + // + // This function would run synchronously, which means it would wait until the response was + // returned and processed appropriately. + dsn::error_s exec_method(std::string *response); + + // Get the last http status code after requesting. + dsn::error_s get_http_status(long &http_status) const; + +private: + using header_field_map = std::unordered_map; + + void clear_error_buf(); + bool is_error_buf_empty() const; + std::string to_error_msg(CURLcode code) const; + + size_t on_response_data(const void *data, size_t length); + + // Specify which http method would be used, such as GET. Enabling POST should not use this + // function (use `with_post_method` instead). + dsn::error_s set_method(http_method method); + + void free_header_list(); + void set_header_field(dsn::string_view key, dsn::string_view val); + dsn::error_s process_header(); + + // The size of a buffer that is used by libcurl to store human readable + // error messages on failures or problems. + static const constexpr size_t kErrorBufferBytes = CURL_ERROR_SIZE; + + CURL *_curl; + http_method _method; + std::string _url; + const recv_callback *_recv_callback; + char _error_buf[kErrorBufferBytes]; + + bool _header_changed; + header_field_map _header_fields; + struct curl_slist *_header_list; + + DISALLOW_COPY_AND_ASSIGN(http_client); +}; + +} // namespace dsn diff --git a/src/http/http_message_parser.cpp b/src/http/http_message_parser.cpp index ce2e0054ce..df873eea2e 100644 --- a/src/http/http_message_parser.cpp +++ b/src/http/http_message_parser.cpp @@ -26,12 +26,13 @@ #include "http_message_parser.h" +#include // IWYU pragma: no_include #include #include #include -#include "http_server.h" +#include "http/http_method.h" #include "nodejs/http_parser.h" #include "runtime/rpc/rpc_message.h" #include "utils/blob.h" @@ -132,10 +133,10 @@ http_message_parser::http_message_parser() message_header *header = msg->header; if (parser->type == HTTP_REQUEST && parser->method == HTTP_GET) { - header->hdr_type = http_method::HTTP_METHOD_GET; + header->hdr_type = static_cast(http_method::GET); header->context.u.is_request = 1; } else if (parser->type == HTTP_REQUEST && parser->method == HTTP_POST) { - header->hdr_type = http_method::HTTP_METHOD_POST; + header->hdr_type = static_cast(http_method::POST); header->context.u.is_request = 1; } else { // Bit fields don't work with "perfect" forwarding, see diff --git a/src/http/http_method.h b/src/http/http_method.h new file mode 100644 index 0000000000..f337d24e78 --- /dev/null +++ b/src/http/http_method.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "utils/enum_helper.h" + +namespace dsn { + +enum class http_method +{ + GET = 1, + POST = 2, + INVALID = 100, +}; + +ENUM_BEGIN(http_method, http_method::INVALID) +ENUM_REG2(http_method, GET) +ENUM_REG2(http_method, POST) +ENUM_END(http_method) + +} // namespace dsn diff --git a/src/http/http_server.cpp b/src/http/http_server.cpp index cbc179c815..4c2f704016 100644 --- a/src/http/http_server.cpp +++ b/src/http/http_server.cpp @@ -25,6 +25,7 @@ #include "builtin_http_calls.h" #include "fmt/core.h" +#include "http/http_method.h" #include "http_call_registry.h" #include "http_message_parser.h" #include "http_server_impl.h" @@ -146,8 +147,8 @@ void http_server::serve(message_ex *msg) resp.body = fmt::format("failed to parse request: {}", res.get_error()); } else { const http_request &req = res.get_value(); - std::shared_ptr call = http_call_registry::instance().find(req.path); - if (call != nullptr) { + auto call = http_call_registry::instance().find(req.path); + if (call) { call->callback(req, resp); } else { resp.status_code = http_status_code::not_found; diff --git a/src/http/http_server.h b/src/http/http_server.h index c84055261c..5ae780b983 100644 --- a/src/http/http_server.h +++ b/src/http/http_server.h @@ -25,6 +25,7 @@ #include #include +#include "http_method.h" #include "runtime/task/task_code.h" #include "utils/blob.h" #include "utils/errors.h" @@ -38,12 +39,6 @@ DSN_DECLARE_bool(enable_http_server); /// The rpc code for all the HTTP RPCs. DEFINE_TASK_CODE_RPC(RPC_HTTP_SERVICE, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT); -enum http_method -{ - HTTP_METHOD_GET = 1, - HTTP_METHOD_POST = 2, -}; - class message_ex; struct http_request diff --git a/src/http/pprof_http_service.cpp b/src/http/pprof_http_service.cpp index f57c57b193..f5aeb8b505 100644 --- a/src/http/pprof_http_service.cpp +++ b/src/http/pprof_http_service.cpp @@ -38,6 +38,7 @@ #include #include +#include "http/http_method.h" #include "http/http_server.h" #include "runtime/api_layer1.h" #include "utils/blob.h" @@ -308,7 +309,7 @@ void pprof_http_service::symbol_handler(const http_request &req, http_response & // Load /proc/self/maps pthread_once(&s_load_symbolmap_once, load_symbols); - if (req.method != http_method::HTTP_METHOD_POST) { + if (req.method != http_method::POST) { char buf[64]; snprintf(buf, sizeof(buf), "num_symbols: %lu\n", symbol_map.size()); resp.body = buf; diff --git a/src/http/test/CMakeLists.txt b/src/http/test/CMakeLists.txt index d4da7e5b3a..85f2c3798a 100644 --- a/src/http/test/CMakeLists.txt +++ b/src/http/test/CMakeLists.txt @@ -24,14 +24,16 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS dsn_http dsn_runtime + curl gtest - gtest_main - rocksdb) + rocksdb + ) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/run.sh" + "${CMAKE_CURRENT_SOURCE_DIR}/config-test.ini" ) dsn_add_test() diff --git a/src/http/test/config-test.ini b/src/http/test/config-test.ini new file mode 100644 index 0000000000..744d8d412a --- /dev/null +++ b/src/http/test/config-test.ini @@ -0,0 +1,75 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +; KIND, either express or implied. See the License for the +; specific language governing permissions and limitations +; under the License. + +[apps..default] +run = true +count = 1 +network.client.RPC_CHANNEL_TCP = dsn::tools::asio_network_provider, 65536 +network.client.RPC_CHANNEL_UDP = dsn::tools::asio_udp_provider, 65536 +network.server.0.RPC_CHANNEL_TCP = dsn::tools::asio_network_provider, 65536 +network.server.0.RPC_CHANNEL_UDP = dsn::tools::asio_udp_provider, 65536 + +[apps.test] +type = test +arguments = +run = true +ports = 20001 +count = 1 +pools = THREAD_POOL_DEFAULT + +[core] +tool = nativerun + +toollets = tracer, profiler +pause_on_start = false + +logging_start_level = LOG_LEVEL_DEBUG +logging_factory_name = dsn::tools::simple_logger + +[tools.simple_logger] +fast_flush = true +short_header = false +stderr_start_level = LOG_LEVEL_DEBUG + +[network] +; how many network threads for network library (used by asio) +io_service_worker_count = 2 + +[task..default] +is_trace = true +is_profile = true +allow_inline = false +rpc_call_channel = RPC_CHANNEL_TCP +rpc_message_header_format = dsn +rpc_timeout_milliseconds = 1000 + +[task.LPC_RPC_TIMEOUT] +is_trace = false +is_profile = false + +[task.RPC_TEST_UDP] +rpc_call_channel = RPC_CHANNEL_UDP +rpc_message_crc_required = true + +; specification for each thread pool +[threadpool..default] +worker_count = 2 + +[threadpool.THREAD_POOL_DEFAULT] +partitioned = false +worker_priority = THREAD_xPRIORITY_NORMAL + diff --git a/src/http/test/http_client_test.cpp b/src/http/test/http_client_test.cpp new file mode 100644 index 0000000000..d15cb7bdd4 --- /dev/null +++ b/src/http/test/http_client_test.cpp @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +#include +#include +#include +#include + +#include "http/http_client.h" +#include "http/http_method.h" +#include "utils/error_code.h" +#include "utils/errors.h" +#include "utils/fmt_logging.h" +#include "utils/test_macros.h" + +namespace dsn { + +void check_expected_description_prefix(const std::string &expected_description_prefix, + const dsn::error_s &err) +{ + const std::string actual_description(err.description()); + std::cout << actual_description << std::endl; + + ASSERT_LT(expected_description_prefix.size(), actual_description.size()); + EXPECT_EQ(expected_description_prefix, + actual_description.substr(0, expected_description_prefix.size())); +} + +TEST(HttpClientTest, Connect) +{ + http_client client; + ASSERT_TRUE(client.init()); + + // No one has listened on port 20000, thus this would lead to "Connection refused". + ASSERT_TRUE(client.set_url("http://127.0.0.1:20000/test/get")); + + const auto &err = client.exec_method(); + ASSERT_EQ(dsn::ERR_CURL_FAILED, err.code()); + + std::cout << "failed to connect: "; + + // "code=7" means CURLE_COULDNT_CONNECT, see https://curl.se/libcurl/c/libcurl-errors.html + // for details. + // + // We just check the prefix of description, including `method`, `url`, `code` and `desc`. + // The `msg` differ in various systems, such as: + // * msg="Failed to connect to 127.0.0.1 port 20000: Connection refused" + // * msg="Failed to connect to 127.0.0.1 port 20000 after 0 ms: Connection refused" + // Thus we don't check if `msg` fields are consistent. + const std::string expected_description_prefix( + "ERR_CURL_FAILED: failed to perform http request(" + "method=GET, url=http://127.0.0.1:20000/test/get): code=7, " + "desc=\"Couldn't connect to server\""); + NO_FATALS(check_expected_description_prefix(expected_description_prefix, err)); +} + +TEST(HttpClientTest, Callback) +{ + http_client client; + ASSERT_TRUE(client.init()); + + ASSERT_TRUE(client.set_url("http://127.0.0.1:20001/test/get")); + ASSERT_TRUE(client.with_get_method()); + + auto callback = [](const void *, size_t) { return false; }; + + const auto &err = client.exec_method(callback); + ASSERT_EQ(dsn::ERR_CURL_FAILED, err.code()); + + long actual_http_status; + ASSERT_TRUE(client.get_http_status(actual_http_status)); + EXPECT_EQ(200, actual_http_status); + + std::cout << "failed for callback: "; + + // "code=23" means CURLE_WRITE_ERROR, see https://curl.se/libcurl/c/libcurl-errors.html + // for details. + // + // We just check the prefix of description, including `method`, `url`, `code` and `desc`. + // The `msg` differ in various systems, such as: + // * msg="Failed writing body (18446744073709551615 != 24)" + // * msg="Failure writing output to destination" + // Thus we don't check if `msg` fields are consistent. + const auto expected_description_prefix = + fmt::format("ERR_CURL_FAILED: failed to perform http request(" + "method=GET, url=http://127.0.0.1:20001/test/get): code=23, " + "desc=\"Failed writing received data to disk/application\""); + NO_FATALS(check_expected_description_prefix(expected_description_prefix, err)); +} + +using http_client_method_case = + std::tuple; + +class HttpClientMethodTest : public testing::TestWithParam +{ +public: + void SetUp() override { ASSERT_TRUE(_client.init()); } + + void test_method_with_response_string(const long expected_http_status, + const std::string &expected_response) + { + std::string actual_response; + ASSERT_TRUE(_client.exec_method(&actual_response)); + + long actual_http_status; + ASSERT_TRUE(_client.get_http_status(actual_http_status)); + + EXPECT_EQ(expected_http_status, actual_http_status); + EXPECT_EQ(expected_response, actual_response); + } + + void test_method_with_response_callback(const long expected_http_status, + const std::string &expected_response) + { + auto callback = [&expected_response](const void *data, size_t length) { + auto compare = [](const char *expected_data, + size_t expected_length, + const void *actual_data, + size_t actual_length) { + if (expected_length != actual_length) { + return false; + } + return std::memcmp(expected_data, actual_data, actual_length) == 0; + }; + EXPECT_PRED4(compare, expected_response.data(), expected_response.size(), data, length); + return true; + }; + ASSERT_TRUE(_client.exec_method(callback)); + + long actual_http_status; + ASSERT_TRUE(_client.get_http_status(actual_http_status)); + EXPECT_EQ(expected_http_status, actual_http_status); + } + + void test_mothod(const std::string &url, + const http_method method, + const std::string &post_data, + const long expected_http_status, + const std::string &expected_response) + { + _client.set_url(url); + + switch (method) { + case http_method::GET: + ASSERT_TRUE(_client.with_get_method()); + break; + case http_method::POST: + ASSERT_TRUE(_client.with_post_method(post_data)); + break; + default: + LOG_FATAL("Unsupported http_method"); + } + + test_method_with_response_string(expected_http_status, expected_response); + test_method_with_response_callback(expected_http_status, expected_response); + } + +private: + http_client _client; +}; + +TEST_P(HttpClientMethodTest, ExecMethod) +{ + const char *url; + http_method method; + const char *post_data; + long expected_http_status; + const char *expected_response; + std::tie(url, method, post_data, expected_http_status, expected_response) = GetParam(); + + http_client _client; + test_mothod(url, method, post_data, expected_http_status, expected_response); +} + +const std::vector http_client_method_tests = { + {"http://127.0.0.1:20001/test/get", + http_method::POST, + "with POST DATA", + 400, + "please use GET method"}, + {"http://127.0.0.1:20001/test/get", http_method::GET, "", 200, "you are using GET method"}, + {"http://127.0.0.1:20001/test/post", + http_method::POST, + "with POST DATA", + 200, + "you are using POST method with POST DATA"}, + {"http://127.0.0.1:20001/test/post", http_method::GET, "", 400, "please use POST method"}, +}; + +INSTANTIATE_TEST_CASE_P(HttpClientTest, + HttpClientMethodTest, + testing::ValuesIn(http_client_method_tests)); + +} // namespace dsn diff --git a/src/http/test/http_server_test.cpp b/src/http/test/http_server_test.cpp index 9aa9f17c39..0c6b9ec6d3 100644 --- a/src/http/test/http_server_test.cpp +++ b/src/http/test/http_server_test.cpp @@ -18,6 +18,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include "http/builtin_http_calls.h" #include "http/http_call_registry.h" #include "http/http_message_parser.h" +#include "http/http_method.h" #include "http/http_server.h" #include "runtime/rpc/message_parser.h" #include "runtime/rpc/rpc_message.h" @@ -87,7 +89,12 @@ TEST(bultin_http_calls_test, meta_query) TEST(bultin_http_calls_test, get_help) { + // Used to save current http calls as backup. + std::vector> backup_calls; + + // Remove all http calls. for (const auto &call : http_call_registry::instance().list_all_calls()) { + backup_calls.push_back(call); http_call_registry::instance().remove(call->path); } @@ -111,9 +118,15 @@ TEST(bultin_http_calls_test, get_help) get_help_handler(req, resp); ASSERT_EQ(resp.body, "{\"/\":\"ip:port/\",\"/recentStartTime\":\"ip:port/recentStartTime\"}\n"); + // Remove all http calls, especially `recentStartTime`. for (const auto &call : http_call_registry::instance().list_all_calls()) { http_call_registry::instance().remove(call->path); } + + // Recover http calls from backup. + for (const auto &call : backup_calls) { + http_call_registry::instance().add(call); + } } class http_message_parser_test : public testing::Test @@ -174,7 +187,7 @@ class http_message_parser_test : public testing::Test message_ptr msg = parser.get_message_on_receive(&reader, read_next); ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_GET); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::GET)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[2].size(), 1); // url @@ -215,7 +228,7 @@ TEST_F(http_message_parser_test, parse_request) ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_POST); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::POST)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[1].to_string(), "Message Body sdfsdf"); // body @@ -266,7 +279,7 @@ TEST_F(http_message_parser_test, eof) ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_GET); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::GET)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[1].to_string(), ""); // body @@ -297,7 +310,7 @@ TEST_F(http_message_parser_test, parse_long_url) message_ptr msg = parser.get_message_on_receive(&reader, read_next); ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_GET); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::GET)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[2].size(), 4097); // url diff --git a/src/http/test/main.cpp b/src/http/test/main.cpp new file mode 100644 index 0000000000..1eeb3d04ef --- /dev/null +++ b/src/http/test/main.cpp @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include + +#include "http/http_method.h" +#include "http/http_server.h" +#include "runtime/app_model.h" +#include "runtime/service_app.h" +#include "utils/blob.h" +#include "utils/error_code.h" +#include "utils/ports.h" + +int gtest_flags = 0; +int gtest_ret = 0; + +class test_http_service : public dsn::http_server_base +{ +public: + test_http_service() + { + register_handler("get", + std::bind(&test_http_service::method_handler, + this, + dsn::http_method::GET, + std::placeholders::_1, + std::placeholders::_2), + "ip:port/test/get"); + register_handler("post", + std::bind(&test_http_service::method_handler, + this, + dsn::http_method::POST, + std::placeholders::_1, + std::placeholders::_2), + "ip:port/test/post"); + } + + ~test_http_service() = default; + + std::string path() const override { return "test"; } + +private: + void method_handler(dsn::http_method target_method, + const dsn::http_request &req, + dsn::http_response &resp) + { + if (req.method != target_method) { + resp.body = fmt::format("please use {} method", enum_to_string(target_method)); + resp.status_code = dsn::http_status_code::bad_request; + return; + } + + std::string postfix; + if (target_method == dsn::http_method::POST) { + postfix = " "; + postfix += req.body.to_string(); + } + + resp.body = + fmt::format("you are using {} method{}", enum_to_string(target_method), postfix); + resp.status_code = dsn::http_status_code::ok; + } + + DISALLOW_COPY_AND_ASSIGN(test_http_service); +}; + +class test_service_app : public dsn::service_app +{ +public: + test_service_app(const dsn::service_app_info *info) : dsn::service_app(info) + { + dsn::register_http_service(new test_http_service); + dsn::start_http_server(); + } + + dsn::error_code start(const std::vector &args) override + { + gtest_ret = RUN_ALL_TESTS(); + gtest_flags = 1; + return dsn::ERR_OK; + } +}; + +GTEST_API_ int main(int argc, char **argv) +{ + testing::InitGoogleTest(&argc, argv); + + // Register test service. + dsn::service_app::register_factory("test"); + + dsn_run_config("config-test.ini", false); + while (gtest_flags == 0) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + +#ifndef ENABLE_GCOV + dsn_exit(gtest_ret); +#endif + return gtest_ret; +} diff --git a/src/utils/error_code.h b/src/utils/error_code.h index 45a6e793cc..04df97947a 100644 --- a/src/utils/error_code.h +++ b/src/utils/error_code.h @@ -180,6 +180,8 @@ DEFINE_ERR_CODE(ERR_RANGER_POLICIES_NO_NEED_UPDATE) DEFINE_ERR_CODE(ERR_RDB_CORRUPTION) DEFINE_ERR_CODE(ERR_DISK_IO_ERROR) + +DEFINE_ERR_CODE(ERR_CURL_FAILED) } // namespace dsn USER_DEFINED_STRUCTURE_FORMATTER(::dsn::error_code); diff --git a/src/utils/errors.h b/src/utils/errors.h index 9cbf70e092..ce36befca9 100644 --- a/src/utils/errors.h +++ b/src/utils/errors.h @@ -97,6 +97,8 @@ class error_s return true; } + explicit operator bool() const noexcept { return is_ok(); } + std::string description() const { if (!_info) { @@ -227,7 +229,7 @@ USER_DEFINED_STRUCTURE_FORMATTER(::dsn::error_s); #define RETURN_NOT_OK(s) \ do { \ const ::dsn::error_s &_s = (s); \ - if (dsn_unlikely(!_s.is_ok())) { \ + if (dsn_unlikely(!_s)) { \ return _s; \ } \ } while (false); diff --git a/src/utils/metrics.cpp b/src/utils/metrics.cpp index 40caace7a4..b5800d3cd1 100644 --- a/src/utils/metrics.cpp +++ b/src/utils/metrics.cpp @@ -23,6 +23,7 @@ #include #include +#include "http/http_method.h" #include "runtime/api_layer1.h" #include "utils/flags.h" #include "utils/rand.h" @@ -275,7 +276,7 @@ const dsn::metric_filters::metric_fields_type kBriefMetricFields = get_brief_met void metrics_http_service::get_metrics_handler(const http_request &req, http_response &resp) { - if (req.method != http_method::HTTP_METHOD_GET) { + if (req.method != http_method::GET) { resp.body = encode_error_as_json("please use 'GET' method while querying for metrics"); resp.status_code = http_status_code::bad_request; return; diff --git a/src/utils/test/metrics_test.cpp b/src/utils/test/metrics_test.cpp index b2118f03ec..8d65c65dae 100644 --- a/src/utils/test/metrics_test.cpp +++ b/src/utils/test/metrics_test.cpp @@ -2513,7 +2513,7 @@ void test_http_get_metrics(const std::string &request_string, ASSERT_TRUE(req_res.is_ok()); const auto &req = req_res.get_value(); - std::cout << "method: " << req.method << std::endl; + std::cout << "method: " << enum_to_string(req.method) << std::endl; http_response resp; test_get_metrics_handler(req, resp); diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 4558aa94f7..f6fd11d4ba 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -276,11 +276,7 @@ ExternalProject_Add(civetweb ExternalProject_Get_property(civetweb SOURCE_DIR) set(civetweb_SRC ${SOURCE_DIR}) -ExternalProject_Add(curl - URL ${OSS_URL_PREFIX}/curl-7.47.0.tar.gz - http://curl.haxx.se/download/curl-7.47.0.tar.gz - URL_MD5 5109d1232d208dfd712c0272b8360393 - CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} +set(CURL_OPTIONS --disable-dict --disable-file --disable-ftp @@ -301,6 +297,19 @@ ExternalProject_Add(curl --without-libssh2 --without-ssl --without-libidn + ) +if (APPLE) + set(CURL_OPTIONS + ${CURL_OPTIONS} + --without-nghttp2 + ) +endif () +ExternalProject_Add(curl + URL ${OSS_URL_PREFIX}/curl-7.47.0.tar.gz + http://curl.haxx.se/download/curl-7.47.0.tar.gz + URL_MD5 5109d1232d208dfd712c0272b8360393 + CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} + ${CURL_OPTIONS} BUILD_IN_SOURCE 1 ) From 70d2467393d9f4152059d7428859e9491776d49b Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 10 Oct 2023 12:30:00 +0800 Subject: [PATCH 36/42] refactor(file_system): remove useless read/write_file() (#1628) https://github.com/apache/incubator-pegasus/issues/887 Remove `read_file()` and `write_file()` from `dsn::utils::filesystem` namespace. --- src/replica/bulk_load/replica_bulk_loader.cpp | 11 +++-- src/utils/filesystem.cpp | 45 ------------------- src/utils/filesystem.h | 6 --- 3 files changed, 7 insertions(+), 55 deletions(-) diff --git a/src/replica/bulk_load/replica_bulk_loader.cpp b/src/replica/bulk_load/replica_bulk_loader.cpp index 5261e4183c..643816d423 100644 --- a/src/replica/bulk_load/replica_bulk_loader.cpp +++ b/src/replica/bulk_load/replica_bulk_loader.cpp @@ -16,6 +16,8 @@ // under the License. #include +#include +#include #include #include #include @@ -48,6 +50,7 @@ #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" +#include "utils/ports.h" #include "utils/string_view.h" #include "utils/thread_access_checker.h" @@ -562,10 +565,10 @@ void replica_bulk_loader::download_sst_file(const std::string &remote_dir, error_code replica_bulk_loader::parse_bulk_load_metadata(const std::string &fname) { std::string buf; - error_code ec = utils::filesystem::read_file(fname, buf); - if (ec != ERR_OK) { - LOG_ERROR_PREFIX("read file {} failed, error = {}", fname, ec); - return ec; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &buf); + if (dsn_unlikely(!s.ok())) { + LOG_ERROR_PREFIX("read file {} failed, error = {}", fname, s.ToString()); + return ERR_FILE_OPERATION_FAILED; } blob bb = blob::create_from_bytes(std::move(buf)); diff --git a/src/utils/filesystem.cpp b/src/utils/filesystem.cpp index 2f80a51d50..a252d823b6 100644 --- a/src/utils/filesystem.cpp +++ b/src/utils/filesystem.cpp @@ -48,7 +48,6 @@ #include // IWYU pragma: no_include #include -#include #include #include "utils/defer.h" @@ -830,36 +829,6 @@ std::pair is_directory_empty(const std::string &dirname) return res; } -error_code read_file(const std::string &fname, std::string &buf) -{ - if (!file_exists(fname)) { - LOG_ERROR("file({}) doesn't exist", fname); - return ERR_FILE_OPERATION_FAILED; - } - - int64_t file_sz = 0; - if (!file_size(fname, file_sz)) { - LOG_ERROR("get file({}) size failed", fname); - return ERR_FILE_OPERATION_FAILED; - } - - buf.resize(file_sz); - std::ifstream fin(fname, std::ifstream::in); - if (!fin.is_open()) { - LOG_ERROR("open file({}) failed", fname); - return ERR_FILE_OPERATION_FAILED; - } - fin.read(&buf[0], file_sz); - CHECK_EQ_MSG(file_sz, - fin.gcount(), - "read file({}) failed, file_size = {} but read size = {}", - fname, - file_sz, - fin.gcount()); - fin.close(); - return ERR_OK; -} - bool verify_file(const std::string &fname, FileDataType type, const std::string &expected_md5, @@ -933,20 +902,6 @@ bool create_directory(const std::string &path, std::string &absolute_path, std:: return true; } -bool write_file(const std::string &fname, std::string &buf) -{ - if (!file_exists(fname)) { - LOG_ERROR("file({}) doesn't exist", fname); - return false; - } - - std::ofstream fstream; - fstream.open(fname.c_str()); - fstream << buf; - fstream.close(); - return true; -} - bool check_dir_rw(const std::string &path, std::string &err_msg) { FAIL_POINT_INJECT_F("filesystem_check_dir_rw", [path](string_view str) { diff --git a/src/utils/filesystem.h b/src/utils/filesystem.h index 440128a97d..be62e76831 100644 --- a/src/utils/filesystem.h +++ b/src/utils/filesystem.h @@ -146,9 +146,6 @@ error_code deprecated_md5sum(const std::string &file_path, /*out*/ std::string & // B is represent wheter the directory is empty, true means empty, otherwise false std::pair is_directory_empty(const std::string &dirname); -// TODO(yingchun): remove it! -error_code read_file(const std::string &fname, /*out*/ std::string &buf); - // compare file metadata calculated by fname with expected md5 and file_size bool verify_file(const std::string &fname, FileDataType type, @@ -162,9 +159,6 @@ bool create_directory(const std::string &path, /*out*/ std::string &absolute_path, /*out*/ std::string &err_msg); -// TODO(yingchun): remove it! -bool write_file(const std::string &fname, std::string &buf); - // check if directory is readable and writable // call `create_directory` before to make `path` exist bool check_dir_rw(const std::string &path, /*out*/ std::string &err_msg); From b551e2627eb13fcd8757ec9c4c70ff937de7657d Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 10 Oct 2023 14:07:49 +0800 Subject: [PATCH 37/42] refactor: some minor refactors without functional changes (#1629) --- .licenserc.yaml | 2 +- src/block_service/block_service.h | 6 +- src/geo/bench/bench.cpp | 6 +- src/meta/test/main.cpp | 6 +- ...{nfs_node_impl.cpp => nfs_node_simple.cpp} | 13 ++- src/nfs/nfs_node_simple.h | 14 ++- src/nfs/nfs_server_impl.cpp | 102 ++++++++---------- src/nfs/nfs_server_impl.h | 12 +-- src/server/test/pegasus_server_impl_test.cpp | 25 ++--- src/test/function_test/base_api/test_copy.cpp | 12 +-- 10 files changed, 104 insertions(+), 94 deletions(-) rename src/nfs/{nfs_node_impl.cpp => nfs_node_simple.cpp} (93%) diff --git a/.licenserc.yaml b/.licenserc.yaml index c6f63afd8b..53d316d23b 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -312,7 +312,7 @@ header: - 'src/nfs/nfs_client_impl.h' - 'src/nfs/nfs_code_definition.h' - 'src/nfs/nfs_node.cpp' - - 'src/nfs/nfs_node_impl.cpp' + - 'src/nfs/nfs_node_simple.cpp' - 'src/nfs/nfs_node_simple.h' - 'src/nfs/nfs_server_impl.cpp' - 'src/nfs/nfs_server_impl.h' diff --git a/src/block_service/block_service.h b/src/block_service/block_service.h index 96f1416445..d351dcf44a 100644 --- a/src/block_service/block_service.h +++ b/src/block_service/block_service.h @@ -238,8 +238,8 @@ struct upload_request */ struct upload_response { - dsn::error_code err; - uint64_t uploaded_size; + dsn::error_code err = ERR_OK; + uint64_t uploaded_size = 0; }; typedef std::function upload_callback; typedef future_task upload_future; @@ -378,6 +378,8 @@ class block_file : public dsn::ref_counter const write_callback &cb, dsn::task_tracker *tracker = nullptr) = 0; + // TODO(yingchun): it seems every read() will read the whole file, consider to read the whole + // file directly. /** * @brief read * @param req, ref {@link #read_request} diff --git a/src/geo/bench/bench.cpp b/src/geo/bench/bench.cpp index e65714b7ad..0ac170b02e 100644 --- a/src/geo/bench/bench.cpp +++ b/src/geo/bench/bench.cpp @@ -76,10 +76,12 @@ int main(int argc, char **argv) } } + // TODO(yingchun): the benchmark can not exit normally, we need to fix it later. pegasus::geo::geo_client my_geo( "config.ini", cluster_name.c_str(), app_name.c_str(), geo_app_name.c_str()); - if (!my_geo.set_max_level(max_level).is_ok()) { - std::cerr << "set_max_level failed" << std::endl; + auto err = my_geo.set_max_level(max_level); + if (!err.is_ok()) { + std::cerr << "set_max_level failed, err: " << err << std::endl; return -1; } diff --git a/src/meta/test/main.cpp b/src/meta/test/main.cpp index fd82dd1779..e410572bcc 100644 --- a/src/meta/test/main.cpp +++ b/src/meta/test/main.cpp @@ -63,7 +63,11 @@ TEST(meta, state_sync) { g_app->state_sync_test(); } TEST(meta, update_configuration) { g_app->update_configuration_test(); } -TEST(meta, balancer_validator) { g_app->balancer_validator(); } +TEST(meta, balancer_validator) +{ + // TODO(yingchun): this test last too long time, optimize it! + g_app->balancer_validator(); +} TEST(meta, apply_balancer) { g_app->apply_balancer_test(); } diff --git a/src/nfs/nfs_node_impl.cpp b/src/nfs/nfs_node_simple.cpp similarity index 93% rename from src/nfs/nfs_node_impl.cpp rename to src/nfs/nfs_node_simple.cpp index 387547fd88..bb334a5085 100644 --- a/src/nfs/nfs_node_impl.cpp +++ b/src/nfs/nfs_node_simple.cpp @@ -80,12 +80,17 @@ void nfs_node_simple::register_async_rpc_handler_for_test() error_code nfs_node_simple::stop() { - delete _server; - _server = nullptr; + if (_server != nullptr) { + _server->close_service(); - delete _client; - _client = nullptr; + delete _server; + _server = nullptr; + } + if (_client != nullptr) { + delete _client; + _client = nullptr; + } return ERR_OK; } diff --git a/src/nfs/nfs_node_simple.h b/src/nfs/nfs_node_simple.h index 2376b1e34e..15e2344168 100644 --- a/src/nfs/nfs_node_simple.h +++ b/src/nfs/nfs_node_simple.h @@ -34,14 +34,24 @@ */ #pragma once -#include "runtime/tool_api.h" +#include + #include "nfs/nfs_node.h" +#include "utils/error_code.h" namespace dsn { +class aio_task; +template +class rpc_replier; + namespace service { -class nfs_service_impl; +class copy_request; +class copy_response; +class get_file_size_request; +class get_file_size_response; class nfs_client_impl; +class nfs_service_impl; class nfs_node_simple : public nfs_node { diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index cadacba1a3..08821042a4 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -26,12 +26,11 @@ #include "nfs/nfs_server_impl.h" -#include #include -#include #include #include #include +#include #include #include "nfs/nfs_code_definition.h" @@ -39,10 +38,10 @@ #include "runtime/api_layer1.h" #include "runtime/task/async_calls.h" #include "utils/TokenBucket.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/ports.h" -#include "utils/safe_strerror_posix.h" #include "utils/string_conv.h" #include "utils/utils.h" @@ -90,38 +89,35 @@ void nfs_service_impl::on_copy(const ::dsn::service::copy_request &request, dsn::utils::filesystem::path_combine(request.source_dir, request.file_name); disk_file *dfile = nullptr; - { + do { zauto_lock l(_handles_map_lock); auto it = _handles_map.find(file_path); // find file handle cache first - if (it == _handles_map.end()) { dfile = file::open(file_path.c_str(), O_RDONLY | O_BINARY, 0); - if (dfile != nullptr) { - auto fh = std::make_shared(); - fh->file_handle = dfile; - fh->file_access_count = 1; - fh->last_access_time = dsn_now_ms(); - _handles_map.insert(std::make_pair(file_path, std::move(fh))); + if (dfile == nullptr) { + LOG_ERROR("[nfs_service] open file {} failed", file_path); + ::dsn::service::copy_response resp; + resp.error = ERR_OBJECT_NOT_FOUND; + reply(resp); + return; } - } else { - dfile = it->second->file_handle; - it->second->file_access_count++; - it->second->last_access_time = dsn_now_ms(); - } - } - - LOG_DEBUG( - "nfs: copy file {} [{}, {}]", file_path, request.offset, request.offset + request.size); - if (dfile == nullptr) { - LOG_ERROR("[nfs_service] open file {} failed", file_path); - ::dsn::service::copy_response resp; - resp.error = ERR_OBJECT_NOT_FOUND; - reply(resp); - return; - } - - std::shared_ptr cp = std::make_shared(std::move(reply)); + auto fh = std::make_shared(); + fh->file_handle = dfile; + it = _handles_map.insert(std::make_pair(file_path, std::move(fh))).first; + } + dfile = it->second->file_handle; + it->second->file_access_count++; + it->second->last_access_time = dsn_now_ms(); + } while (false); + + CHECK_NOTNULL(dfile, ""); + LOG_DEBUG("nfs: copy from file {} [{}, {}]", + file_path, + request.offset, + request.offset + request.size); + + auto cp = std::make_shared(std::move(reply)); cp->bb = blob(dsn::utils::make_shared_array(request.size), request.size); cp->dst_dir = request.dst_dir; cp->source_disk_tag = request.source_disk_tag; @@ -182,58 +178,53 @@ void nfs_service_impl::on_get_file_size( { get_file_size_response resp; error_code err = ERR_OK; - std::vector file_list; std::string folder = request.source_dir; + // TODO(yingchun): refactor the following code! if (request.file_list.size() == 0) // return all file size in the destination file folder { if (!dsn::utils::filesystem::directory_exists(folder)) { LOG_ERROR("[nfs_service] directory {} not exist", folder); err = ERR_OBJECT_NOT_FOUND; } else { + std::vector file_list; if (!dsn::utils::filesystem::get_subfiles(folder, file_list, true)) { LOG_ERROR("[nfs_service] get subfiles of directory {} failed", folder); err = ERR_FILE_OPERATION_FAILED; } else { - for (auto &fpath : file_list) { - // TODO: using uint64 instead as file ma - // Done + for (const auto &fpath : file_list) { int64_t sz; - if (!dsn::utils::filesystem::file_size(fpath, sz)) { + // TODO(yingchun): check if there are any files that are not sensitive (not + // encrypted). + if (!dsn::utils::filesystem::file_size( + fpath, dsn::utils::FileDataType::kSensitive, sz)) { LOG_ERROR("[nfs_service] get size of file {} failed", fpath); err = ERR_FILE_OPERATION_FAILED; break; } - resp.size_list.push_back((uint64_t)sz); + resp.size_list.push_back(sz); resp.file_list.push_back( fpath.substr(request.source_dir.length(), fpath.length() - 1)); } - file_list.clear(); } } } else // return file size in the request file folder { - for (size_t i = 0; i < request.file_list.size(); i++) { - std::string file_path = - dsn::utils::filesystem::path_combine(folder, request.file_list[i]); - - struct stat st; - if (0 != ::stat(file_path.c_str(), &st)) { - LOG_ERROR("[nfs_service] get stat of file {} failed, err = {}", - file_path, - dsn::utils::safe_strerror(errno)); - err = ERR_OBJECT_NOT_FOUND; + for (const auto &file_name : request.file_list) { + std::string file_path = dsn::utils::filesystem::path_combine(folder, file_name); + int64_t sz; + // TODO(yingchun): check if there are any files that are not sensitive (not encrypted). + if (!dsn::utils::filesystem::file_size( + file_path, dsn::utils::FileDataType::kSensitive, sz)) { + LOG_ERROR("[nfs_service] get size of file {} failed", file_path); + err = ERR_FILE_OPERATION_FAILED; break; } - // TODO: using int64 instead as file may exceed the size of 32bit - // Done - uint64_t size = st.st_size; - - resp.size_list.push_back(size); - resp.file_list.push_back((folder + request.file_list[i]) - .substr(request.source_dir.length(), - (folder + request.file_list[i]).length() - 1)); + resp.size_list.push_back(sz); + resp.file_list.push_back( + (folder + file_name) + .substr(request.source_dir.length(), (folder + file_name).length() - 1)); } } @@ -253,8 +244,9 @@ void nfs_service_impl::close_file() // release out-of-date file handle dsn_now_ms() - fptr->last_access_time > (uint64_t)FLAGS_file_close_expire_time_ms) { LOG_DEBUG("nfs: close file handle {}", it->first); it = _handles_map.erase(it); - } else + } else { it++; + } } } diff --git a/src/nfs/nfs_server_impl.h b/src/nfs/nfs_server_impl.h index 9ba1134040..ece68ecb33 100644 --- a/src/nfs/nfs_server_impl.h +++ b/src/nfs/nfs_server_impl.h @@ -66,7 +66,6 @@ class nfs_service_impl : public ::dsn::serverlet void register_cli_commands(); - // TODO(yingchun): seems nobody call it, can be removed? void close_service() { unregister_rpc_handler(RPC_NFS_COPY); @@ -107,14 +106,9 @@ class nfs_service_impl : public ::dsn::serverlet struct file_handle_info_on_server { - disk_file *file_handle; - int32_t file_access_count; // concurrent r/w count - uint64_t last_access_time; // last touch time - - file_handle_info_on_server() - : file_handle(nullptr), file_access_count(0), last_access_time(0) - { - } + disk_file *file_handle = nullptr; + int32_t file_access_count = 0; // concurrent r/w count + uint64_t last_access_time = 0; // last touch time ~file_handle_info_on_server() { diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 9776571ae9..718446821d 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -21,6 +21,7 @@ #include #include #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -124,10 +125,10 @@ class pegasus_server_impl_test : public pegasus_server_test_base } } - start(all_test_envs); + ASSERT_EQ(dsn::ERR_OK, start(all_test_envs)); if (is_restart) { - _server->stop(false); - start(); + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); + ASSERT_EQ(dsn::ERR_OK, start()); } std::map query_envs; @@ -145,20 +146,20 @@ class pegasus_server_impl_test : public pegasus_server_test_base TEST_F(pegasus_server_impl_test, test_table_level_slow_query) { - start(); + ASSERT_EQ(dsn::ERR_OK, start()); test_table_level_slow_query(); } TEST_F(pegasus_server_impl_test, default_data_version) { - start(); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_EQ(_server->_pegasus_data_version, 1); } TEST_F(pegasus_server_impl_test, test_open_db_with_latest_options) { // open a new db with no app env. - start(); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_NORMAL, _server->_usage_scenario); // set bulk_load scenario for the db. ASSERT_TRUE(_server->set_usage_scenario(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD)); @@ -167,8 +168,8 @@ TEST_F(pegasus_server_impl_test, test_open_db_with_latest_options) ASSERT_EQ(1000000000, opts.level0_file_num_compaction_trigger); ASSERT_EQ(true, opts.disable_auto_compactions); // reopen the db. - _server->stop(false); - start(); + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario); ASSERT_EQ(opts.level0_file_num_compaction_trigger, _server->_db->GetOptions().level0_file_num_compaction_trigger); @@ -179,7 +180,7 @@ TEST_F(pegasus_server_impl_test, test_open_db_with_app_envs) { std::map envs; envs[ROCKSDB_ENV_USAGE_SCENARIO_KEY] = ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD; - start(envs); + ASSERT_EQ(dsn::ERR_OK, start(envs)); ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario); } @@ -197,16 +198,16 @@ TEST_F(pegasus_server_impl_test, test_restart_db_with_rocksdb_envs) TEST_F(pegasus_server_impl_test, test_stop_db_twice) { - start(); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_TRUE(_server->_is_open); ASSERT_TRUE(_server->_db != nullptr); - _server->stop(false); + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); ASSERT_FALSE(_server->_is_open); ASSERT_TRUE(_server->_db == nullptr); // stop again - _server->stop(false); + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); ASSERT_FALSE(_server->_is_open); ASSERT_TRUE(_server->_db == nullptr); } diff --git a/src/test/function_test/base_api/test_copy.cpp b/src/test/function_test/base_api/test_copy.cpp index 910a07fff6..d2155f12f0 100644 --- a/src/test/function_test/base_api/test_copy.cpp +++ b/src/test/function_test/base_api/test_copy.cpp @@ -98,9 +98,9 @@ class copy_data_test : public test_util ASSERT_EQ(dsn::ERR_OK, ddl_client_->create_app( destination_app_name, "pegasus", default_partitions, 3, {}, false)); - srouce_client_ = + source_client_ = pegasus_client_factory::get_client(cluster_name_.c_str(), source_app_name.c_str()); - ASSERT_NE(nullptr, srouce_client_); + ASSERT_NE(nullptr, source_client_); destination_client_ = pegasus_client_factory::get_client(cluster_name_.c_str(), destination_app_name.c_str()); ASSERT_NE(nullptr, destination_client_); @@ -132,7 +132,7 @@ class copy_data_test : public test_util while (expect_data_[empty_hash_key].size() < 1000) { sort_key = random_string(); value = random_string(); - ASSERT_EQ(PERR_OK, srouce_client_->set(empty_hash_key, sort_key, value)) + ASSERT_EQ(PERR_OK, source_client_->set(empty_hash_key, sort_key, value)) << "hash_key=" << hash_key << ", sort_key=" << sort_key; expect_data_[empty_hash_key][sort_key] = value; } @@ -142,7 +142,7 @@ class copy_data_test : public test_util while (expect_data_[hash_key].size() < 10) { sort_key = random_string(); value = random_string(); - ASSERT_EQ(PERR_OK, srouce_client_->set(hash_key, sort_key, value)) + ASSERT_EQ(PERR_OK, source_client_->set(hash_key, sort_key, value)) << "hash_key=" << hash_key << ", sort_key=" << sort_key; expect_data_[hash_key][sort_key] = value; } @@ -163,7 +163,7 @@ class copy_data_test : public test_util char buffer_[256]; map> expect_data_; - pegasus_client *srouce_client_; + pegasus_client *source_client_; pegasus_client *destination_client_; }; const char copy_data_test::CCH[] = @@ -176,7 +176,7 @@ TEST_F(copy_data_test, EMPTY_HASH_KEY_COPY) pegasus_client::scan_options options; options.return_expire_ts = true; vector raw_scanners; - ASSERT_EQ(PERR_OK, srouce_client_->get_unordered_scanners(INT_MAX, options, raw_scanners)); + ASSERT_EQ(PERR_OK, source_client_->get_unordered_scanners(INT_MAX, options, raw_scanners)); LOG_INFO("open source app scanner succeed, partition_count = {}", raw_scanners.size()); From 4cb68284cf4931b0ade34c31ce18156ad618d158 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 11 Oct 2023 15:41:17 +0800 Subject: [PATCH 38/42] fix(ut): fix flaky unit test test_rename_path_while_writing (#1630) https://github.com/apache/incubator-pegasus/issues/1631 Add retry logic for test `HDFSClientTest.test_rename_path_while_writing`, fix an used-after-free ASAN issue and remove tar files after downloading from artifacts to reduce space consumption. --- .github/workflows/lint_and_test_cpp.yaml | 4 + src/block_service/test/hdfs_service_test.cpp | 79 +++++++++++--------- src/test_util/test_util.h | 2 +- 3 files changed, 49 insertions(+), 36 deletions(-) diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 6c125727c2..f3dc28d389 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -247,6 +247,7 @@ jobs: - name: Tar files run: | tar -zxvf release__builder.tar + rm -f release__builder.tar - name: Unit Testing run: | export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server @@ -386,6 +387,7 @@ jobs: - name: Tar files run: | tar -zxvf release_address_builder.tar + rm -f release_address_builder.tar - name: Unit Testing run: | export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server @@ -527,6 +529,7 @@ jobs: # - name: Tar files # run: | # tar -zxvf release_undefined_builder.tar +# rm -f release_undefined_builder.tar # - name: Unit Testing # run: | # export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server @@ -639,6 +642,7 @@ jobs: - name: Tar files run: | tar -zxvf release_jemalloc_builder.tar + rm -f release_jemalloc_builder.tar - name: Unit Testing run: | export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server diff --git a/src/block_service/test/hdfs_service_test.cpp b/src/block_service/test/hdfs_service_test.cpp index d7b33a8daf..0eaec29e88 100644 --- a/src/block_service/test/hdfs_service_test.cpp +++ b/src/block_service/test/hdfs_service_test.cpp @@ -69,11 +69,21 @@ DEFINE_TASK_CODE(LPC_TEST_HDFS, TASK_PRIORITY_HIGH, dsn::THREAD_POOL_DEFAULT) class HDFSClientTest : public pegasus::encrypt_data_test_base { protected: + HDFSClientTest() : pegasus::encrypt_data_test_base() + { + for (int i = 0; i < FLAGS_num_test_file_lines; ++i) { + _local_test_data += "test"; + } + } + void generate_test_file(const std::string &filename); void write_test_files_async(const std::string &local_test_path, int total_files, int *success_count, task_tracker *tracker); + +private: + std::string _local_test_data; }; void HDFSClientTest::generate_test_file(const std::string &filename) @@ -97,29 +107,24 @@ void HDFSClientTest::write_test_files_async(const std::string &local_test_path, task_tracker *tracker) { dsn::utils::filesystem::create_directory(local_test_path); - std::string local_test_data; - for (int i = 0; i < FLAGS_num_test_file_lines; ++i) { - local_test_data += "test"; - } for (int i = 0; i < total_files; ++i) { - tasking::enqueue( - LPC_TEST_HDFS, tracker, [&local_test_path, &local_test_data, i, success_count]() { - // mock the writing process in hdfs_file_object::download(). - std::string test_file_name = local_test_path + "/test_file_" + std::to_string(i); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice(local_test_data), - test_file_name, - /* should_sync */ true); - if (s.ok()) { - ++(*success_count); - } else { - CHECK(s.IsIOError(), "{}", s.ToString()); - auto pos1 = s.ToString().find( - "IO error: No such file or directory: While open a file for appending: "); - auto pos2 = s.ToString().find("IO error: While appending to file: "); - CHECK(pos1 == 0 || pos2 == 0, "{}", s.ToString()); - } - }); + tasking::enqueue(LPC_TEST_HDFS, tracker, [this, &local_test_path, i, success_count]() { + // mock the writing process in hdfs_file_object::download(). + std::string test_file_name = local_test_path + "/test_file_" + std::to_string(i); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(_local_test_data), + test_file_name, + /* should_sync */ true); + if (s.ok()) { + ++(*success_count); + } else { + CHECK(s.IsIOError(), "{}", s.ToString()); + auto pos1 = s.ToString().find( + "IO error: No such file or directory: While open a file for appending: "); + auto pos2 = s.ToString().find("IO error: While appending to file: "); + CHECK(pos1 == 0 || pos2 == 0, "{}", s.ToString()); + } + }); } } @@ -443,17 +448,21 @@ TEST_P(HDFSClientTest, test_rename_path_while_writing) std::string kLocalTestPath = "test_dir"; const int kTotalFiles = 100; - task_tracker tracker; - int success_count = 0; - write_test_files_async(kLocalTestPath, kTotalFiles, &success_count, &tracker); - usleep(100); - - std::string kLocalRenamedTestPath = "rename_dir." + std::to_string(dsn_now_ms()); - // Rename succeed but some files write failed. - ASSERT_TRUE(dsn::utils::filesystem::rename_path(kLocalTestPath, kLocalRenamedTestPath)); - tracker.cancel_outstanding_tasks(); - // Generally, we can assume partial files are written failed. - // It maybe flaky, please retry if it failed. - ASSERT_GT(success_count, 0) << success_count; - ASSERT_LT(success_count, kTotalFiles) << success_count; + // The test is flaky, retry if it failed in 300 seconds. + ASSERT_IN_TIME_WITH_FIXED_INTERVAL( + [&] { + task_tracker tracker; + int success_count = 0; + write_test_files_async(kLocalTestPath, kTotalFiles, &success_count, &tracker); + usleep(100); + + std::string kLocalRenamedTestPath = "rename_dir." + std::to_string(dsn_now_ms()); + // Rename succeed but some files write failed. + ASSERT_TRUE(dsn::utils::filesystem::rename_path(kLocalTestPath, kLocalRenamedTestPath)); + tracker.cancel_outstanding_tasks(); + // Generally, we can assume partial files are written failed. + ASSERT_GT(success_count, 0) << success_count; + ASSERT_LT(success_count, kTotalFiles) << success_count; + }, + 300); } diff --git a/src/test_util/test_util.h b/src/test_util/test_util.h index 762d154367..debeb7fa90 100644 --- a/src/test_util/test_util.h +++ b/src/test_util/test_util.h @@ -59,7 +59,7 @@ void create_local_test_file(const std::string &full_name, dsn::replication::file #define ASSERT_IN_TIME_WITH_FIXED_INTERVAL(expr, sec) \ do { \ - AssertEventually(expr, sec, WaitBackoff::NONE); \ + AssertEventually(expr, sec, ::pegasus::WaitBackoff::NONE); \ NO_PENDING_FATALS(); \ } while (0) From 0ccf1108f71252816d16fd23523adf197d00dd83 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Thu, 12 Oct 2023 10:52:43 +0800 Subject: [PATCH 39/42] fix(jemalloc): enlarge the buffer size to dump jemalloc stats (#1636) https://github.com/apache/incubator-pegasus/issues/1635 - Enlarge the buffer size to dump jemalloc stats - Improve the jemalloc unit tests --- src/utils/je_ctl.cpp | 2 +- src/utils/test/je_ctl_test.cpp | 33 +++++++++++++++++---------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/utils/je_ctl.cpp b/src/utils/je_ctl.cpp index fa6a7168d4..38f5bb1be0 100644 --- a/src/utils/je_ctl.cpp +++ b/src/utils/je_ctl.cpp @@ -75,7 +75,7 @@ const char *je_stats_type_to_opts(je_stats_type type) size_t je_stats_type_to_default_buf_sz(je_stats_type type) { static const size_t buf_sz_map[] = { - 2 * 1024, 4 * 1024, 1024 * 1024, 2 * 1024 * 1024, + 2 * 1024, 4 * 1024, 8 * 1024 * 1024, 8 * 1024 * 1024, }; RETURN_ARRAY_ELEM_BY_ENUM_TYPE(type, buf_sz_map); diff --git a/src/utils/test/je_ctl_test.cpp b/src/utils/test/je_ctl_test.cpp index 06e64be17c..b0e0db576e 100644 --- a/src/utils/test/je_ctl_test.cpp +++ b/src/utils/test/je_ctl_test.cpp @@ -17,10 +17,11 @@ #ifdef DSN_USE_JEMALLOC -#include "utils/je_ctl.h" - #include +#include "utils/je_ctl.h" +#include "utils/test_macros.h" + namespace dsn { namespace { @@ -53,10 +54,10 @@ void check_configs_marks(const std::string &stats) void check_arena_marks(const std::string &stats) { // Marks for merged arenas. - ASSERT_NE(stats.find("Merged arenas stats:"), std::string::npos); + ASSERT_NE(stats.find("Merged arenas stats:"), std::string::npos) << stats; // Marks for each arena. - ASSERT_NE(stats.find("arenas[0]:"), std::string::npos); + ASSERT_NE(stats.find("arenas[0]:"), std::string::npos) << stats; } } // anonymous namespace @@ -64,41 +65,41 @@ void check_arena_marks(const std::string &stats) TEST(je_ctl_test, dump_summary_stats) { std::string stats; - je_dump_stats(je_stats_type::SUMMARY_STATS, stats); + NO_FATALS(je_dump_stats(je_stats_type::SUMMARY_STATS, stats)); - check_base_stats_marks_with_end(stats); + NO_FATALS(check_base_stats_marks_with_end(stats)); } TEST(je_ctl_test, dump_configs) { std::string stats; - je_dump_stats(je_stats_type::CONFIGS, stats); + NO_FATALS(je_dump_stats(je_stats_type::CONFIGS, stats)); - check_base_stats_marks_with_end(stats); - check_configs_marks(stats); + NO_FATALS(check_base_stats_marks_with_end(stats)); + NO_FATALS(check_configs_marks(stats)); } TEST(je_ctl_test, dump_brief_arena_stats) { std::string stats; - je_dump_stats(je_stats_type::BRIEF_ARENA_STATS, stats); + NO_FATALS(je_dump_stats(je_stats_type::BRIEF_ARENA_STATS, stats)); // Since there may be many arenas, "End" mark is not required to be checked here. - check_base_stats_marks(stats); - check_arena_marks(stats); + NO_FATALS(check_base_stats_marks(stats)); + NO_FATALS(check_arena_marks(stats)); } TEST(je_ctl_test, dump_detailed_stats) { std::string stats; - je_dump_stats(je_stats_type::DETAILED_STATS, stats); + NO_FATALS(je_dump_stats(je_stats_type::DETAILED_STATS, stats)); // Since there may be many arenas, "End" mark is not required to be checked here. - check_base_stats_marks(stats); + NO_FATALS(check_base_stats_marks(stats)); // Detailed stats will contain all information, therefore everything should be checked. - check_configs_marks(stats); - check_arena_marks(stats); + NO_FATALS(check_configs_marks(stats)); + NO_FATALS(check_arena_marks(stats)); ASSERT_NE(stats.find("bins:"), std::string::npos); ASSERT_NE(stats.find("extents:"), std::string::npos); } From 6064d7085f47fed4ca7d5bce0610153974d0092f Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Thu, 12 Oct 2023 11:14:53 +0800 Subject: [PATCH 40/42] build(thirdparty): bump libcurl from 7.47.0 to 8.4.0 (#1633) #1604 In pegasus, both prometheus-cpp and http client are based on libcurl. Above all, an important version 8.4.0 for libcurl, which has fixed severity HIGH security problem(curl/curl#12026), would be released in Oct 11. Therefore, we should bump libcurl from 7.47.0 to 8.4.0 --- thirdparty/CMakeLists.txt | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index f6fd11d4ba..f005f321aa 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -297,17 +297,20 @@ set(CURL_OPTIONS --without-libssh2 --without-ssl --without-libidn + --without-zstd ) if (APPLE) set(CURL_OPTIONS - ${CURL_OPTIONS} - --without-nghttp2 - ) + ${CURL_OPTIONS} + --without-nghttp2 + --without-libidn2 + --without-brotli + ) endif () ExternalProject_Add(curl - URL ${OSS_URL_PREFIX}/curl-7.47.0.tar.gz - http://curl.haxx.se/download/curl-7.47.0.tar.gz - URL_MD5 5109d1232d208dfd712c0272b8360393 + URL ${OSS_URL_PREFIX}/curl-8.4.0.tar.gz + http://curl.haxx.se/download/curl-8.4.0.tar.gz + URL_MD5 533e8a3b1228d5945a6a512537bea4c7 CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} ${CURL_OPTIONS} BUILD_IN_SOURCE 1 From ed3d31ef7f7a85d4220abebd65c637788b49b94e Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Thu, 12 Oct 2023 11:17:50 +0800 Subject: [PATCH 41/42] refactor(filesystem): refactor file_utils unit tests (#1634) https://github.com/apache/incubator-pegasus/issues/887 - Remove the useless return value of `get_normalized_path()` - Simplify the code of file_utils.cpp --- src/utils/filesystem.cpp | 102 +-- src/utils/filesystem.h | 2 +- src/utils/test/file_utils.cpp | 1132 +++++++++------------------------ 3 files changed, 320 insertions(+), 916 deletions(-) diff --git a/src/utils/filesystem.cpp b/src/utils/filesystem.cpp index a252d823b6..211aefbecb 100644 --- a/src/utils/filesystem.cpp +++ b/src/utils/filesystem.cpp @@ -94,8 +94,7 @@ static inline int get_stat_internal(const std::string &npath, struct stat_ &st) return err; } -// TODO(yingchun): remove the return value because it's always 0. -int get_normalized_path(const std::string &path, std::string &npath) +void get_normalized_path(const std::string &path, std::string &npath) { char sep; size_t i; @@ -105,7 +104,7 @@ int get_normalized_path(const std::string &path, std::string &npath) if (path.empty()) { npath = ""; - return 0; + return; } len = path.length(); @@ -131,8 +130,6 @@ int get_normalized_path(const std::string &path, std::string &npath) CHECK_NE_MSG(tls_path_buffer[0], _FS_NULL, "Normalized path cannot be empty!"); npath = tls_path_buffer; - - return 0; } static __thread struct @@ -210,44 +207,31 @@ bool path_exists(const std::string &path) return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + get_normalized_path(path, npath); return dsn::utils::filesystem::path_exists_internal(npath, FTW_NS); } bool directory_exists(const std::string &path) { - std::string npath; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); return dsn::utils::filesystem::path_exists_internal(npath, FTW_D); } bool file_exists(const std::string &path) { - std::string npath; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); return dsn::utils::filesystem::path_exists_internal(npath, FTW_F); } @@ -257,23 +241,18 @@ static bool get_subpaths(const std::string &path, bool recursive, int typeflags) { - std::string npath; - bool ret; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); if (!dsn::utils::filesystem::path_exists_internal(npath, FTW_D)) { return false; } + bool ret; switch (typeflags) { case FTW_F: ret = dsn::utils::filesystem::file_tree_walk( @@ -351,17 +330,12 @@ static bool remove_directory(const std::string &npath) bool remove_path(const std::string &path) { - std::string npath; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); if (dsn::utils::filesystem::path_exists_internal(npath, FTW_F)) { bool ret = (::remove(npath.c_str()) == 0); @@ -391,20 +365,15 @@ bool rename_path(const std::string &path1, const std::string &path2) bool deprecated_file_size(const std::string &path, int64_t &sz) { - struct stat_ st; - std::string npath; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); - err = dsn::utils::filesystem::get_stat_internal(npath, st); + struct stat_ st; + int err = dsn::utils::filesystem::get_stat_internal(npath, st); if (err != 0) { return false; } @@ -458,18 +427,14 @@ bool create_directory(const std::string &path) std::string npath; std::string cpath; size_t len; - int err; if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + get_normalized_path(path, npath); - err = dsn::utils::filesystem::create_directory_component(npath); + int err = dsn::utils::filesystem::create_directory_component(npath); if (err == 0) { return true; } else if (err != ENOENT) { @@ -514,10 +479,7 @@ bool create_directory(const std::string &path) bool create_file(const std::string &path) { std::string npath; - int err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + get_normalized_path(path, npath); auto pos = npath.find_last_of("\\/"); if ((pos != std::string::npos) && (pos > 0)) { @@ -599,23 +561,20 @@ std::string get_file_name(const std::string &path) std::string path_combine(const std::string &path1, const std::string &path2) { - int err; - std::string path3; std::string npath; - if (path1.empty()) { - err = dsn::utils::filesystem::get_normalized_path(path2, npath); + get_normalized_path(path2, npath); } else if (path2.empty()) { - err = dsn::utils::filesystem::get_normalized_path(path1, npath); + get_normalized_path(path1, npath); } else { - path3 = path1; + std::string path3 = path1; path3.append(1, _FS_SLASH); path3.append(path2); - err = dsn::utils::filesystem::get_normalized_path(path3, npath); + get_normalized_path(path3, npath); } - return ((err == 0) ? npath : ""); + return npath; } bool get_current_directory(std::string &path) @@ -632,20 +591,15 @@ bool get_current_directory(std::string &path) bool last_write_time(const std::string &path, time_t &tm) { - struct stat_ st; - std::string npath; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); - err = dsn::utils::filesystem::get_stat_internal(npath, st); + struct stat_ st; + int err = dsn::utils::filesystem::get_stat_internal(npath, st); if (err != 0) { return false; } diff --git a/src/utils/filesystem.h b/src/utils/filesystem.h index be62e76831..2142d1be1a 100644 --- a/src/utils/filesystem.h +++ b/src/utils/filesystem.h @@ -67,7 +67,7 @@ namespace filesystem { // TODO(yingchun): Consider using rocksdb APIs to rewrite the following functions. -int get_normalized_path(const std::string &path, std::string &npath); +void get_normalized_path(const std::string &path, std::string &npath); bool get_absolute_path(const std::string &path1, std::string &path2); diff --git a/src/utils/test/file_utils.cpp b/src/utils/test/file_utils.cpp index 0cc2173e8e..c4cadeb959 100644 --- a/src/utils/test/file_utils.cpp +++ b/src/utils/test/file_utils.cpp @@ -25,877 +25,327 @@ */ // IWYU pragma: no_include +// IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include #include #include #include #include +#include "test_util/test_util.h" #include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" -static void file_utils_test_setup() +class file_utils : public pegasus::encrypt_data_test_base { - std::string path; - bool ret; - - path = "./file_utils_temp.txt"; - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_FALSE(ret); - - path = "./file_utils_temp"; - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_FALSE(ret); -} +public: + void file_utils_test_setup() + { + std::string path = "./file_utils_temp.txt"; + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + ASSERT_FALSE(dsn::utils::filesystem::file_exists(path)); + + path = "./file_utils_temp"; + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + ASSERT_FALSE(dsn::utils::filesystem::directory_exists(path)); + } -static void file_utils_test_get_process_image_path() -{ - std::string path; - std::string imagepath; - dsn::error_code ret; - // int pid; + void file_utils_test_get_process_image_path() + { + std::string imagepath; + ASSERT_TRUE(dsn::utils::filesystem::get_current_directory(imagepath)); + imagepath = dsn::utils::filesystem::path_combine(imagepath, "dsn_utils_tests"); - if (!dsn::utils::filesystem::get_current_directory(imagepath)) { - EXPECT_TRUE(false); + std::string path; + ASSERT_EQ(dsn::ERR_OK, dsn::utils::filesystem::get_current_process_image_path(path)); } - imagepath = dsn::utils::filesystem::path_combine(imagepath, "dsn_utils_tests"); - ret = dsn::utils::filesystem::get_current_process_image_path(path); - EXPECT_TRUE(ret == dsn::ERR_OK); - // TODO: not always true when running dir is not where the test resides - // EXPECT_TRUE(path == imagepath); // e: vs E: -} + void file_utils_test_get_normalized_path() + { + struct same_normalized_paths + { + std::string path; + } same_normalized_path_tests[] = {{"\\\\?\\"}, + {"c:\\"}, + {"c:"}, + {"\\\\?\\c:\\"}, + {"\\\\?\\c:"}, + {"c:\\a"}, + {"c:\\\\a"}, + {"c:\\\\a\\"}, + {"c:\\\\a\\\\"}, + {"\\\\?\\c:\\a"}, + {"\\\\?\\c:\\\\a"}, + {"\\\\?\\c:\\\\a\\"}, + {"\\\\?\\c:\\\\a\\\\"}, + {"\\"}, + {"\\\\"}, + {"\\\\\\"}, + {"\\\\a"}, + {"\\\\\\a"}, + {"\\\\a\\"}, + {"\\\\\\a\\"}, + {"\\\\\\a\\\\"}, + {"/"}, + {"c:/a"}, + {"."}, + {"./a"}, + {"./a/b"}, + {".."}, + {"../a"}, + {"../a/b"}}; + for (const auto &test : same_normalized_path_tests) { + std::string npath; + dsn::utils::filesystem::get_normalized_path(test.path, npath); + ASSERT_EQ(test.path, npath); + } + + struct normalized_paths + { + std::string path; + std::string normalized_path; + } normalized_path_tests[] = { + {"//", "/"}, + {"//?/", "/?"}, + {"//a", "/a"}, + {"//a/", "/a"}, + {"//a//", "/a"}, + {"c:/", "c:"}, + {"c://", "c:"}, + {"c:/a/", "c:/a"}, + {"c://a/", "c:/a"}, + {"c://a//", "c:/a"}, + {"/////////////////////////////////////////////////////////////////", "/"}, + {"/////////////////////////////////////////////////////////////////a/////////////////" + "b///" + "////////", + "/a/b"}, + {"./", "."}, + {".//a", "./a"}, + {"./a/", "./a"}, + {"./a/b/", "./a/b"}, + {".///a////b///", "./a/b"}, + {"../", ".."}, + {"..//a", "../a"}, + {"../a/", "../a"}, + {"../a/b/", "../a/b"}, + {"..///a////b///", "../a/b"}}; + for (const auto &test : normalized_path_tests) { + std::string npath; + dsn::utils::filesystem::get_normalized_path(test.path, npath); + ASSERT_EQ(test.normalized_path, npath) << test.path; + } + } -static void file_utils_test_get_normalized_path() -{ - int ret; - std::string path; - std::string npath; - - path = "\\\\?\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "c:\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "c:"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\?\\c:\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\?\\c:"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "c:\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "c:\\\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "c:\\\\a\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "c:\\\\a\\\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\?\\c:\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\?\\c:\\\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\?\\c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\?\\c:\\\\a\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\?\\c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\?\\c:\\\\a\\\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\?\\c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\a\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\\\a\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\\\a\\\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "//"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\"); -#else - EXPECT_TRUE(npath == "/"); -#endif - - path = "//?/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\?\\"); -#else - EXPECT_TRUE(npath == "/?"); -#endif - - path = "//a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == "/a"); -#endif - - path = "//a/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == "/a"); -#endif - - path = "//a//"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == "/a"); -#endif - - path = "c:/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\"); -#else - EXPECT_TRUE(npath == "c:"); -#endif - - path = "c://"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\"); -#else - EXPECT_TRUE(npath == "c:"); -#endif - - path = "c:/a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "c:/a/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == "c:/a"); -#endif - - path = "c://a/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == "c:/a"); -#endif - - path = "c://a//"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == "c:/a"); -#endif - - path = "/////////////////////////////////////////////////////////////////"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\"); -#else - EXPECT_TRUE(npath == "/"); -#endif - - path = "/////////////////////////////////////////////////////////////////a/////////////////b///" - "////////"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a\\b"); -#else - EXPECT_TRUE(npath == "/a/b"); -#endif - - path = "."; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "./"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == "."); - - path = "./a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = ".//a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a"); -#else - EXPECT_TRUE(npath == "./a"); -#endif - - path = "./a/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a"); -#else - EXPECT_TRUE(npath == "./a"); -#endif - - path = "./a/b"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a\\b"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "./a/b/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a\\b"); -#else - EXPECT_TRUE(npath == "./a/b"); -#endif - - path = ".///a////b///"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a\\b"); -#else - EXPECT_TRUE(npath == "./a/b"); -#endif - - path = ".."; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "../"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == ".."); - - path = "../a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "..//a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a"); -#else - EXPECT_TRUE(npath == "../a"); -#endif - - path = "../a/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a"); -#else - EXPECT_TRUE(npath == "../a"); -#endif - - path = "../a/b"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a\\b"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "../a/b/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a\\b"); -#else - EXPECT_TRUE(npath == "../a/b"); -#endif - - path = "..///a////b///"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a\\b"); -#else - EXPECT_TRUE(npath == "../a/b"); -#endif -} + void file_utils_test_get_current_directory() + { + std::string path; + ASSERT_TRUE(dsn::utils::filesystem::get_current_directory(path)); + ASSERT_TRUE(!path.empty()); + } -static void file_utils_test_get_current_directory() -{ - std::string path; - bool ret; + void file_utils_test_path_combine() + { + struct combine_paths + { + std::string path1; + std::string path2; + std::string combined_path; + } tests[] = {{"", "", ""}, + {"c:", "Windows\\explorer.exe", "c:/Windows\\explorer.exe"}, + {"c:", "\\Windows\\explorer.exe", "c:/Windows\\explorer.exe"}, + {"c:\\", "\\Windows\\explorer.exe", "c:\\/Windows\\explorer.exe"}, + {"/bin", "ls", "/bin/ls"}, + {"/bin/", "ls", "/bin/ls"}, + {"/bin", "/ls", "/bin/ls"}, + {"/bin/", "/ls", "/bin/ls"}}; + for (const auto &test : tests) { + std::string path = dsn::utils::filesystem::path_combine(test.path1, test.path2); + ASSERT_EQ(test.combined_path, path) << test.path1 << " + " << test.path2; + } + } - path = ""; - ret = dsn::utils::filesystem::get_current_directory(path); - EXPECT_TRUE(ret); - EXPECT_TRUE(!path.empty()); -} + void file_utils_test_get_file_name() + { + struct combine_paths + { + std::string path; + std::string file_name; + } tests[] = {{"", ""}, + {"c:", "c:"}, + {"c:\\", ""}, + {"c:1.txt", "c:1.txt"}, + {"c:\\1.txt", "1.txt"}, + {"c:\\Windows\\1.txt", "1.txt"}, + {"/bin/", ""}, + {"/bin/ls", "ls"}}; + for (const auto &test : tests) { + std::string file_name = dsn::utils::filesystem::get_file_name(test.path); + ASSERT_EQ(test.file_name, file_name) << test.path; + } + } -static void file_utils_test_path_combine() -{ - std::string path; - std::string path1; - std::string path2; - - path1 = ""; - path2 = ""; - path = dsn::utils::filesystem::path_combine(path1, path2); - EXPECT_TRUE(path == ""); - - path1 = "c:"; - path2 = "Windows\\explorer.exe"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "c:Windows\\explorer.exe"); -#else - EXPECT_TRUE(path == "c:/Windows\\explorer.exe"); -#endif - - path1 = "c:"; - path2 = "\\Windows\\explorer.exe"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "c:\\Windows\\explorer.exe"); -#else - EXPECT_TRUE(path == "c:/Windows\\explorer.exe"); -#endif - - path1 = "c:\\"; - path2 = "\\Windows\\explorer.exe"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "c:\\Windows\\explorer.exe"); -#else - EXPECT_TRUE(path == "c:\\/Windows\\explorer.exe"); -#endif - - path1 = "/bin"; - path2 = "ls"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "\\bin\\ls"); -#else - EXPECT_TRUE(path == "/bin/ls"); -#endif - - path1 = "/bin/"; - path2 = "ls"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "\\bin\\ls"); -#else - EXPECT_TRUE(path == "/bin/ls"); -#endif - - path1 = "/bin"; - path2 = "/ls"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "\\bin\\ls"); -#else - EXPECT_TRUE(path == "/bin/ls"); -#endif - - path1 = "/bin/"; - path2 = "/ls"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "\\bin\\ls"); -#else - EXPECT_TRUE(path == "/bin/ls"); -#endif -} + void file_utils_test_create() + { + std::string path = "./file_utils_temp.txt"; + ASSERT_TRUE(dsn::utils::filesystem::create_file(path)); + ASSERT_TRUE(dsn::utils::filesystem::file_exists(path)); + + time_t current_time = ::time(nullptr); + ASSERT_NE(current_time, 1); + + auto s = rocksdb::WriteStringToFile( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice("Hello world!"), + path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + time_t last_write_time; + ASSERT_TRUE(dsn::utils::filesystem::last_write_time(path, last_write_time)); + ASSERT_NE(last_write_time, -1); + ASSERT_GE(last_write_time, current_time); + + path = "./file_utils_temp"; + ASSERT_TRUE(dsn::utils::filesystem::create_directory(path)); + ASSERT_TRUE(dsn::utils::filesystem::directory_exists(path)); + + path = "./file_utils_temp/a/b/c/d//"; + ASSERT_TRUE(dsn::utils::filesystem::create_directory(path)); + ASSERT_TRUE(dsn::utils::filesystem::directory_exists(path)); + + struct create_files + { + std::string filename; + } tests[] = {{"./file_utils_temp/a/1.txt"}, + {"./file_utils_temp/a/2.txt"}, + {"./file_utils_temp/b/c/d/1.txt"}}; + for (const auto &test : tests) { + ASSERT_TRUE(dsn::utils::filesystem::create_file(test.filename)) << test.filename; + ASSERT_TRUE(dsn::utils::filesystem::file_exists(test.filename)) << test.filename; + } + } -static void file_utils_test_get_file_name() -{ - std::string path1; - std::string path2; - - path1 = ""; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == ""); - - path1 = "c:"; - path2 = dsn::utils::filesystem::get_file_name(path1); -#ifdef _WIN32 - EXPECT_TRUE(path2 == ""); -#else - EXPECT_TRUE(path2 == "c:"); -#endif - - path1 = "c:\\"; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == ""); - - path1 = "c:1.txt"; - path2 = dsn::utils::filesystem::get_file_name(path1); -#ifdef _WIN32 - EXPECT_TRUE(path2 == "1.txt"); -#else - EXPECT_TRUE(path2 == "c:1.txt"); -#endif - - path1 = "c:\\1.txt"; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == "1.txt"); - - path1 = "c:\\Windows\\1.txt"; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == "1.txt"); - - path1 = "/bin/"; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == ""); - - path1 = "/bin/ls"; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == "ls"); -} + void file_utils_test_file_size() + { + std::string path = "./file_utils_temp.txt"; + int64_t sz; + ASSERT_TRUE( + dsn::utils::filesystem::file_size(path, dsn::utils::FileDataType::kNonSensitive, sz)); + ASSERT_EQ(12, sz); + + path = "./file_utils_temp2.txt"; + ASSERT_FALSE( + dsn::utils::filesystem::file_size(path, dsn::utils::FileDataType::kNonSensitive, sz)); + } -static void file_utils_test_create() -{ - std::string path; - bool ret; - - path = "./file_utils_temp.txt"; - ret = dsn::utils::filesystem::create_file(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_TRUE(ret); - - time_t current_time = ::time(nullptr); - EXPECT_TRUE(current_time != 1); - - std::ofstream myfile(path.c_str(), std::ios::out | std::ios::app | std::ios::binary); - EXPECT_TRUE(myfile.is_open()); - myfile << "Hello world!"; - myfile.close(); - - time_t last_write_time; - ret = dsn::utils::filesystem::last_write_time(path, last_write_time); - EXPECT_TRUE(ret); - EXPECT_TRUE((last_write_time != -1) && (last_write_time >= current_time)); - - path = "./file_utils_temp"; - ret = dsn::utils::filesystem::create_directory(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/a/b/c/d//"; - ret = dsn::utils::filesystem::create_directory(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/a/1.txt"; - ret = dsn::utils::filesystem::create_file(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/a/1.txt"; - ret = dsn::utils::filesystem::create_file(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/a/2.txt"; - ret = dsn::utils::filesystem::create_file(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/b/c/d/1.txt"; - ret = dsn::utils::filesystem::create_file(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_TRUE(ret); -} + void file_utils_test_path_exists() + { + std::string path = "/"; + ASSERT_TRUE(dsn::utils::filesystem::path_exists(path)); + ASSERT_TRUE(dsn::utils::filesystem::directory_exists(path)); + ASSERT_FALSE(dsn::utils::filesystem::file_exists(path)); -static void file_utils_test_file_size() -{ - std::string path; - int64_t sz; - bool ret; - - path = "./file_utils_temp.txt"; - ret = dsn::utils::filesystem::file_size(path, dsn::utils::FileDataType::kNonSensitive, sz); - ASSERT_TRUE(ret); - ASSERT_EQ(12, sz); - - path = "./file_utils_temp2.txt"; - ret = dsn::utils::filesystem::file_size(path, dsn::utils::FileDataType::kNonSensitive, sz); - EXPECT_FALSE(ret); -} + path = "./not_exists_not_exists"; + ASSERT_FALSE(dsn::utils::filesystem::path_exists(path)); -static void file_utils_test_path_exists() -{ - std::string path; - bool ret; - - path = "c:\\"; - ret = dsn::utils::filesystem::path_exists(path); -#ifdef _WIN32 - EXPECT_TRUE(ret); -#else - EXPECT_FALSE(ret); -#endif - - path = "c:\\"; - ret = dsn::utils::filesystem::directory_exists(path); -#ifdef _WIN32 - EXPECT_TRUE(ret); -#else - EXPECT_FALSE(ret); -#endif - - path = "c:\\"; - ret = dsn::utils::filesystem::file_exists(path); -#ifdef _WIN32 - EXPECT_FALSE(ret); -#else - EXPECT_FALSE(ret); -#endif - - path = "/"; - ret = dsn::utils::filesystem::path_exists(path); - EXPECT_TRUE(ret); - - path = "/"; - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_TRUE(ret); - - path = "/"; - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_FALSE(ret); - - path = "./not_exists_not_exists"; - ret = dsn::utils::filesystem::path_exists(path); - EXPECT_FALSE(ret); - - path = "c:\\Windows\\System32\\notepad.exe"; - ret = dsn::utils::filesystem::path_exists(path); -#ifdef _WIN32 - EXPECT_TRUE(ret); -#else - EXPECT_FALSE(ret); -#endif - - path = "c:\\Windows\\System32\\notepad.exe"; - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_FALSE(ret); - - path = "c:\\Windows\\System32\\notepad.exe"; - ret = dsn::utils::filesystem::file_exists(path); -#ifdef _WIN32 - EXPECT_TRUE(ret); -#else - EXPECT_FALSE(ret); -#endif - - path = "/bin/ls"; - ret = dsn::utils::filesystem::path_exists(path); -#ifdef _WIN32 - EXPECT_FALSE(ret); -#else - EXPECT_TRUE(ret); -#endif - - path = "/bin/ls"; - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_FALSE(ret); - - path = "/bin/ls"; - ret = dsn::utils::filesystem::file_exists(path); -#ifdef _WIN32 - EXPECT_FALSE(ret); -#else - EXPECT_TRUE(ret); -#endif -} + path = "/bin/ls"; + ASSERT_TRUE(dsn::utils::filesystem::path_exists(path)); + ASSERT_FALSE(dsn::utils::filesystem::directory_exists(path)); + ASSERT_TRUE(dsn::utils::filesystem::file_exists(path)); + } -static void file_utils_test_get_paths() -{ - std::string path; - bool ret; - std::vector file_list; - - path = "."; - ret = dsn::utils::filesystem::get_subfiles(path, file_list, false); - EXPECT_TRUE(ret); -#ifdef _WIN32 - EXPECT_TRUE(file_list.size() >= 3); -#else - EXPECT_TRUE(file_list.size() >= 2); -#endif - file_list.clear(); - - path = "."; - ret = dsn::utils::filesystem::get_subfiles(path, file_list, true); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() >= 3); - file_list.clear(); - - path = "../../"; - ret = dsn::utils::filesystem::get_subfiles(path, file_list, true); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() >= 3); - file_list.clear(); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::get_subfiles(path, file_list, true); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 3); - file_list.clear(); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::get_subdirectories(path, file_list, true); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 7); - file_list.clear(); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::get_subdirectories(path, file_list, false); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 2); - file_list.clear(); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::get_subpaths(path, file_list, true); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 10); - file_list.clear(); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::get_subpaths(path, file_list, false); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 2); - file_list.clear(); - - path = "./file_utils_temp/a/"; - ret = dsn::utils::filesystem::get_subfiles(path, file_list, false); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 2); - file_list.clear(); - - path = "./file_utils_temp/a/"; - ret = dsn::utils::filesystem::get_subpaths(path, file_list, false); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 3); - file_list.clear(); -} + void file_utils_test_get_paths() + { + std::string path = "."; + std::vector file_list; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(path, file_list, false)); + ASSERT_GE(file_list.size(), 2); + file_list.clear(); + + path = "."; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(path, file_list, true)); + ASSERT_GE(file_list.size(), 3); + file_list.clear(); + + path = "../../"; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(path, file_list, true)); + ASSERT_GE(file_list.size(), 3); + file_list.clear(); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(path, file_list, true)); + ASSERT_EQ(file_list.size(), 3); + file_list.clear(); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subdirectories(path, file_list, true)); + ASSERT_EQ(file_list.size(), 7); + file_list.clear(); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subdirectories(path, file_list, false)); + ASSERT_EQ(file_list.size(), 2); + file_list.clear(); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subpaths(path, file_list, true)); + ASSERT_EQ(file_list.size(), 10); + file_list.clear(); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subpaths(path, file_list, false)); + ASSERT_EQ(file_list.size(), 2); + file_list.clear(); + + path = "./file_utils_temp/a/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(path, file_list, false)); + ASSERT_EQ(file_list.size(), 2); + file_list.clear(); + + path = "./file_utils_temp/a/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subpaths(path, file_list, false)); + ASSERT_EQ(file_list.size(), 3); + file_list.clear(); + } -static void file_utils_test_rename() -{ - std::string path; - std::string path2; - bool ret; - - path = "./file_utils_temp/b/c/d/1.txt"; - path2 = "./file_utils_temp/b/c/d/2.txt"; - ret = dsn::utils::filesystem::rename_path(path, path2); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_FALSE(ret); - ret = dsn::utils::filesystem::file_exists(path2); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::rename_path(path, path2); - EXPECT_FALSE(ret); -} + void file_utils_test_rename() + { + std::string path = "./file_utils_temp/b/c/d/1.txt"; + std::string path2 = "./file_utils_temp/b/c/d/2.txt"; + ASSERT_TRUE(dsn::utils::filesystem::rename_path(path, path2)); + ASSERT_FALSE(dsn::utils::filesystem::file_exists(path)); + ASSERT_TRUE(dsn::utils::filesystem::file_exists(path2)); + ASSERT_FALSE(dsn::utils::filesystem::rename_path(path, path2)); + } -static void file_utils_test_remove() -{ - std::string path; - std::vector file_list; - bool ret; - - path = "./file_utils_temp.txt"; - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_FALSE(ret); - - path = "./file_utils_temp/a/2.txt"; - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_FALSE(ret); -} + void file_utils_test_remove() + { + std::string path = "./file_utils_temp.txt"; + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + ASSERT_FALSE(dsn::utils::filesystem::file_exists(path)); + + path = "./file_utils_temp/a/2.txt"; + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + ASSERT_FALSE(dsn::utils::filesystem::directory_exists(path)); + } + + void file_utils_test_cleanup() {} +}; -static void file_utils_test_cleanup() {} +INSTANTIATE_TEST_CASE_P(, file_utils, ::testing::Values(false)); -TEST(core, file_utils) +TEST_P(file_utils, basic) { file_utils_test_setup(); file_utils_test_get_process_image_path(); From 0f92716dd3bfaec85009ffad1fe3efa541e620b5 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 16 Oct 2023 12:30:37 +0800 Subject: [PATCH 42/42] feat(aio): use rocksdb APIs to re-implement the aio module (#1637) https://github.com/apache/incubator-pegasus/issues/1575 This is a dependent work to implement encryption at rest, we can use the capacity of rocksdb encryption after this patch. - Use rocksdb APIs to implement class `native_linux_aio_provider`. Both of the implementations are using `pread()` and `pwrite()` system calls, so there isn't significant performance changes, see the newly added simple benchmark performance comparation below. - Separate the file read and write operations for class `aio_provider` --- src/aio/CMakeLists.txt | 2 +- src/aio/aio_provider.h | 16 ++- src/aio/disk_engine.cpp | 7 +- src/aio/disk_engine.h | 11 +- src/aio/file_io.cpp | 48 +++++-- src/aio/file_io.h | 10 +- src/aio/native_linux_aio_provider.cpp | 132 ++++++++++-------- src/aio/native_linux_aio_provider.h | 19 ++- src/aio/test/CMakeLists.txt | 2 +- src/aio/test/aio.cpp | 75 ++++++---- src/aio/test/config.ini | 5 + src/meta/meta_state_service_simple.cpp | 4 +- src/nfs/nfs_client_impl.cpp | 9 +- src/nfs/nfs_server_impl.cpp | 4 +- .../test/load_from_private_log_test.cpp | 5 +- src/replica/log_file.cpp | 8 +- .../simple_kv/simple_kv.server.impl.cpp | 4 +- .../simple_kv/test/simple_kv.server.impl.cpp | 4 +- src/replica/test/mutation_log_test.cpp | 3 +- src/runtime/test/CMakeLists.txt | 1 + src/runtime/test/task_test.cpp | 4 +- src/test_util/test_util.cpp | 1 - src/test_util/test_util.h | 24 +++- .../long_adder_bench/long_adder_bench.cpp | 23 +-- src/utils/test/env.cpp | 2 +- 25 files changed, 265 insertions(+), 158 deletions(-) diff --git a/src/aio/CMakeLists.txt b/src/aio/CMakeLists.txt index 45d24cf687..3754361d06 100644 --- a/src/aio/CMakeLists.txt +++ b/src/aio/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime) +set(MY_PROJ_LIBS dsn_runtime rocksdb) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/aio/aio_provider.h b/src/aio/aio_provider.h index 73848d87be..74fb410bd7 100644 --- a/src/aio/aio_provider.h +++ b/src/aio/aio_provider.h @@ -27,10 +27,17 @@ #pragma once #include +#include +#include #include "utils/error_code.h" #include "utils/factory_store.h" +namespace rocksdb { +class RandomAccessFile; +class RandomRWFile; +} // namespace rocksdb + namespace dsn { class aio_context; @@ -60,12 +67,13 @@ class aio_provider explicit aio_provider(disk_engine *disk); virtual ~aio_provider() = default; - virtual linux_fd_t open(const char *file_name, int flag, int pmode) = 0; + virtual std::unique_ptr open_read_file(const std::string &fname) = 0; + virtual error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0; - virtual error_code close(linux_fd_t fd) = 0; - virtual error_code flush(linux_fd_t fd) = 0; + virtual std::unique_ptr open_write_file(const std::string &fname) = 0; virtual error_code write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0; - virtual error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0; + virtual error_code flush(rocksdb::RandomRWFile *rwf) = 0; + virtual error_code close(rocksdb::RandomRWFile *rwf) = 0; // Submits the aio_task to the underlying disk-io executor. // This task may not be executed immediately, call `aio_task::wait` diff --git a/src/aio/disk_engine.cpp b/src/aio/disk_engine.cpp index 0e25bfaac6..1b104be301 100644 --- a/src/aio/disk_engine.cpp +++ b/src/aio/disk_engine.cpp @@ -102,22 +102,26 @@ aio_task *disk_write_queue::unlink_next_workload(void *plength) return first; } -disk_file::disk_file(linux_fd_t fd) : _fd(fd) {} +disk_file::disk_file(std::unique_ptr rf) : _read_file(std::move(rf)) {} +disk_file::disk_file(std::unique_ptr wf) : _write_file(std::move(wf)) {} aio_task *disk_file::read(aio_task *tsk) { + CHECK(_read_file, ""); tsk->add_ref(); // release on completion, see `on_read_completed`. return _read_queue.add_work(tsk, nullptr); } aio_task *disk_file::write(aio_task *tsk, void *ctx) { + CHECK(_write_file, ""); tsk->add_ref(); // release on completion return _write_queue.add_work(tsk, ctx); } aio_task *disk_file::on_read_completed(aio_task *wk, error_code err, size_t size) { + CHECK(_read_file, ""); CHECK(wk->next == nullptr, ""); auto ret = _read_queue.on_work_completed(wk, nullptr); wk->enqueue(err, size); @@ -128,6 +132,7 @@ aio_task *disk_file::on_read_completed(aio_task *wk, error_code err, size_t size aio_task *disk_file::on_write_completed(aio_task *wk, void *ctx, error_code err, size_t size) { + CHECK(_write_file, ""); auto ret = _write_queue.on_work_completed(wk, ctx); while (wk) { diff --git a/src/aio/disk_engine.h b/src/aio/disk_engine.h index 8eb7846359..70cce466a1 100644 --- a/src/aio/disk_engine.h +++ b/src/aio/disk_engine.h @@ -32,6 +32,7 @@ #include "aio/aio_task.h" #include "aio_provider.h" +#include "rocksdb/env.h" #include "utils/singleton.h" #include "utils/work_queue.h" @@ -56,17 +57,21 @@ class disk_write_queue : public work_queue class disk_file { public: - explicit disk_file(linux_fd_t fd); + explicit disk_file(std::unique_ptr rf); + explicit disk_file(std::unique_ptr wf); aio_task *read(aio_task *tsk); aio_task *write(aio_task *tsk, void *ctx); aio_task *on_read_completed(aio_task *wk, error_code err, size_t size); aio_task *on_write_completed(aio_task *wk, void *ctx, error_code err, size_t size); - linux_fd_t native_handle() const { return _fd; } + rocksdb::RandomAccessFile *rfile() const { return _read_file.get(); } + rocksdb::RandomRWFile *wfile() const { return _write_file.get(); } private: - linux_fd_t _fd; + // TODO(yingchun): unify to use a single RandomRWFile member variable. + std::unique_ptr _read_file; + std::unique_ptr _write_file; disk_write_queue _write_queue; work_queue _read_queue; }; diff --git a/src/aio/file_io.cpp b/src/aio/file_io.cpp index 4f3c10bb54..a4bf26ba85 100644 --- a/src/aio/file_io.cpp +++ b/src/aio/file_io.cpp @@ -26,32 +26,51 @@ #include "aio/file_io.h" +#include // IWYU pragma: no_include #include #include "aio/aio_provider.h" #include "disk_engine.h" +#include "rocksdb/env.h" +#include "utils/fmt_logging.h" namespace dsn { class task_tracker; namespace file { -/*extern*/ disk_file *open(const char *file_name, int flag, int pmode) +/*extern*/ disk_file *open(const std::string &fname, FileOpenType type) { - auto fd = disk_engine::provider().open(file_name, flag, pmode); - if (fd.is_invalid()) { - return nullptr; + switch (type) { + case FileOpenType::kReadOnly: { + auto sf = disk_engine::provider().open_read_file(fname); + if (!sf) { + return nullptr; + } + return new disk_file(std::move(sf)); } - - return new disk_file(fd); + case FileOpenType::kWriteOnly: { + auto wf = disk_engine::provider().open_write_file(fname); + if (!wf) { + return nullptr; + } + return new disk_file(std::move(wf)); + } + default: + CHECK(false, ""); + } + return nullptr; } /*extern*/ error_code close(disk_file *file) { - error_code result = ERR_INVALID_HANDLE; + error_code result = ERR_OK; if (file != nullptr) { - result = disk_engine::provider().close(file->native_handle()); + // A read file is not needed to close. + if (file->wfile()) { + result = disk_engine::provider().close(file->wfile()); + } delete file; file = nullptr; } @@ -60,11 +79,11 @@ namespace file { /*extern*/ error_code flush(disk_file *file) { - if (nullptr != file) { - return disk_engine::provider().flush(file->native_handle()); - } else { + if (file == nullptr || file->wfile() == nullptr) { return ERR_INVALID_HANDLE; } + + return disk_engine::provider().flush(file->wfile()); } /*extern*/ aio_task_ptr read(disk_file *file, @@ -84,7 +103,8 @@ namespace file { cb->get_aio_context()->engine = &disk_engine::instance(); cb->get_aio_context()->dfile = file; - if (!cb->spec().on_aio_call.execute(task::get_current_task(), cb, true)) { + if (!cb->spec().on_aio_call.execute(task::get_current_task(), cb, true) || + file->rfile() == nullptr) { cb->enqueue(ERR_FILE_OPERATION_FAILED, 0); return cb; } @@ -110,6 +130,10 @@ namespace file { cb->get_aio_context()->file_offset = offset; cb->get_aio_context()->type = AIO_Write; cb->get_aio_context()->dfile = file; + if (file->wfile() == nullptr) { + cb->enqueue(ERR_FILE_OPERATION_FAILED, 0); + return cb; + } disk_engine::instance().write(cb); return cb; diff --git a/src/aio/file_io.h b/src/aio/file_io.h index f0b6ffc420..0cef3f1b80 100644 --- a/src/aio/file_io.h +++ b/src/aio/file_io.h @@ -28,6 +28,7 @@ #include #include +#include #include #include "aio/aio_task.h" @@ -47,6 +48,13 @@ class task_tracker; namespace file { +enum class FileOpenType +{ + kReadOnly = 0, + kWriteOnly +}; + +// TODO(yingchun): consider to return a smart pointer /// open file /// /// \param file_name filename of the file. @@ -55,7 +63,7 @@ namespace file { /// /// \return file handle /// -extern disk_file *open(const char *file_name, int flag, int pmode); +extern disk_file *open(const std::string &fname, FileOpenType type); /// close the file handle extern error_code close(disk_file *file); diff --git a/src/aio/native_linux_aio_provider.cpp b/src/aio/native_linux_aio_provider.cpp index 56a0efa1e1..7270dff733 100644 --- a/src/aio/native_linux_aio_provider.cpp +++ b/src/aio/native_linux_aio_provider.cpp @@ -26,19 +26,17 @@ #include "native_linux_aio_provider.h" -#include -#include -#include -#include - #include "aio/aio_provider.h" #include "aio/disk_engine.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/service_engine.h" #include "runtime/task/async_calls.h" +#include "utils/env.h" #include "utils/fmt_logging.h" #include "utils/latency_tracer.h" #include "utils/ports.h" -#include "utils/safe_strerror_posix.h" namespace dsn { @@ -46,87 +44,101 @@ native_linux_aio_provider::native_linux_aio_provider(disk_engine *disk) : aio_pr native_linux_aio_provider::~native_linux_aio_provider() {} -linux_fd_t native_linux_aio_provider::open(const char *file_name, int flag, int pmode) +std::unique_ptr +native_linux_aio_provider::open_read_file(const std::string &fname) { - auto fd = ::open(file_name, flag, pmode); - if (fd == DSN_INVALID_FILE_HANDLE) { - LOG_ERROR("create file '{}' failed, err = {}", file_name, utils::safe_strerror(errno)); + std::unique_ptr rfile; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewRandomAccessFile(fname, &rfile, rocksdb::EnvOptions()); + if (!s.ok()) { + LOG_ERROR("open read file '{}' failed, err = {}", fname, s.ToString()); } - return linux_fd_t(fd); + return rfile; } -error_code native_linux_aio_provider::close(linux_fd_t fd) +std::unique_ptr +native_linux_aio_provider::open_write_file(const std::string &fname) { - if (fd.is_invalid() || ::close(fd.fd) == 0) { - return ERR_OK; + // rocksdb::NewRandomRWFile() doesn't act as the docs described, it will not create the + // file if it not exists, and an error Status will be returned, so we try to create the + // file by ReopenWritableFile() if it not exist. + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive)->FileExists(fname); + if (!s.ok() && !s.IsNotFound()) { + LOG_ERROR("failed to check whether the file '{}' exist, err = {}", fname, s.ToString()); + return nullptr; + } + + if (s.IsNotFound()) { + std::unique_ptr cfile; + s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->ReopenWritableFile(fname, &cfile, rocksdb::EnvOptions()); + if (!s.ok()) { + LOG_ERROR("failed to create file '{}', err = {}", fname, s.ToString()); + return nullptr; + } } - LOG_ERROR("close file failed, err = {}", utils::safe_strerror(errno)); - return ERR_FILE_OPERATION_FAILED; + // Open the file for write as RandomRWFile, to support un-sequential write. + std::unique_ptr wfile; + s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewRandomRWFile(fname, &wfile, rocksdb::EnvOptions()); + if (!s.ok()) { + LOG_ERROR("open write file '{}' failed, err = {}", fname, s.ToString()); + } + return wfile; } -error_code native_linux_aio_provider::flush(linux_fd_t fd) +error_code native_linux_aio_provider::close(rocksdb::RandomRWFile *wf) { - if (fd.is_invalid() || ::fsync(fd.fd) == 0) { - return ERR_OK; + auto s = wf->Close(); + if (!s.ok()) { + LOG_ERROR("close file failed, err = {}", s.ToString()); + return ERR_FILE_OPERATION_FAILED; } - LOG_ERROR("flush file failed, err = {}", utils::safe_strerror(errno)); - return ERR_FILE_OPERATION_FAILED; + return ERR_OK; +} + +error_code native_linux_aio_provider::flush(rocksdb::RandomRWFile *wf) +{ + auto s = wf->Fsync(); + if (!s.ok()) { + LOG_ERROR("flush file failed, err = {}", s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } + + return ERR_OK; } error_code native_linux_aio_provider::write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) { - dsn::error_code resp = ERR_OK; - uint64_t buffer_offset = 0; - do { - // ret is the written data size - auto ret = ::pwrite(aio_ctx.dfile->native_handle().fd, - (char *)aio_ctx.buffer + buffer_offset, - aio_ctx.buffer_size - buffer_offset, - aio_ctx.file_offset + buffer_offset); - if (dsn_unlikely(ret < 0)) { - if (errno == EINTR) { - LOG_WARNING("write failed with errno={} and will retry it.", - utils::safe_strerror(errno)); - continue; - } - resp = ERR_FILE_OPERATION_FAILED; - LOG_ERROR("write failed with errno={}, return {}.", utils::safe_strerror(errno), resp); - return resp; - } - - buffer_offset += ret; - if (dsn_unlikely(buffer_offset != aio_ctx.buffer_size)) { - LOG_WARNING( - "write incomplete, request_size={}, total_write_size={}, this_write_size={}, " - "and will retry it.", - aio_ctx.buffer_size, - buffer_offset, - ret); - } - } while (dsn_unlikely(buffer_offset < aio_ctx.buffer_size)); + rocksdb::Slice data((const char *)(aio_ctx.buffer), aio_ctx.buffer_size); + auto s = aio_ctx.dfile->wfile()->Write(aio_ctx.file_offset, data); + if (!s.ok()) { + LOG_ERROR("write file failed, err = {}", s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } - *processed_bytes = buffer_offset; - return resp; + *processed_bytes = aio_ctx.buffer_size; + return ERR_OK; } error_code native_linux_aio_provider::read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) { - auto ret = ::pread(aio_ctx.dfile->native_handle().fd, - aio_ctx.buffer, - aio_ctx.buffer_size, - aio_ctx.file_offset); - if (dsn_unlikely(ret < 0)) { - LOG_WARNING("write failed with errno={} and will retry it.", utils::safe_strerror(errno)); + rocksdb::Slice result; + auto s = aio_ctx.dfile->rfile()->Read( + aio_ctx.file_offset, aio_ctx.buffer_size, &result, (char *)(aio_ctx.buffer)); + if (!s.ok()) { + LOG_ERROR("read file failed, err = {}", s.ToString()); return ERR_FILE_OPERATION_FAILED; } - if (ret == 0) { + + if (result.empty()) { return ERR_HANDLE_EOF; } - *processed_bytes = static_cast(ret); + *processed_bytes = result.size(); return ERR_OK; } diff --git a/src/aio/native_linux_aio_provider.h b/src/aio/native_linux_aio_provider.h index bdb1339b9c..538b808dfa 100644 --- a/src/aio/native_linux_aio_provider.h +++ b/src/aio/native_linux_aio_provider.h @@ -27,11 +27,18 @@ #pragma once #include +#include +#include #include "aio/aio_task.h" #include "aio_provider.h" #include "utils/error_code.h" +namespace rocksdb { +class RandomAccessFile; +class RandomRWFile; +} // namespace rocksdb + namespace dsn { class disk_engine; @@ -41,16 +48,18 @@ class native_linux_aio_provider : public aio_provider explicit native_linux_aio_provider(disk_engine *disk); ~native_linux_aio_provider() override; - linux_fd_t open(const char *file_name, int flag, int pmode) override; - error_code close(linux_fd_t fd) override; - error_code flush(linux_fd_t fd) override; - error_code write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) override; + std::unique_ptr open_read_file(const std::string &fname) override; error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) override; + std::unique_ptr open_write_file(const std::string &fname) override; + error_code write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) override; + error_code flush(rocksdb::RandomRWFile *wf) override; + error_code close(rocksdb::RandomRWFile *wf) override; + void submit_aio_task(aio_task *aio) override; aio_context *prepare_aio_context(aio_task *tsk) override { return new aio_context; } -protected: +private: error_code aio_internal(aio_task *aio); }; diff --git a/src/aio/test/CMakeLists.txt b/src/aio/test/CMakeLists.txt index c1b0a44e46..357499a9c8 100644 --- a/src/aio/test/CMakeLists.txt +++ b/src/aio/test/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio rocksdb) +set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio test_utils rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/aio/test/aio.cpp b/src/aio/test/aio.cpp index a95cbeb8f0..fa6b0114ae 100644 --- a/src/aio/test/aio.cpp +++ b/src/aio/test/aio.cpp @@ -24,7 +24,7 @@ * THE SOFTWARE. */ -#include +#include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include @@ -48,11 +48,25 @@ #include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" #include "utils/test_macros.h" #include "utils/threadpool_code.h" +DSN_DEFINE_uint32(aio_test, + op_buffer_size, + 12, + "The buffer size of each aio read or write operation for the aio_test.basic"); +DSN_DEFINE_uint32(aio_test, + total_op_count, + 100, + "The total count of read or write operations for the aio_test.basic"); +DSN_DEFINE_uint32( + aio_test, + op_count_per_batch, + 10, + "The operation count of per read or write batch operation for the aio_test.basic"); + using namespace ::dsn; DEFINE_THREAD_POOL_CODE(THREAD_POOL_TEST_SERVER) @@ -62,6 +76,7 @@ class aio_test : public pegasus::encrypt_data_test_base { public: void SetUp() override { utils::filesystem::remove_path(kTestFileName); } + void TearDown() override { utils::filesystem::remove_path(kTestFileName); } const std::string kTestFileName = "aio_test.txt"; }; @@ -71,10 +86,10 @@ INSTANTIATE_TEST_CASE_P(, aio_test, ::testing::Values(false)); TEST_P(aio_test, basic) { - const char *kUnitBuffer = "hello, world"; - const size_t kUnitBufferLength = strlen(kUnitBuffer); - const int kTotalBufferCount = 100; - const int kBufferCountPerBatch = 10; + const size_t kUnitBufferLength = FLAGS_op_buffer_size; + const std::string kUnitBuffer(kUnitBufferLength, 'x'); + const int kTotalBufferCount = FLAGS_total_op_count; + const int kBufferCountPerBatch = FLAGS_op_count_per_batch; const int64_t kFileSize = kUnitBufferLength * kTotalBufferCount; ASSERT_EQ(0, kTotalBufferCount % kBufferCountPerBatch); @@ -86,15 +101,16 @@ TEST_P(aio_test, basic) auto verify_data = [=]() { int64_t file_size; ASSERT_TRUE(utils::filesystem::file_size( - kTestFileName.c_str(), dsn::utils::FileDataType::kSensitive, file_size)); + kTestFileName, dsn::utils::FileDataType::kSensitive, file_size)); ASSERT_EQ(kFileSize, file_size); // Create a read file handler. - auto rfile = file::open(kTestFileName.c_str(), O_RDONLY | O_BINARY, 0); + auto rfile = file::open(kTestFileName, file::FileOpenType::kReadOnly); ASSERT_NE(rfile, nullptr); // 1. Check sequential read. { + pegasus::stop_watch sw; uint64_t offset = 0; std::list tasks; for (int i = 0; i < kTotalBufferCount; i++) { @@ -111,12 +127,14 @@ TEST_P(aio_test, basic) t->wait(); ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); - ASSERT_STREQ(kUnitBuffer, read_buffer); + ASSERT_STREQ(kUnitBuffer.c_str(), read_buffer); } + sw.stop_and_output(fmt::format("sequential read")); } // 2. Check concurrent read. { + pegasus::stop_watch sw; uint64_t offset = 0; std::list tasks; char read_buffers[kTotalBufferCount][kUnitBufferLength + 1]; @@ -137,22 +155,24 @@ TEST_P(aio_test, basic) ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); } for (int i = 0; i < kTotalBufferCount; i++) { - ASSERT_STREQ(kUnitBuffer, read_buffers[i]); + ASSERT_STREQ(kUnitBuffer.c_str(), read_buffers[i]); } + sw.stop_and_output(fmt::format("concurrent read")); } ASSERT_EQ(ERR_OK, file::close(rfile)); }; // 1. Sequential write. { - auto wfile = file::open(kTestFileName.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + pegasus::stop_watch sw; + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); ASSERT_NE(wfile, nullptr); uint64_t offset = 0; std::list tasks; for (int i = 0; i < kTotalBufferCount; i++) { auto t = ::dsn::file::write(wfile, - kUnitBuffer, + kUnitBuffer.c_str(), kUnitBufferLength, offset, LPC_AIO_TEST, @@ -167,12 +187,14 @@ TEST_P(aio_test, basic) } ASSERT_EQ(ERR_OK, file::flush(wfile)); ASSERT_EQ(ERR_OK, file::close(wfile)); + sw.stop_and_output(fmt::format("sequential write")); } NO_FATALS(verify_data()); // 2. Un-sequential write. { - auto wfile = file::open(kTestFileName.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + pegasus::stop_watch sw; + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); ASSERT_NE(wfile, nullptr); std::vector offsets; @@ -188,7 +210,7 @@ TEST_P(aio_test, basic) std::list tasks; for (const auto &offset : offsets) { auto t = ::dsn::file::write(wfile, - kUnitBuffer, + kUnitBuffer.c_str(), kUnitBufferLength, offset, LPC_AIO_TEST, @@ -202,19 +224,21 @@ TEST_P(aio_test, basic) } ASSERT_EQ(ERR_OK, file::flush(wfile)); ASSERT_EQ(ERR_OK, file::close(wfile)); + sw.stop_and_output(fmt::format("un-sequential write")); } NO_FATALS(verify_data()); // 3. Overwrite. { - auto wfile = file::open(kTestFileName.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + pegasus::stop_watch sw; + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); ASSERT_NE(wfile, nullptr); uint64_t offset = 0; std::list tasks; for (int i = 0; i < kTotalBufferCount; i++) { auto t = ::dsn::file::write(wfile, - kUnitBuffer, + kUnitBuffer.c_str(), kUnitBufferLength, offset, LPC_AIO_TEST, @@ -229,19 +253,21 @@ TEST_P(aio_test, basic) } ASSERT_EQ(ERR_OK, file::flush(wfile)); ASSERT_EQ(ERR_OK, file::close(wfile)); + sw.stop_and_output(fmt::format("overwrite")); } NO_FATALS(verify_data()); // 4. Vector write. { - auto wfile = file::open(kTestFileName.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + pegasus::stop_watch sw; + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); ASSERT_NE(wfile, nullptr); uint64_t offset = 0; std::list tasks; std::unique_ptr buffers(new dsn_file_buffer_t[kBufferCountPerBatch]); for (int i = 0; i < kBufferCountPerBatch; i++) { - buffers[i].buffer = static_cast(const_cast(kUnitBuffer)); + buffers[i].buffer = static_cast(const_cast(kUnitBuffer.c_str())); buffers[i].size = kUnitBufferLength; } for (int i = 0; i < kTotalBufferCount / kBufferCountPerBatch; i++) { @@ -264,16 +290,17 @@ TEST_P(aio_test, basic) } ASSERT_EQ(ERR_OK, file::flush(wfile)); ASSERT_EQ(ERR_OK, file::close(wfile)); + sw.stop_and_output(fmt::format("vector write")); } NO_FATALS(verify_data()); } TEST_P(aio_test, aio_share) { - auto wfile = file::open(kTestFileName.c_str(), O_WRONLY | O_CREAT | O_BINARY, 0666); + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); ASSERT_NE(wfile, nullptr); - auto rfile = file::open(kTestFileName.c_str(), O_RDONLY | O_BINARY, 0); + auto rfile = file::open(kTestFileName, file::FileOpenType::kReadOnly); ASSERT_NE(rfile, nullptr); ASSERT_EQ(ERR_OK, file::close(wfile)); @@ -289,7 +316,7 @@ TEST_P(aio_test, operation_failed) *count = n; }; - auto wfile = file::open(kTestFileName.c_str(), O_WRONLY | O_CREAT | O_BINARY, 0666); + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); ASSERT_NE(wfile, nullptr); char buff[512] = {0}; @@ -305,7 +332,7 @@ TEST_P(aio_test, operation_failed) t->wait(); ASSERT_EQ(ERR_FILE_OPERATION_FAILED, *err); - auto rfile = file::open(kTestFileName.c_str(), O_RDONLY | O_BINARY, 0); + auto rfile = file::open(kTestFileName, file::FileOpenType::kReadOnly); ASSERT_NE(nullptr, rfile); t = ::dsn::file::read(rfile, buff, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); @@ -357,9 +384,9 @@ TEST_P(aio_test, dsn_file) ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(src_file, src_file_md5)); ASSERT_FALSE(src_file_md5.empty()); - auto fin = file::open(src_file.c_str(), O_RDONLY | O_BINARY, 0); + auto fin = file::open(src_file, file::FileOpenType::kReadOnly); ASSERT_NE(nullptr, fin); - auto fout = file::open(dst_file.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0666); + auto fout = file::open(dst_file, file::FileOpenType::kWriteOnly); ASSERT_NE(nullptr, fout); char kUnitBuffer[1024]; uint64_t offset = 0; diff --git a/src/aio/test/config.ini b/src/aio/test/config.ini index 47bc9cf7fb..fd46a38d75 100644 --- a/src/aio/test/config.ini +++ b/src/aio/test/config.ini @@ -43,3 +43,8 @@ tool = nativerun pause_on_start = false logging_start_level = LOG_LEVEL_DEBUG logging_factory_name = dsn::tools::simple_logger + +[aio_test] +op_buffer_size = 12 +total_op_count = 100 +op_count_per_batch = 10 diff --git a/src/meta/meta_state_service_simple.cpp b/src/meta/meta_state_service_simple.cpp index 6ba00176ea..776cfba342 100644 --- a/src/meta/meta_state_service_simple.cpp +++ b/src/meta/meta_state_service_simple.cpp @@ -26,7 +26,6 @@ #include "meta_state_service_simple.h" -#include #include #include #include @@ -44,7 +43,6 @@ #include "utils/binary_reader.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" #include "utils/strings.h" #include "utils/utils.h" @@ -314,7 +312,7 @@ error_code meta_state_service_simple::initialize(const std::vector } } - _log = file::open(log_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + _log = file::open(log_path, file::FileOpenType::kWriteOnly); if (!_log) { LOG_ERROR("open file failed: {}", log_path); return ERR_FILE_OPERATION_FAILED; diff --git a/src/nfs/nfs_client_impl.cpp b/src/nfs/nfs_client_impl.cpp index c0ced7b8c6..910bae8ec0 100644 --- a/src/nfs/nfs_client_impl.cpp +++ b/src/nfs/nfs_client_impl.cpp @@ -27,7 +27,6 @@ #include "nfs_client_impl.h" // IWYU pragma: no_include -#include #include #include "nfs/nfs_code_definition.h" @@ -38,7 +37,6 @@ #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" #include "utils/string_conv.h" #include "utils/token_buckets.h" @@ -460,8 +458,7 @@ void nfs_client_impl::continue_write() // double check zauto_lock l(fc->user_req->user_req_lock); if (!fc->file_holder->file_handle) { - fc->file_holder->file_handle = - file::open(file_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + fc->file_holder->file_handle = file::open(file_path, file::FileOpenType::kWriteOnly); } } @@ -470,6 +467,10 @@ void nfs_client_impl::continue_write() LOG_ERROR("open file {} failed", file_path); handle_completion(fc->user_req, ERR_FILE_OPERATION_FAILED); } else { + LOG_DEBUG("nfs: copy to file {} [{}, {}]", + file_path, + reqc->response.offset, + reqc->response.offset + reqc->response.size); zauto_lock l(reqc->lock); if (reqc->is_valid) { reqc->local_write_task = file::write(fc->file_holder->file_handle, diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index 08821042a4..c4bcf7a2e4 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -26,7 +26,6 @@ #include "nfs/nfs_server_impl.h" -#include #include #include #include @@ -41,7 +40,6 @@ #include "utils/env.h" #include "utils/filesystem.h" #include "utils/flags.h" -#include "utils/ports.h" #include "utils/string_conv.h" #include "utils/utils.h" @@ -93,7 +91,7 @@ void nfs_service_impl::on_copy(const ::dsn::service::copy_request &request, zauto_lock l(_handles_map_lock); auto it = _handles_map.find(file_path); // find file handle cache first if (it == _handles_map.end()) { - dfile = file::open(file_path.c_str(), O_RDONLY | O_BINARY, 0); + dfile = file::open(file_path, file::FileOpenType::kReadOnly); if (dfile == nullptr) { LOG_ERROR("[nfs_service] open file {} failed", file_path); ::dsn::service::copy_response resp; diff --git a/src/replica/duplication/test/load_from_private_log_test.cpp b/src/replica/duplication/test/load_from_private_log_test.cpp index 2782f55f11..b215be60f0 100644 --- a/src/replica/duplication/test/load_from_private_log_test.cpp +++ b/src/replica/duplication/test/load_from_private_log_test.cpp @@ -16,8 +16,8 @@ // under the License. // IWYU pragma: no_include -#include #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include @@ -54,7 +54,6 @@ #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" #define BOOST_NO_CXX11_SCOPED_ENUMS #include @@ -459,7 +458,7 @@ TEST_F(load_fail_mode_test, fail_skip_real_corrupted_file) int64_t file_size; ASSERT_TRUE(utils::filesystem::file_size( log_path, dsn::utils::FileDataType::kSensitive, file_size)); - auto wfile = file::open(log_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + auto wfile = file::open(log_path, file::FileOpenType::kWriteOnly); ASSERT_NE(wfile, nullptr); const char buf[] = "xxxxxx"; diff --git a/src/replica/log_file.cpp b/src/replica/log_file.cpp index e239d4dae1..973213b3a4 100644 --- a/src/replica/log_file.cpp +++ b/src/replica/log_file.cpp @@ -26,7 +26,6 @@ #include "log_file.h" -#include #include #include #include @@ -99,7 +98,7 @@ log_file::~log_file() { close(); } return nullptr; } - disk_file *hfile = file::open(path, O_RDONLY | O_BINARY, 0); + disk_file *hfile = file::open(path, file::FileOpenType::kReadOnly); if (!hfile) { err = ERR_FILE_OPERATION_FAILED; LOG_WARNING("open log file {} failed", path); @@ -155,7 +154,7 @@ log_file::~log_file() { close(); } return nullptr; } - disk_file *hfile = file::open(path, O_RDWR | O_CREAT | O_BINARY, 0666); + disk_file *hfile = file::open(path, file::FileOpenType::kWriteOnly); if (!hfile) { LOG_WARNING("create log {} failed", path); return nullptr; @@ -268,6 +267,7 @@ aio_task_ptr log_file::commit_log_block(log_block &block, log_appender pending(offset, block); return commit_log_blocks(pending, evt, tracker, std::move(callback), hash); } + aio_task_ptr log_file::commit_log_blocks(log_appender &pending, dsn::task_code evt, dsn::task_tracker *tracker, @@ -333,7 +333,7 @@ aio_task_ptr log_file::commit_log_blocks(log_appender &pending, hash); } - if (utils::FLAGS_enable_latency_tracer) { + if (dsn_unlikely(utils::FLAGS_enable_latency_tracer)) { tsk->_tracer->set_parent_point_name("commit_pending_mutations"); tsk->_tracer->set_description("log"); for (const auto &mutation : pending.mutations()) { diff --git a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp index 81a718a45f..e78ae81efb 100644 --- a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp @@ -35,7 +35,6 @@ #include "simple_kv.server.impl.h" -#include #include #include #include @@ -61,7 +60,6 @@ #include "utils/blob.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" #include "utils/utils.h" namespace dsn { @@ -249,7 +247,7 @@ ::dsn::error_code simple_kv_service_impl::sync_checkpoint() return ERR_OK; } - auto wfile = file::open(fname.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + auto wfile = file::open(fname, file::FileOpenType::kWriteOnly); CHECK_NOTNULL(wfile, ""); #define WRITE_DATA_SIZE(data, size) \ diff --git a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp index ae04a7f4e6..021496194b 100644 --- a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp @@ -25,7 +25,6 @@ */ #include "simple_kv.server.impl.h" -#include #include #include #include @@ -49,7 +48,6 @@ #include "utils/blob.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" #include "utils/threadpool_code.h" #include "utils/utils.h" @@ -254,7 +252,7 @@ ::dsn::error_code simple_kv_service_impl::sync_checkpoint() } std::string fname = fmt::format("{}/checkpoint.{}", data_dir(), last_commit); - auto wfile = file::open(fname.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + auto wfile = file::open(fname, file::FileOpenType::kWriteOnly); CHECK_NOTNULL(wfile, ""); #define WRITE_DATA_SIZE(data, size) \ diff --git a/src/replica/test/mutation_log_test.cpp b/src/replica/test/mutation_log_test.cpp index 980a538bf1..4f96d7d57e 100644 --- a/src/replica/test/mutation_log_test.cpp +++ b/src/replica/test/mutation_log_test.cpp @@ -26,7 +26,6 @@ #include "replica/mutation_log.h" -#include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include @@ -70,7 +69,7 @@ using namespace ::dsn::replication; static void overwrite_file(const char *file, int offset, const void *buf, int size) { - auto wfile = file::open(file, O_RDWR | O_CREAT | O_BINARY, 0666); + auto wfile = file::open(file, file::FileOpenType::kWriteOnly); ASSERT_NE(wfile, nullptr); auto t = ::dsn::file::write(wfile, (const char *)buf, diff --git a/src/runtime/test/CMakeLists.txt b/src/runtime/test/CMakeLists.txt index c0146eb73b..01f17e8c50 100644 --- a/src/runtime/test/CMakeLists.txt +++ b/src/runtime/test/CMakeLists.txt @@ -33,6 +33,7 @@ set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio dsn_meta_server + rocksdb ) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/runtime/test/task_test.cpp b/src/runtime/test/task_test.cpp index ba4f3d8875..5716c8ce41 100644 --- a/src/runtime/test/task_test.cpp +++ b/src/runtime/test/task_test.cpp @@ -17,7 +17,6 @@ #include "runtime/task/task.h" -#include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -66,7 +65,7 @@ class task_test : public ::testing::Test static void test_signal_finished_task() { - disk_file *fp = file::open("config-test.ini", O_RDONLY | O_BINARY, 0); + disk_file *fp = file::open("config-test.ini", file::FileOpenType::kReadOnly); // this aio task is enqueued into read-queue of disk_engine char buffer[128]; @@ -80,6 +79,7 @@ class task_test : public ::testing::Test // signal a finished task won't cause failure t->signal_waiters(); // signal_waiters may return false t->signal_waiters(); + ASSERT_EQ(ERR_OK, file::close(fp)); } }; diff --git a/src/test_util/test_util.cpp b/src/test_util/test_util.cpp index 0789c4678b..935d50631a 100644 --- a/src/test_util/test_util.cpp +++ b/src/test_util/test_util.cpp @@ -18,7 +18,6 @@ #include "test_util.h" #include -#include #include #include #include diff --git a/src/test_util/test_util.h b/src/test_util/test_util.h index debeb7fa90..23b1624be7 100644 --- a/src/test_util/test_util.h +++ b/src/test_util/test_util.h @@ -19,10 +19,15 @@ #pragma once -#include +#include +#include +#include #include +#include #include +#include "fmt/core.h" +#include "runtime/api_layer1.h" #include "utils/flags.h" #include "utils/test_macros.h" @@ -43,6 +48,23 @@ class encrypt_data_test_base : public testing::TestWithParam encrypt_data_test_base() { FLAGS_encrypt_data_at_rest = GetParam(); } }; +class stop_watch +{ +public: + stop_watch() { _start_ms = dsn_now_ms(); } + void stop_and_output(const std::string &msg) + { + auto duration_ms = + std::chrono::duration_cast>( + std::chrono::milliseconds(static_cast(dsn_now_ms() - _start_ms))) + .count(); + fmt::print(stdout, "{}, cost {} ms\n", msg, duration_ms); + } + +private: + uint64_t _start_ms = 0; +}; + void create_local_test_file(const std::string &full_name, dsn::replication::file_meta *fm); #define ASSERT_EVENTUALLY(expr) \ diff --git a/src/utils/long_adder_bench/long_adder_bench.cpp b/src/utils/long_adder_bench/long_adder_bench.cpp index 93ec649c7b..0c2a12deed 100644 --- a/src/utils/long_adder_bench/long_adder_bench.cpp +++ b/src/utils/long_adder_bench/long_adder_bench.cpp @@ -20,12 +20,11 @@ #include #include #include -#include #include #include #include -#include "runtime/api_layer1.h" +#include "test_util/test_util.h" #include "utils/long_adder.h" #include "utils/ports.h" #include "utils/process_utils.h" @@ -133,7 +132,7 @@ void run_bench(int64_t num_operations, int64_t num_threads, const char *name) std::vector threads; - uint64_t start = dsn_now_ns(); + pegasus::stop_watch sw; for (int64_t i = 0; i < num_threads; i++) { threads.emplace_back([num_operations, &adder]() { for (int64_t i = 0; i < num_operations; ++i) { @@ -144,19 +143,11 @@ void run_bench(int64_t num_operations, int64_t num_threads, const char *name) for (auto &t : threads) { t.join(); } - uint64_t end = dsn_now_ns(); - - auto duration_ns = static_cast(end - start); - std::chrono::nanoseconds nano(duration_ns); - auto duration_s = std::chrono::duration_cast>(nano).count(); - - fmt::print(stdout, - "Running {} operations of {} with {} threads took {} seconds, result = {}.\n", - num_operations, - name, - num_threads, - duration_s, - adder.value()); + sw.stop_and_output(fmt::format("Running {} operations of {} with {} threads, result = {}", + num_operations, + name, + num_threads, + adder.value())); } int main(int argc, char **argv) diff --git a/src/utils/test/env.cpp b/src/utils/test/env.cpp index 813465de26..49e389729f 100644 --- a/src/utils/test/env.cpp +++ b/src/utils/test/env.cpp @@ -47,8 +47,8 @@ #include #include "test_util/test_util.h" -#include "utils/enum_helper.h" #include "utils/env.h" +#include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/rand.h"