diff --git a/.github/workflows/free_disk_space.sh b/.github/workflows/free_disk_space.sh new file mode 100755 index 0000000000..87f0cdef8e --- /dev/null +++ b/.github/workflows/free_disk_space.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +echo "==============================================================================" +echo "Freeing up disk space on Github workflows" +echo "==============================================================================" + +echo "Before freeing, the space of each disk:" +df -h + +echo "Listing 100 largest packages ..." +dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 + +echo "Removing large packages ..." +apt-get -s autoremove +apt-get remove -y openjdk-11-jre-headless + +echo "After removing large packages, the space of each disk:" +df -h + +echo "Listing directories ..." +mount +ls -lrt / +du -csh /* +du -csh /__e/*/* +du -csh /__t/*/* +du -csh /__w/*/* +ls -lrt /github +du -csh /github/* +du -csh /opt/* +du -csh /usr/local/* +du -csh /usr/local/lib/* +du -csh /usr/local/share/* +du -csh /usr/share/* +echo "AGENT_TOOLSDIRECTORY is $AGENT_TOOLSDIRECTORY" + +echo "Removing large directories ..." +rm -rf /__t/CodeQL +rm -rf /__t/PyPy +rm -rf /__t/Python +rm -rf /__t/Ruby +rm -rf /__t/go +rm -rf /__t/node +rm -rf /opt/ghc +rm -rf /usr/local/.ghcup +rm -rf /usr/local/graalvm +rm -rf /usr/local/lib/android +rm -rf /usr/local/lib/node_modules +rm -rf /usr/local/share/boost +rm -rf /usr/local/share/chromium +rm -rf /usr/local/share/powershell +rm -rf /usr/share/dotnet + +echo "After freeing, the space of each disk:" +df -h diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 5c320eda5a..f3dc28d389 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -49,8 +49,8 @@ on: env: # Update the options to reduce the consumption of the disk space ONEBOX_OPTS: disk_min_available_space_ratio=5 - TEST_OPTS: throttle_test_medium_value_kb=10,throttle_test_large_value_kb=25 - + TEST_OPTS: disk_min_available_space_ratio=5,throttle_test_medium_value_kb=10,throttle_test_large_value_kb=25 + jobs: cpp_clang_format_linter: name: Lint @@ -70,6 +70,9 @@ jobs: image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }} steps: - uses: actions/checkout@v3 + - name: Free Disk Space (Ubuntu) + run: | + .github/workflows/free_disk_space.sh - uses: dorny/paths-filter@v2 id: changes with: @@ -88,7 +91,7 @@ jobs: # Build thirdparties and leave some necessary libraries and source run: | mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* ../scripts/download_hadoop.sh hadoop-bin @@ -120,6 +123,9 @@ jobs: path: | /github/home/.ccache key: release_ccache + - name: Free Disk Space (Ubuntu) + run: | + .github/workflows/free_disk_space.sh - uses: dorny/paths-filter@v2 id: changes with: @@ -132,25 +138,36 @@ jobs: - name: Unpack prebuilt third-parties if: steps.changes.outputs.thirdparty == 'false' run: | + rm -f /root/thirdparties-src.zip unzip /root/thirdparties-bin.zip -d ./thirdparty rm -f /root/thirdparties-bin.zip + find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + rm -rf ./thirdparty/hadoop-bin/share/doc + rm -rf ./thirdparty/zookeeper-bin/docs - name: Rebuild third-parties if: steps.changes.outputs.thirdparty == 'true' working-directory: thirdparty # Build thirdparties and leave some necessary libraries and source run: | + rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* + find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + ../scripts/download_hadoop.sh hadoop-bin ../scripts/download_zk.sh zookeeper-bin + rm -rf hadoop-bin/share/doc + rm -rf zookeeper-bin/docs - name: Compilation run: | ccache -p ccache -z ./run.sh build --test --skip_thirdparty -j $(nproc) -t release ccache -s + - name: Clear Build Files + run: | + find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - name: Pack Server run: | ./run.sh pack_server @@ -164,7 +181,6 @@ jobs: mv thirdparty/hadoop-bin ./ mv thirdparty/zookeeper-bin ./ rm -rf thirdparty - find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + tar -zcvhf release__builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - name: Upload Artifact uses: actions/upload-artifact@v3 @@ -231,6 +247,7 @@ jobs: - name: Tar files run: | tar -zxvf release__builder.tar + rm -f release__builder.tar - name: Unit Testing run: | export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server @@ -243,7 +260,7 @@ jobs: needs: cpp_clang_format_linter runs-on: ubuntu-latest container: - image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }} + image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - uses: actions/checkout@v3 - name: Setup cache @@ -252,6 +269,9 @@ jobs: path: | /github/home/.ccache key: asan_ccache + - name: Free Disk Space (Ubuntu) + run: | + .github/workflows/free_disk_space.sh - uses: dorny/paths-filter@v2 id: changes with: @@ -264,31 +284,41 @@ jobs: - name: Unpack prebuilt third-parties if: steps.changes.outputs.thirdparty == 'false' run: | + rm -f /root/thirdparties-src.zip unzip /root/thirdparties-bin.zip -d ./thirdparty rm -f /root/thirdparties-bin.zip + find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + rm -rf ./thirdparty/hadoop-bin/share/doc + rm -rf ./thirdparty/zookeeper-bin/docs - name: Rebuild third-parties if: steps.changes.outputs.thirdparty == 'true' working-directory: thirdparty # Build thirdparties and leave some necessary libraries and source run: | + rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* + find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + ../scripts/download_hadoop.sh hadoop-bin ../scripts/download_zk.sh zookeeper-bin + rm -rf hadoop-bin/share/doc + rm -rf zookeeper-bin/docs - name: Compilation run: | ccache -p ccache -z ./run.sh build --test --sanitizer address --skip_thirdparty --disable_gperf -j $(nproc) ccache -s + - name: Clear Build Files + run: | + find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - name: Tar files run: | mv thirdparty/hadoop-bin ./ mv thirdparty/zookeeper-bin ./ rm -rf thirdparty - find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + tar -zcvhf release_address_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - name: Upload Artifact uses: actions/upload-artifact@v3 @@ -341,7 +371,7 @@ jobs: needs: build_ASAN runs-on: ubuntu-latest container: - image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }} + image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} options: --cap-add=SYS_PTRACE steps: - uses: actions/checkout@v3 @@ -357,6 +387,7 @@ jobs: - name: Tar files run: | tar -zxvf release_address_builder.tar + rm -f release_address_builder.tar - name: Unit Testing run: | export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server @@ -372,7 +403,7 @@ jobs: # needs: cpp_clang_format_linter # runs-on: ubuntu-latest # container: -# image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }} +# image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} # steps: # - uses: actions/checkout@v3 # - name: Setup cache @@ -381,6 +412,9 @@ jobs: # path: | # /github/home/.ccache # key: ubsan_ccache +# - name: Free Disk Space (Ubuntu) +# run: | +# .github/workflows/free_disk_space.sh # - uses: dorny/paths-filter@v2 # id: changes # with: @@ -393,31 +427,41 @@ jobs: # - name: Unpack prebuilt third-parties # if: steps.changes.outputs.thirdparty == 'false' # run: | +# rm -f /root/thirdparties-src.zip # unzip /root/thirdparties-bin.zip -d ./thirdparty # rm -f /root/thirdparties-bin.zip +# find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + +# rm -rf ./thirdparty/hadoop-bin/share/doc +# rm -rf ./thirdparty/zookeeper-bin/docs # - name: Rebuild third-parties # if: steps.changes.outputs.thirdparty == 'true' # working-directory: thirdparty # # Build thirdparties and leave some necessary libraries and source # run: | +# rm -f /root/thirdparties-src.zip # mkdir build -# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ +# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ # cmake --build build/ -j $(nproc) # rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* +# find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + # ../scripts/download_hadoop.sh hadoop-bin # ../scripts/download_zk.sh zookeeper-bin +# rm -rf hadoop-bin/share/doc +# rm -rf zookeeper-bin/docs # - name: Compilation # run: | # ccache -p # ccache -z # ./run.sh build --test --sanitizer undefined --skip_thirdparty --disable_gperf -j $(nproc) # ccache -s +# - name: Clear Build Files +# run: | +# find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + # - name: Tar files # run: | # mv thirdparty/hadoop-bin ./ # mv thirdparty/zookeeper-bin ./ # rm -rf thirdparty -# find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + # tar -zcvhf release_undefined_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin # - name: Upload Artifact # uses: actions/upload-artifact@v3 @@ -466,10 +510,13 @@ jobs: # needs: build_UBSAN # runs-on: ubuntu-latest # container: -# image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }} +# image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} # options: --cap-add=SYS_PTRACE # steps: # - uses: actions/checkout@v3 +# - name: Free Disk Space (Ubuntu) +# run: | +# .github/workflows/free_disk_space.sh # - name: Unpack prebuilt third-parties # run: | # unzip /root/thirdparties-bin.zip -d ./thirdparty @@ -482,6 +529,7 @@ jobs: # - name: Tar files # run: | # tar -zxvf release_undefined_builder.tar +# rm -f release_undefined_builder.tar # - name: Unit Testing # run: | # export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server @@ -503,6 +551,9 @@ jobs: path: | /github/home/.ccache key: jemalloc_ccache + - name: Free Disk Space (Ubuntu) + run: | + .github/workflows/free_disk_space.sh - uses: dorny/paths-filter@v2 id: changes with: @@ -515,35 +566,49 @@ jobs: - name: Unpack prebuilt third-parties if: steps.changes.outputs.thirdparty == 'false' run: | + rm -f /root/thirdparties-src.zip unzip /root/thirdparties-bin.zip -d ./thirdparty rm -f /root/thirdparties-bin.zip + find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + rm -rf ./thirdparty/hadoop-bin/share/doc + rm -rf ./thirdparty/zookeeper-bin/docs - name: Rebuild third-parties if: steps.changes.outputs.thirdparty == 'true' working-directory: thirdparty # Build thirdparties and leave some necessary libraries and source run: | + rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* + find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + ../scripts/download_hadoop.sh hadoop-bin ../scripts/download_zk.sh zookeeper-bin + rm -rf hadoop-bin/share/doc + rm -rf zookeeper-bin/docs - name: Compilation run: | ccache -p ccache -z ./run.sh build --test --skip_thirdparty -j $(nproc) -t release --use_jemalloc ccache -s + - name: Clear Build Files + run: | + find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - name: Pack Server - run: ./run.sh pack_server -j + run: | + ./run.sh pack_server -j + rm -rf pegasus-server-* - name: Pack Tools - run: ./run.sh pack_tools -j + run: | + ./run.sh pack_tools -j + rm -rf pegasus-tools-* - name: Tar files run: | mv thirdparty/hadoop-bin ./ mv thirdparty/zookeeper-bin ./ rm -rf thirdparty - find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + tar -zcvhf release_jemalloc_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - name: Upload Artifact uses: actions/upload-artifact@v3 @@ -577,6 +642,7 @@ jobs: - name: Tar files run: | tar -zxvf release_jemalloc_builder.tar + rm -f release_jemalloc_builder.tar - name: Unit Testing run: | export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server @@ -614,7 +680,7 @@ jobs: run: | export JAVA_HOME="${JAVA_HOME_8_X64}" mkdir -p build - cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} + cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DROCKSDB_PORTABLE=1 cmake --build build/ -j $(sysctl -n hw.physicalcpu) - name: Compilation run: | diff --git a/.github/workflows/regular-build.yml b/.github/workflows/regular-build.yml index 798b28e3a9..9e9a0d06fb 100644 --- a/.github/workflows/regular-build.yml +++ b/.github/workflows/regular-build.yml @@ -75,14 +75,27 @@ jobs: working-directory: /root run: | git clone -b ${{ github.ref_name }} --depth=1 https://github.com/apache/incubator-pegasus.git + - name: Free Disk Space (Ubuntu) + run: | + .github/workflows/free_disk_space.sh - name: Unpack prebuilt third-parties run: | + rm -f /root/thirdparties-src.zip unzip /root/thirdparties-bin.zip -d ./thirdparty rm -f /root/thirdparties-bin.zip + find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + rm -rf ./thirdparty/hadoop-bin/share/doc + rm -rf ./thirdparty/zookeeper-bin/docs - name: Compilation Pegasus - run: ./run.sh build --test --compiler ${{ matrix.compiler }} --skip_thirdparty -j $(nproc) + run: | + ./run.sh build --test --compiler ${{ matrix.compiler }} --skip_thirdparty -j $(nproc) + - name: Clear Build Files + run: | + find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - name: Packaging Server - run: ./run.sh pack_server + run: | + ./run.sh pack_server + rm -rf pegasus-server-* build_and_lint_go: name: Build and Lint Golang diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml index 152fe15a78..f56b3f4c2d 100644 --- a/.github/workflows/thirdparty-regular-push.yml +++ b/.github/workflows/thirdparty-regular-push.yml @@ -167,7 +167,7 @@ jobs: build-args: | GITHUB_BRANCH=${{ github.ref_name }} OS_VERSION=${{ matrix.osversion }} - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 HADOOP_BIN_PATH=hadoop-bin ZOOKEEPER_BIN_PATH=zookeeper-bin - name: Image digest @@ -203,7 +203,7 @@ jobs: build-args: | GITHUB_BRANCH=${{ github.ref_name }} OS_VERSION=${{ matrix.osversion }} - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 USE_JEMALLOC=ON HADOOP_BIN_PATH=hadoop-bin ZOOKEEPER_BIN_PATH=zookeeper-bin diff --git a/.gitignore b/.gitignore index 457dbf3098..2bfd1ed34d 100644 --- a/.gitignore +++ b/.gitignore @@ -115,7 +115,7 @@ scripts/py_utils/*.pyc cmake-build-debug packages -src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files/ +src/test/function_test/bulk_load/pegasus-bulk-load-function-test-files/ config-shell.ini.* *.tar.gz pegasus-server* diff --git a/.licenserc.yaml b/.licenserc.yaml index c6f63afd8b..53d316d23b 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -312,7 +312,7 @@ header: - 'src/nfs/nfs_client_impl.h' - 'src/nfs/nfs_code_definition.h' - 'src/nfs/nfs_node.cpp' - - 'src/nfs/nfs_node_impl.cpp' + - 'src/nfs/nfs_node_simple.cpp' - 'src/nfs/nfs_node_simple.h' - 'src/nfs/nfs_server_impl.cpp' - 'src/nfs/nfs_server_impl.h' diff --git a/CMakeLists.txt b/CMakeLists.txt index bc451ae269..fee1807892 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,9 +19,9 @@ project(pegasus) cmake_minimum_required(VERSION 3.11.0) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - # require at least gcc 5.4.0 - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.4.0) - message(FATAL_ERROR "GCC version must be at least 5.4.0!") + # require at least gcc 7.0.0 + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0.0) + message(FATAL_ERROR "GCC version must be at least 7.0.0!") endif () endif () diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 79aba27cb7..0147b50298 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -147,6 +147,7 @@ function(dsn_add_project) endif() ms_add_project("${MY_PROJ_TYPE}" "${MY_PROJ_NAME}" "${MY_PROJ_SRC}" "${MY_PROJ_LIBS}" "${MY_BINPLACES}") define_file_basename_for_sources(${MY_PROJ_NAME}) + target_compile_features(${MY_PROJ_NAME} PRIVATE cxx_std_17) endfunction(dsn_add_project) function(dsn_add_static_library) @@ -204,7 +205,7 @@ function(dsn_setup_compiler_flags) # We want access to the PRI* print format macros. add_definitions(-D__STDC_FORMAT_MACROS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -gdwarf-4" CACHE STRING "" FORCE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -gdwarf-4" CACHE STRING "" FORCE) # -Wall: Enable all warnings. add_compile_options(-Wall) @@ -221,9 +222,6 @@ function(dsn_setup_compiler_flags) # use frame pointers to allow simple stack frame walking for backtraces. # This has a small perf hit but worth it for the ability to profile in production add_compile_options( -fno-omit-frame-pointer) - # -Wno-deprecated-register - # kbr5.h uses the legacy 'register' keyword. - add_compile_options(-Wno-deprecated-register) # -Wno-implicit-float-conversion # Poco/Dynamic/VarHolder.h uses 'unsigned long' to 'float' conversion add_compile_options(-Wno-implicit-float-conversion) @@ -388,11 +386,9 @@ function(dsn_common_setup) set(BUILD_SHARED_LIBS OFF) - include(CheckCXXCompilerFlag) - CHECK_CXX_COMPILER_FLAG("-std=c++1y" COMPILER_SUPPORTS_CXX1Y) - if(NOT ${COMPILER_SUPPORTS_CXX1Y}) - message(FATAL_ERROR "You need a compiler with C++1y support.") - endif() + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) dsn_setup_system_libs() dsn_setup_compiler_flags() diff --git a/docker/pegasus-build-env/centos7/Dockerfile b/docker/pegasus-build-env/centos7/Dockerfile index 5c536166c1..0e89f316b1 100644 --- a/docker/pegasus-build-env/centos7/Dockerfile +++ b/docker/pegasus-build-env/centos7/Dockerfile @@ -50,6 +50,8 @@ RUN yum -y install centos-release-scl \ lz4-devel \ bison \ flex \ + krb5-devel \ + cyrus-sasl-devel \ patch; \ yum clean all; \ rm -rf /var/cache/yum; diff --git a/docker/pegasus-build-env/ubuntu1804/Dockerfile b/docker/pegasus-build-env/ubuntu1804/Dockerfile index c1e53891af..667b64120e 100644 --- a/docker/pegasus-build-env/ubuntu1804/Dockerfile +++ b/docker/pegasus-build-env/ubuntu1804/Dockerfile @@ -50,6 +50,8 @@ RUN apt-get update -y; \ libtool \ libssl-dev \ bison \ + libkrb5-dev \ + libsasl2-dev \ maven \ flex \ python3-setuptools; \ diff --git a/docker/pegasus-build-env/ubuntu2004/Dockerfile b/docker/pegasus-build-env/ubuntu2004/Dockerfile index 6438fb03bf..604b5a0b1e 100644 --- a/docker/pegasus-build-env/ubuntu2004/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2004/Dockerfile @@ -50,6 +50,8 @@ RUN apt-get update -y; \ libtool \ libssl-dev \ bison \ + libkrb5-dev \ + libsasl2-dev \ maven \ flex; \ rm -rf /var/lib/apt/lists/* diff --git a/docker/pegasus-build-env/ubuntu2204/Dockerfile b/docker/pegasus-build-env/ubuntu2204/Dockerfile index bab697c786..0eed16c6cf 100644 --- a/docker/pegasus-build-env/ubuntu2204/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2204/Dockerfile @@ -51,6 +51,8 @@ RUN apt-get update -y; \ libtool \ libssl-dev \ bison \ + libkrb5-dev \ + libsasl2-dev \ maven \ flex; \ rm -rf /var/lib/apt/lists/* diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile index ab91710632..36f75fe9c5 100644 --- a/docker/thirdparties-bin/Dockerfile +++ b/docker/thirdparties-bin/Dockerfile @@ -26,7 +26,7 @@ COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip ARG GITHUB_BRANCH=master ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git -ARG ROCKSDB_PORTABLE=OFF +ARG ROCKSDB_PORTABLE=native ARG USE_JEMALLOC=OFF ARG HADOOP_BIN_PATH ARG ZOOKEEPER_BIN_PATH diff --git a/docker/thirdparties-src/Dockerfile b/docker/thirdparties-src/Dockerfile index 48070659e6..a11538e2e6 100644 --- a/docker/thirdparties-src/Dockerfile +++ b/docker/thirdparties-src/Dockerfile @@ -23,9 +23,10 @@ ARG GITHUB_BRANCH=master ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} +ARG ROCKSDB_PORTABLE=native RUN cd incubator-pegasus/thirdparty \ && mkdir -p build \ - && cmake -DCMAKE_BUILD_TYPE=Release -B build/ . \ + && cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=${ROCKSDB_PORTABLE} -B build/ . \ && cmake --build build/ -j $(($(nproc)/2+1)) RUN cd incubator-pegasus/thirdparty \ diff --git a/run.sh b/run.sh index 75b688a8ae..2672094f7c 100755 --- a/run.sh +++ b/run.sh @@ -26,6 +26,8 @@ export BUILD_LATEST_DIR=${BUILD_ROOT_DIR}/latest export REPORT_DIR="$ROOT/test_report" export THIRDPARTY_ROOT=$ROOT/thirdparty export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:${BUILD_LATEST_DIR}/output/lib:${THIRDPARTY_ROOT}/output/lib:${LD_LIBRARY_PATH} +# Disable AddressSanitizerOneDefinitionRuleViolation, see https://github.com/google/sanitizers/issues/1017 for details. +export ASAN_OPTIONS=detect_odr_violation=0 function usage() { @@ -118,7 +120,7 @@ function run_build() ENABLE_GPERF=ON SKIP_THIRDPARTY=NO SANITIZER="" - ROCKSDB_PORTABLE=OFF + ROCKSDB_PORTABLE=0 USE_JEMALLOC=OFF BUILD_TEST=OFF IWYU="" @@ -177,7 +179,7 @@ function run_build() SKIP_THIRDPARTY=YES ;; --enable_rocksdb_portable) - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 ;; --use_jemalloc) ENABLE_GPERF=OFF @@ -435,10 +437,10 @@ function run_test() echo "test_modules=$test_modules" # download bulk load test data - if [[ "$test_modules" =~ "bulk_load_test" && ! -d "$ROOT/src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files" ]]; then + if [[ "$test_modules" =~ "bulk_load_test" && ! -d "$ROOT/src/test/function_test/bulk_load/pegasus-bulk-load-function-test-files" ]]; then echo "Start to download files used for bulk load function test" wget "https://github.com/XiaoMi/pegasus-common/releases/download/deps/pegasus-bulk-load-function-test-files.zip" - unzip "pegasus-bulk-load-function-test-files.zip" -d "$ROOT/src/test/function_test/bulk_load_test" + unzip "pegasus-bulk-load-function-test-files.zip" -d "$ROOT/src/test/function_test/bulk_load" rm "pegasus-bulk-load-function-test-files.zip" echo "Prepare files used for bulk load function test succeed" fi @@ -482,7 +484,7 @@ function run_test() run_stop_zk run_start_zk fi - pushd ${BUILD_LATEST_DIR}/bin/$module + pushd ${BUILD_LATEST_DIR}/bin/${module} REPORT_DIR=${REPORT_DIR} TEST_BIN=${module} TEST_OPTS=${test_opts} ./run.sh if [ $? != 0 ]; then echo "run test \"$module\" in `pwd` failed" diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh index 391f7822f5..d80d6437e2 100755 --- a/scripts/pack_server.sh +++ b/scripts/pack_server.sh @@ -125,8 +125,6 @@ fi copy_file ./thirdparty/output/lib/libboost*.so.1.69.0 ${pack}/bin copy_file ./thirdparty/output/lib/libhdfs* ${pack}/bin -copy_file ./thirdparty/output/lib/libsasl*.so.* ${pack}/bin -copy_file ./thirdparty/output/lib/libcom_err*.so.* ${pack}/bin copy_file ./scripts/sendmail.sh ${pack}/bin copy_file ./src/server/config.ini ${pack}/bin copy_file ./src/server/config.min.ini ${pack}/bin diff --git a/src/aio/CMakeLists.txt b/src/aio/CMakeLists.txt index 45d24cf687..3754361d06 100644 --- a/src/aio/CMakeLists.txt +++ b/src/aio/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime) +set(MY_PROJ_LIBS dsn_runtime rocksdb) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/aio/aio_provider.h b/src/aio/aio_provider.h index 73848d87be..74fb410bd7 100644 --- a/src/aio/aio_provider.h +++ b/src/aio/aio_provider.h @@ -27,10 +27,17 @@ #pragma once #include +#include +#include #include "utils/error_code.h" #include "utils/factory_store.h" +namespace rocksdb { +class RandomAccessFile; +class RandomRWFile; +} // namespace rocksdb + namespace dsn { class aio_context; @@ -60,12 +67,13 @@ class aio_provider explicit aio_provider(disk_engine *disk); virtual ~aio_provider() = default; - virtual linux_fd_t open(const char *file_name, int flag, int pmode) = 0; + virtual std::unique_ptr open_read_file(const std::string &fname) = 0; + virtual error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0; - virtual error_code close(linux_fd_t fd) = 0; - virtual error_code flush(linux_fd_t fd) = 0; + virtual std::unique_ptr open_write_file(const std::string &fname) = 0; virtual error_code write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0; - virtual error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0; + virtual error_code flush(rocksdb::RandomRWFile *rwf) = 0; + virtual error_code close(rocksdb::RandomRWFile *rwf) = 0; // Submits the aio_task to the underlying disk-io executor. // This task may not be executed immediately, call `aio_task::wait` diff --git a/src/aio/disk_engine.cpp b/src/aio/disk_engine.cpp index 82748de500..1b104be301 100644 --- a/src/aio/disk_engine.cpp +++ b/src/aio/disk_engine.cpp @@ -26,7 +26,6 @@ #include "disk_engine.h" -#include #include // IWYU pragma: no_include #include @@ -103,22 +102,26 @@ aio_task *disk_write_queue::unlink_next_workload(void *plength) return first; } -disk_file::disk_file(linux_fd_t fd) : _fd(fd) {} +disk_file::disk_file(std::unique_ptr rf) : _read_file(std::move(rf)) {} +disk_file::disk_file(std::unique_ptr wf) : _write_file(std::move(wf)) {} aio_task *disk_file::read(aio_task *tsk) { + CHECK(_read_file, ""); tsk->add_ref(); // release on completion, see `on_read_completed`. return _read_queue.add_work(tsk, nullptr); } aio_task *disk_file::write(aio_task *tsk, void *ctx) { + CHECK(_write_file, ""); tsk->add_ref(); // release on completion return _write_queue.add_work(tsk, ctx); } aio_task *disk_file::on_read_completed(aio_task *wk, error_code err, size_t size) { + CHECK(_read_file, ""); CHECK(wk->next == nullptr, ""); auto ret = _read_queue.on_work_completed(wk, nullptr); wk->enqueue(err, size); @@ -129,6 +132,7 @@ aio_task *disk_file::on_read_completed(aio_task *wk, error_code err, size_t size aio_task *disk_file::on_write_completed(aio_task *wk, void *ctx, error_code err, size_t size) { + CHECK(_write_file, ""); auto ret = _write_queue.on_work_completed(wk, ctx); while (wk) { diff --git a/src/aio/disk_engine.h b/src/aio/disk_engine.h index 8eb7846359..70cce466a1 100644 --- a/src/aio/disk_engine.h +++ b/src/aio/disk_engine.h @@ -32,6 +32,7 @@ #include "aio/aio_task.h" #include "aio_provider.h" +#include "rocksdb/env.h" #include "utils/singleton.h" #include "utils/work_queue.h" @@ -56,17 +57,21 @@ class disk_write_queue : public work_queue class disk_file { public: - explicit disk_file(linux_fd_t fd); + explicit disk_file(std::unique_ptr rf); + explicit disk_file(std::unique_ptr wf); aio_task *read(aio_task *tsk); aio_task *write(aio_task *tsk, void *ctx); aio_task *on_read_completed(aio_task *wk, error_code err, size_t size); aio_task *on_write_completed(aio_task *wk, void *ctx, error_code err, size_t size); - linux_fd_t native_handle() const { return _fd; } + rocksdb::RandomAccessFile *rfile() const { return _read_file.get(); } + rocksdb::RandomRWFile *wfile() const { return _write_file.get(); } private: - linux_fd_t _fd; + // TODO(yingchun): unify to use a single RandomRWFile member variable. + std::unique_ptr _read_file; + std::unique_ptr _write_file; disk_write_queue _write_queue; work_queue _read_queue; }; diff --git a/src/aio/file_io.cpp b/src/aio/file_io.cpp index 4f3c10bb54..a4bf26ba85 100644 --- a/src/aio/file_io.cpp +++ b/src/aio/file_io.cpp @@ -26,32 +26,51 @@ #include "aio/file_io.h" +#include // IWYU pragma: no_include #include #include "aio/aio_provider.h" #include "disk_engine.h" +#include "rocksdb/env.h" +#include "utils/fmt_logging.h" namespace dsn { class task_tracker; namespace file { -/*extern*/ disk_file *open(const char *file_name, int flag, int pmode) +/*extern*/ disk_file *open(const std::string &fname, FileOpenType type) { - auto fd = disk_engine::provider().open(file_name, flag, pmode); - if (fd.is_invalid()) { - return nullptr; + switch (type) { + case FileOpenType::kReadOnly: { + auto sf = disk_engine::provider().open_read_file(fname); + if (!sf) { + return nullptr; + } + return new disk_file(std::move(sf)); } - - return new disk_file(fd); + case FileOpenType::kWriteOnly: { + auto wf = disk_engine::provider().open_write_file(fname); + if (!wf) { + return nullptr; + } + return new disk_file(std::move(wf)); + } + default: + CHECK(false, ""); + } + return nullptr; } /*extern*/ error_code close(disk_file *file) { - error_code result = ERR_INVALID_HANDLE; + error_code result = ERR_OK; if (file != nullptr) { - result = disk_engine::provider().close(file->native_handle()); + // A read file is not needed to close. + if (file->wfile()) { + result = disk_engine::provider().close(file->wfile()); + } delete file; file = nullptr; } @@ -60,11 +79,11 @@ namespace file { /*extern*/ error_code flush(disk_file *file) { - if (nullptr != file) { - return disk_engine::provider().flush(file->native_handle()); - } else { + if (file == nullptr || file->wfile() == nullptr) { return ERR_INVALID_HANDLE; } + + return disk_engine::provider().flush(file->wfile()); } /*extern*/ aio_task_ptr read(disk_file *file, @@ -84,7 +103,8 @@ namespace file { cb->get_aio_context()->engine = &disk_engine::instance(); cb->get_aio_context()->dfile = file; - if (!cb->spec().on_aio_call.execute(task::get_current_task(), cb, true)) { + if (!cb->spec().on_aio_call.execute(task::get_current_task(), cb, true) || + file->rfile() == nullptr) { cb->enqueue(ERR_FILE_OPERATION_FAILED, 0); return cb; } @@ -110,6 +130,10 @@ namespace file { cb->get_aio_context()->file_offset = offset; cb->get_aio_context()->type = AIO_Write; cb->get_aio_context()->dfile = file; + if (file->wfile() == nullptr) { + cb->enqueue(ERR_FILE_OPERATION_FAILED, 0); + return cb; + } disk_engine::instance().write(cb); return cb; diff --git a/src/aio/file_io.h b/src/aio/file_io.h index f0b6ffc420..0cef3f1b80 100644 --- a/src/aio/file_io.h +++ b/src/aio/file_io.h @@ -28,6 +28,7 @@ #include #include +#include #include #include "aio/aio_task.h" @@ -47,6 +48,13 @@ class task_tracker; namespace file { +enum class FileOpenType +{ + kReadOnly = 0, + kWriteOnly +}; + +// TODO(yingchun): consider to return a smart pointer /// open file /// /// \param file_name filename of the file. @@ -55,7 +63,7 @@ namespace file { /// /// \return file handle /// -extern disk_file *open(const char *file_name, int flag, int pmode); +extern disk_file *open(const std::string &fname, FileOpenType type); /// close the file handle extern error_code close(disk_file *file); diff --git a/src/aio/native_linux_aio_provider.cpp b/src/aio/native_linux_aio_provider.cpp index 470f3d63ac..7270dff733 100644 --- a/src/aio/native_linux_aio_provider.cpp +++ b/src/aio/native_linux_aio_provider.cpp @@ -26,21 +26,17 @@ #include "native_linux_aio_provider.h" -#include -#include -#include -#include - #include "aio/aio_provider.h" #include "aio/disk_engine.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/service_engine.h" #include "runtime/task/async_calls.h" -#include "utils/fail_point.h" +#include "utils/env.h" #include "utils/fmt_logging.h" #include "utils/latency_tracer.h" #include "utils/ports.h" -#include "utils/safe_strerror_posix.h" -#include "utils/string_view.h" namespace dsn { @@ -48,94 +44,101 @@ native_linux_aio_provider::native_linux_aio_provider(disk_engine *disk) : aio_pr native_linux_aio_provider::~native_linux_aio_provider() {} -linux_fd_t native_linux_aio_provider::open(const char *file_name, int flag, int pmode) +std::unique_ptr +native_linux_aio_provider::open_read_file(const std::string &fname) { - auto fd = ::open(file_name, flag, pmode); - if (fd == DSN_INVALID_FILE_HANDLE) { - LOG_ERROR("create file '{}' failed, err = {}", file_name, utils::safe_strerror(errno)); + std::unique_ptr rfile; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewRandomAccessFile(fname, &rfile, rocksdb::EnvOptions()); + if (!s.ok()) { + LOG_ERROR("open read file '{}' failed, err = {}", fname, s.ToString()); } - return linux_fd_t(fd); + return rfile; } -error_code native_linux_aio_provider::close(linux_fd_t fd) +std::unique_ptr +native_linux_aio_provider::open_write_file(const std::string &fname) { - if (fd.is_invalid() || ::close(fd.fd) == 0) { - return ERR_OK; + // rocksdb::NewRandomRWFile() doesn't act as the docs described, it will not create the + // file if it not exists, and an error Status will be returned, so we try to create the + // file by ReopenWritableFile() if it not exist. + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive)->FileExists(fname); + if (!s.ok() && !s.IsNotFound()) { + LOG_ERROR("failed to check whether the file '{}' exist, err = {}", fname, s.ToString()); + return nullptr; + } + + if (s.IsNotFound()) { + std::unique_ptr cfile; + s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->ReopenWritableFile(fname, &cfile, rocksdb::EnvOptions()); + if (!s.ok()) { + LOG_ERROR("failed to create file '{}', err = {}", fname, s.ToString()); + return nullptr; + } } - LOG_ERROR("close file failed, err = {}", utils::safe_strerror(errno)); - return ERR_FILE_OPERATION_FAILED; + // Open the file for write as RandomRWFile, to support un-sequential write. + std::unique_ptr wfile; + s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewRandomRWFile(fname, &wfile, rocksdb::EnvOptions()); + if (!s.ok()) { + LOG_ERROR("open write file '{}' failed, err = {}", fname, s.ToString()); + } + return wfile; } -error_code native_linux_aio_provider::flush(linux_fd_t fd) +error_code native_linux_aio_provider::close(rocksdb::RandomRWFile *wf) { - if (fd.is_invalid() || ::fsync(fd.fd) == 0) { - return ERR_OK; + auto s = wf->Close(); + if (!s.ok()) { + LOG_ERROR("close file failed, err = {}", s.ToString()); + return ERR_FILE_OPERATION_FAILED; } - LOG_ERROR("flush file failed, err = {}", utils::safe_strerror(errno)); - return ERR_FILE_OPERATION_FAILED; + return ERR_OK; +} + +error_code native_linux_aio_provider::flush(rocksdb::RandomRWFile *wf) +{ + auto s = wf->Fsync(); + if (!s.ok()) { + LOG_ERROR("flush file failed, err = {}", s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } + + return ERR_OK; } error_code native_linux_aio_provider::write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) { - dsn::error_code resp = ERR_OK; - uint64_t buffer_offset = 0; - do { - // ret is the written data size - auto ret = ::pwrite(aio_ctx.dfile->native_handle().fd, - (char *)aio_ctx.buffer + buffer_offset, - aio_ctx.buffer_size - buffer_offset, - aio_ctx.file_offset + buffer_offset); - if (dsn_unlikely(ret < 0)) { - if (errno == EINTR) { - LOG_WARNING("write failed with errno={} and will retry it.", - utils::safe_strerror(errno)); - continue; - } - resp = ERR_FILE_OPERATION_FAILED; - LOG_ERROR("write failed with errno={}, return {}.", utils::safe_strerror(errno), resp); - return resp; - } - - // mock the `ret` to reproduce the `write incomplete` case in the first write - FAIL_POINT_INJECT_NOT_RETURN_F("aio_pwrite_incomplete", [&](string_view s) -> void { - if (dsn_unlikely(buffer_offset == 0)) { - --ret; - } - }); - - buffer_offset += ret; - if (dsn_unlikely(buffer_offset != aio_ctx.buffer_size)) { - LOG_WARNING( - "write incomplete, request_size={}, total_write_size={}, this_write_size={}, " - "and will retry it.", - aio_ctx.buffer_size, - buffer_offset, - ret); - } - } while (dsn_unlikely(buffer_offset < aio_ctx.buffer_size)); + rocksdb::Slice data((const char *)(aio_ctx.buffer), aio_ctx.buffer_size); + auto s = aio_ctx.dfile->wfile()->Write(aio_ctx.file_offset, data); + if (!s.ok()) { + LOG_ERROR("write file failed, err = {}", s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } - *processed_bytes = buffer_offset; - return resp; + *processed_bytes = aio_ctx.buffer_size; + return ERR_OK; } error_code native_linux_aio_provider::read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) { - auto ret = ::pread(aio_ctx.dfile->native_handle().fd, - aio_ctx.buffer, - aio_ctx.buffer_size, - aio_ctx.file_offset); - if (dsn_unlikely(ret < 0)) { - LOG_WARNING("write failed with errno={} and will retry it.", utils::safe_strerror(errno)); + rocksdb::Slice result; + auto s = aio_ctx.dfile->rfile()->Read( + aio_ctx.file_offset, aio_ctx.buffer_size, &result, (char *)(aio_ctx.buffer)); + if (!s.ok()) { + LOG_ERROR("read file failed, err = {}", s.ToString()); return ERR_FILE_OPERATION_FAILED; } - if (ret == 0) { + + if (result.empty()) { return ERR_HANDLE_EOF; } - *processed_bytes = static_cast(ret); + *processed_bytes = result.size(); return ERR_OK; } diff --git a/src/aio/native_linux_aio_provider.h b/src/aio/native_linux_aio_provider.h index bdb1339b9c..538b808dfa 100644 --- a/src/aio/native_linux_aio_provider.h +++ b/src/aio/native_linux_aio_provider.h @@ -27,11 +27,18 @@ #pragma once #include +#include +#include #include "aio/aio_task.h" #include "aio_provider.h" #include "utils/error_code.h" +namespace rocksdb { +class RandomAccessFile; +class RandomRWFile; +} // namespace rocksdb + namespace dsn { class disk_engine; @@ -41,16 +48,18 @@ class native_linux_aio_provider : public aio_provider explicit native_linux_aio_provider(disk_engine *disk); ~native_linux_aio_provider() override; - linux_fd_t open(const char *file_name, int flag, int pmode) override; - error_code close(linux_fd_t fd) override; - error_code flush(linux_fd_t fd) override; - error_code write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) override; + std::unique_ptr open_read_file(const std::string &fname) override; error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) override; + std::unique_ptr open_write_file(const std::string &fname) override; + error_code write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) override; + error_code flush(rocksdb::RandomRWFile *wf) override; + error_code close(rocksdb::RandomRWFile *wf) override; + void submit_aio_task(aio_task *aio) override; aio_context *prepare_aio_context(aio_task *tsk) override { return new aio_context; } -protected: +private: error_code aio_internal(aio_task *aio); }; diff --git a/src/aio/test/CMakeLists.txt b/src/aio/test/CMakeLists.txt index 4228a01481..357499a9c8 100644 --- a/src/aio/test/CMakeLists.txt +++ b/src/aio/test/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio) +set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio test_utils rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/aio/test/aio.cpp b/src/aio/test/aio.cpp index 7037996526..fa6b0114ae 100644 --- a/src/aio/test/aio.cpp +++ b/src/aio/test/aio.cpp @@ -24,156 +24,291 @@ * THE SOFTWARE. */ -#include -#include +#include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include +#include #include +#include +#include #include #include +#include #include +#include #include "aio/aio_task.h" #include "aio/file_io.h" #include "runtime/task/task_code.h" #include "runtime/tool_api.h" +#include "test_util/test_util.h" #include "utils/autoref_ptr.h" +#include "utils/env.h" #include "utils/error_code.h" -#include "utils/fail_point.h" #include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" -#include "utils/strings.h" +#include "utils/test_macros.h" #include "utils/threadpool_code.h" -namespace dsn { -class disk_file; -} // namespace dsn +DSN_DEFINE_uint32(aio_test, + op_buffer_size, + 12, + "The buffer size of each aio read or write operation for the aio_test.basic"); +DSN_DEFINE_uint32(aio_test, + total_op_count, + 100, + "The total count of read or write operations for the aio_test.basic"); +DSN_DEFINE_uint32( + aio_test, + op_count_per_batch, + 10, + "The operation count of per read or write batch operation for the aio_test.basic"); using namespace ::dsn; DEFINE_THREAD_POOL_CODE(THREAD_POOL_TEST_SERVER) DEFINE_TASK_CODE_AIO(LPC_AIO_TEST, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER); -TEST(core, aio) +class aio_test : public pegasus::encrypt_data_test_base { - fail::setup(); - fail::cfg("aio_pwrite_incomplete", "void()"); - const char *buffer = "hello, world"; - int len = (int)strlen(buffer); +public: + void SetUp() override { utils::filesystem::remove_path(kTestFileName); } + void TearDown() override { utils::filesystem::remove_path(kTestFileName); } - // write - auto fp = file::open("tmp", O_RDWR | O_CREAT | O_BINARY, 0666); + const std::string kTestFileName = "aio_test.txt"; +}; - std::list tasks; - uint64_t offset = 0; +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, aio_test, ::testing::Values(false)); - // new write - for (int i = 0; i < 100; i++) { - auto t = ::dsn::file::write(fp, buffer, len, offset, LPC_AIO_TEST, nullptr, nullptr); - tasks.push_back(t); - offset += len; - } +TEST_P(aio_test, basic) +{ + const size_t kUnitBufferLength = FLAGS_op_buffer_size; + const std::string kUnitBuffer(kUnitBufferLength, 'x'); + const int kTotalBufferCount = FLAGS_total_op_count; + const int kBufferCountPerBatch = FLAGS_op_count_per_batch; + const int64_t kFileSize = kUnitBufferLength * kTotalBufferCount; + ASSERT_EQ(0, kTotalBufferCount % kBufferCountPerBatch); + + auto check_callback = [kUnitBufferLength](::dsn::error_code err, size_t n) { + // Use CHECK_* instead of ASSERT_* to exit the tests immediately when error occurs. + CHECK_EQ(ERR_OK, err); + CHECK_EQ(kUnitBufferLength, n); + }; + auto verify_data = [=]() { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + kTestFileName, dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(kFileSize, file_size); + + // Create a read file handler. + auto rfile = file::open(kTestFileName, file::FileOpenType::kReadOnly); + ASSERT_NE(rfile, nullptr); + + // 1. Check sequential read. + { + pegasus::stop_watch sw; + uint64_t offset = 0; + std::list tasks; + for (int i = 0; i < kTotalBufferCount; i++) { + char read_buffer[kUnitBufferLength + 1]; + read_buffer[kUnitBufferLength] = 0; + auto t = ::dsn::file::read(rfile, + read_buffer, + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + ASSERT_STREQ(kUnitBuffer.c_str(), read_buffer); + } + sw.stop_and_output(fmt::format("sequential read")); + } - for (auto &t : tasks) { - t->wait(); - } + // 2. Check concurrent read. + { + pegasus::stop_watch sw; + uint64_t offset = 0; + std::list tasks; + char read_buffers[kTotalBufferCount][kUnitBufferLength + 1]; + for (int i = 0; i < kTotalBufferCount; i++) { + read_buffers[i][kUnitBufferLength] = 0; + auto t = ::dsn::file::read(rfile, + read_buffers[i], + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + for (int i = 0; i < kTotalBufferCount; i++) { + ASSERT_STREQ(kUnitBuffer.c_str(), read_buffers[i]); + } + sw.stop_and_output(fmt::format("concurrent read")); + } + ASSERT_EQ(ERR_OK, file::close(rfile)); + }; - // overwrite - offset = 0; - tasks.clear(); - for (int i = 0; i < 100; i++) { - auto t = ::dsn::file::write(fp, buffer, len, offset, LPC_AIO_TEST, nullptr, nullptr); - tasks.push_back(t); - offset += len; + // 1. Sequential write. + { + pegasus::stop_watch sw; + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + + uint64_t offset = 0; + std::list tasks; + for (int i = 0; i < kTotalBufferCount; i++) { + auto t = ::dsn::file::write(wfile, + kUnitBuffer.c_str(), + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); + sw.stop_and_output(fmt::format("sequential write")); } + NO_FATALS(verify_data()); + + // 2. Un-sequential write. + { + pegasus::stop_watch sw; + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + + std::vector offsets; + offsets.reserve(kTotalBufferCount); + for (int i = 0; i < kTotalBufferCount; i++) { + offsets.push_back(i * kUnitBufferLength); + } - for (auto &t : tasks) { - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == (size_t)len); - } + std::random_device rd; + std::mt19937 gen(rd()); + std::shuffle(offsets.begin(), offsets.end(), gen); - // vector write - tasks.clear(); - std::unique_ptr buffers(new dsn_file_buffer_t[100]); - for (int i = 0; i < 10; i++) { - buffers[i].buffer = static_cast(const_cast(buffer)); - buffers[i].size = len; - } - for (int i = 0; i < 10; i++) { - tasks.push_back(::dsn::file::write_vector( - fp, buffers.get(), 10, offset, LPC_AIO_TEST, nullptr, nullptr)); - offset += 10 * len; - } - for (auto &t : tasks) { - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == 10 * len); - } - auto err = file::close(fp); - EXPECT_TRUE(err == ERR_OK); - - // read - char *buffer2 = (char *)alloca((size_t)len); - fp = file::open("tmp", O_RDONLY | O_BINARY, 0); - - // concurrent read - offset = 0; - tasks.clear(); - for (int i = 0; i < 100; i++) { - auto t = ::dsn::file::read(fp, buffer2, len, offset, LPC_AIO_TEST, nullptr, nullptr); - tasks.push_back(t); - offset += len; + std::list tasks; + for (const auto &offset : offsets) { + auto t = ::dsn::file::write(wfile, + kUnitBuffer.c_str(), + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); + sw.stop_and_output(fmt::format("un-sequential write")); } - - for (auto &t : tasks) { - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == (size_t)len); + NO_FATALS(verify_data()); + + // 3. Overwrite. + { + pegasus::stop_watch sw; + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + + uint64_t offset = 0; + std::list tasks; + for (int i = 0; i < kTotalBufferCount; i++) { + auto t = ::dsn::file::write(wfile, + kUnitBuffer.c_str(), + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); + sw.stop_and_output(fmt::format("overwrite")); } - - // sequential read - offset = 0; - tasks.clear(); - for (int i = 0; i < 200; i++) { - buffer2[0] = 'x'; - auto t = ::dsn::file::read(fp, buffer2, len, offset, LPC_AIO_TEST, nullptr, nullptr); - offset += len; - - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == (size_t)len); - EXPECT_TRUE(dsn::utils::mequals(buffer, buffer2, len)); + NO_FATALS(verify_data()); + + // 4. Vector write. + { + pegasus::stop_watch sw; + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + + uint64_t offset = 0; + std::list tasks; + std::unique_ptr buffers(new dsn_file_buffer_t[kBufferCountPerBatch]); + for (int i = 0; i < kBufferCountPerBatch; i++) { + buffers[i].buffer = static_cast(const_cast(kUnitBuffer.c_str())); + buffers[i].size = kUnitBufferLength; + } + for (int i = 0; i < kTotalBufferCount / kBufferCountPerBatch; i++) { + tasks.push_back( + ::dsn::file::write_vector(wfile, + buffers.get(), + kBufferCountPerBatch, + offset, + LPC_AIO_TEST, + nullptr, + [=](::dsn::error_code err, size_t n) { + CHECK_EQ(ERR_OK, err); + CHECK_EQ(kBufferCountPerBatch * kUnitBufferLength, n); + })); + offset += kBufferCountPerBatch * kUnitBufferLength; + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kBufferCountPerBatch * kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); + sw.stop_and_output(fmt::format("vector write")); } - - err = file::close(fp); - fail::teardown(); - EXPECT_TRUE(err == ERR_OK); - - utils::filesystem::remove_path("tmp"); + NO_FATALS(verify_data()); } -TEST(core, aio_share) +TEST_P(aio_test, aio_share) { - auto fp = file::open("tmp", O_WRONLY | O_CREAT | O_BINARY, 0666); - EXPECT_TRUE(fp != nullptr); - - auto fp2 = file::open("tmp", O_RDONLY | O_BINARY, 0); - EXPECT_TRUE(fp2 != nullptr); + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); - file::close(fp); - file::close(fp2); + auto rfile = file::open(kTestFileName, file::FileOpenType::kReadOnly); + ASSERT_NE(rfile, nullptr); - utils::filesystem::remove_path("tmp"); + ASSERT_EQ(ERR_OK, file::close(wfile)); + ASSERT_EQ(ERR_OK, file::close(rfile)); } -TEST(core, operation_failed) +TEST_P(aio_test, operation_failed) { - fail::setup(); - fail::cfg("aio_pwrite_incomplete", "void()"); - - auto fp = file::open("tmp_test_file", O_WRONLY, 0600); - EXPECT_TRUE(fp == nullptr); - auto err = std::make_unique(); auto count = std::make_unique(); auto io_callback = [&err, &count](::dsn::error_code e, size_t n) { @@ -181,39 +316,42 @@ TEST(core, operation_failed) *count = n; }; - fp = file::open("tmp_test_file", O_WRONLY | O_CREAT | O_BINARY, 0666); - EXPECT_TRUE(fp != nullptr); - char buffer[512]; - const char *str = "hello file"; - auto t = ::dsn::file::write(fp, str, strlen(str), 0, LPC_AIO_TEST, nullptr, io_callback, 0); + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + + char buff[512] = {0}; + const char *kUnitBuffer = "hello file"; + const size_t kUnitBufferLength = strlen(kUnitBuffer); + auto t = ::dsn::file::write( + wfile, kUnitBuffer, kUnitBufferLength, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_OK && *count == strlen(str)); + ASSERT_EQ(ERR_OK, *err); + ASSERT_EQ(kUnitBufferLength, *count); - t = ::dsn::file::read(fp, buffer, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(wfile, buff, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_FILE_OPERATION_FAILED); + ASSERT_EQ(ERR_FILE_OPERATION_FAILED, *err); - auto fp2 = file::open("tmp_test_file", O_RDONLY | O_BINARY, 0); - EXPECT_TRUE(fp2 != nullptr); + auto rfile = file::open(kTestFileName, file::FileOpenType::kReadOnly); + ASSERT_NE(nullptr, rfile); - t = ::dsn::file::read(fp2, buffer, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(rfile, buff, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_OK && *count == strlen(str)); - EXPECT_TRUE(dsn::utils::equals(buffer, str, 10)); + ASSERT_EQ(ERR_OK, *err); + ASSERT_EQ(kUnitBufferLength, *count); + ASSERT_STREQ(kUnitBuffer, buff); - t = ::dsn::file::read(fp2, buffer, 5, 0, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(rfile, buff, 5, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_OK && *count == 5); - EXPECT_TRUE(dsn::utils::equals(buffer, str, 5)); + ASSERT_EQ(ERR_OK, *err); + ASSERT_EQ(5, *count); + ASSERT_STREQ(kUnitBuffer, buff); - t = ::dsn::file::read(fp2, buffer, 512, 100, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(rfile, buff, 512, 100, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - LOG_INFO("error code: {}", *err); - file::close(fp); - file::close(fp2); - fail::teardown(); - - EXPECT_TRUE(utils::filesystem::remove_path("tmp_test_file")); + ASSERT_EQ(ERR_HANDLE_EOF, *err); + ASSERT_EQ(ERR_OK, file::close(wfile)); + ASSERT_EQ(ERR_OK, file::close(rfile)); } DEFINE_TASK_CODE_AIO(LPC_AIO_TEST_READ, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) @@ -223,22 +361,39 @@ struct aio_result dsn::error_code err; size_t sz; }; -TEST(core, dsn_file) + +TEST_P(aio_test, dsn_file) { - int64_t fin_size, fout_size; - ASSERT_TRUE(utils::filesystem::file_size("copy_source.txt", fin_size)); - ASSERT_LT(0, fin_size); + std::string src_file = "copy_source.txt"; + std::string dst_file = "copy_dest.txt"; + if (FLAGS_encrypt_data_at_rest) { + auto s = dsn::utils::encrypt_file(src_file, src_file + ".encrypted"); + ASSERT_TRUE(s.ok()) << s.ToString(); + src_file += ".encrypted"; + + s = dsn::utils::encrypt_file(dst_file, dst_file + ".encrypted"); + ASSERT_TRUE(s.ok()) << s.ToString(); + dst_file += ".encrypted"; + } + + int64_t src_file_size; + ASSERT_TRUE(utils::filesystem::file_size( + src_file, dsn::utils::FileDataType::kSensitive, src_file_size)); + ASSERT_LT(0, src_file_size); + std::string src_file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(src_file, src_file_md5)); + ASSERT_FALSE(src_file_md5.empty()); - dsn::disk_file *fin = file::open("copy_source.txt", O_RDONLY, 0); + auto fin = file::open(src_file, file::FileOpenType::kReadOnly); ASSERT_NE(nullptr, fin); - dsn::disk_file *fout = file::open("copy_dest.txt", O_RDWR | O_CREAT | O_TRUNC, 0666); + auto fout = file::open(dst_file, file::FileOpenType::kWriteOnly); ASSERT_NE(nullptr, fout); - char buffer[1024]; + char kUnitBuffer[1024]; uint64_t offset = 0; while (true) { aio_result rin; aio_task_ptr tin = file::read(fin, - buffer, + kUnitBuffer, 1024, offset, LPC_AIO_TEST_READ, @@ -270,7 +425,7 @@ TEST(core, dsn_file) aio_result rout; aio_task_ptr tout = file::write(fout, - buffer, + kUnitBuffer, rin.sz, offset, LPC_AIO_TEST_WRITE, @@ -296,10 +451,15 @@ TEST(core, dsn_file) offset += rin.sz; } - ASSERT_EQ((uint64_t)fin_size, offset); + ASSERT_EQ(static_cast(src_file_size), offset); ASSERT_EQ(ERR_OK, file::close(fout)); ASSERT_EQ(ERR_OK, file::close(fin)); - ASSERT_TRUE(utils::filesystem::file_size("copy_dest.txt", fout_size)); - ASSERT_EQ(fin_size, fout_size); + int64_t dst_file_size; + ASSERT_TRUE(utils::filesystem::file_size( + dst_file, dsn::utils::FileDataType::kSensitive, dst_file_size)); + ASSERT_EQ(src_file_size, dst_file_size); + std::string dst_file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(src_file, dst_file_md5)); + ASSERT_EQ(src_file_md5, dst_file_md5); } diff --git a/src/aio/test/config.ini b/src/aio/test/config.ini index 47bc9cf7fb..fd46a38d75 100644 --- a/src/aio/test/config.ini +++ b/src/aio/test/config.ini @@ -43,3 +43,8 @@ tool = nativerun pause_on_start = false logging_start_level = LOG_LEVEL_DEBUG logging_factory_name = dsn::tools::simple_logger + +[aio_test] +op_buffer_size = 12 +total_op_count = 100 +op_count_per_batch = 10 diff --git a/src/base/idl_utils.h b/src/base/idl_utils.h index 5b8d93c8a9..bb04018102 100644 --- a/src/base/idl_utils.h +++ b/src/base/idl_utils.h @@ -22,6 +22,7 @@ #include #include "rrdb/rrdb_types.h" +#include "utils/fmt_utils.h" namespace pegasus { @@ -40,3 +41,11 @@ inline bool cas_is_check_operand_needed(dsn::apps::cas_check_type::type type) } } // namespace pegasus + +namespace dsn { +namespace apps { +USER_DEFINED_ENUM_FORMATTER(cas_check_type::type) +USER_DEFINED_ENUM_FORMATTER(filter_type::type) +USER_DEFINED_ENUM_FORMATTER(mutate_operation::type) +} // namespace apps +} // namespace dsn diff --git a/src/base/pegasus_const.cpp b/src/base/pegasus_const.cpp index c93c0baf58..2a788cd24d 100644 --- a/src/base/pegasus_const.cpp +++ b/src/base/pegasus_const.cpp @@ -19,6 +19,12 @@ #include "pegasus_const.h" +#include +#include +#include + +#include "utils/string_conv.h" + namespace pegasus { // should be same with items in dsn::backup_restore_constant @@ -101,4 +107,47 @@ const std::string USER_SPECIFIED_COMPACTION("user_specified_compaction"); const std::string READ_SIZE_THROTTLING("replica.read_throttling_by_size"); const std::string ROCKSDB_ALLOW_INGEST_BEHIND("rocksdb.allow_ingest_behind"); + +const std::string ROCKSDB_WRITE_BUFFER_SIZE("rocksdb.write_buffer_size"); + +const std::string ROCKSDB_NUM_LEVELS("rocksdb.num_levels"); + +const std::set ROCKSDB_DYNAMIC_OPTIONS = { + ROCKSDB_WRITE_BUFFER_SIZE, +}; +const std::set ROCKSDB_STATIC_OPTIONS = { + ROCKSDB_NUM_LEVELS, +}; + +const std::unordered_map cf_opts_setters = { + {ROCKSDB_WRITE_BUFFER_SIZE, + [](const std::string &str, rocksdb::ColumnFamilyOptions &option) -> bool { + uint64_t val = 0; + if (!dsn::buf2uint64(str, val)) { + return false; + } + option.write_buffer_size = static_cast(val); + return true; + }}, + {ROCKSDB_NUM_LEVELS, + [](const std::string &str, rocksdb::ColumnFamilyOptions &option) -> bool { + int32_t val = 0; + if (!dsn::buf2int32(str, val)) { + return false; + } + option.num_levels = val; + return true; + }}, +}; + +const std::unordered_map cf_opts_getters = { + {ROCKSDB_WRITE_BUFFER_SIZE, + [](const rocksdb::ColumnFamilyOptions &option, /*out*/ std::string &str) { + str = std::to_string(option.write_buffer_size); + }}, + {ROCKSDB_NUM_LEVELS, + [](const rocksdb::ColumnFamilyOptions &option, /*out*/ std::string &str) { + str = std::to_string(option.num_levels); + }}, +}; } // namespace pegasus diff --git a/src/base/pegasus_const.h b/src/base/pegasus_const.h index c2a47ce519..e9326dfaa9 100644 --- a/src/base/pegasus_const.h +++ b/src/base/pegasus_const.h @@ -19,7 +19,14 @@ #pragma once +#include +#include #include +#include + +namespace rocksdb { +struct ColumnFamilyOptions; +} // namespace rocksdb namespace pegasus { @@ -72,4 +79,19 @@ extern const std::string USER_SPECIFIED_COMPACTION; extern const std::string READ_SIZE_THROTTLING; extern const std::string ROCKSDB_ALLOW_INGEST_BEHIND; + +extern const std::string ROCKSDB_WRITE_BUFFER_SIZE; + +extern const std::string ROCKSDB_NUM_LEVELS; + +extern const std::set ROCKSDB_DYNAMIC_OPTIONS; + +extern const std::set ROCKSDB_STATIC_OPTIONS; + +using cf_opts_setter = std::function; +extern const std::unordered_map cf_opts_setters; + +using cf_opts_getter = + std::function; +extern const std::unordered_map cf_opts_getters; } // namespace pegasus diff --git a/src/base/test/utils_test.cpp b/src/base/test/utils_test.cpp index d9922c2f0d..ad7dcc337c 100644 --- a/src/base/test/utils_test.cpp +++ b/src/base/test/utils_test.cpp @@ -20,7 +20,6 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include diff --git a/src/base/value_field.h b/src/base/value_field.h index ecb2b766a9..eaa287c632 100644 --- a/src/base/value_field.h +++ b/src/base/value_field.h @@ -19,6 +19,8 @@ #pragma once +#include "utils/fmt_utils.h" + namespace pegasus { enum value_field_type @@ -28,6 +30,7 @@ enum value_field_type USER_DATA, FIELD_COUNT, }; +USER_DEFINED_ENUM_FORMATTER(value_field_type) struct value_field { diff --git a/src/block_service/block_service.h b/src/block_service/block_service.h index 96f1416445..d351dcf44a 100644 --- a/src/block_service/block_service.h +++ b/src/block_service/block_service.h @@ -238,8 +238,8 @@ struct upload_request */ struct upload_response { - dsn::error_code err; - uint64_t uploaded_size; + dsn::error_code err = ERR_OK; + uint64_t uploaded_size = 0; }; typedef std::function upload_callback; typedef future_task upload_future; @@ -378,6 +378,8 @@ class block_file : public dsn::ref_counter const write_callback &cb, dsn::task_tracker *tracker = nullptr) = 0; + // TODO(yingchun): it seems every read() will read the whole file, consider to read the whole + // file directly. /** * @brief read * @param req, ref {@link #read_request} diff --git a/src/block_service/directio_writable_file.cpp b/src/block_service/directio_writable_file.cpp deleted file mode 100644 index 2c74ae05d2..0000000000 --- a/src/block_service/directio_writable_file.cpp +++ /dev/null @@ -1,169 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "block_service/directio_writable_file.h" - -#include -#include -#include // posix_memalign -#include -#include // getpagesize -#include -#include -#include - -#include "utils/flags.h" -#include "utils/fmt_logging.h" -#include "utils/safe_strerror_posix.h" - -namespace dsn { -namespace dist { -namespace block_service { - -DSN_DEFINE_uint32(replication, - direct_io_buffer_pages, - 64, - "Number of pages we need to set to direct io buffer"); -DSN_TAG_VARIABLE(direct_io_buffer_pages, FT_MUTABLE); - -DSN_DEFINE_bool(replication, - enable_direct_io, - false, - "Whether to enable direct I/O when download files"); -DSN_TAG_VARIABLE(enable_direct_io, FT_MUTABLE); - -const uint32_t g_page_size = getpagesize(); - -direct_io_writable_file::direct_io_writable_file(const std::string &file_path) - : _file_path(file_path), - _fd(-1), - _file_size(0), - _buffer(nullptr), - _buffer_size(FLAGS_direct_io_buffer_pages * g_page_size), - _offset(0) -{ -} - -direct_io_writable_file::~direct_io_writable_file() -{ - if (!_buffer || _fd < 0) { - return; - } - // Here is an ensurance, users shuold call finalize manually - CHECK_EQ_MSG(_offset, 0, "finalize() should be called before destructor"); - - ::free(_buffer); - CHECK_EQ_MSG( - 0, ::close(_fd), "Failed to close {}, err = {}", _file_path, utils::safe_strerror(errno)); -} - -bool direct_io_writable_file::initialize() -{ - if (posix_memalign(&_buffer, g_page_size, _buffer_size) != 0) { - LOG_ERROR("Allocate memaligned buffer failed, err = {}", utils::safe_strerror(errno)); - return false; - } - - int flag = O_WRONLY | O_TRUNC | O_CREAT; -#if !defined(__APPLE__) - flag |= O_DIRECT; -#endif - // TODO(yingchun): there maybe serious error of the disk driver when these system call failed, - // maybe just terminate the process or mark the disk as failed would be better - _fd = ::open(_file_path.c_str(), flag, S_IRUSR | S_IWUSR | S_IRGRP); - if (_fd < 0) { - LOG_ERROR("Failed to open {} with flag {}, err = {}", - _file_path, - flag, - utils::safe_strerror(errno)); - ::free(_buffer); - _buffer = nullptr; - return false; - } - return true; -} - -bool direct_io_writable_file::finalize() -{ - CHECK(_buffer && _fd >= 0, "Initialize the instance first"); - - if (_offset > 0) { - ssize_t written_bytes = ::write(_fd, _buffer, _buffer_size); - if (dsn_unlikely(written_bytes < 0)) { - LOG_ERROR("Failed to write the last chunk, file_path = {}, err = {}", - _file_path, - utils::safe_strerror(errno)); - return false; - } - // TODO(yingchun): would better to retry - if (dsn_unlikely(written_bytes != _buffer_size)) { - LOG_ERROR("Failed to write the last chunk, file_path = {}, data bytes = {}, written " - "bytes = {}", - _file_path, - _buffer_size, - written_bytes); - return false; - } - _offset = 0; - if (::ftruncate(_fd, _file_size) < 0) { - LOG_ERROR("Failed to truncate {}, err = {}", _file_path, utils::safe_strerror(errno)); - return false; - } - } - return true; -} - -bool direct_io_writable_file::write(const char *s, size_t n) -{ - CHECK(_buffer && _fd >= 0, "Initialize the instance first"); - - uint32_t remaining = n; - while (remaining > 0) { - uint32_t bytes = std::min((_buffer_size - _offset), remaining); - memcpy((char *)_buffer + _offset, s, bytes); - _offset += bytes; - remaining -= bytes; - s += bytes; - // buffer is full, flush to file - if (_offset == _buffer_size) { - ssize_t written_bytes = ::write(_fd, _buffer, _buffer_size); - if (dsn_unlikely(written_bytes < 0)) { - LOG_ERROR("Failed to write chunk, file_path = {}, err = {}", - _file_path, - utils::safe_strerror(errno)); - return false; - } - // TODO(yingchun): would better to retry - if (dsn_unlikely(written_bytes != _buffer_size)) { - LOG_ERROR( - "Failed to write chunk, file_path = {}, data bytes = {}, written bytes = {}", - _file_path, - _buffer_size, - written_bytes); - return false; - } - // reset offset - _offset = 0; - } - } - _file_size += n; - return true; -} - -} // namespace block_service -} // namespace dist -} // namespace dsn diff --git a/src/block_service/hdfs/CMakeLists.txt b/src/block_service/hdfs/CMakeLists.txt index 803e85bec3..2bba8b96bb 100644 --- a/src/block_service/hdfs/CMakeLists.txt +++ b/src/block_service/hdfs/CMakeLists.txt @@ -17,20 +17,16 @@ set(MY_PROJ_NAME dsn.block_service.hdfs) -set(DIRECTIO_SRC - ../directio_writable_file.cpp - ) - #Source files under CURRENT project directory will be automatically included. #You can manually set MY_PROJ_SRC to include source files under other directories. -set(MY_PROJ_SRC "${DIRECTIO_SRC}") +set(MY_PROJ_SRC "") #Search mode for source files under CURRENT project directory ? #"GLOB_RECURSE" for recursive search #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS hdfs) +set(MY_PROJ_LIBS hdfs rocksdb) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/hdfs/hdfs_service.cpp b/src/block_service/hdfs/hdfs_service.cpp index 459108e111..5909f40de0 100644 --- a/src/block_service/hdfs/hdfs_service.cpp +++ b/src/block_service/hdfs/hdfs_service.cpp @@ -17,19 +17,21 @@ #include #include +#include #include -#include #include #include -#include "block_service/directio_writable_file.h" #include "hdfs/hdfs.h" #include "hdfs_service.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/task/async_calls.h" #include "runtime/task/task.h" #include "utils/TokenBucket.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -37,6 +39,8 @@ #include "utils/safe_strerror_posix.h" #include "utils/strings.h" +DSN_DECLARE_bool(enable_direct_io); + struct hdfsBuilder; namespace dsn { @@ -65,8 +69,6 @@ DSN_DEFINE_uint64(replication, "hdfs write batch size, the default value is 64MB"); DSN_TAG_VARIABLE(hdfs_write_batch_size_bytes, FT_MUTABLE); -DSN_DECLARE_bool(enable_direct_io); - hdfs_service::hdfs_service() { _read_token_bucket.reset(new folly::DynamicTokenBucket()); @@ -108,12 +110,12 @@ error_code hdfs_service::create_fs() hdfsBuilderSetNameNode(builder, _hdfs_name_node.c_str()); _fs = hdfsBuilderConnect(builder); if (!_fs) { - LOG_ERROR("Fail to connect hdfs name node {}, error: {}.", + LOG_ERROR("Fail to connect HDFS name node {}, error: {}.", _hdfs_name_node, utils::safe_strerror(errno)); return ERR_FS_INTERNAL; } - LOG_INFO("Succeed to connect hdfs name node {}.", _hdfs_name_node); + LOG_INFO("Succeed to connect HDFS name node {}.", _hdfs_name_node); return ERR_OK; } @@ -122,10 +124,10 @@ void hdfs_service::close() // This method should be carefully called. // Calls to hdfsDisconnect() by individual threads would terminate // all other connections handed out via hdfsConnect() to the same URI. - LOG_INFO("Try to disconnect hdfs."); + LOG_INFO("Try to disconnect HDFS."); int result = hdfsDisconnect(_fs); if (result == -1) { - LOG_ERROR("Fail to disconnect from the hdfs file system, error: {}.", + LOG_ERROR("Fail to disconnect from the HDFS file system, error: {}.", utils::safe_strerror(errno)); } // Even if there is an error, the resources associated with the hdfsFS will be freed. @@ -134,7 +136,7 @@ void hdfs_service::close() std::string hdfs_service::get_hdfs_entry_name(const std::string &hdfs_path) { - // get exact file name from an hdfs path. + // get exact file name from an HDFS path. int pos = hdfs_path.find_last_of("/"); return hdfs_path.substr(pos + 1); } @@ -305,7 +307,7 @@ error_code hdfs_file_object::write_data_in_batches(const char *data, hdfsFile write_file = hdfsOpenFile(_service->get_fs(), file_name().c_str(), O_WRONLY | O_CREAT, 0, 0, 0); if (!write_file) { - LOG_ERROR("Failed to open hdfs file {} for writting, error: {}.", + LOG_ERROR("Failed to open HDFS file {} for writting, error: {}.", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; @@ -323,7 +325,7 @@ error_code hdfs_file_object::write_data_in_batches(const char *data, (void *)(data + cur_pos), static_cast(write_len)); if (num_written_bytes == -1) { - LOG_ERROR("Failed to write hdfs file {}, error: {}.", + LOG_ERROR("Failed to write HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); hdfsCloseFile(_service->get_fs(), write_file); @@ -333,18 +335,18 @@ error_code hdfs_file_object::write_data_in_batches(const char *data, } if (hdfsHFlush(_service->get_fs(), write_file) != 0) { LOG_ERROR( - "Failed to flush hdfs file {}, error: {}.", file_name(), utils::safe_strerror(errno)); + "Failed to flush HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); hdfsCloseFile(_service->get_fs(), write_file); return ERR_FS_INTERNAL; } written_size = cur_pos; if (hdfsCloseFile(_service->get_fs(), write_file) != 0) { LOG_ERROR( - "Failed to close hdfs file {}, error: {}", file_name(), utils::safe_strerror(errno)); + "Failed to close HDFS file {}, error: {}", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; } - LOG_INFO("start to synchronize meta data after successfully wrote data to hdfs"); + LOG_INFO("start to synchronize meta data after successfully wrote data to HDFS"); return get_file_meta(); } @@ -376,23 +378,51 @@ dsn::task_ptr hdfs_file_object::upload(const upload_request &req, add_ref(); auto upload_background = [this, req, t]() { + LOG_INFO("start to upload from '{}' to '{}'", req.input_local_name, file_name()); + upload_response resp; - resp.uploaded_size = 0; - std::ifstream is(req.input_local_name, std::ios::binary | std::ios::in); - if (is.is_open()) { - int64_t file_sz = 0; - dsn::utils::filesystem::file_size(req.input_local_name, file_sz); - std::unique_ptr buffer(new char[file_sz]); - is.read(buffer.get(), file_sz); - is.close(); - resp.err = write_data_in_batches(buffer.get(), file_sz, resp.uploaded_size); - } else { - LOG_ERROR("HDFS upload failed: open local file {} failed when upload to {}, error: {}", - req.input_local_name, - file_name(), - utils::safe_strerror(errno)); - resp.err = dsn::ERR_FILE_OPERATION_FAILED; - } + do { + rocksdb::EnvOptions env_options; + env_options.use_direct_reads = FLAGS_enable_direct_io; + std::unique_ptr rfile; + auto s = rocksdb::Env::Default()->NewSequentialFile( + req.input_local_name, &rfile, env_options); + if (!s.ok()) { + LOG_ERROR( + "open local file '{}' failed, err = {}", req.input_local_name, s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + int64_t file_size; + if (!dsn::utils::filesystem::file_size( + req.input_local_name, dsn::utils::FileDataType::kSensitive, file_size)) { + LOG_ERROR("get size of local file '{}' failed", req.input_local_name); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + rocksdb::Slice result; + char scratch[file_size]; + s = rfile->Read(file_size, &result, scratch); + if (!s.ok()) { + LOG_ERROR( + "read local file '{}' failed, err = {}", req.input_local_name, s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + resp.err = write_data_in_batches(result.data(), result.size(), resp.uploaded_size); + if (resp.err != ERR_OK) { + LOG_ERROR("write data to remote '{}' failed, err = {}", file_name(), resp.err); + break; + } + + LOG_INFO("finish to upload from '{}' to '{}', size = {}", + req.input_local_name, + file_name(), + resp.uploaded_size); + } while (false); t->enqueue_with(resp); release_ref(); }; @@ -417,7 +447,7 @@ error_code hdfs_file_object::read_data_in_batches(uint64_t start_pos, hdfsFile read_file = hdfsOpenFile(_service->get_fs(), file_name().c_str(), O_RDONLY, 0, 0, 0); if (!read_file) { - LOG_ERROR("Failed to open hdfs file {} for reading, error: {}.", + LOG_ERROR("Failed to open HDFS file {} for reading, error: {}.", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; @@ -446,7 +476,7 @@ error_code hdfs_file_object::read_data_in_batches(uint64_t start_pos, cur_pos += num_read_bytes; dst_buf += num_read_bytes; } else if (num_read_bytes == -1) { - LOG_ERROR("Failed to read hdfs file {}, error: {}.", + LOG_ERROR("Failed to read HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); read_success = false; @@ -455,7 +485,7 @@ error_code hdfs_file_object::read_data_in_batches(uint64_t start_pos, } if (hdfsCloseFile(_service->get_fs(), read_file) != 0) { LOG_ERROR( - "Failed to close hdfs file {}, error: {}.", file_name(), utils::safe_strerror(errno)); + "Failed to close HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; } if (read_success) { @@ -504,48 +534,53 @@ dsn::task_ptr hdfs_file_object::download(const download_request &req, auto download_background = [this, req, t]() { download_response resp; resp.downloaded_size = 0; - std::string read_buffer; - size_t read_length = 0; - resp.err = - read_data_in_batches(req.remote_pos, req.remote_length, read_buffer, read_length); - if (resp.err == ERR_OK) { - bool write_succ = false; - if (FLAGS_enable_direct_io) { - auto dio_file = std::make_unique(req.output_local_name); - do { - if (!dio_file->initialize()) { - break; - } - bool wr_ret = dio_file->write(read_buffer.c_str(), read_length); - if (!wr_ret) { - break; - } - if (dio_file->finalize()) { - resp.downloaded_size = read_length; - resp.file_md5 = utils::string_md5(read_buffer.c_str(), read_length); - write_succ = true; - } - } while (0); - } else { - std::ofstream out(req.output_local_name, - std::ios::binary | std::ios::out | std::ios::trunc); - if (out.is_open()) { - out.write(read_buffer.c_str(), read_length); - out.close(); - resp.downloaded_size = read_length; - resp.file_md5 = utils::string_md5(read_buffer.c_str(), read_length); - write_succ = true; - } + resp.err = ERR_OK; + bool write_succ = false; + std::string target_file = req.output_local_name; + do { + LOG_INFO("start to download from '{}' to '{}'", file_name(), target_file); + + std::string read_buffer; + size_t read_length = 0; + resp.err = + read_data_in_batches(req.remote_pos, req.remote_length, read_buffer, read_length); + if (resp.err != ERR_OK) { + LOG_ERROR("read data from remote '{}' failed, err = {}", file_name(), resp.err); + break; } - if (!write_succ) { - LOG_ERROR("HDFS download failed: fail to open localfile {} when download {}, " - "error: {}", - req.output_local_name, - file_name(), - utils::safe_strerror(errno)); - resp.err = ERR_FILE_OPERATION_FAILED; - resp.downloaded_size = 0; + + rocksdb::EnvOptions env_options; + env_options.use_direct_writes = FLAGS_enable_direct_io; + std::unique_ptr wfile; + auto s = rocksdb::Env::Default()->NewWritableFile(target_file, &wfile, env_options); + if (!s.ok()) { + LOG_ERROR("create local file '{}' failed, err = {}", target_file, s.ToString()); + break; + } + + s = wfile->Append(rocksdb::Slice(read_buffer.data(), read_length)); + if (!s.ok()) { + LOG_ERROR("append local file '{}' failed, err = {}", target_file, s.ToString()); + break; + } + + s = wfile->Fsync(); + if (!s.ok()) { + LOG_ERROR("fsync local file '{}' failed, err = {}", target_file, s.ToString()); + break; } + + resp.downloaded_size = read_length; + resp.file_md5 = utils::string_md5(read_buffer.c_str(), read_length); + write_succ = true; + } while (false); + + if (!write_succ) { + LOG_ERROR("HDFS download failed: fail to write local file {} when download {}", + target_file, + file_name()); + resp.err = ERR_FILE_OPERATION_FAILED; + resp.downloaded_size = 0; } t->enqueue_with(resp); release_ref(); diff --git a/src/block_service/local/CMakeLists.txt b/src/block_service/local/CMakeLists.txt index 0886bece9a..9d830f7825 100644 --- a/src/block_service/local/CMakeLists.txt +++ b/src/block_service/local/CMakeLists.txt @@ -26,7 +26,7 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS "") +set(MY_PROJ_LIBS rocksdb) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/local/local_service.cpp b/src/block_service/local/local_service.cpp index 5eb4b3fba4..9175961340 100644 --- a/src/block_service/local/local_service.cpp +++ b/src/block_service/local/local_service.cpp @@ -15,10 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include -#include +#include #include #include #include @@ -26,20 +23,24 @@ #include #include "local_service.h" -#include "nlohmann/detail/macro_scope.hpp" +#include "nlohmann/json.hpp" #include "nlohmann/json_fwd.hpp" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/task/async_calls.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" -#include "utils/defer.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/fail_point.h" #include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" -#include "utils/safe_strerror_posix.h" #include "utils/string_view.h" #include "utils/strings.h" +DSN_DECLARE_bool(enable_direct_io); + namespace dsn { class task_tracker; } // namespace dsn @@ -52,22 +53,13 @@ namespace block_service { DEFINE_TASK_CODE(LPC_LOCAL_SERVICE_CALL, TASK_PRIORITY_COMMON, THREAD_POOL_BLOCK_SERVICE) -struct file_metadata -{ - uint64_t size; - std::string md5; -}; -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(file_metadata, size, md5) - -bool file_metadata_from_json(std::ifstream &fin, file_metadata &fmeta) noexcept +bool file_metadata_from_json(const std::string &data, file_metadata &fmeta) noexcept { - std::string data; - fin >> data; try { nlohmann::json::parse(data).get_to(fmeta); return true; } catch (nlohmann::json::exception &exp) { - LOG_WARNING("decode meta data from json failed: {} [{}]", exp.what(), data); + LOG_WARNING("decode metadata from json failed: {} [{}]", exp.what(), data); return false; } } @@ -199,7 +191,7 @@ dsn::task_ptr local_service::create_file(const create_file_request &req, if (utils::filesystem::file_exists(file_path) && utils::filesystem::file_exists(meta_file_path)) { - LOG_DEBUG("file({}) already exist", file_path); + LOG_INFO("file({}) already exist", file_path); resp.err = f->load_metadata(); } @@ -276,17 +268,17 @@ error_code local_file_object::load_metadata() return ERR_OK; std::string metadata_path = local_service::get_metafile(file_name()); - std::ifstream is(metadata_path, std::ios::in); - if (!is.is_open()) { - LOG_WARNING( - "load meta data from {} failed, err = {}", metadata_path, utils::safe_strerror(errno)); + std::string data; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), metadata_path, &data); + if (!s.ok()) { + LOG_WARNING("read file '{}' failed, err = {}", metadata_path, s.ToString()); return ERR_FS_INTERNAL; } - auto cleanup = dsn::defer([&is]() { is.close(); }); file_metadata meta; - bool ans = file_metadata_from_json(is, meta); + bool ans = file_metadata_from_json(data, meta); if (!ans) { + LOG_WARNING("decode metadata '{}' file content failed", metadata_path); return ERR_FS_INTERNAL; } _size = meta.size; @@ -300,16 +292,16 @@ error_code local_file_object::store_metadata() file_metadata meta; meta.md5 = _md5_value; meta.size = _size; - + std::string data = nlohmann::json(meta).dump(); std::string metadata_path = local_service::get_metafile(file_name()); - std::ofstream os(metadata_path, std::ios::out | std::ios::trunc); - if (!os.is_open()) { - LOG_WARNING( - "store to metadata file {} failed, err={}", metadata_path, utils::safe_strerror(errno)); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(data), + metadata_path, + /* should_sync */ true); + if (!s.ok()) { + LOG_WARNING("store to metadata file {} failed, err={}", metadata_path, s.ToString()); return ERR_FS_INTERNAL; } - auto cleanup = dsn::defer([&os]() { os.close(); }); - os << nlohmann::json(meta); return ERR_OK; } @@ -345,24 +337,36 @@ dsn::task_ptr local_file_object::write(const write_request &req, } if (resp.err == ERR_OK) { - LOG_DEBUG("start write file, file = {}", file_name()); + LOG_INFO("start write file, file = {}", file_name()); + + do { + auto s = rocksdb::WriteStringToFile( + rocksdb::Env::Default(), + rocksdb::Slice(req.buffer.data(), req.buffer.length()), + file_name(), + /* should_sync */ true); + if (!s.ok()) { + LOG_WARNING("write file '{}' failed, err = {}", file_name(), s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; + } - std::ofstream fout(file_name(), std::ifstream::out | std::ifstream::trunc); - if (!fout.is_open()) { - resp.err = ERR_FS_INTERNAL; - } else { - fout.write(req.buffer.data(), req.buffer.length()); resp.written_size = req.buffer.length(); - fout.close(); // Currently we calc the meta data from source data, which save the io bandwidth // a lot, but it is somewhat not correct. _size = resp.written_size; _md5_value = utils::string_md5(req.buffer.data(), req.buffer.length()); + // TODO(yingchun): make store_metadata as a local function, do not depend on the + // member variables (i.e. _size and _md5_value). + auto err = store_metadata(); + if (err != ERR_OK) { + LOG_WARNING("store_metadata failed"); + resp.err = ERR_FS_INTERNAL; + break; + } _has_meta_synced = true; - - store_metadata(); - } + } while (false); } tsk->enqueue_with(resp); release_ref(); @@ -383,38 +387,65 @@ dsn::task_ptr local_file_object::read(const read_request &req, auto read_func = [this, req, tsk]() { read_response resp; - resp.err = ERR_OK; - if (!utils::filesystem::file_exists(file_name()) || - !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { - resp.err = ERR_OBJECT_NOT_FOUND; - } else { - if ((resp.err = load_metadata()) != ERR_OK) { - LOG_WARNING("load meta data of {} failed", file_name()); + do { + if (!utils::filesystem::file_exists(file_name()) || + !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { + LOG_WARNING("data file '{}' or metadata file '{}' not exist", + file_name(), + local_service::get_metafile(file_name())); + resp.err = ERR_OBJECT_NOT_FOUND; + break; + } + + resp.err = load_metadata(); + if (resp.err != ERR_OK) { + LOG_WARNING("load metadata of {} failed", file_name()); + break; + } + + int64_t file_sz = _size; + int64_t total_sz = 0; + if (req.remote_length == -1 || req.remote_length + req.remote_pos > file_sz) { + total_sz = file_sz - req.remote_pos; } else { - int64_t file_sz = _size; - int64_t total_sz = 0; - if (req.remote_length == -1 || req.remote_length + req.remote_pos > file_sz) { - total_sz = file_sz - req.remote_pos; - } else { - total_sz = req.remote_length; - } + total_sz = req.remote_length; + } - LOG_DEBUG("read file({}), size = {}", file_name(), total_sz); - std::string buf; - buf.resize(total_sz + 1); - std::ifstream fin(file_name(), std::ifstream::in); - if (!fin.is_open()) { - resp.err = ERR_FS_INTERNAL; - } else { - fin.seekg(static_cast(req.remote_pos), fin.beg); - fin.read((char *)buf.c_str(), total_sz); - buf[fin.gcount()] = '\0'; - resp.buffer = blob::create_from_bytes(std::move(buf)); - } - fin.close(); + LOG_INFO("start to read file '{}', offset = {}, size = {}", + file_name(), + req.remote_pos, + total_sz); + rocksdb::EnvOptions env_options; + env_options.use_direct_reads = FLAGS_enable_direct_io; + std::unique_ptr sfile; + auto s = rocksdb::Env::Default()->NewSequentialFile(file_name(), &sfile, env_options); + if (!s.ok()) { + LOG_WARNING("open file '{}' failed, err = {}", file_name(), s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; + } + + s = sfile->Skip(req.remote_pos); + if (!s.ok()) { + LOG_WARNING( + "skip '{}' for {} failed, err = {}", file_name(), req.remote_pos, s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; } - } + rocksdb::Slice result; + std::string buf; + buf.resize(total_sz + 1); + s = sfile->Read(total_sz, &result, buf.data()); + if (!s.ok()) { + LOG_WARNING("read file '{}' failed, err = {}", file_name(), s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; + } + + buf[result.size()] = 0; + resp.buffer = blob::create_from_bytes(std::move(buf)); + } while (false); tsk->enqueue_with(resp); release_ref(); }; @@ -432,58 +463,50 @@ dsn::task_ptr local_file_object::upload(const upload_request &req, upload_future_ptr tsk(new upload_future(code, cb, 0)); tsk->set_tracker(tracker); auto upload_file_func = [this, req, tsk]() { - upload_response resp; - resp.err = ERR_OK; - std::ifstream fin(req.input_local_name, std::ios_base::in); - if (!fin.is_open()) { - LOG_WARNING("open source file {} for read failed, err({})", - req.input_local_name, - utils::safe_strerror(errno)); - resp.err = ERR_FILE_OPERATION_FAILED; - } - - utils::filesystem::create_file(file_name()); - std::ofstream fout(file_name(), std::ios_base::out | std::ios_base::trunc); - if (!fout.is_open()) { - LOG_WARNING("open target file {} for write failed, err({})", - file_name(), - utils::safe_strerror(errno)); - resp.err = ERR_FS_INTERNAL; - } + LOG_INFO("start to upload from '{}' to '{}'", req.input_local_name, file_name()); - if (resp.err == ERR_OK) { - LOG_DEBUG("start to transfer from src_file({}) to dst_file({})", - req.input_local_name, - file_name()); - int64_t total_sz = 0; - char buf[max_length] = {'\0'}; - while (!fin.eof()) { - fin.read(buf, max_length); - total_sz += fin.gcount(); - fout.write(buf, fin.gcount()); + upload_response resp; + do { + // Create the directory. + std::string path = dsn::utils::filesystem::remove_file_name(file_name()); + if (!dsn::utils::filesystem::create_directory(path)) { + LOG_WARNING("create directory '{}' failed", path); + resp.err = ERR_FILE_OPERATION_FAILED; + break; } - LOG_DEBUG("finish upload file, file = {}, total_size = {}", file_name(), total_sz); - fout.close(); - fin.close(); - - resp.uploaded_size = static_cast(total_sz); - // calc the md5sum by source file for simplicity - _size = total_sz; - error_code res = utils::filesystem::md5sum(req.input_local_name, _md5_value); - if (res == dsn::ERR_OK) { - _has_meta_synced = true; - store_metadata(); - } else { + uint64_t file_size; + auto s = dsn::utils::copy_file(req.input_local_name, file_name(), &file_size); + if (!s.ok()) { + LOG_WARNING("upload from '{}' to '{}' failed, err = {}", + req.input_local_name, + file_name(), + s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + LOG_INFO("finish to upload from '{}' to '{}', size = {}", + req.input_local_name, + file_name(), + file_size); + + resp.uploaded_size = file_size; + _size = file_size; + auto res = utils::filesystem::md5sum(file_name(), _md5_value); + if (res != dsn::ERR_OK) { + LOG_WARNING("calculate md5sum for '{}' failed", file_name()); resp.err = ERR_FS_INTERNAL; + break; } - } else { - if (fin.is_open()) - fin.close(); - if (fout.is_open()) - fout.close(); - } + auto err = store_metadata(); + if (err != ERR_OK) { + LOG_ERROR("store_metadata failed"); + resp.err = ERR_FS_INTERNAL; + break; + } + _has_meta_synced = true; + } while (false); tsk->enqueue_with(resp); release_ref(); }; @@ -505,65 +528,61 @@ dsn::task_ptr local_file_object::download(const download_request &req, download_response resp; resp.err = ERR_OK; std::string target_file = req.output_local_name; - if (target_file.empty()) { - LOG_ERROR( - "download {} failed, because target name({}) is invalid", file_name(), target_file); - resp.err = ERR_INVALID_PARAMETERS; - } - if (resp.err == ERR_OK && !_has_meta_synced) { - if (!utils::filesystem::file_exists(file_name()) || - !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { - resp.err = ERR_OBJECT_NOT_FOUND; + do { + if (target_file.empty()) { + LOG_WARNING("download {} failed, because target name({}) is invalid", + file_name(), + target_file); + resp.err = ERR_INVALID_PARAMETERS; + break; } - } - if (resp.err == ERR_OK) { - std::ifstream fin(file_name(), std::ifstream::in); - if (!fin.is_open()) { - LOG_ERROR("open block file({}) failed, err({})", - file_name(), - utils::safe_strerror(errno)); - resp.err = ERR_FS_INTERNAL; + if (!_has_meta_synced) { + if (!utils::filesystem::file_exists(file_name()) || + !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { + LOG_WARNING("file '{}' or metadata file '{}' not found", + file_name(), + local_service::get_metafile(file_name())); + resp.err = ERR_OBJECT_NOT_FOUND; + break; + } } - std::ofstream fout(target_file, std::ios_base::out | std::ios_base::trunc); - if (!fout.is_open()) { - if (fin.is_open()) - fin.close(); - LOG_ERROR("open target file({}) failed, err({})", - target_file, - utils::safe_strerror(errno)); + LOG_INFO("start to download from '{}' to '{}'", file_name(), target_file); + + // Create the directory. + std::string path = dsn::utils::filesystem::remove_file_name(file_name()); + if (!dsn::utils::filesystem::create_directory(path)) { + LOG_WARNING("create directory '{}' failed", path); resp.err = ERR_FILE_OPERATION_FAILED; + break; } - if (resp.err == ERR_OK) { - LOG_DEBUG( - "start to transfer, src_file({}), dst_file({})", file_name(), target_file); - int64_t total_sz = 0; - char buf[max_length] = {'\0'}; - while (!fin.eof()) { - fin.read(buf, max_length); - total_sz += fin.gcount(); - fout.write(buf, fin.gcount()); - } - LOG_DEBUG("finish download file({}), total_size = {}", target_file, total_sz); - fout.close(); - fin.close(); - resp.downloaded_size = static_cast(total_sz); - - _size = total_sz; - if ((resp.err = utils::filesystem::md5sum(target_file, _md5_value)) != ERR_OK) { - LOG_WARNING("download {} failed when calculate the md5sum of {}", - file_name(), - target_file); - } else { - _has_meta_synced = true; - resp.file_md5 = _md5_value; - } + uint64_t file_size; + auto s = dsn::utils::copy_file(file_name(), target_file, &file_size); + if (!s.ok()) { + LOG_WARNING("download from '{}' to '{}' failed, err = {}", + file_name(), + target_file, + s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + auto res = utils::filesystem::md5sum(target_file, _md5_value); + if (res != dsn::ERR_OK) { + LOG_WARNING("calculate md5sum for {} failed", target_file); + resp.err = ERR_FILE_OPERATION_FAILED; + break; } - } + LOG_INFO("finish download file '{}', size = {}", target_file, file_size); + resp.downloaded_size = file_size; + resp.file_md5 = _md5_value; + _size = file_size; + _has_meta_synced = true; + } while (false); tsk->enqueue_with(resp); release_ref(); }; diff --git a/src/block_service/local/local_service.h b/src/block_service/local/local_service.h index 9816734cf0..9c944e1d0e 100644 --- a/src/block_service/local/local_service.h +++ b/src/block_service/local/local_service.h @@ -17,6 +17,9 @@ #pragma once +#include +#include // IWYU pragma: keep +#include // IWYU pragma: keep #include #include #include @@ -32,6 +35,13 @@ class task_tracker; namespace dist { namespace block_service { +struct file_metadata +{ + int64_t size = 0; + std::string md5; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(file_metadata, size, md5) + class local_service : public block_filesystem { public: diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt index a8e47d597e..19dd06808b 100644 --- a/src/block_service/test/CMakeLists.txt +++ b/src/block_service/test/CMakeLists.txt @@ -36,7 +36,8 @@ set(MY_PROJ_LIBS gtest gtest_main hdfs - ) + test_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/block_service/test/block_service_manager_test.cpp b/src/block_service/test/block_service_manager_test.cpp index 8ead04fd2a..ef93ed1962 100644 --- a/src/block_service/test/block_service_manager_test.cpp +++ b/src/block_service/test/block_service_manager_test.cpp @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include #include -#include #include #include #include @@ -29,14 +29,16 @@ #include "block_service/local/local_service.h" #include "block_service_mock.h" #include "metadata_types.h" +#include "test_util/test_util.h" #include "utils/error_code.h" #include "utils/filesystem.h" +#include "utils/test_macros.h" namespace dsn { namespace dist { namespace block_service { -class block_service_manager_test : public ::testing::Test +class block_service_manager_test : public pegasus::encrypt_data_test_base { public: block_service_manager_test() @@ -54,20 +56,6 @@ class block_service_manager_test : public ::testing::Test PROVIDER, LOCAL_DIR, FILE_NAME, _fs.get(), download_size); } - void create_local_file(const std::string &file_name) - { - std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, file_name); - utils::filesystem::create_file(whole_name); - std::ofstream test_file; - test_file.open(whole_name); - test_file << "write some data.\n"; - test_file.close(); - - _file_meta.name = whole_name; - utils::filesystem::md5sum(whole_name, _file_meta.md5); - utils::filesystem::file_size(whole_name, _file_meta.size); - } - void create_remote_file(const std::string &file_name, int64_t size, const std::string &md5) { std::string whole_file_name = utils::filesystem::path_combine(PROVIDER, file_name); @@ -84,8 +72,10 @@ class block_service_manager_test : public ::testing::Test std::string FILE_NAME = "test_file"; }; -// download_file unit tests -TEST_F(block_service_manager_test, do_download_remote_file_not_exist) +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, block_service_manager_test, ::testing::Values(false)); + +TEST_P(block_service_manager_test, remote_file_not_exist) { utils::filesystem::remove_path(LOCAL_DIR); auto fs = std::make_unique(); @@ -93,31 +83,35 @@ TEST_F(block_service_manager_test, do_download_remote_file_not_exist) uint64_t download_size = 0; error_code err = _block_service_manager.download_file( PROVIDER, LOCAL_DIR, FILE_NAME, fs.get(), download_size); - ASSERT_EQ(err, ERR_CORRUPTION); // file does not exist + ASSERT_EQ(ERR_CORRUPTION, err); } -TEST_F(block_service_manager_test, do_download_same_name_file) +TEST_P(block_service_manager_test, local_file_exist) { - // local file exists, but md5 not matched with remote file - create_local_file(FILE_NAME); - create_remote_file(FILE_NAME, 2333, "md5_not_match"); - uint64_t download_size = 0; - ASSERT_EQ(test_download_file(download_size), ERR_PATH_ALREADY_EXIST); - ASSERT_EQ(download_size, 0); -} - -TEST_F(block_service_manager_test, do_download_file_exist) -{ - create_local_file(FILE_NAME); - create_remote_file(FILE_NAME, _file_meta.size, _file_meta.md5); - uint64_t download_size = 0; - ASSERT_EQ(test_download_file(download_size), ERR_PATH_ALREADY_EXIST); - ASSERT_EQ(download_size, 0); + NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME), + &_file_meta)); + struct remote_file_info + { + int64_t size; + std::string md5; + } tests[]{ + {2333, "bad_md5"}, // wrong size and md5 + {2333, _file_meta.md5}, // wrong size + {_file_meta.size, "bad_md5"} // wrong md5 + }; + for (const auto &test : tests) { + // The remote file will be overwritten when repeatedly created. + create_remote_file(FILE_NAME, test.size, test.md5); + uint64_t download_size = 0; + ASSERT_EQ(ERR_PATH_ALREADY_EXIST, test_download_file(download_size)); + ASSERT_EQ(0, download_size); + } } -TEST_F(block_service_manager_test, do_download_succeed) +TEST_P(block_service_manager_test, do_download_succeed) { - create_local_file(FILE_NAME); + NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME), + &_file_meta)); create_remote_file(FILE_NAME, _file_meta.size, _file_meta.md5); // remove local file to mock condition that file not existed std::string file_name = utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME); diff --git a/src/block_service/test/config-test.ini b/src/block_service/test/config-test.ini index c1996d5518..2acb86e2bb 100644 --- a/src/block_service/test/config-test.ini +++ b/src/block_service/test/config-test.ini @@ -53,7 +53,7 @@ max_size = 150 worker_count = 8 [hdfs_test] -test_name_node = -test_backup_path = +test_name_node = +test_backup_path = num_test_file_lines = 4096 num_total_files_for_hdfs_concurrent_test = 64 diff --git a/src/block_service/test/hdfs_service_test.cpp b/src/block_service/test/hdfs_service_test.cpp index 6cfcfb379c..0eaec29e88 100644 --- a/src/block_service/test/hdfs_service_test.cpp +++ b/src/block_service/test/hdfs_service_test.cpp @@ -15,15 +15,20 @@ // specific language governing permissions and limitations // under the License. +#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include #include #include #include #include -#include +#include #include #include #include @@ -35,27 +40,23 @@ #include "runtime/task/task.h" #include "runtime/task/task_code.h" #include "runtime/task/task_tracker.h" +#include "test_util/test_util.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" #include "utils/enum_helper.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" -#include "utils/strings.h" +#include "utils/fmt_logging.h" +#include "utils/test_macros.h" #include "utils/threadpool_code.h" using namespace dsn; using namespace dsn::dist::block_service; -static std::string example_name_node = ""; -static std::string example_backup_path = ""; -// Please modify following paras in 'config-test.ini' to enable hdfs_service_test, -// or hdfs_service_test will be skipped and return true. -DSN_DEFINE_string(hdfs_test, test_name_node, "", "hdfs name node"); -DSN_DEFINE_string(hdfs_test, - test_backup_path, - "", - "path for uploading and downloading test files"); +DSN_DEFINE_string(hdfs_test, test_name_node, "", "hdfs name node"); +DSN_DEFINE_string(hdfs_test, test_backup_path, "", "path for uploading and downloading test files"); DSN_DEFINE_uint32(hdfs_test, num_test_file_lines, 4096, "number of lines in test file"); DSN_DEFINE_uint32(hdfs_test, @@ -65,206 +66,262 @@ DSN_DEFINE_uint32(hdfs_test, DEFINE_TASK_CODE(LPC_TEST_HDFS, TASK_PRIORITY_HIGH, dsn::THREAD_POOL_DEFAULT) -class HDFSClientTest : public testing::Test +class HDFSClientTest : public pegasus::encrypt_data_test_base { protected: - virtual void SetUp() override; - virtual void TearDown() override; - void generate_test_file(const char *filename); - void write_test_files_async(task_tracker *tracker); - std::string name_node; - std::string backup_path; - std::string local_test_dir; - std::string test_data_str; -}; - -void HDFSClientTest::SetUp() -{ - name_node = FLAGS_test_name_node; - backup_path = FLAGS_test_backup_path; - local_test_dir = "test_dir"; - test_data_str = ""; - for (int i = 0; i < FLAGS_num_test_file_lines; ++i) { - test_data_str += "test"; + HDFSClientTest() : pegasus::encrypt_data_test_base() + { + for (int i = 0; i < FLAGS_num_test_file_lines; ++i) { + _local_test_data += "test"; + } } -} -void HDFSClientTest::TearDown() {} + void generate_test_file(const std::string &filename); + void write_test_files_async(const std::string &local_test_path, + int total_files, + int *success_count, + task_tracker *tracker); -void HDFSClientTest::generate_test_file(const char *filename) +private: + std::string _local_test_data; +}; + +void HDFSClientTest::generate_test_file(const std::string &filename) { - // generate a local test file. int lines = FLAGS_num_test_file_lines; - FILE *fp = fopen(filename, "wb"); + std::unique_ptr wfile; + auto s = rocksdb::Env::Default()->NewWritableFile(filename, &wfile, rocksdb::EnvOptions()); + ASSERT_TRUE(s.ok()) << s.ToString(); for (int i = 0; i < lines; ++i) { - fprintf(fp, "%04d_this_is_a_simple_test_file\n", i); + rocksdb::Slice data(fmt::format("{:04}d_this_is_a_simple_test_file\n", i)); + s = wfile->Append(data); + ASSERT_TRUE(s.ok()) << s.ToString(); } - fclose(fp); + s = wfile->Fsync(); + ASSERT_TRUE(s.ok()) << s.ToString(); } -void HDFSClientTest::write_test_files_async(task_tracker *tracker) +void HDFSClientTest::write_test_files_async(const std::string &local_test_path, + int total_files, + int *success_count, + task_tracker *tracker) { - dsn::utils::filesystem::create_directory(local_test_dir); - for (int i = 0; i < 100; ++i) { - tasking::enqueue(LPC_TEST_HDFS, tracker, [this, i]() { + dsn::utils::filesystem::create_directory(local_test_path); + for (int i = 0; i < total_files; ++i) { + tasking::enqueue(LPC_TEST_HDFS, tracker, [this, &local_test_path, i, success_count]() { // mock the writing process in hdfs_file_object::download(). - std::string test_file_name = local_test_dir + "/test_file_" + std::to_string(i); - std::ofstream out(test_file_name, std::ios::binary | std::ios::out | std::ios::trunc); - if (out.is_open()) { - out.write(test_data_str.c_str(), test_data_str.length()); + std::string test_file_name = local_test_path + "/test_file_" + std::to_string(i); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(_local_test_data), + test_file_name, + /* should_sync */ true); + if (s.ok()) { + ++(*success_count); + } else { + CHECK(s.IsIOError(), "{}", s.ToString()); + auto pos1 = s.ToString().find( + "IO error: No such file or directory: While open a file for appending: "); + auto pos2 = s.ToString().find("IO error: While appending to file: "); + CHECK(pos1 == 0 || pos2 == 0, "{}", s.ToString()); } - out.close(); }); } } -TEST_F(HDFSClientTest, test_basic_operation) +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, HDFSClientTest, ::testing::Values(false)); + +TEST_P(HDFSClientTest, test_hdfs_read_write) { - if (name_node == example_name_node || backup_path == example_backup_path) { + if (strlen(FLAGS_test_name_node) == 0 || strlen(FLAGS_test_backup_path) == 0) { + // TODO(yingchun): use GTEST_SKIP after upgrading gtest. + std::cout << "Set hdfs_test.* configs in config-test.ini to enable hdfs_service_test." + << std::endl; return; } - std::vector args = {name_node, backup_path}; - std::shared_ptr s = std::make_shared(); - ASSERT_EQ(dsn::ERR_OK, s->initialize(args)); + auto s = std::make_shared(); + ASSERT_EQ(dsn::ERR_OK, s->initialize({FLAGS_test_name_node, FLAGS_test_backup_path})); + + const std::string kRemoteTestPath = "hdfs_client_test"; + const std::string kRemoteTestRWFile = kRemoteTestPath + "/test_write_file"; + const std::string kTestBuffer = "write_hello_world_for_test"; + const int kTestBufferLength = kTestBuffer.size(); + + // 1. clean up all old file in remote test directory. + printf("clean up all old files.\n"); + remove_path_response rem_resp; + s->remove_path(remove_path_request{kRemoteTestPath, true}, + LPC_TEST_HDFS, + [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, + nullptr) + ->wait(); + ASSERT_TRUE(dsn::ERR_OK == rem_resp.err || dsn::ERR_OBJECT_NOT_FOUND == rem_resp.err); + + // 2. create file. + printf("test write operation.\n"); + create_file_response cf_resp; + s->create_file(create_file_request{kRemoteTestRWFile, false}, + LPC_TEST_HDFS, + [&cf_resp](const create_file_response &r) { cf_resp = r; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - std::string local_test_file = "test_file"; - std::string remote_test_file = "hdfs_client_test/test_file"; - int64_t test_file_size = 0; + // 3. write file. + dsn::blob bb(kTestBuffer.c_str(), 0, kTestBufferLength); + write_response w_resp; + cf_resp.file_handle + ->write(write_request{bb}, + LPC_TEST_HDFS, + [&w_resp](const write_response &w) { w_resp = w; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, w_resp.err); + ASSERT_EQ(kTestBufferLength, w_resp.written_size); + ASSERT_EQ(kTestBufferLength, cf_resp.file_handle->get_size()); - generate_test_file(local_test_file.c_str()); - dsn::utils::filesystem::file_size(local_test_file, test_file_size); + // 4. read file. + printf("test read just written contents.\n"); + read_response r_resp; + cf_resp.file_handle + ->read(read_request{0, -1}, + LPC_TEST_HDFS, + [&r_resp](const read_response &r) { r_resp = r; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, r_resp.err); + ASSERT_EQ(kTestBufferLength, r_resp.buffer.length()); + ASSERT_EQ(kTestBuffer, r_resp.buffer.to_string()); - // fisrt clean up all old file in test directory. + // 5. partial read. + const uint64_t kOffset = 5; + const int64_t kSize = 10; + cf_resp.file_handle + ->read(read_request{kOffset, kSize}, + LPC_TEST_HDFS, + [&r_resp](const read_response &r) { r_resp = r; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, r_resp.err); + ASSERT_EQ(kSize, r_resp.buffer.length()); + ASSERT_EQ(kTestBuffer.substr(kOffset, kSize), r_resp.buffer.to_string()); +} + +TEST_P(HDFSClientTest, test_upload_and_download) +{ + if (strlen(FLAGS_test_name_node) == 0 || strlen(FLAGS_test_backup_path) == 0) { + // TODO(yingchun): use GTEST_SKIP after upgrading gtest. + std::cout << "Set hdfs_test.* configs in config-test.ini to enable hdfs_service_test." + << std::endl; + return; + } + + auto s = std::make_shared(); + ASSERT_EQ(dsn::ERR_OK, s->initialize({FLAGS_test_name_node, FLAGS_test_backup_path})); + + const std::string kLocalFile = "test_file"; + const std::string kRemoteTestPath = "hdfs_client_test"; + const std::string kRemoteTestFile = kRemoteTestPath + "/test_file"; + + // 0. generate test file. + NO_FATALS(generate_test_file(kLocalFile)); + int64_t local_file_size = 0; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kLocalFile, dsn::utils::FileDataType::kSensitive, local_file_size)); + std::string local_file_md5sum; + dsn::utils::filesystem::md5sum(kLocalFile, local_file_md5sum); + + // 1. clean up all old file in remote test directory. printf("clean up all old files.\n"); remove_path_response rem_resp; - s->remove_path(remove_path_request{"hdfs_client_test", true}, + s->remove_path(remove_path_request{kRemoteTestPath, true}, LPC_TEST_HDFS, [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, nullptr) ->wait(); ASSERT_TRUE(dsn::ERR_OK == rem_resp.err || dsn::ERR_OBJECT_NOT_FOUND == rem_resp.err); - // test upload file. - printf("create and upload: %s.\n", remote_test_file.c_str()); + // 2. create file. + printf("create and upload: %s.\n", kRemoteTestFile.c_str()); create_file_response cf_resp; - s->create_file(create_file_request{remote_test_file, true}, + s->create_file(create_file_request{kRemoteTestFile, true}, LPC_TEST_HDFS, [&cf_resp](const create_file_response &r) { cf_resp = r; }, nullptr) ->wait(); - ASSERT_EQ(cf_resp.err, dsn::ERR_OK); + ASSERT_EQ(dsn::ERR_OK, cf_resp.err); + + // 3. upload file. upload_response u_resp; cf_resp.file_handle - ->upload(upload_request{local_test_file}, + ->upload(upload_request{kLocalFile}, LPC_TEST_HDFS, [&u_resp](const upload_response &r) { u_resp = r; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, u_resp.err); - ASSERT_EQ(test_file_size, cf_resp.file_handle->get_size()); + ASSERT_EQ(local_file_size, cf_resp.file_handle->get_size()); - // test list directory. + // 4. list directory. ls_response l_resp; - s->list_dir(ls_request{"hdfs_client_test"}, + s->list_dir(ls_request{kRemoteTestPath}, LPC_TEST_HDFS, [&l_resp](const ls_response &resp) { l_resp = resp; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, l_resp.err); ASSERT_EQ(1, l_resp.entries->size()); - ASSERT_EQ("test_file", l_resp.entries->at(0).entry_name); + ASSERT_EQ(kLocalFile, l_resp.entries->at(0).entry_name); ASSERT_EQ(false, l_resp.entries->at(0).is_directory); - // test download file. + // 5. download file. download_response d_resp; - printf("test download %s.\n", remote_test_file.c_str()); - s->create_file(create_file_request{remote_test_file, false}, + printf("test download %s.\n", kRemoteTestFile.c_str()); + s->create_file(create_file_request{kRemoteTestFile, false}, LPC_TEST_HDFS, [&cf_resp](const create_file_response &resp) { cf_resp = resp; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_EQ(test_file_size, cf_resp.file_handle->get_size()); - std::string local_file_for_download = "test_file_d"; + ASSERT_EQ(local_file_size, cf_resp.file_handle->get_size()); + std::string kLocalDownloadFile = "test_file_d"; cf_resp.file_handle - ->download(download_request{local_file_for_download, 0, -1}, + ->download(download_request{kLocalDownloadFile, 0, -1}, LPC_TEST_HDFS, [&d_resp](const download_response &resp) { d_resp = resp; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, d_resp.err); - ASSERT_EQ(test_file_size, d_resp.downloaded_size); - - // compare local_test_file and local_file_for_download. - int64_t file_size = 0; - dsn::utils::filesystem::file_size(local_file_for_download, file_size); - ASSERT_EQ(test_file_size, file_size); - std::string test_file_md5sum; - dsn::utils::filesystem::md5sum(local_test_file, test_file_md5sum); - std::string downloaded_file_md5sum; - dsn::utils::filesystem::md5sum(local_file_for_download, downloaded_file_md5sum); - ASSERT_EQ(test_file_md5sum, downloaded_file_md5sum); - - // test read and write. - printf("test read write operation.\n"); - std::string test_write_file = "hdfs_client_test/test_write_file"; - s->create_file(create_file_request{test_write_file, false}, - LPC_TEST_HDFS, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - const char *test_buffer = "write_hello_world_for_test"; - int length = strlen(test_buffer); - dsn::blob bb(test_buffer, 0, length); - write_response w_resp; - cf_resp.file_handle - ->write(write_request{bb}, - LPC_TEST_HDFS, - [&w_resp](const write_response &w) { w_resp = w; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, w_resp.err); - ASSERT_EQ(length, w_resp.written_size); - ASSERT_EQ(length, cf_resp.file_handle->get_size()); - printf("test read just written contents.\n"); - read_response r_resp; - cf_resp.file_handle - ->read(read_request{0, -1}, - LPC_TEST_HDFS, - [&r_resp](const read_response &r) { r_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, r_resp.err); - ASSERT_EQ(length, r_resp.buffer.length()); - ASSERT_TRUE(dsn::utils::mequals(r_resp.buffer.data(), test_buffer, length)); - - // test partitial read. - cf_resp.file_handle - ->read(read_request{5, 10}, - LPC_TEST_HDFS, - [&r_resp](const read_response &r) { r_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, r_resp.err); - ASSERT_EQ(10, r_resp.buffer.length()); - ASSERT_TRUE(dsn::utils::mequals(r_resp.buffer.data(), test_buffer + 5, 10)); - - // clean up local test files. - utils::filesystem::remove_path(local_test_file); - utils::filesystem::remove_path(local_file_for_download); + ASSERT_EQ(local_file_size, d_resp.downloaded_size); + + // 6. compare kLocalFile and kLocalDownloadFile. + // 6.1 check file size. + int64_t local_download_file_size = 0; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kLocalDownloadFile, dsn::utils::FileDataType::kSensitive, local_download_file_size)); + ASSERT_EQ(local_file_size, local_download_file_size); + // 6.2 check file md5sum. + std::string local_download_file_md5sum; + dsn::utils::filesystem::md5sum(kLocalDownloadFile, local_download_file_md5sum); + ASSERT_EQ(local_file_md5sum, local_download_file_md5sum); + + // 7. clean up local test files. + utils::filesystem::remove_path(kLocalFile); + utils::filesystem::remove_path(kLocalDownloadFile); } -TEST_F(HDFSClientTest, test_concurrent_upload_download) +TEST_P(HDFSClientTest, test_concurrent_upload_download) { - if (name_node == example_name_node || backup_path == example_backup_path) { + if (strlen(FLAGS_test_name_node) == 0 || strlen(FLAGS_test_backup_path) == 0) { + // TODO(yingchun): use GTEST_SKIP after upgrading gtest. + std::cout << "Set hdfs_test.* configs in config-test.ini to enable hdfs_service_test." + << std::endl; return; } - std::vector args = {name_node, backup_path}; - std::shared_ptr s = std::make_shared(); - ASSERT_EQ(dsn::ERR_OK, s->initialize(args)); + auto s = std::make_shared(); + ASSERT_EQ(dsn::ERR_OK, s->initialize({FLAGS_test_name_node, FLAGS_test_backup_path})); int total_files = FLAGS_num_total_files_for_hdfs_concurrent_test; std::vector local_file_names; @@ -282,11 +339,12 @@ TEST_F(HDFSClientTest, test_concurrent_upload_download) // generate test files. for (int i = 0; i < total_files; ++i) { std::string file_name = "randomfile" + std::to_string(i); - generate_test_file(file_name.c_str()); + NO_FATALS(generate_test_file(file_name)); int64_t file_size = 0; - dsn::utils::filesystem::file_size(file_name, file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + file_name, dsn::utils::FileDataType::kSensitive, file_size)); std::string md5sum; - dsn::utils::filesystem::md5sum(file_name, md5sum); + ASSERT_EQ(ERR_OK, dsn::utils::filesystem::md5sum(file_name, md5sum)); local_file_names.emplace_back(file_name); remote_file_names.emplace_back("hdfs_concurrent_test/" + file_name); @@ -385,13 +443,26 @@ TEST_F(HDFSClientTest, test_concurrent_upload_download) } } -TEST_F(HDFSClientTest, test_rename_path_while_writing) +TEST_P(HDFSClientTest, test_rename_path_while_writing) { - task_tracker tracker; - write_test_files_async(&tracker); - usleep(100); - std::string rename_dir = "rename_dir." + std::to_string(dsn_now_ms()); - // rename succeed but writing failed. - ASSERT_TRUE(dsn::utils::filesystem::rename_path(local_test_dir, rename_dir)); - tracker.cancel_outstanding_tasks(); + std::string kLocalTestPath = "test_dir"; + const int kTotalFiles = 100; + + // The test is flaky, retry if it failed in 300 seconds. + ASSERT_IN_TIME_WITH_FIXED_INTERVAL( + [&] { + task_tracker tracker; + int success_count = 0; + write_test_files_async(kLocalTestPath, kTotalFiles, &success_count, &tracker); + usleep(100); + + std::string kLocalRenamedTestPath = "rename_dir." + std::to_string(dsn_now_ms()); + // Rename succeed but some files write failed. + ASSERT_TRUE(dsn::utils::filesystem::rename_path(kLocalTestPath, kLocalRenamedTestPath)); + tracker.cancel_outstanding_tasks(); + // Generally, we can assume partial files are written failed. + ASSERT_GT(success_count, 0) << success_count; + ASSERT_LT(success_count, kTotalFiles) << success_count; + }, + 300); } diff --git a/src/block_service/test/local_service_test.cpp b/src/block_service/test/local_service_test.cpp index e355a1b281..e4e6aeb2ee 100644 --- a/src/block_service/test/local_service_test.cpp +++ b/src/block_service/test/local_service_test.cpp @@ -19,16 +19,23 @@ #include // IWYU pragma: no_include +// IWYU pragma: no_include // IWYU pragma: no_include #include #include #include #include -#include +#include +#include +#include #include +#include +#include +#include #include #include "block_service/local/local_service.h" +#include "test_util/test_util.h" #include "utils/error_code.h" namespace dsn { @@ -36,53 +43,69 @@ namespace dist { namespace block_service { // Simple tests for nlohmann::json serialization, via NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE. +class local_service_test : public pegasus::encrypt_data_test_base +{ +}; + +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, local_service_test, ::testing::Values(false)); -TEST(local_service, store_metadata) +TEST_P(local_service_test, store_metadata) { local_file_object file("a.txt"); error_code ec = file.store_metadata(); - ASSERT_EQ(ec, ERR_OK); + ASSERT_EQ(ERR_OK, ec); auto meta_file_path = local_service::get_metafile(file.file_name()); ASSERT_TRUE(boost::filesystem::exists(meta_file_path)); - std::ifstream ifs(meta_file_path); - nlohmann::json j; - ifs >> j; - ASSERT_EQ(j["md5"], ""); - ASSERT_EQ(j["size"], 0); + std::string data; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), meta_file_path, &data); + ASSERT_TRUE(s.ok()) << s.ToString(); + + nlohmann::json j = nlohmann::json::parse(data); + ASSERT_EQ("", j["md5"]); + ASSERT_EQ(0, j["size"]); } -TEST(local_service, load_metadata) +TEST_P(local_service_test, load_metadata) { local_file_object file("a.txt"); auto meta_file_path = local_service::get_metafile(file.file_name()); { - std::ofstream ofs(meta_file_path); nlohmann::json j({{"md5", "abcde"}, {"size", 5}}); - ofs << j; - ofs.close(); + std::string data = j.dump(); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(data), + meta_file_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); - ASSERT_EQ(file.load_metadata(), ERR_OK); - ASSERT_EQ(file.get_md5sum(), "abcde"); - ASSERT_EQ(file.get_size(), 5); + ASSERT_EQ(ERR_OK, file.load_metadata()); + ASSERT_EQ("abcde", file.get_md5sum()); + ASSERT_EQ(5, file.get_size()); } { - std::ofstream ofs(meta_file_path); - ofs << "invalid json string"; - ofs.close(); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice("invalid json string"), + meta_file_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); local_file_object file2("a.txt"); ASSERT_EQ(file2.load_metadata(), ERR_FS_INTERNAL); } { - std::ofstream ofs(meta_file_path); nlohmann::json j({{"md5", "abcde"}, {"no such key", "illegal"}}); - ofs << j; - ofs.close(); + std::string data = j.dump(); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(data), + meta_file_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); local_file_object file2("a.txt"); ASSERT_EQ(file2.load_metadata(), ERR_FS_INTERNAL); diff --git a/src/client/replication_ddl_client.cpp b/src/client/replication_ddl_client.cpp index 00276e9861..a8a6cf7db6 100644 --- a/src/client/replication_ddl_client.cpp +++ b/src/client/replication_ddl_client.cpp @@ -46,7 +46,6 @@ #include "common/replication_common.h" #include "common/replication_enums.h" #include "fmt/core.h" -#include "fmt/ostream.h" #include "meta/meta_rpc_types.h" #include "runtime/api_layer1.h" #include "runtime/rpc/group_address.h" diff --git a/src/client/test/CMakeLists.txt b/src/client/test/CMakeLists.txt index 90ce5d5e84..bcae3897aa 100644 --- a/src/client/test/CMakeLists.txt +++ b/src/client/test/CMakeLists.txt @@ -27,7 +27,7 @@ set(MY_PROJ_LIBS dsn_runtime dsn_utils gtest -) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/client/test/ddl_client_test.cpp b/src/client/test/ddl_client_test.cpp index c489bce373..414c1f029f 100644 --- a/src/client/test/ddl_client_test.cpp +++ b/src/client/test/ddl_client_test.cpp @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. -#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include #include #include #include -#include #include #include diff --git a/src/client_lib/pegasus_client_impl.cpp b/src/client_lib/pegasus_client_impl.cpp index 22f30a3b54..a390e23c32 100644 --- a/src/client_lib/pegasus_client_impl.cpp +++ b/src/client_lib/pegasus_client_impl.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include #include #include #include diff --git a/src/common/duplication_common.h b/src/common/duplication_common.h index c1e77450df..6b953f6e96 100644 --- a/src/common/duplication_common.h +++ b/src/common/duplication_common.h @@ -28,6 +28,7 @@ #include "runtime/rpc/rpc_holder.h" #include "utils/errors.h" #include "utils/flags.h" +#include "utils/fmt_utils.h" namespace dsn { namespace replication { @@ -87,5 +88,7 @@ struct duplication_constants const static std::string kDuplicationEnvMasterMetasKey; }; +USER_DEFINED_ENUM_FORMATTER(duplication_fail_mode::type) +USER_DEFINED_ENUM_FORMATTER(duplication_status::type) } // namespace replication } // namespace dsn diff --git a/src/common/fs_manager.cpp b/src/common/fs_manager.cpp index 6fed98132b..362b29ba87 100644 --- a/src/common/fs_manager.cpp +++ b/src/common/fs_manager.cpp @@ -37,13 +37,13 @@ #include #include #include -#include #include +#include // IWYU pragma: keep + #include "common/gpid.h" #include "common/replication_enums.h" #include "fmt/core.h" -#include "fmt/ostream.h" #include "perf_counter/perf_counter.h" #include "replica_admin_types.h" #include "runtime/api_layer1.h" diff --git a/src/common/gpid.h b/src/common/gpid.h index 1792df3106..ba2f9ee5bc 100644 --- a/src/common/gpid.h +++ b/src/common/gpid.h @@ -29,6 +29,8 @@ #include #include +#include "utils/fmt_utils.h" + namespace dsn { // Group-Partition-ID. @@ -91,6 +93,8 @@ class gpid } // namespace dsn +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::gpid); + namespace std { template <> struct hash<::dsn::gpid> diff --git a/src/common/replica_envs.h b/src/common/replica_envs.h index 4db367a7fa..1db2e5399f 100644 --- a/src/common/replica_envs.h +++ b/src/common/replica_envs.h @@ -28,6 +28,7 @@ #include #include +#include namespace dsn { namespace replication { @@ -64,6 +65,11 @@ class replica_envs static const std::string USER_SPECIFIED_COMPACTION; static const std::string ROCKSDB_ALLOW_INGEST_BEHIND; static const std::string UPDATE_MAX_REPLICA_COUNT; + static const std::string ROCKSDB_WRITE_BUFFER_SIZE; + static const std::string ROCKSDB_NUM_LEVELS; + + static const std::set ROCKSDB_DYNAMIC_OPTIONS; + static const std::set ROCKSDB_STATIC_OPTIONS; }; } // namespace replication diff --git a/src/common/replication_common.cpp b/src/common/replication_common.cpp index 63617797ac..1afe9d49fe 100644 --- a/src/common/replication_common.cpp +++ b/src/common/replication_common.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include "common/gpid.h" #include "common/replica_envs.h" @@ -394,6 +395,14 @@ const std::string replica_envs::USER_SPECIFIED_COMPACTION("user_specified_compac const std::string replica_envs::BACKUP_REQUEST_QPS_THROTTLING("replica.backup_request_throttling"); const std::string replica_envs::ROCKSDB_ALLOW_INGEST_BEHIND("rocksdb.allow_ingest_behind"); const std::string replica_envs::UPDATE_MAX_REPLICA_COUNT("max_replica_count.update"); +const std::string replica_envs::ROCKSDB_WRITE_BUFFER_SIZE("rocksdb.write_buffer_size"); +const std::string replica_envs::ROCKSDB_NUM_LEVELS("rocksdb.num_levels"); +const std::set replica_envs::ROCKSDB_DYNAMIC_OPTIONS = { + replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, +}; +const std::set replica_envs::ROCKSDB_STATIC_OPTIONS = { + replica_envs::ROCKSDB_NUM_LEVELS, +}; } // namespace replication } // namespace dsn diff --git a/src/common/replication_enums.h b/src/common/replication_enums.h index 5eef1710e6..bd16d81415 100644 --- a/src/common/replication_enums.h +++ b/src/common/replication_enums.h @@ -32,6 +32,7 @@ #include "consensus_types.h" #include "meta_admin_types.h" #include "replica_admin_types.h" +#include "utils/fmt_utils.h" namespace dsn { ENUM_BEGIN2(app_status::type, app_status, app_status::AS_INVALID) @@ -163,4 +164,18 @@ ENUM_REG(replication::manual_compaction_status::QUEUING) ENUM_REG(replication::manual_compaction_status::RUNNING) ENUM_REG(replication::manual_compaction_status::FINISHED) ENUM_END2(replication::manual_compaction_status::type, manual_compaction_status) + +USER_DEFINED_ENUM_FORMATTER(app_status::type) +namespace replication { +USER_DEFINED_ENUM_FORMATTER(bulk_load_status::type) +USER_DEFINED_ENUM_FORMATTER(config_type::type) +USER_DEFINED_ENUM_FORMATTER(detect_action::type) +USER_DEFINED_ENUM_FORMATTER(disk_migration_status::type) +USER_DEFINED_ENUM_FORMATTER(disk_status::type) +USER_DEFINED_ENUM_FORMATTER(learner_status::type) +USER_DEFINED_ENUM_FORMATTER(learn_type::type) +USER_DEFINED_ENUM_FORMATTER(manual_compaction_status::type) +USER_DEFINED_ENUM_FORMATTER(meta_function_level::type) +USER_DEFINED_ENUM_FORMATTER(partition_status::type) +} // namespace replication } // namespace dsn diff --git a/src/common/test/CMakeLists.txt b/src/common/test/CMakeLists.txt index 78d94000c3..1cdf58407f 100644 --- a/src/common/test/CMakeLists.txt +++ b/src/common/test/CMakeLists.txt @@ -30,7 +30,7 @@ set(MY_PROJ_LIBS dsn_replication_common dsn_runtime gtest - ) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/common/test/common_test.cpp b/src/common/test/common_test.cpp index dc2bdccaac..c2bf31713a 100644 --- a/src/common/test/common_test.cpp +++ b/src/common/test/common_test.cpp @@ -22,6 +22,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include namespace dsn { TEST(duplication_common, get_current_cluster_name) diff --git a/src/common/test/config-test.ini b/src/common/test/config-test.ini index 774ac82cd3..2775e06bd4 100644 --- a/src/common/test/config-test.ini +++ b/src/common/test/config-test.ini @@ -92,6 +92,7 @@ rpc_message_header_format = dsn rpc_timeout_milliseconds = 5000 [replication] +disk_min_available_space_ratio = 10 cluster_name = master-cluster [duplication-group] diff --git a/src/common/test/run.sh b/src/common/test/run.sh index ce89cc4eac..affb98db8b 100755 --- a/src/common/test/run.sh +++ b/src/common/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -31,6 +31,25 @@ exit_if_fail() { fi } +if [ -n ${TEST_OPTS} ]; then + if [ ! -f ./config-test.ini ]; then + echo "./config-test.ini does not exists !" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi + ./dsn_replication_common_test exit_if_fail $? "run unit test failed" diff --git a/src/failure_detector/test/CMakeLists.txt b/src/failure_detector/test/CMakeLists.txt index d529e50ea4..ed4a9703d6 100644 --- a/src/failure_detector/test/CMakeLists.txt +++ b/src/failure_detector/test/CMakeLists.txt @@ -41,7 +41,7 @@ set(MY_PROJ_LIBS dsn.failure_detector gtest hashtable - ) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/geo/bench/bench.cpp b/src/geo/bench/bench.cpp index e65714b7ad..0ac170b02e 100644 --- a/src/geo/bench/bench.cpp +++ b/src/geo/bench/bench.cpp @@ -76,10 +76,12 @@ int main(int argc, char **argv) } } + // TODO(yingchun): the benchmark can not exit normally, we need to fix it later. pegasus::geo::geo_client my_geo( "config.ini", cluster_name.c_str(), app_name.c_str(), geo_app_name.c_str()); - if (!my_geo.set_max_level(max_level).is_ok()) { - std::cerr << "set_max_level failed" << std::endl; + auto err = my_geo.set_max_level(max_level); + if (!err.is_ok()) { + std::cerr << "set_max_level failed, err: " << err << std::endl; return -1; } diff --git a/src/geo/lib/geo_client.cpp b/src/geo/lib/geo_client.cpp index 335afda62e..0d4927695e 100644 --- a/src/geo/lib/geo_client.cpp +++ b/src/geo/lib/geo_client.cpp @@ -32,7 +32,6 @@ #include #include #include -#include #include #include #include diff --git a/src/geo/lib/latlng_codec.cpp b/src/geo/lib/latlng_codec.cpp index 8a74618bc6..171db59c58 100644 --- a/src/geo/lib/latlng_codec.cpp +++ b/src/geo/lib/latlng_codec.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include "utils/error_code.h" #include "utils/errors.h" diff --git a/src/http/http_call_registry.h b/src/http/http_call_registry.h index fef3b0c467..2e944463ce 100644 --- a/src/http/http_call_registry.h +++ b/src/http/http_call_registry.h @@ -35,11 +35,11 @@ class http_call_registry : public utils::singleton std::shared_ptr find(const std::string &path) const { std::lock_guard guard(_mu); - auto it = _call_map.find(path); - if (it == _call_map.end()) { + const auto &iter = _call_map.find(path); + if (iter == _call_map.end()) { return nullptr; } - return it->second; + return iter->second; } void remove(const std::string &path) @@ -48,14 +48,19 @@ class http_call_registry : public utils::singleton _call_map.erase(path); } - void add(std::unique_ptr call_uptr) + void add(const std::shared_ptr &call) { - auto call = std::shared_ptr(call_uptr.release()); std::lock_guard guard(_mu); - CHECK_EQ_MSG(_call_map.count(call->path), 0, call->path); + CHECK_EQ_MSG(_call_map.count(call->path), 0, "{} has been added", call->path); _call_map[call->path] = call; } + void add(std::unique_ptr call_uptr) + { + auto call = std::shared_ptr(call_uptr.release()); + add(call); + } + std::vector> list_all_calls() const { std::lock_guard guard(_mu); diff --git a/src/http/http_client.cpp b/src/http/http_client.cpp new file mode 100644 index 0000000000..32ae5eaea1 --- /dev/null +++ b/src/http/http_client.cpp @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "http/http_client.h" + +#include +#include +#include + +#include "curl/curl.h" +#include "utils/error_code.h" +#include "utils/flags.h" +#include "utils/fmt_logging.h" + +namespace dsn { + +DSN_DEFINE_uint32(http, + curl_timeout_ms, + 10000, + "The maximum time in milliseconds that you allow the libcurl transfer operation " + "to complete"); + +http_client::http_client() + : _curl(nullptr), + _method(http_method::GET), + _recv_callback(nullptr), + _header_changed(true), + _header_list(nullptr) +{ + // Since `kErrorBufferBytes` is private, `static_assert` have to be put in constructor. + static_assert(http_client::kErrorBufferBytes >= CURL_ERROR_SIZE, + "The error buffer used by libcurl must be at least CURL_ERROR_SIZE bytes big"); + + clear_error_buf(); +} + +http_client::~http_client() +{ + if (_curl != nullptr) { + curl_easy_cleanup(_curl); + _curl = nullptr; + } + + free_header_list(); +} + +namespace { + +inline dsn::error_code to_error_code(CURLcode code) +{ + switch (code) { + case CURLE_OK: + return dsn::ERR_OK; + case CURLE_OPERATION_TIMEDOUT: + return dsn::ERR_TIMEOUT; + default: + return dsn::ERR_CURL_FAILED; + } +} + +} // anonymous namespace + +#define RETURN_IF_CURL_NOT_OK(expr, ...) \ + do { \ + const auto code = (expr); \ + if (dsn_unlikely(code != CURLE_OK)) { \ + std::string msg(fmt::format("{}: {}", fmt::format(__VA_ARGS__), to_error_msg(code))); \ + return dsn::error_s::make(to_error_code(code), msg); \ + } \ + } while (0) + +#define RETURN_IF_SETOPT_NOT_OK(opt, input) \ + RETURN_IF_CURL_NOT_OK(curl_easy_setopt(_curl, opt, input), \ + "failed to set " #opt " to" \ + " " #input) + +#define RETURN_IF_GETINFO_NOT_OK(info, output) \ + RETURN_IF_CURL_NOT_OK(curl_easy_getinfo(_curl, info, output), "failed to get from " #info) + +#define RETURN_IF_EXEC_METHOD_NOT_OK() \ + RETURN_IF_CURL_NOT_OK(curl_easy_perform(_curl), \ + "failed to perform http request(method={}, url={})", \ + enum_to_string(_method), \ + _url) + +dsn::error_s http_client::init() +{ + if (_curl == nullptr) { + _curl = curl_easy_init(); + if (_curl == nullptr) { + return dsn::error_s::make(dsn::ERR_CURL_FAILED, "fail to initialize curl"); + } + } else { + curl_easy_reset(_curl); + } + + clear_header_fields(); + free_header_list(); + + // Additional messages for errors are needed. + clear_error_buf(); + RETURN_IF_SETOPT_NOT_OK(CURLOPT_ERRORBUFFER, _error_buf); + + // Set with NOSIGNAL since we are multi-threaded. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_NOSIGNAL, 1L); + + // Redirects are supported. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_FOLLOWLOCATION, 1L); + + // Before 8.3.0, CURLOPT_MAXREDIRS was unlimited. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_MAXREDIRS, 20); + + // Set common timeout for transfer operation. Users could also change it with their + // custom values by `set_timeout`. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_TIMEOUT_MS, static_cast(FLAGS_curl_timeout_ms)); + + // A lambda can only be converted to a function pointer if it does not capture: + // https://stackoverflow.com/questions/28746744/passing-capturing-lambda-as-function-pointer + // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2012/n3337.pdf + curl_write_callback callback = [](char *buffer, size_t size, size_t nmemb, void *param) { + http_client *client = reinterpret_cast(param); + return client->on_response_data(buffer, size * nmemb); + }; + RETURN_IF_SETOPT_NOT_OK(CURLOPT_WRITEFUNCTION, callback); + + // This http_client object itself is passed to the callback function. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_WRITEDATA, reinterpret_cast(this)); + + return dsn::error_s::ok(); +} + +void http_client::clear_error_buf() { _error_buf[0] = 0; } + +bool http_client::is_error_buf_empty() const { return _error_buf[0] == 0; } + +std::string http_client::to_error_msg(CURLcode code) const +{ + std::string err_msg = + fmt::format("code={}, desc=\"{}\"", static_cast(code), curl_easy_strerror(code)); + if (is_error_buf_empty()) { + return err_msg; + } + + err_msg += fmt::format(", msg=\"{}\"", _error_buf); + return err_msg; +} + +// `data` passed to this function is NOT null-terminated. +// `length` might be zero. +size_t http_client::on_response_data(const void *data, size_t length) +{ + if (_recv_callback == nullptr) { + return length; + } + + if (!(*_recv_callback)) { + // callback function is empty. + return length; + } + + // According to libcurl, callback should return the number of bytes actually taken care of. + // If that amount differs from the amount passed to callback function, it would signals an + // error condition. This causes the transfer to get aborted and the libcurl function used + // returns CURLE_WRITE_ERROR. Therefore, here we just return the max limit of size_t for + // failure. + // + // See https://curl.se/libcurl/c/CURLOPT_WRITEFUNCTION.html for details. + return (*_recv_callback)(data, length) ? length : std::numeric_limits::max(); +} + +dsn::error_s http_client::set_url(const std::string &url) +{ + RETURN_IF_SETOPT_NOT_OK(CURLOPT_URL, url.c_str()); + + _url = url; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::with_post_method(const std::string &data) +{ + // No need to enable CURLOPT_POST by `RETURN_IF_SETOPT_NOT_OK(CURLOPT_POST, 1L)`, since using + // either of CURLOPT_POSTFIELDS or CURLOPT_COPYPOSTFIELDS implies setting CURLOPT_POST to 1. + // See https://curl.se/libcurl/c/CURLOPT_POSTFIELDS.html for details. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_POSTFIELDSIZE, static_cast(data.size())); + RETURN_IF_SETOPT_NOT_OK(CURLOPT_COPYPOSTFIELDS, data.data()); + _method = http_method::POST; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::with_get_method() { return set_method(http_method::GET); } + +dsn::error_s http_client::set_method(http_method method) +{ + // No need to process the case of http_method::POST, since it should be enabled by + // `with_post_method`. + switch (method) { + case http_method::GET: + RETURN_IF_SETOPT_NOT_OK(CURLOPT_HTTPGET, 1L); + break; + default: + LOG_FATAL("Unsupported http_method"); + } + + _method = method; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::set_timeout(long timeout_ms) +{ + RETURN_IF_SETOPT_NOT_OK(CURLOPT_TIMEOUT_MS, timeout_ms); + return dsn::error_s::ok(); +} + +void http_client::clear_header_fields() +{ + _header_fields.clear(); + + _header_changed = true; +} + +void http_client::free_header_list() +{ + if (_header_list == nullptr) { + return; + } + + curl_slist_free_all(_header_list); + _header_list = nullptr; +} + +void http_client::set_header_field(dsn::string_view key, dsn::string_view val) +{ + _header_fields[std::string(key)] = std::string(val); + _header_changed = true; +} + +void http_client::set_accept(dsn::string_view val) { set_header_field("Accept", val); } + +void http_client::set_content_type(dsn::string_view val) { set_header_field("Content-Type", val); } + +dsn::error_s http_client::process_header() +{ + if (!_header_changed) { + return dsn::error_s::ok(); + } + + free_header_list(); + + for (const auto &field : _header_fields) { + auto str = fmt::format("{}: {}", field.first, field.second); + + // A null pointer is returned if anything went wrong, otherwise the new list pointer is + // returned. To avoid overwriting an existing non-empty list on failure, the new list + // should be returned to a temporary variable which can be tested for NULL before updating + // the original list pointer. (https://curl.se/libcurl/c/curl_slist_append.html) + struct curl_slist *temp = curl_slist_append(_header_list, str.c_str()); + if (temp == nullptr) { + free_header_list(); + return dsn::error_s::make(dsn::ERR_CURL_FAILED, "curl_slist_append failed"); + } + _header_list = temp; + } + + // This would work well even if `_header_list` is NULL pointer. Pass a NULL to this option + // to reset back to no custom headers. (https://curl.se/libcurl/c/CURLOPT_HTTPHEADER.html) + RETURN_IF_SETOPT_NOT_OK(CURLOPT_HTTPHEADER, _header_list); + + // New header has been built successfully, thus mark it unchanged. + _header_changed = false; + + return dsn::error_s::ok(); +} + +dsn::error_s http_client::exec_method(const http_client::recv_callback &callback) +{ + // `curl_easy_perform` would run synchronously, thus it is safe to use the pointer to + // `callback`. + _recv_callback = &callback; + + RETURN_NOT_OK(process_header()); + + RETURN_IF_EXEC_METHOD_NOT_OK(); + return dsn::error_s::ok(); +} + +dsn::error_s http_client::exec_method(std::string *response) +{ + if (response == nullptr) { + return exec_method(); + } + + auto callback = [response](const void *data, size_t length) { + response->append(reinterpret_cast(data), length); + return true; + }; + + return exec_method(callback); +} + +dsn::error_s http_client::get_http_status(long &http_status) const +{ + RETURN_IF_GETINFO_NOT_OK(CURLINFO_RESPONSE_CODE, &http_status); + return dsn::error_s::ok(); +} + +#undef RETURN_IF_EXEC_METHOD_NOT_OK +#undef RETURN_IF_SETOPT_NOT_OK +#undef RETURN_IF_CURL_NOT_OK + +} // namespace dsn diff --git a/src/http/http_client.h b/src/http/http_client.h new file mode 100644 index 0000000000..190af11f2b --- /dev/null +++ b/src/http/http_client.h @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "http/http_method.h" +#include "utils/errors.h" +#include "utils/ports.h" +#include "utils/string_view.h" + +namespace dsn { + +// A library for http client that provides convenient APIs to access http services, implemented +// based on libcurl (https://curl.se/libcurl/c/). +// +// This class is not thread-safe. Thus maintain one instance for each thread. +// +// Example of submitting GET request to remote http service +// -------------------------------------------------------- +// Create an instance of http_client: +// http_client client; +// +// It's necessary to initialize the new instance before coming into use: +// auto err = client.init(); +// +// Specify the target url that you would request for: +// err = client.set_url(method); +// +// If you would use GET method, call `with_get_method`: +// err = client.with_get_method(); +// +// If you would use POST method, call `with_post_method` with post data: +// err = client.with_post_method(post_data); +// +// Submit the request to remote http service: +// err = client.exec_method(); +// +// If response data should be processed, use callback function: +// auto callback = [...](const void *data, size_t length) { +// ...... +// return true; +// }; +// err = client.exec_method(callback); +// +// Or just provide a string pointer: +// std::string response; +// err = client.exec_method(&response); +// +// Get the http status code after requesting: +// long http_status; +// err = client.get_http_status(http_status); +class http_client +{ +public: + using recv_callback = std::function; + + http_client(); + ~http_client(); + + // Before coming into use, init() must be called to initialize http client. It could also be + // called to reset the http clients that have been initialized previously. + dsn::error_s init(); + + // Specify the target url that the request would be sent for. + dsn::error_s set_url(const std::string &url); + + // Using post method, with `data` as the payload for post body. + dsn::error_s with_post_method(const std::string &data); + + // Using get method. + dsn::error_s with_get_method(); + + // Specify the maximum time in milliseconds that a request is allowed to complete. + dsn::error_s set_timeout(long timeout_ms); + + // Operations for the header fields. + void clear_header_fields(); + void set_accept(dsn::string_view val); + void set_content_type(dsn::string_view val); + + // Submit request to remote http service, with response processed by callback function. + // + // `callback` function gets called by libcurl as soon as there is data received that needs + // to be saved. For most transfers, this callback gets called many times and each invoke + // delivers another chunk of data. + // + // This function would run synchronously, which means it would wait until the response was + // returned and processed appropriately. + dsn::error_s exec_method(const recv_callback &callback = {}); + + // Submit request to remote http service, with response data returned in a string. + // + // This function would run synchronously, which means it would wait until the response was + // returned and processed appropriately. + dsn::error_s exec_method(std::string *response); + + // Get the last http status code after requesting. + dsn::error_s get_http_status(long &http_status) const; + +private: + using header_field_map = std::unordered_map; + + void clear_error_buf(); + bool is_error_buf_empty() const; + std::string to_error_msg(CURLcode code) const; + + size_t on_response_data(const void *data, size_t length); + + // Specify which http method would be used, such as GET. Enabling POST should not use this + // function (use `with_post_method` instead). + dsn::error_s set_method(http_method method); + + void free_header_list(); + void set_header_field(dsn::string_view key, dsn::string_view val); + dsn::error_s process_header(); + + // The size of a buffer that is used by libcurl to store human readable + // error messages on failures or problems. + static const constexpr size_t kErrorBufferBytes = CURL_ERROR_SIZE; + + CURL *_curl; + http_method _method; + std::string _url; + const recv_callback *_recv_callback; + char _error_buf[kErrorBufferBytes]; + + bool _header_changed; + header_field_map _header_fields; + struct curl_slist *_header_list; + + DISALLOW_COPY_AND_ASSIGN(http_client); +}; + +} // namespace dsn diff --git a/src/http/http_message_parser.cpp b/src/http/http_message_parser.cpp index c6186de30a..df873eea2e 100644 --- a/src/http/http_message_parser.cpp +++ b/src/http/http_message_parser.cpp @@ -26,12 +26,13 @@ #include "http_message_parser.h" +#include // IWYU pragma: no_include #include #include #include -#include "http_server.h" +#include "http/http_method.h" #include "nodejs/http_parser.h" #include "runtime/rpc/rpc_message.h" #include "utils/blob.h" @@ -132,13 +133,15 @@ http_message_parser::http_message_parser() message_header *header = msg->header; if (parser->type == HTTP_REQUEST && parser->method == HTTP_GET) { - header->hdr_type = http_method::HTTP_METHOD_GET; + header->hdr_type = static_cast(http_method::GET); header->context.u.is_request = 1; } else if (parser->type == HTTP_REQUEST && parser->method == HTTP_POST) { - header->hdr_type = http_method::HTTP_METHOD_POST; + header->hdr_type = static_cast(http_method::POST); header->context.u.is_request = 1; } else { - LOG_ERROR("invalid http type {} and method {}", parser->type, parser->method); + // Bit fields don't work with "perfect" forwarding, see + // https://github.com/fmtlib/fmt/issues/1284 + LOG_ERROR("invalid http type {} and method {}", +parser->type, +parser->method); return 1; } return 0; diff --git a/src/block_service/directio_writable_file.h b/src/http/http_method.h similarity index 56% rename from src/block_service/directio_writable_file.h rename to src/http/http_method.h index d4f99949ff..f337d24e78 100644 --- a/src/block_service/directio_writable_file.h +++ b/src/http/http_method.h @@ -17,41 +17,20 @@ #pragma once -#include -#include -#include - -#include "utils/ports.h" +#include "utils/enum_helper.h" namespace dsn { -namespace dist { -namespace block_service { -class direct_io_writable_file +enum class http_method { -public: - explicit direct_io_writable_file(const std::string &file_path); - ~direct_io_writable_file(); - - bool initialize(); - bool write(const char *s, size_t n); - bool finalize(); - -private: - DISALLOW_COPY_AND_ASSIGN(direct_io_writable_file); - - std::string _file_path; - int _fd; - uint32_t _file_size; - - // page size aligned buffer - void *_buffer; - // buffer size - uint32_t _buffer_size; - // buffer offset - uint32_t _offset; + GET = 1, + POST = 2, + INVALID = 100, }; -} // namespace block_service -} // namespace dist +ENUM_BEGIN(http_method, http_method::INVALID) +ENUM_REG2(http_method, GET) +ENUM_REG2(http_method, POST) +ENUM_END(http_method) + } // namespace dsn diff --git a/src/http/http_server.cpp b/src/http/http_server.cpp index db66f8e6ce..4c2f704016 100644 --- a/src/http/http_server.cpp +++ b/src/http/http_server.cpp @@ -17,7 +17,6 @@ #include "http_server.h" -#include #include #include #include @@ -26,6 +25,7 @@ #include "builtin_http_calls.h" #include "fmt/core.h" +#include "http/http_method.h" #include "http_call_registry.h" #include "http_message_parser.h" #include "http_server_impl.h" @@ -147,8 +147,8 @@ void http_server::serve(message_ex *msg) resp.body = fmt::format("failed to parse request: {}", res.get_error()); } else { const http_request &req = res.get_value(); - std::shared_ptr call = http_call_registry::instance().find(req.path); - if (call != nullptr) { + auto call = http_call_registry::instance().find(req.path); + if (call) { call->callback(req, resp); } else { resp.status_code = http_status_code::not_found; diff --git a/src/http/http_server.h b/src/http/http_server.h index b182947e41..5ae780b983 100644 --- a/src/http/http_server.h +++ b/src/http/http_server.h @@ -25,6 +25,7 @@ #include #include +#include "http_method.h" #include "runtime/task/task_code.h" #include "utils/blob.h" #include "utils/errors.h" @@ -38,12 +39,6 @@ DSN_DECLARE_bool(enable_http_server); /// The rpc code for all the HTTP RPCs. DEFINE_TASK_CODE_RPC(RPC_HTTP_SERVICE, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT); -enum http_method -{ - HTTP_METHOD_GET = 1, - HTTP_METHOD_POST = 2, -}; - class message_ex; struct http_request @@ -148,6 +143,8 @@ class http_server_base : public http_service // ``` extern http_call ®ister_http_call(std::string full_path); +extern void deregister_http_call(const std::string &full_path); + // Starts serving HTTP requests. // The internal HTTP server will reuse the rDSN server port. extern void start_http_server(); diff --git a/src/http/pprof_http_service.cpp b/src/http/pprof_http_service.cpp index f57c57b193..f5aeb8b505 100644 --- a/src/http/pprof_http_service.cpp +++ b/src/http/pprof_http_service.cpp @@ -38,6 +38,7 @@ #include #include +#include "http/http_method.h" #include "http/http_server.h" #include "runtime/api_layer1.h" #include "utils/blob.h" @@ -308,7 +309,7 @@ void pprof_http_service::symbol_handler(const http_request &req, http_response & // Load /proc/self/maps pthread_once(&s_load_symbolmap_once, load_symbols); - if (req.method != http_method::HTTP_METHOD_POST) { + if (req.method != http_method::POST) { char buf[64]; snprintf(buf, sizeof(buf), "num_symbols: %lu\n", symbol_map.size()); resp.body = buf; diff --git a/src/http/test/CMakeLists.txt b/src/http/test/CMakeLists.txt index 6a09e42dc5..85f2c3798a 100644 --- a/src/http/test/CMakeLists.txt +++ b/src/http/test/CMakeLists.txt @@ -24,14 +24,16 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS dsn_http dsn_runtime + curl gtest - gtest_main + rocksdb ) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/run.sh" + "${CMAKE_CURRENT_SOURCE_DIR}/config-test.ini" ) dsn_add_test() diff --git a/src/http/test/config-test.ini b/src/http/test/config-test.ini new file mode 100644 index 0000000000..744d8d412a --- /dev/null +++ b/src/http/test/config-test.ini @@ -0,0 +1,75 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +; KIND, either express or implied. See the License for the +; specific language governing permissions and limitations +; under the License. + +[apps..default] +run = true +count = 1 +network.client.RPC_CHANNEL_TCP = dsn::tools::asio_network_provider, 65536 +network.client.RPC_CHANNEL_UDP = dsn::tools::asio_udp_provider, 65536 +network.server.0.RPC_CHANNEL_TCP = dsn::tools::asio_network_provider, 65536 +network.server.0.RPC_CHANNEL_UDP = dsn::tools::asio_udp_provider, 65536 + +[apps.test] +type = test +arguments = +run = true +ports = 20001 +count = 1 +pools = THREAD_POOL_DEFAULT + +[core] +tool = nativerun + +toollets = tracer, profiler +pause_on_start = false + +logging_start_level = LOG_LEVEL_DEBUG +logging_factory_name = dsn::tools::simple_logger + +[tools.simple_logger] +fast_flush = true +short_header = false +stderr_start_level = LOG_LEVEL_DEBUG + +[network] +; how many network threads for network library (used by asio) +io_service_worker_count = 2 + +[task..default] +is_trace = true +is_profile = true +allow_inline = false +rpc_call_channel = RPC_CHANNEL_TCP +rpc_message_header_format = dsn +rpc_timeout_milliseconds = 1000 + +[task.LPC_RPC_TIMEOUT] +is_trace = false +is_profile = false + +[task.RPC_TEST_UDP] +rpc_call_channel = RPC_CHANNEL_UDP +rpc_message_crc_required = true + +; specification for each thread pool +[threadpool..default] +worker_count = 2 + +[threadpool.THREAD_POOL_DEFAULT] +partitioned = false +worker_priority = THREAD_xPRIORITY_NORMAL + diff --git a/src/http/test/http_client_test.cpp b/src/http/test/http_client_test.cpp new file mode 100644 index 0000000000..d15cb7bdd4 --- /dev/null +++ b/src/http/test/http_client_test.cpp @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +#include +#include +#include +#include + +#include "http/http_client.h" +#include "http/http_method.h" +#include "utils/error_code.h" +#include "utils/errors.h" +#include "utils/fmt_logging.h" +#include "utils/test_macros.h" + +namespace dsn { + +void check_expected_description_prefix(const std::string &expected_description_prefix, + const dsn::error_s &err) +{ + const std::string actual_description(err.description()); + std::cout << actual_description << std::endl; + + ASSERT_LT(expected_description_prefix.size(), actual_description.size()); + EXPECT_EQ(expected_description_prefix, + actual_description.substr(0, expected_description_prefix.size())); +} + +TEST(HttpClientTest, Connect) +{ + http_client client; + ASSERT_TRUE(client.init()); + + // No one has listened on port 20000, thus this would lead to "Connection refused". + ASSERT_TRUE(client.set_url("http://127.0.0.1:20000/test/get")); + + const auto &err = client.exec_method(); + ASSERT_EQ(dsn::ERR_CURL_FAILED, err.code()); + + std::cout << "failed to connect: "; + + // "code=7" means CURLE_COULDNT_CONNECT, see https://curl.se/libcurl/c/libcurl-errors.html + // for details. + // + // We just check the prefix of description, including `method`, `url`, `code` and `desc`. + // The `msg` differ in various systems, such as: + // * msg="Failed to connect to 127.0.0.1 port 20000: Connection refused" + // * msg="Failed to connect to 127.0.0.1 port 20000 after 0 ms: Connection refused" + // Thus we don't check if `msg` fields are consistent. + const std::string expected_description_prefix( + "ERR_CURL_FAILED: failed to perform http request(" + "method=GET, url=http://127.0.0.1:20000/test/get): code=7, " + "desc=\"Couldn't connect to server\""); + NO_FATALS(check_expected_description_prefix(expected_description_prefix, err)); +} + +TEST(HttpClientTest, Callback) +{ + http_client client; + ASSERT_TRUE(client.init()); + + ASSERT_TRUE(client.set_url("http://127.0.0.1:20001/test/get")); + ASSERT_TRUE(client.with_get_method()); + + auto callback = [](const void *, size_t) { return false; }; + + const auto &err = client.exec_method(callback); + ASSERT_EQ(dsn::ERR_CURL_FAILED, err.code()); + + long actual_http_status; + ASSERT_TRUE(client.get_http_status(actual_http_status)); + EXPECT_EQ(200, actual_http_status); + + std::cout << "failed for callback: "; + + // "code=23" means CURLE_WRITE_ERROR, see https://curl.se/libcurl/c/libcurl-errors.html + // for details. + // + // We just check the prefix of description, including `method`, `url`, `code` and `desc`. + // The `msg` differ in various systems, such as: + // * msg="Failed writing body (18446744073709551615 != 24)" + // * msg="Failure writing output to destination" + // Thus we don't check if `msg` fields are consistent. + const auto expected_description_prefix = + fmt::format("ERR_CURL_FAILED: failed to perform http request(" + "method=GET, url=http://127.0.0.1:20001/test/get): code=23, " + "desc=\"Failed writing received data to disk/application\""); + NO_FATALS(check_expected_description_prefix(expected_description_prefix, err)); +} + +using http_client_method_case = + std::tuple; + +class HttpClientMethodTest : public testing::TestWithParam +{ +public: + void SetUp() override { ASSERT_TRUE(_client.init()); } + + void test_method_with_response_string(const long expected_http_status, + const std::string &expected_response) + { + std::string actual_response; + ASSERT_TRUE(_client.exec_method(&actual_response)); + + long actual_http_status; + ASSERT_TRUE(_client.get_http_status(actual_http_status)); + + EXPECT_EQ(expected_http_status, actual_http_status); + EXPECT_EQ(expected_response, actual_response); + } + + void test_method_with_response_callback(const long expected_http_status, + const std::string &expected_response) + { + auto callback = [&expected_response](const void *data, size_t length) { + auto compare = [](const char *expected_data, + size_t expected_length, + const void *actual_data, + size_t actual_length) { + if (expected_length != actual_length) { + return false; + } + return std::memcmp(expected_data, actual_data, actual_length) == 0; + }; + EXPECT_PRED4(compare, expected_response.data(), expected_response.size(), data, length); + return true; + }; + ASSERT_TRUE(_client.exec_method(callback)); + + long actual_http_status; + ASSERT_TRUE(_client.get_http_status(actual_http_status)); + EXPECT_EQ(expected_http_status, actual_http_status); + } + + void test_mothod(const std::string &url, + const http_method method, + const std::string &post_data, + const long expected_http_status, + const std::string &expected_response) + { + _client.set_url(url); + + switch (method) { + case http_method::GET: + ASSERT_TRUE(_client.with_get_method()); + break; + case http_method::POST: + ASSERT_TRUE(_client.with_post_method(post_data)); + break; + default: + LOG_FATAL("Unsupported http_method"); + } + + test_method_with_response_string(expected_http_status, expected_response); + test_method_with_response_callback(expected_http_status, expected_response); + } + +private: + http_client _client; +}; + +TEST_P(HttpClientMethodTest, ExecMethod) +{ + const char *url; + http_method method; + const char *post_data; + long expected_http_status; + const char *expected_response; + std::tie(url, method, post_data, expected_http_status, expected_response) = GetParam(); + + http_client _client; + test_mothod(url, method, post_data, expected_http_status, expected_response); +} + +const std::vector http_client_method_tests = { + {"http://127.0.0.1:20001/test/get", + http_method::POST, + "with POST DATA", + 400, + "please use GET method"}, + {"http://127.0.0.1:20001/test/get", http_method::GET, "", 200, "you are using GET method"}, + {"http://127.0.0.1:20001/test/post", + http_method::POST, + "with POST DATA", + 200, + "you are using POST method with POST DATA"}, + {"http://127.0.0.1:20001/test/post", http_method::GET, "", 400, "please use POST method"}, +}; + +INSTANTIATE_TEST_CASE_P(HttpClientTest, + HttpClientMethodTest, + testing::ValuesIn(http_client_method_tests)); + +} // namespace dsn diff --git a/src/http/test/http_server_test.cpp b/src/http/test/http_server_test.cpp index 9aa9f17c39..0c6b9ec6d3 100644 --- a/src/http/test/http_server_test.cpp +++ b/src/http/test/http_server_test.cpp @@ -18,6 +18,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include "http/builtin_http_calls.h" #include "http/http_call_registry.h" #include "http/http_message_parser.h" +#include "http/http_method.h" #include "http/http_server.h" #include "runtime/rpc/message_parser.h" #include "runtime/rpc/rpc_message.h" @@ -87,7 +89,12 @@ TEST(bultin_http_calls_test, meta_query) TEST(bultin_http_calls_test, get_help) { + // Used to save current http calls as backup. + std::vector> backup_calls; + + // Remove all http calls. for (const auto &call : http_call_registry::instance().list_all_calls()) { + backup_calls.push_back(call); http_call_registry::instance().remove(call->path); } @@ -111,9 +118,15 @@ TEST(bultin_http_calls_test, get_help) get_help_handler(req, resp); ASSERT_EQ(resp.body, "{\"/\":\"ip:port/\",\"/recentStartTime\":\"ip:port/recentStartTime\"}\n"); + // Remove all http calls, especially `recentStartTime`. for (const auto &call : http_call_registry::instance().list_all_calls()) { http_call_registry::instance().remove(call->path); } + + // Recover http calls from backup. + for (const auto &call : backup_calls) { + http_call_registry::instance().add(call); + } } class http_message_parser_test : public testing::Test @@ -174,7 +187,7 @@ class http_message_parser_test : public testing::Test message_ptr msg = parser.get_message_on_receive(&reader, read_next); ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_GET); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::GET)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[2].size(), 1); // url @@ -215,7 +228,7 @@ TEST_F(http_message_parser_test, parse_request) ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_POST); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::POST)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[1].to_string(), "Message Body sdfsdf"); // body @@ -266,7 +279,7 @@ TEST_F(http_message_parser_test, eof) ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_GET); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::GET)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[1].to_string(), ""); // body @@ -297,7 +310,7 @@ TEST_F(http_message_parser_test, parse_long_url) message_ptr msg = parser.get_message_on_receive(&reader, read_next); ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_GET); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::GET)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[2].size(), 4097); // url diff --git a/src/http/test/main.cpp b/src/http/test/main.cpp new file mode 100644 index 0000000000..1eeb3d04ef --- /dev/null +++ b/src/http/test/main.cpp @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include + +#include "http/http_method.h" +#include "http/http_server.h" +#include "runtime/app_model.h" +#include "runtime/service_app.h" +#include "utils/blob.h" +#include "utils/error_code.h" +#include "utils/ports.h" + +int gtest_flags = 0; +int gtest_ret = 0; + +class test_http_service : public dsn::http_server_base +{ +public: + test_http_service() + { + register_handler("get", + std::bind(&test_http_service::method_handler, + this, + dsn::http_method::GET, + std::placeholders::_1, + std::placeholders::_2), + "ip:port/test/get"); + register_handler("post", + std::bind(&test_http_service::method_handler, + this, + dsn::http_method::POST, + std::placeholders::_1, + std::placeholders::_2), + "ip:port/test/post"); + } + + ~test_http_service() = default; + + std::string path() const override { return "test"; } + +private: + void method_handler(dsn::http_method target_method, + const dsn::http_request &req, + dsn::http_response &resp) + { + if (req.method != target_method) { + resp.body = fmt::format("please use {} method", enum_to_string(target_method)); + resp.status_code = dsn::http_status_code::bad_request; + return; + } + + std::string postfix; + if (target_method == dsn::http_method::POST) { + postfix = " "; + postfix += req.body.to_string(); + } + + resp.body = + fmt::format("you are using {} method{}", enum_to_string(target_method), postfix); + resp.status_code = dsn::http_status_code::ok; + } + + DISALLOW_COPY_AND_ASSIGN(test_http_service); +}; + +class test_service_app : public dsn::service_app +{ +public: + test_service_app(const dsn::service_app_info *info) : dsn::service_app(info) + { + dsn::register_http_service(new test_http_service); + dsn::start_http_server(); + } + + dsn::error_code start(const std::vector &args) override + { + gtest_ret = RUN_ALL_TESTS(); + gtest_flags = 1; + return dsn::ERR_OK; + } +}; + +GTEST_API_ int main(int argc, char **argv) +{ + testing::InitGoogleTest(&argc, argv); + + // Register test service. + dsn::service_app::register_factory("test"); + + dsn_run_config("config-test.ini", false); + while (gtest_flags == 0) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + +#ifndef ENABLE_GCOV + dsn_exit(gtest_ret); +#endif + return gtest_ret; +} diff --git a/src/include/pegasus/client.h b/src/include/pegasus/client.h index 8f92b5933c..60e36c55b5 100644 --- a/src/include/pegasus/client.h +++ b/src/include/pegasus/client.h @@ -28,6 +28,8 @@ #include #include +#include "utils/fmt_utils.h" + namespace pegasus { class rrdb_client; @@ -1216,4 +1218,6 @@ class pegasus_client_factory static pegasus_client *get_client(const char *cluster_name, const char *app_name); }; +USER_DEFINED_ENUM_FORMATTER(pegasus_client::filter_type) +USER_DEFINED_ENUM_FORMATTER(pegasus_client::cas_check_type) } // namespace pegasus diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index b38ced7228..def16ec26c 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -54,7 +54,7 @@ set(MY_PROJ_LIBS crypto hashtable hdfs - ) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/meta/app_env_validator.cpp b/src/meta/app_env_validator.cpp index 41f9c299ce..7f197f5a0f 100644 --- a/src/meta/app_env_validator.cpp +++ b/src/meta/app_env_validator.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,26 @@ namespace dsn { namespace replication { +bool validate_app_envs(const std::map &envs) +{ + // only check rocksdb app envs currently + + for (const auto &it : envs) { + if (replica_envs::ROCKSDB_STATIC_OPTIONS.find(it.first) == + replica_envs::ROCKSDB_STATIC_OPTIONS.end() && + replica_envs::ROCKSDB_DYNAMIC_OPTIONS.find(it.first) == + replica_envs::ROCKSDB_DYNAMIC_OPTIONS.end()) { + continue; + } + std::string hint_message; + if (!validate_app_env(it.first, it.second, hint_message)) { + LOG_WARNING( + "app env {}={} is invaild, hint_message:{}", it.first, it.second, hint_message); + return false; + } + } + return true; +} bool validate_app_env(const std::string &env_name, const std::string &env_value, @@ -151,6 +172,36 @@ bool check_bool_value(const std::string &env_value, std::string &hint_message) return true; } +bool check_rocksdb_write_buffer_size(const std::string &env_value, std::string &hint_message) +{ + uint64_t val = 0; + + if (!dsn::buf2uint64(env_value, val)) { + hint_message = fmt::format("rocksdb.write_buffer_size cannot set this val: {}", env_value); + return false; + } + if (val < (16 << 20) || val > (512 << 20)) { + hint_message = + fmt::format("rocksdb.write_buffer_size suggest set val in range [16777216, 536870912]"); + return false; + } + return true; +} +bool check_rocksdb_num_levels(const std::string &env_value, std::string &hint_message) +{ + int32_t val = 0; + + if (!dsn::buf2int32(env_value, val)) { + hint_message = fmt::format("rocksdb.num_levels cannot set this val: {}", env_value); + return false; + } + if (val < 1 || val > 10) { + hint_message = fmt::format("rocksdb.num_levels suggest set val in range [1 , 10]"); + return false; + } + return true; +} + bool app_env_validator::validate_app_env(const std::string &env_name, const std::string &env_value, std::string &hint_message) @@ -198,6 +249,10 @@ void app_env_validator::register_all_validators() std::bind(&check_bool_value, std::placeholders::_1, std::placeholders::_2)}, {replica_envs::DENY_CLIENT_REQUEST, std::bind(&check_deny_client, std::placeholders::_1, std::placeholders::_2)}, + {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, + std::bind(&check_rocksdb_write_buffer_size, std::placeholders::_1, std::placeholders::_2)}, + {replica_envs::ROCKSDB_NUM_LEVELS, + std::bind(&check_rocksdb_num_levels, std::placeholders::_1, std::placeholders::_2)}, // TODO(zhaoliwei): not implemented {replica_envs::BUSINESS_INFO, nullptr}, {replica_envs::TABLE_LEVEL_DEFAULT_TTL, nullptr}, @@ -213,7 +268,8 @@ void app_env_validator::register_all_validators() {replica_envs::MANUAL_COMPACT_PERIODIC_TARGET_LEVEL, nullptr}, {replica_envs::MANUAL_COMPACT_PERIODIC_BOTTOMMOST_LEVEL_COMPACTION, nullptr}, {replica_envs::REPLICA_ACCESS_CONTROLLER_ALLOWED_USERS, nullptr}, - {replica_envs::REPLICA_ACCESS_CONTROLLER_RANGER_POLICIES, nullptr}}; + {replica_envs::REPLICA_ACCESS_CONTROLLER_RANGER_POLICIES, nullptr}, + }; } } // namespace replication diff --git a/src/meta/app_env_validator.h b/src/meta/app_env_validator.h index c401c39598..d963df0513 100644 --- a/src/meta/app_env_validator.h +++ b/src/meta/app_env_validator.h @@ -25,6 +25,8 @@ namespace dsn { namespace replication { +bool validate_app_envs(const std::map &envs); + bool validate_app_env(const std::string &env_name, const std::string &env_value, std::string &hint_message); diff --git a/src/meta/distributed_lock_service_simple.h b/src/meta/distributed_lock_service_simple.h index a2da463b49..59b2bde5e6 100644 --- a/src/meta/distributed_lock_service_simple.h +++ b/src/meta/distributed_lock_service_simple.h @@ -53,6 +53,7 @@ namespace dsn { namespace dist { +// Only for test purpose. class distributed_lock_service_simple : public distributed_lock_service { public: diff --git a/src/meta/dump_file.h b/src/meta/dump_file.h index 4e8018ca58..062ffd1fcc 100644 --- a/src/meta/dump_file.h +++ b/src/meta/dump_file.h @@ -63,6 +63,8 @@ struct block_header uint32_t crc32; }; +// TODO(yingchun): use rocksdb APIs to unify the file operations. +// A tool to dump app_states of meta server to local file, used by remote command "meta.dump". class dump_file { public: diff --git a/src/meta/duplication/meta_duplication_service.cpp b/src/meta/duplication/meta_duplication_service.cpp index 663c35d1c9..4d51bac70c 100644 --- a/src/meta/duplication/meta_duplication_service.cpp +++ b/src/meta/duplication/meta_duplication_service.cpp @@ -16,10 +16,8 @@ // under the License. #include -#include #include #include -#include #include #include diff --git a/src/meta/meta_backup_service.cpp b/src/meta/meta_backup_service.cpp index db2afe1d90..8a28ffaf35 100644 --- a/src/meta/meta_backup_service.cpp +++ b/src/meta/meta_backup_service.cpp @@ -1036,10 +1036,7 @@ void policy_context::sync_remove_backup_info(const backup_info &info, dsn::task_ 0, _backup_service->backup_option().meta_retry_delay_ms); } else { - CHECK(false, - "{}: we can't handle this right now, error({})", - _policy.policy_name, - err.to_string()); + CHECK(false, "{}: we can't handle this right now, error({})", _policy.policy_name, err); } }; @@ -1373,9 +1370,7 @@ void backup_service::do_add_policy(dsn::message_ex *req, _opt.meta_retry_delay_ms); return; } else { - CHECK(false, - "we can't handle this when create backup policy, err({})", - err.to_string()); + CHECK(false, "we can't handle this when create backup policy, err({})", err); } }, value); @@ -1411,9 +1406,7 @@ void backup_service::do_update_policy_to_remote_storage( 0, _opt.meta_retry_delay_ms); } else { - CHECK(false, - "we can't handle this when create backup policy, err({})", - err.to_string()); + CHECK(false, "we can't handle this when create backup policy, err({})", err); } }); } diff --git a/src/meta/meta_backup_service.h b/src/meta/meta_backup_service.h index e6563dbcc2..3c7138cafb 100644 --- a/src/meta/meta_backup_service.h +++ b/src/meta/meta_backup_service.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include #include @@ -41,11 +42,13 @@ #include "runtime/task/task_tracker.h" #include "utils/api_utilities.h" #include "utils/error_code.h" +#include "utils/fmt_utils.h" #include "utils/zlocks.h" namespace dsn { class message_ex; class rpc_address; + namespace dist { namespace block_service { class block_filesystem; @@ -420,3 +423,5 @@ class backup_service }; } // namespace replication } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::backup_start_time); diff --git a/src/meta/meta_bulk_load_service.h b/src/meta/meta_bulk_load_service.h index ba151757a8..c411d87f44 100644 --- a/src/meta/meta_bulk_load_service.h +++ b/src/meta/meta_bulk_load_service.h @@ -97,6 +97,10 @@ struct bulk_load_info int32_t app_id; std::string app_name; int32_t partition_count; + bulk_load_info(int32_t id = 0, const std::string &name = "", int32_t pcount = 0) + : app_id(id), app_name(name), partition_count(pcount) + { + } DEFINE_JSON_SERIALIZATION(app_id, app_name, partition_count) }; diff --git a/src/meta/meta_data.cpp b/src/meta/meta_data.cpp index 48ab6ccd3f..23dba0c929 100644 --- a/src/meta/meta_data.cpp +++ b/src/meta/meta_data.cpp @@ -159,10 +159,7 @@ bool construct_replica(meta_view view, const gpid &pid, int max_replica_count) // we put max_replica_count-1 recent replicas to last_drops, in case of the DDD-state when the // only primary dead // when add node to pc.last_drops, we don't remove it from our cc.drop_list - CHECK(pc.last_drops.empty(), - "last_drops of partition({}.{}) must be empty", - pid.get_app_id(), - pid.get_partition_index()); + CHECK(pc.last_drops.empty(), "last_drops of partition({}) must be empty", pid); for (auto iter = drop_list.rbegin(); iter != drop_list.rend(); ++iter) { if (pc.last_drops.size() + 1 >= max_replica_count) break; @@ -402,10 +399,9 @@ int config_context::collect_drop_replica(const rpc_address &node, const replica_ iter = find_from_dropped(node); if (iter == dropped.end()) { CHECK(!in_dropped, - "adjust position of existing node({}) failed, this is a bug, partition({}.{})", - node.to_string(), - config_owner->pid.get_app_id(), - config_owner->pid.get_partition_index()); + "adjust position of existing node({}) failed, this is a bug, partition({})", + node, + config_owner->pid); return -1; } return in_dropped ? 1 : 0; diff --git a/src/meta/meta_options.cpp b/src/meta/meta_options.cpp index 7d9864a354..ebc39ee597 100644 --- a/src/meta/meta_options.cpp +++ b/src/meta/meta_options.cpp @@ -38,6 +38,7 @@ #include #include +#include "common/replication_enums.h" // IWYU pragma: keep #include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/strings.h" diff --git a/src/meta/meta_service.cpp b/src/meta/meta_service.cpp index 4b65d7b299..5ea615c61d 100644 --- a/src/meta/meta_service.cpp +++ b/src/meta/meta_service.cpp @@ -25,9 +25,8 @@ */ // IWYU pragma: no_include -#include // IWYU pragma: no_include -#include +#include #include // for std::remove_if #include #include diff --git a/src/meta/meta_state_service_simple.cpp b/src/meta/meta_state_service_simple.cpp index f2db0da0a7..776cfba342 100644 --- a/src/meta/meta_state_service_simple.cpp +++ b/src/meta/meta_state_service_simple.cpp @@ -26,8 +26,6 @@ #include "meta_state_service_simple.h" -#include -#include #include #include #include @@ -35,6 +33,9 @@ #include #include "aio/file_io.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/service_app.h" #include "runtime/task/async_calls.h" #include "runtime/task/task.h" @@ -42,7 +43,6 @@ #include "utils/binary_reader.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" #include "utils/strings.h" #include "utils/utils.h" @@ -228,7 +228,7 @@ error_code meta_state_service_simple::apply_transaction( default: CHECK(false, "unsupported operation"); } - CHECK_EQ_MSG(ec, ERR_OK, "unexpected error when applying"); + CHECK_EQ_MSG(ERR_OK, ec, "unexpected error when applying"); } return ERR_OK; @@ -242,57 +242,77 @@ error_code meta_state_service_simple::initialize(const std::vector _offset = 0; std::string log_path = dsn::utils::filesystem::path_combine(work_dir, "meta_state_service.log"); if (utils::filesystem::file_exists(log_path)) { - if (FILE *fd = fopen(log_path.c_str(), "rb")) { - for (;;) { - log_header header; - if (fread(&header, sizeof(log_header), 1, fd) != 1) { - break; - } - if (header.magic != log_header::default_magic) { - break; - } - std::shared_ptr buffer(dsn::utils::make_shared_array(header.size)); - if (fread(buffer.get(), header.size, 1, fd) != 1) { - break; - } - _offset += sizeof(header) + header.size; - binary_reader reader(blob(buffer, (int)header.size)); - int op_type = 0; - reader.read(op_type); - - switch (static_cast(op_type)) { - case operation_type::create_node: { - std::string node; - blob data; - create_node_log::parse(reader, node, data); - create_node_internal(node, data); - break; - } - case operation_type::delete_node: { - std::string node; - bool recursively_delete; - delete_node_log::parse(reader, node, recursively_delete); - delete_node_internal(node, recursively_delete); - break; - } - case operation_type::set_data: { - std::string node; - blob data; - set_data_log::parse(reader, node, data); - set_data_internal(node, data); - break; - } - default: - // The log is complete but its content is modified by cosmic ray. This is - // unacceptable - CHECK(false, "meta state server log corrupted"); - } + std::unique_ptr log_file; + auto s = + rocksdb::Env::Default()->NewSequentialFile(log_path, &log_file, rocksdb::EnvOptions()); + CHECK(s.ok(), "open log file '{}' failed, err = {}", log_path, s.ToString()); + + while (true) { + static const int kLogHeaderSize = sizeof(log_header); + static const int kDefaultMagic = 0xdeadbeef; + rocksdb::Slice result; + + // Read header. + char scratch[kLogHeaderSize] = {0}; + s = log_file->PositionedRead(_offset, kLogHeaderSize, &result, scratch); + CHECK(s.ok(), "read log file '{}' header failed, err = {}", log_path, s.ToString()); + if (result.empty()) { + LOG_INFO("read EOF of log file '{}'", log_path); + break; + } + log_header *header = reinterpret_cast(scratch); + if (header->magic != kDefaultMagic) { + LOG_WARNING("read log file '{}' header with bad magic {}, skip the left!", + log_path, + header->magic); + break; + } + _offset += kLogHeaderSize; + + // Read body. + std::shared_ptr buffer(dsn::utils::make_shared_array(header->size)); + s = log_file->PositionedRead(_offset, header->size, &result, buffer.get()); + CHECK(s.ok(), + "read log file '{}' header with bad body, err = {}", + log_path, + s.ToString()); + _offset += header->size; + + binary_reader reader(blob(buffer, header->size)); + int op_type = 0; + CHECK_EQ(sizeof(op_type), reader.read(op_type)); + + switch (static_cast(op_type)) { + case operation_type::create_node: { + std::string node; + blob data; + create_node_log::parse(reader, node, data); + create_node_internal(node, data); + break; + } + case operation_type::delete_node: { + std::string node; + bool recursively_delete; + delete_node_log::parse(reader, node, recursively_delete); + delete_node_internal(node, recursively_delete); + break; + } + case operation_type::set_data: { + std::string node; + blob data; + set_data_log::parse(reader, node, data); + set_data_internal(node, data); + break; + } + default: + // The log is complete but its content is modified by cosmic ray. This is + // unacceptable + CHECK(false, "meta state server log corrupted"); } - fclose(fd); } } - _log = file::open(log_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + _log = file::open(log_path, file::FileOpenType::kWriteOnly); if (!_log) { LOG_ERROR("open file failed: {}", log_path); return ERR_FILE_OPERATION_FAILED; @@ -507,8 +527,9 @@ task_ptr meta_state_service_simple::get_children(const std::string &node, meta_state_service_simple::~meta_state_service_simple() { - _tracker.cancel_outstanding_tasks(); - file::close(_log); + _tracker.wait_outstanding_tasks(); + CHECK_EQ(ERR_OK, file::flush(_log)); + CHECK_EQ(ERR_OK, file::close(_log)); for (const auto &kv : _quick_map) { if ("/" != kv.first) { diff --git a/src/meta/meta_state_service_simple.h b/src/meta/meta_state_service_simple.h index f26e33e174..697b5b159a 100644 --- a/src/meta/meta_state_service_simple.h +++ b/src/meta/meta_state_service_simple.h @@ -66,6 +66,7 @@ DEFINE_TASK_CODE_AIO(LPC_META_STATE_SERVICE_SIMPLE_INTERNAL, TASK_PRIORITY_HIGH, THREAD_POOL_DEFAULT); +// Only for test purpose. class meta_state_service_simple : public meta_state_service { public: diff --git a/src/meta/meta_state_service_utils_impl.h b/src/meta/meta_state_service_utils_impl.h index afe4200ce5..f6a1b75918 100644 --- a/src/meta/meta_state_service_utils_impl.h +++ b/src/meta/meta_state_service_utils_impl.h @@ -17,12 +17,12 @@ #pragma once +#include "common/replication.codes.h" +#include "meta/meta_state_service.h" +#include "meta_state_service_utils.h" #include "runtime/pipeline.h" #include "utils/fmt_logging.h" - -#include "meta_state_service_utils.h" -#include "meta/meta_state_service.h" -#include "common/replication.codes.h" +#include "utils/fmt_utils.h" namespace dsn { namespace replication { @@ -61,6 +61,7 @@ struct op_type return op_type_to_string_map[v - 1]; } }; +USER_DEFINED_ENUM_FORMATTER(op_type::type) /// Base class for all operations. struct operation : pipeline::environment diff --git a/src/meta/meta_state_service_zookeeper.cpp b/src/meta/meta_state_service_zookeeper.cpp index 03717f7f6a..a367c1897e 100644 --- a/src/meta/meta_state_service_zookeeper.cpp +++ b/src/meta/meta_state_service_zookeeper.cpp @@ -390,8 +390,9 @@ void meta_state_service_zookeeper::visit_zookeeper_internal(ref_this, { zookeeper_session::zoo_opcontext *op = reinterpret_cast(result); - LOG_DEBUG( - "visit zookeeper internal: ans({}), call type({})", zerror(op->_output.error), op->_optype); + LOG_DEBUG("visit zookeeper internal: ans({}), call type({})", + zerror(op->_output.error), + static_cast(op->_optype)); switch (op->_optype) { case zookeeper_session::ZOO_OPERATION::ZOO_CREATE: diff --git a/src/meta/partition_guardian.cpp b/src/meta/partition_guardian.cpp index 09b0862c4c..44e0017f69 100644 --- a/src/meta/partition_guardian.cpp +++ b/src/meta/partition_guardian.cpp @@ -102,9 +102,8 @@ void partition_guardian::reconfig(meta_view view, const configuration_update_req if (!cc->lb_actions.empty()) { const configuration_proposal_action *current = cc->lb_actions.front(); CHECK(current != nullptr && current->type != config_type::CT_INVALID, - "invalid proposal for gpid({}.{})", - gpid.get_app_id(), - gpid.get_partition_index()); + "invalid proposal for gpid({})", + gpid); // if the valid proposal is from cure if (!cc->lb_actions.is_from_balancer()) { finish_cure_proposal(view, gpid, *current); @@ -132,7 +131,7 @@ void partition_guardian::reconfig(meta_view view, const configuration_update_req CHECK(cc->record_drop_history(request.node), "node({}) has been in the dropped", - request.node.to_string()); + request.node); } }); } @@ -241,8 +240,7 @@ pc_status partition_guardian::on_missing_primary(meta_view &view, const dsn::gpi for (int i = 0; i < pc.secondaries.size(); ++i) { node_state *ns = get_node_state(*(view.nodes), pc.secondaries[i], false); - CHECK_NOTNULL( - ns, "invalid secondary address, address = {}", pc.secondaries[i].to_string()); + CHECK_NOTNULL(ns, "invalid secondary address, address = {}", pc.secondaries[i]); if (!ns->alive()) continue; @@ -607,9 +605,7 @@ pc_status partition_guardian::on_missing_secondary(meta_view &view, const dsn::g // if not emergency, only try to recover last dropped server const dropped_replica &server = cc.dropped.back(); if (is_node_alive(*view.nodes, server.node)) { - CHECK(!server.node.is_invalid(), - "invalid server address, address = {}", - server.node.to_string()); + CHECK(!server.node.is_invalid(), "invalid server address, address = {}", server.node); action.node = server.node; } diff --git a/src/meta/server_state.cpp b/src/meta/server_state.cpp index 38aba0c5b9..02722568ef 100644 --- a/src/meta/server_state.cpp +++ b/src/meta/server_state.cpp @@ -1163,6 +1163,9 @@ void server_state::create_app(dsn::message_ex *msg) !validate_target_max_replica_count(request.options.replica_count)) { response.err = ERR_INVALID_PARAMETERS; will_create_app = false; + } else if (!validate_app_envs(request.options.envs)) { + response.err = ERR_INVALID_PARAMETERS; + will_create_app = false; } else { zauto_write_lock l(_lock); app = get_app(request.app_name); diff --git a/src/meta/test/config-test.ini b/src/meta/test/config-test.ini index 26bf34f759..d0a8656567 100644 --- a/src/meta/test/config-test.ini +++ b/src/meta/test/config-test.ini @@ -106,6 +106,7 @@ only_move_primary = false cold_backup_disabled = false [replication] +disk_min_available_space_ratio = 10 cluster_name = master-cluster duplication_enabled = true diff --git a/src/meta/test/main.cpp b/src/meta/test/main.cpp index 5165352006..cd100d4b41 100644 --- a/src/meta/test/main.cpp +++ b/src/meta/test/main.cpp @@ -63,7 +63,11 @@ TEST(meta, state_sync) { g_app->state_sync_test(); } TEST(meta, update_configuration) { g_app->update_configuration_test(); } -TEST(meta, balancer_validator) { g_app->balancer_validator(); } +TEST(meta, balancer_validator) +{ + // TODO(yingchun): this test last too long time, optimize it! + g_app->balancer_validator(); +} TEST(meta, apply_balancer) { g_app->apply_balancer_test(); } diff --git a/src/meta/test/meta_app_envs_test.cpp b/src/meta/test/meta_app_envs_test.cpp index 255930320d..d5dc21edac 100644 --- a/src/meta/test/meta_app_envs_test.cpp +++ b/src/meta/test/meta_app_envs_test.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include "common/replica_envs.h" @@ -142,6 +143,17 @@ TEST_F(meta_app_envs_test, update_app_envs_test) {replica_envs::MANUAL_COMPACT_ONCE_TARGET_LEVEL, "80", ERR_OK, "", "80"}, {replica_envs::MANUAL_COMPACT_PERIODIC_TRIGGER_TIME, "90", ERR_OK, "", "90"}, {replica_envs::MANUAL_COMPACT_PERIODIC_TARGET_LEVEL, "100", ERR_OK, "", "100"}, + {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, + "100", + ERR_INVALID_PARAMETERS, + "rocksdb.write_buffer_size suggest set val in range [16777216, 536870912]", + "67108864"}, + {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, + "636870912", + ERR_INVALID_PARAMETERS, + "rocksdb.write_buffer_size suggest set val in range [16777216, 536870912]", + "536870912"}, + {replica_envs::ROCKSDB_WRITE_BUFFER_SIZE, "67108864", ERR_OK, "", "67108864"}, {replica_envs::MANUAL_COMPACT_PERIODIC_BOTTOMMOST_LEVEL_COMPACTION, "200", ERR_OK, @@ -192,6 +204,18 @@ TEST_F(meta_app_envs_test, update_app_envs_test) ASSERT_EQ(app->envs.at(test.env_key), test.expect_value); } } + + { + // Make sure all rocksdb options of ROCKSDB_DYNAMIC_OPTIONS are tested. + // Hint: Mainly verify the update_rocksdb_dynamic_options function. + std::map all_test_envs; + for (const auto &test : tests) { + all_test_envs[test.env_key] = test.env_value; + } + for (const auto &option : replica_envs::ROCKSDB_DYNAMIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + } } } // namespace replication diff --git a/src/meta/test/meta_app_operation_test.cpp b/src/meta/test/meta_app_operation_test.cpp index c3ddafa72c..e0afc1baa5 100644 --- a/src/meta/test/meta_app_operation_test.cpp +++ b/src/meta/test/meta_app_operation_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -68,7 +69,8 @@ class meta_app_operation_test : public meta_test_base error_code create_app_test(int32_t partition_count, int32_t replica_count, bool success_if_exist, - const std::string &app_name) + const std::string &app_name, + const std::map &envs = {}) { configuration_create_app_request create_request; configuration_create_app_response create_response; @@ -78,6 +80,7 @@ class meta_app_operation_test : public meta_test_base create_request.options.replica_count = replica_count; create_request.options.success_if_exist = success_if_exist; create_request.options.is_stateful = true; + create_request.options.envs = envs; auto result = fake_create_app(_ss.get(), create_request); fake_wait_rpc(result, create_response); @@ -362,6 +365,14 @@ TEST_F(meta_app_operation_test, create_app) // - wrong app_status dropping // - create succeed with app_status dropped // - create succeed with success_if_exist=true + // - wrong rocksdb.num_levels (< 1) + // - wrong rocksdb.num_levels (> 10) + // - wrong rocksdb.num_levels (non-digital character) + // - create app with rocksdb.num_levels (= 5) succeed + // - wrong rocksdb.write_buffer_size (< (16<<20)) + // - wrong rocksdb.write_buffer_size (> (512<<20)) + // - wrong rocksdb.write_buffer_size (non-digital character) + // - create app with rocksdb.write_buffer_size (= (32<<20)) succeed struct create_test { std::string app_name; @@ -373,43 +384,126 @@ TEST_F(meta_app_operation_test, create_app) bool success_if_exist; app_status::type before_status; error_code expected_err; - } tests[] = {{APP_NAME, -1, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 0, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, -1, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 0, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 7, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 5, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 5, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 4, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 6, 2, 7, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME + "_1", 4, 5, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_2", 4, 5, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_3", 4, 4, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_4", 4, 4, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_5", 4, 3, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_6", 4, 4, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 3, 2, 5, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 4, 5, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 4, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 2, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 3, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME, 4, 4, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, - {APP_NAME + "_7", 4, 3, 2, 4, 3, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 1, 1, 0, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, - {APP_NAME, 4, 2, 2, 1, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, - {APP_NAME, 4, 3, 3, 2, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, - {APP_NAME + "_8", 4, 3, 3, 3, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_9", 4, 1, 1, 1, 1, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME + "_10", 4, 2, 1, 2, 2, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_OK}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_APP_EXIST}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_CREATING, ERR_BUSY_CREATING}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_RECALLING, ERR_BUSY_CREATING}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPING, ERR_BUSY_DROPPING}, - {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPED, ERR_OK}, - {APP_NAME, 4, 3, 2, 3, 3, true, app_status::AS_INVALID, ERR_OK}}; + std::map envs = {}; + } tests[] = { + {APP_NAME, -1, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 0, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, -1, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 0, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 7, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 5, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 5, 2, 4, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 4, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 6, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 6, 2, 7, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME + "_1", 4, 5, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_2", 4, 5, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_3", 4, 4, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_4", 4, 4, 2, 6, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_5", 4, 3, 2, 4, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_6", 4, 4, 2, 5, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 3, 2, 5, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 4, 5, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 4, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 2, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 3, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME, 4, 4, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS}, + {APP_NAME + "_7", 4, 3, 2, 4, 3, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 1, 1, 0, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, + {APP_NAME, 4, 2, 2, 1, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, + {APP_NAME, 4, 3, 3, 2, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED}, + {APP_NAME + "_8", 4, 3, 3, 3, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_9", 4, 1, 1, 1, 1, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME + "_10", 4, 2, 1, 2, 2, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_APP_EXIST}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_CREATING, ERR_BUSY_CREATING}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_RECALLING, ERR_BUSY_CREATING}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPING, ERR_BUSY_DROPPING}, + {APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPED, ERR_OK}, + {APP_NAME, 4, 3, 2, 3, 3, true, app_status::AS_INVALID, ERR_OK}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.num_levels", "0"}}}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.num_levels", "11"}}}, + {APP_NAME + "_11", + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.num_levels", "5i"}}}, + {APP_NAME + "_11", + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_OK, + {{"rocksdb.num_levels", "5"}}}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.write_buffer_size", "1000"}}}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.write_buffer_size", "1073741824"}}}, + {APP_NAME, + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_INVALID_PARAMETERS, + {{"rocksdb.write_buffer_size", "n33554432"}}}, + {APP_NAME + "_12", + 4, + 3, + 2, + 3, + 3, + false, + app_status::AS_INVALID, + ERR_OK, + {{"rocksdb.write_buffer_size", "33554432"}}}, + }; clear_nodes(); @@ -453,13 +547,30 @@ TEST_F(meta_app_operation_test, create_app) } else if (test.before_status != app_status::AS_INVALID) { update_app_status(test.before_status); } - auto err = create_app_test( - test.partition_count, test.replica_count, test.success_if_exist, test.app_name); + auto err = create_app_test(test.partition_count, + test.replica_count, + test.success_if_exist, + test.app_name, + test.envs); ASSERT_EQ(err, test.expected_err); _ms->set_node_state(nodes, true); } + { + // Make sure all rocksdb options of ROCKSDB_DYNAMIC_OPTIONS and ROCKSDB_STATIC_OPTIONS are + // tested. Hint: Mainly verify the validate_app_envs function. + std::map all_test_envs; + for (const auto &test : tests) { + all_test_envs.insert(test.envs.begin(), test.envs.end()); + } + for (const auto &option : replica_envs::ROCKSDB_DYNAMIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + for (const auto &option : replica_envs::ROCKSDB_STATIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + } // set FLAGS_min_allowed_replica_count successfully res = update_flag("min_allowed_replica_count", "2"); ASSERT_TRUE(res.is_ok()); diff --git a/src/meta/test/meta_service_test.cpp b/src/meta/test/meta_service_test.cpp index bf993c0dde..2bb7ee2995 100644 --- a/src/meta/test/meta_service_test.cpp +++ b/src/meta/test/meta_service_test.cpp @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. -#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include @@ -28,7 +27,6 @@ #include "meta/meta_service.h" #include "meta_admin_types.h" #include "meta_test_base.h" -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_holder.h" diff --git a/src/meta/test/meta_state/meta_state_service.cpp b/src/meta/test/meta_state/meta_state_service.cpp index a286c5067a..4959670b29 100644 --- a/src/meta/test/meta_state/meta_state_service.cpp +++ b/src/meta/test/meta_state/meta_state_service.cpp @@ -27,6 +27,7 @@ #include "meta/meta_state_service.h" #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include @@ -36,12 +37,18 @@ #include "meta/meta_state_service_simple.h" #include "meta/meta_state_service_zookeeper.h" +#include "runtime/service_app.h" #include "runtime/task/task_tracker.h" +#include "test_util/test_util.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" +#include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/threadpool_code.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + using namespace dsn; using namespace dsn::dist; @@ -50,8 +57,8 @@ DEFINE_TASK_CODE(META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, TASK_PRIORITY_HIGH, TH typedef std::function service_creator_func; typedef std::function service_deleter_func; -#define expect_ok [](error_code ec) { EXPECT_TRUE(ec == ERR_OK); } -#define expect_err [](error_code ec) { EXPECT_FALSE(ec == ERR_OK); } +#define expect_ok [](error_code ec) { CHECK_EQ(ERR_OK, ec); } +#define expect_err [](error_code ec) { CHECK_NE(ERR_OK, ec); } void provider_basic_test(const service_creator_func &service_creator, const service_deleter_func &service_deleter) @@ -70,9 +77,9 @@ void provider_basic_test(const service_creator_func &service_creator, service->get_children("/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const std::vector &children) { - CHECK(ec == ERR_OK && children.size() == 1 && - *children.begin() == "1", - "unexpected child"); + CHECK_EQ(ERR_OK, ec); + CHECK_EQ(1, children.size()); + CHECK_EQ("1", *children.begin()); }); service->node_exist("/1/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok)->wait(); service->delete_node("/1", false, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_err) @@ -107,11 +114,11 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_data("/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const dsn::blob &value) { - expect_ok(ec); + CHECK_EQ(ERR_OK, ec); dsn::binary_reader reader(value); int read_value = 0; reader.read(read_value); - CHECK_EQ(read_value, 0xdeadbeef); + CHECK_EQ(0xdeadbeef, read_value); }) ->wait(); writer = dsn::binary_writer(); @@ -124,27 +131,26 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_data("/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const dsn::blob &value) { - expect_ok(ec); + CHECK_EQ(ERR_OK, ec); dsn::binary_reader reader(value); int read_value = 0; reader.read(read_value); - CHECK_EQ(read_value, 0xbeefdead); + CHECK_EQ(0xbeefdead, read_value); }) ->wait(); } - // clean the node created in previos code-block, to support test in next round + // clean the node created in previous code-block, to support test in next round { service->delete_node("/1", false, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok) ->wait(); } - typedef dsn::dist::meta_state_service::transaction_entries TEntries; // transaction op { // basic dsn::binary_writer writer; writer.write(0xdeadbeef); - std::shared_ptr entries = service->new_transaction_entries(5); + auto entries = service->new_transaction_entries(5); entries->create_node("/2"); entries->create_node("/2/2"); entries->create_node("/2/3"); @@ -155,11 +161,11 @@ void provider_basic_test(const service_creator_func &service_creator, entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok); tsk->wait(); for (unsigned int i = 0; i < 5; ++i) { - EXPECT_TRUE(entries->get_result(i) == ERR_OK); + ASSERT_EQ(ERR_OK, entries->get_result(i)); } // an invalid operation will stop whole transaction - entries = service->new_transaction_entries(5); + entries = service->new_transaction_entries(4); entries->create_node("/3"); entries->create_node("/4"); entries->delete_node("/2"); // delete a non empty dir @@ -168,11 +174,12 @@ void provider_basic_test(const service_creator_func &service_creator, service->submit_transaction(entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_err) ->wait(); error_code err[4] = {ERR_OK, ERR_OK, ERR_INVALID_PARAMETERS, ERR_INCONSISTENT_STATE}; - for (unsigned int i = 0; i < 4; ++i) - EXPECT_EQ(err[i], entries->get_result(i)); + for (unsigned int i = 0; i < 4; ++i) { + ASSERT_EQ(err[i], entries->get_result(i)); + } // another invalid transaction - entries = service->new_transaction_entries(5); + entries = service->new_transaction_entries(4); entries->create_node("/3"); entries->create_node("/4"); entries->delete_node("/5"); // delete a non exist dir @@ -182,8 +189,9 @@ void provider_basic_test(const service_creator_func &service_creator, err[2] = ERR_OBJECT_NOT_FOUND; service->submit_transaction(entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_err) ->wait(); - for (unsigned int i = 0; i < 4; ++i) - EXPECT_EQ(err[i], entries->get_result(i)); + for (unsigned int i = 0; i < 4; ++i) { + ASSERT_EQ(err[i], entries->get_result(i)); + } } // check replay with transaction @@ -195,7 +203,9 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_children("/2", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const std::vector &children) { - ASSERT_TRUE(children.size() == 1 && children[0] == "2"); + CHECK_EQ(ERR_OK, ec); + CHECK_EQ(1, children.size()); + CHECK_EQ("2", children[0]); }) ->wait(); @@ -203,27 +213,27 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_data("/2", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const blob &value) { - ASSERT_TRUE(ec == ERR_OK); + CHECK_EQ(ERR_OK, ec); binary_reader reader(value); int content_value; reader.read(content_value); - ASSERT_TRUE(content_value == 0xdeadbeef); + CHECK_EQ(0xdeadbeef, content_value); }) ->wait(); } // delete the nodes created just now, using transaction delete { - std::shared_ptr entries = service->new_transaction_entries(2); + auto entries = service->new_transaction_entries(2); entries->delete_node("/2/2"); entries->delete_node("/2"); service->submit_transaction(entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok) ->wait(); - error_code err[2] = {ERR_OK, ERR_OK}; - for (unsigned int i = 0; i < 2; ++i) - EXPECT_EQ(err[i], entries->get_result(i)); + for (unsigned int i = 0; i < 2; ++i) { + ASSERT_EQ(ERR_OK, entries->get_result(i)); + } } service_deleter(service); @@ -235,7 +245,7 @@ void recursively_create_node_callback(meta_state_service *service, int current_layer, error_code ec) { - ASSERT_TRUE(ec == ERR_OK); + ASSERT_EQ(ERR_OK, ec); if (current_layer <= 0) return; @@ -279,31 +289,41 @@ void provider_recursively_create_delete_test(const service_creator_func &creator deleter(service); } -#undef expect_ok -#undef expect_err +class meta_state_service_test : public pegasus::encrypt_data_test_base +{ +}; + +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, meta_state_service_test, ::testing::Values(false)); -TEST(meta_state_service, simple) +TEST_P(meta_state_service_test, simple) { auto simple_service_creator = [] { meta_state_service_simple *svc = new meta_state_service_simple(); - svc->initialize({}); + auto err = svc->initialize({}); + CHECK_EQ(ERR_OK, err); return svc; }; auto simple_service_deleter = [](meta_state_service *simple_svc) { delete simple_svc; }; provider_basic_test(simple_service_creator, simple_service_deleter); provider_recursively_create_delete_test(simple_service_creator, simple_service_deleter); + + std::string log_path = dsn::utils::filesystem::path_combine( + service_app::current_service_app_info().data_dir, "meta_state_service.log"); + ASSERT_TRUE(dsn::utils::filesystem::remove_path(log_path)); } -TEST(meta_state_service, zookeeper) +TEST_P(meta_state_service_test, zookeeper) { auto zookeeper_service_creator = [] { meta_state_service_zookeeper *svc = new meta_state_service_zookeeper(); - svc->initialize({}); + auto err = svc->initialize({}); + CHECK_EQ(ERR_OK, err); return svc; }; auto zookeeper_service_deleter = [](meta_state_service *zookeeper_svc) { - ASSERT_EQ(zookeeper_svc->finalize(), ERR_OK); + ASSERT_EQ(ERR_OK, zookeeper_svc->finalize()); }; provider_basic_test(zookeeper_service_creator, zookeeper_service_deleter); diff --git a/src/meta/test/run.sh b/src/meta/test/run.sh index ad104aa6b4..207a9c7f80 100755 --- a/src/meta/test/run.sh +++ b/src/meta/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -28,6 +28,25 @@ if [ -z "${REPORT_DIR}" ]; then REPORT_DIR="." fi +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi + ./clear.sh output_xml="${REPORT_DIR}/dsn.meta.test.1.xml" GTEST_OUTPUT="xml:${output_xml}" ./dsn.meta.test diff --git a/src/nfs/nfs_client_impl.cpp b/src/nfs/nfs_client_impl.cpp index c0ced7b8c6..910bae8ec0 100644 --- a/src/nfs/nfs_client_impl.cpp +++ b/src/nfs/nfs_client_impl.cpp @@ -27,7 +27,6 @@ #include "nfs_client_impl.h" // IWYU pragma: no_include -#include #include #include "nfs/nfs_code_definition.h" @@ -38,7 +37,6 @@ #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" #include "utils/string_conv.h" #include "utils/token_buckets.h" @@ -460,8 +458,7 @@ void nfs_client_impl::continue_write() // double check zauto_lock l(fc->user_req->user_req_lock); if (!fc->file_holder->file_handle) { - fc->file_holder->file_handle = - file::open(file_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + fc->file_holder->file_handle = file::open(file_path, file::FileOpenType::kWriteOnly); } } @@ -470,6 +467,10 @@ void nfs_client_impl::continue_write() LOG_ERROR("open file {} failed", file_path); handle_completion(fc->user_req, ERR_FILE_OPERATION_FAILED); } else { + LOG_DEBUG("nfs: copy to file {} [{}, {}]", + file_path, + reqc->response.offset, + reqc->response.offset + reqc->response.size); zauto_lock l(reqc->lock); if (reqc->is_valid) { reqc->local_write_task = file::write(fc->file_holder->file_handle, diff --git a/src/nfs/nfs_node_impl.cpp b/src/nfs/nfs_node_simple.cpp similarity index 93% rename from src/nfs/nfs_node_impl.cpp rename to src/nfs/nfs_node_simple.cpp index 387547fd88..bb334a5085 100644 --- a/src/nfs/nfs_node_impl.cpp +++ b/src/nfs/nfs_node_simple.cpp @@ -80,12 +80,17 @@ void nfs_node_simple::register_async_rpc_handler_for_test() error_code nfs_node_simple::stop() { - delete _server; - _server = nullptr; + if (_server != nullptr) { + _server->close_service(); - delete _client; - _client = nullptr; + delete _server; + _server = nullptr; + } + if (_client != nullptr) { + delete _client; + _client = nullptr; + } return ERR_OK; } diff --git a/src/nfs/nfs_node_simple.h b/src/nfs/nfs_node_simple.h index 2376b1e34e..15e2344168 100644 --- a/src/nfs/nfs_node_simple.h +++ b/src/nfs/nfs_node_simple.h @@ -34,14 +34,24 @@ */ #pragma once -#include "runtime/tool_api.h" +#include + #include "nfs/nfs_node.h" +#include "utils/error_code.h" namespace dsn { +class aio_task; +template +class rpc_replier; + namespace service { -class nfs_service_impl; +class copy_request; +class copy_response; +class get_file_size_request; +class get_file_size_response; class nfs_client_impl; +class nfs_service_impl; class nfs_node_simple : public nfs_node { diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index cadacba1a3..c4bcf7a2e4 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -26,12 +26,10 @@ #include "nfs/nfs_server_impl.h" -#include -#include -#include #include #include #include +#include #include #include "nfs/nfs_code_definition.h" @@ -39,10 +37,9 @@ #include "runtime/api_layer1.h" #include "runtime/task/async_calls.h" #include "utils/TokenBucket.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/flags.h" -#include "utils/ports.h" -#include "utils/safe_strerror_posix.h" #include "utils/string_conv.h" #include "utils/utils.h" @@ -90,38 +87,35 @@ void nfs_service_impl::on_copy(const ::dsn::service::copy_request &request, dsn::utils::filesystem::path_combine(request.source_dir, request.file_name); disk_file *dfile = nullptr; - { + do { zauto_lock l(_handles_map_lock); auto it = _handles_map.find(file_path); // find file handle cache first - if (it == _handles_map.end()) { - dfile = file::open(file_path.c_str(), O_RDONLY | O_BINARY, 0); - if (dfile != nullptr) { - auto fh = std::make_shared(); - fh->file_handle = dfile; - fh->file_access_count = 1; - fh->last_access_time = dsn_now_ms(); - _handles_map.insert(std::make_pair(file_path, std::move(fh))); + dfile = file::open(file_path, file::FileOpenType::kReadOnly); + if (dfile == nullptr) { + LOG_ERROR("[nfs_service] open file {} failed", file_path); + ::dsn::service::copy_response resp; + resp.error = ERR_OBJECT_NOT_FOUND; + reply(resp); + return; } - } else { - dfile = it->second->file_handle; - it->second->file_access_count++; - it->second->last_access_time = dsn_now_ms(); - } - } - - LOG_DEBUG( - "nfs: copy file {} [{}, {}]", file_path, request.offset, request.offset + request.size); - if (dfile == nullptr) { - LOG_ERROR("[nfs_service] open file {} failed", file_path); - ::dsn::service::copy_response resp; - resp.error = ERR_OBJECT_NOT_FOUND; - reply(resp); - return; - } - - std::shared_ptr cp = std::make_shared(std::move(reply)); + auto fh = std::make_shared(); + fh->file_handle = dfile; + it = _handles_map.insert(std::make_pair(file_path, std::move(fh))).first; + } + dfile = it->second->file_handle; + it->second->file_access_count++; + it->second->last_access_time = dsn_now_ms(); + } while (false); + + CHECK_NOTNULL(dfile, ""); + LOG_DEBUG("nfs: copy from file {} [{}, {}]", + file_path, + request.offset, + request.offset + request.size); + + auto cp = std::make_shared(std::move(reply)); cp->bb = blob(dsn::utils::make_shared_array(request.size), request.size); cp->dst_dir = request.dst_dir; cp->source_disk_tag = request.source_disk_tag; @@ -182,58 +176,53 @@ void nfs_service_impl::on_get_file_size( { get_file_size_response resp; error_code err = ERR_OK; - std::vector file_list; std::string folder = request.source_dir; + // TODO(yingchun): refactor the following code! if (request.file_list.size() == 0) // return all file size in the destination file folder { if (!dsn::utils::filesystem::directory_exists(folder)) { LOG_ERROR("[nfs_service] directory {} not exist", folder); err = ERR_OBJECT_NOT_FOUND; } else { + std::vector file_list; if (!dsn::utils::filesystem::get_subfiles(folder, file_list, true)) { LOG_ERROR("[nfs_service] get subfiles of directory {} failed", folder); err = ERR_FILE_OPERATION_FAILED; } else { - for (auto &fpath : file_list) { - // TODO: using uint64 instead as file ma - // Done + for (const auto &fpath : file_list) { int64_t sz; - if (!dsn::utils::filesystem::file_size(fpath, sz)) { + // TODO(yingchun): check if there are any files that are not sensitive (not + // encrypted). + if (!dsn::utils::filesystem::file_size( + fpath, dsn::utils::FileDataType::kSensitive, sz)) { LOG_ERROR("[nfs_service] get size of file {} failed", fpath); err = ERR_FILE_OPERATION_FAILED; break; } - resp.size_list.push_back((uint64_t)sz); + resp.size_list.push_back(sz); resp.file_list.push_back( fpath.substr(request.source_dir.length(), fpath.length() - 1)); } - file_list.clear(); } } } else // return file size in the request file folder { - for (size_t i = 0; i < request.file_list.size(); i++) { - std::string file_path = - dsn::utils::filesystem::path_combine(folder, request.file_list[i]); - - struct stat st; - if (0 != ::stat(file_path.c_str(), &st)) { - LOG_ERROR("[nfs_service] get stat of file {} failed, err = {}", - file_path, - dsn::utils::safe_strerror(errno)); - err = ERR_OBJECT_NOT_FOUND; + for (const auto &file_name : request.file_list) { + std::string file_path = dsn::utils::filesystem::path_combine(folder, file_name); + int64_t sz; + // TODO(yingchun): check if there are any files that are not sensitive (not encrypted). + if (!dsn::utils::filesystem::file_size( + file_path, dsn::utils::FileDataType::kSensitive, sz)) { + LOG_ERROR("[nfs_service] get size of file {} failed", file_path); + err = ERR_FILE_OPERATION_FAILED; break; } - // TODO: using int64 instead as file may exceed the size of 32bit - // Done - uint64_t size = st.st_size; - - resp.size_list.push_back(size); - resp.file_list.push_back((folder + request.file_list[i]) - .substr(request.source_dir.length(), - (folder + request.file_list[i]).length() - 1)); + resp.size_list.push_back(sz); + resp.file_list.push_back( + (folder + file_name) + .substr(request.source_dir.length(), (folder + file_name).length() - 1)); } } @@ -253,8 +242,9 @@ void nfs_service_impl::close_file() // release out-of-date file handle dsn_now_ms() - fptr->last_access_time > (uint64_t)FLAGS_file_close_expire_time_ms) { LOG_DEBUG("nfs: close file handle {}", it->first); it = _handles_map.erase(it); - } else + } else { it++; + } } } diff --git a/src/nfs/nfs_server_impl.h b/src/nfs/nfs_server_impl.h index 9ba1134040..ece68ecb33 100644 --- a/src/nfs/nfs_server_impl.h +++ b/src/nfs/nfs_server_impl.h @@ -66,7 +66,6 @@ class nfs_service_impl : public ::dsn::serverlet void register_cli_commands(); - // TODO(yingchun): seems nobody call it, can be removed? void close_service() { unregister_rpc_handler(RPC_NFS_COPY); @@ -107,14 +106,9 @@ class nfs_service_impl : public ::dsn::serverlet struct file_handle_info_on_server { - disk_file *file_handle; - int32_t file_access_count; // concurrent r/w count - uint64_t last_access_time; // last touch time - - file_handle_info_on_server() - : file_handle(nullptr), file_access_count(0), last_access_time(0) - { - } + disk_file *file_handle = nullptr; + int32_t file_access_count = 0; // concurrent r/w count + uint64_t last_access_time = 0; // last touch time ~file_handle_info_on_server() { diff --git a/src/nfs/test/CMakeLists.txt b/src/nfs/test/CMakeLists.txt index 735bb29bd6..64c7967ace 100644 --- a/src/nfs/test/CMakeLists.txt +++ b/src/nfs/test/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio) +set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/nfs/test/main.cpp b/src/nfs/test/main.cpp index 445c4732fe..0788129928 100644 --- a/src/nfs/test/main.cpp +++ b/src/nfs/test/main.cpp @@ -24,13 +24,16 @@ * THE SOFTWARE. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include #include -#include +#include +#include #include #include +#include #include #include "aio/aio_task.h" @@ -40,7 +43,9 @@ #include "runtime/rpc/rpc_address.h" #include "runtime/task/task_code.h" #include "runtime/tool_api.h" +#include "test_util/test_util.h" #include "utils/autoref_ptr.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/threadpool_code.h" @@ -54,36 +59,55 @@ struct aio_result size_t sz; }; -TEST(nfs, basic) +class nfs_test : public pegasus::encrypt_data_test_base { - std::unique_ptr nfs(dsn::nfs_node::create()); +}; + +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, nfs_test, ::testing::Values(false)); + +TEST_P(nfs_test, basic) +{ + auto nfs = dsn::nfs_node::create(); nfs->start(); nfs->register_async_rpc_handler_for_test(); dsn::gpid fake_pid = gpid(1, 0); - utils::filesystem::remove_path("nfs_test_dir"); - utils::filesystem::remove_path("nfs_test_dir_copy"); - - ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir")); - ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir_copy")); - - ASSERT_TRUE(utils::filesystem::create_directory("nfs_test_dir")); - ASSERT_TRUE(utils::filesystem::directory_exists("nfs_test_dir")); + // Prepare the destination directory. + const std::string kDstDir = "nfs_test_dir"; + ASSERT_TRUE(utils::filesystem::remove_path(kDstDir)); + ASSERT_FALSE(utils::filesystem::directory_exists(kDstDir)); + ASSERT_TRUE(utils::filesystem::create_directory(kDstDir)); + ASSERT_TRUE(utils::filesystem::directory_exists(kDstDir)); + + // Prepare the source files information. + std::vector kSrcFilenames({"nfs_test_file1", "nfs_test_file2"}); + std::vector src_file_sizes; + std::vector src_file_md5s; + for (const auto &src_filename : kSrcFilenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + src_filename, dsn::utils::FileDataType::kSensitive, file_size)); + src_file_sizes.push_back(file_size); + std::string src_file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(src_filename, src_file_md5)); + src_file_md5s.emplace_back(std::move(src_file_md5)); + } + // copy files to the destination directory. { - // copy nfs_test_file1 nfs_test_file2 nfs_test_dir - ASSERT_FALSE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1")); - ASSERT_FALSE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2")); - - std::vector files{"nfs_test_file1", "nfs_test_file2"}; + // The destination directory is empty before copying. + std::vector dst_filenames; + ASSERT_TRUE(utils::filesystem::get_subfiles(kDstDir, dst_filenames, true)); + ASSERT_TRUE(dst_filenames.empty()); aio_result r; dsn::aio_task_ptr t = nfs->copy_remote_files(dsn::rpc_address("localhost", 20101), "default", ".", - files, + kSrcFilenames, "default", - "nfs_test_dir", + kDstDir, fake_pid, false, false, @@ -100,32 +124,32 @@ TEST(nfs, basic) ASSERT_EQ(ERR_OK, r.err); ASSERT_EQ(r.sz, t->get_transferred_size()); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2")); - - int64_t sz1, sz2; - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_file1", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file1", sz2)); - ASSERT_EQ(sz1, sz2); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_file2", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file2", sz2)); - ASSERT_EQ(sz1, sz2); + // The destination files equal to the source files after copying. + ASSERT_TRUE(utils::filesystem::get_subfiles(kDstDir, dst_filenames, true)); + std::sort(dst_filenames.begin(), dst_filenames.end()); + ASSERT_EQ(kSrcFilenames.size(), dst_filenames.size()); + int i = 0; + for (const auto &dst_filename : dst_filenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + dst_filename, dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(src_file_sizes[i], file_size); + std::string file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(dst_filename, file_md5)); + ASSERT_EQ(src_file_md5s[i], file_md5); + i++; + } } + // copy files to the destination directory, files will be overwritten. { - // copy files again, overwrite - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2")); - - std::vector files{"nfs_test_file1", "nfs_test_file2"}; - aio_result r; dsn::aio_task_ptr t = nfs->copy_remote_files(dsn::rpc_address("localhost", 20101), "default", ".", - files, + kSrcFilenames, "default", - "nfs_test_dir", + kDstDir, fake_pid, true, false, @@ -141,22 +165,42 @@ TEST(nfs, basic) ASSERT_EQ(r.err, t->error()); ASSERT_EQ(ERR_OK, r.err); ASSERT_EQ(r.sz, t->get_transferred_size()); + // this is only true for simulator if (dsn::tools::get_current_tool()->name() == "simulator") { ASSERT_EQ(1, t->get_count()); } + + // The destination files equal to the source files after overwrite copying. + std::vector dst_filenames; + ASSERT_TRUE(utils::filesystem::get_subfiles(kDstDir, dst_filenames, true)); + std::sort(dst_filenames.begin(), dst_filenames.end()); + ASSERT_EQ(kSrcFilenames.size(), dst_filenames.size()); + int i = 0; + for (const auto &dst_filename : dst_filenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + dst_filename, dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(src_file_sizes[i], file_size); + std::string file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(dst_filename, file_md5)); + ASSERT_EQ(src_file_md5s[i], file_md5); + i++; + } } + // copy files from kDstDir to kNewDstDir. { - // copy nfs_test_dir nfs_test_dir_copy - ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir_copy")); + const std::string kNewDstDir = "nfs_test_dir_copy"; + ASSERT_TRUE(utils::filesystem::remove_path(kNewDstDir)); + ASSERT_FALSE(utils::filesystem::directory_exists(kNewDstDir)); aio_result r; dsn::aio_task_ptr t = nfs->copy_remote_directory(dsn::rpc_address("localhost", 20101), "default", - "nfs_test_dir", + kDstDir, "default", - "nfs_test_dir_copy", + kNewDstDir, fake_pid, false, false, @@ -173,22 +217,25 @@ TEST(nfs, basic) ASSERT_EQ(ERR_OK, r.err); ASSERT_EQ(r.sz, t->get_transferred_size()); - ASSERT_TRUE(utils::filesystem::directory_exists("nfs_test_dir_copy")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir_copy/nfs_test_file1")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir_copy/nfs_test_file2")); - - std::vector sub1, sub2; - ASSERT_TRUE(utils::filesystem::get_subfiles("nfs_test_dir", sub1, true)); - ASSERT_TRUE(utils::filesystem::get_subfiles("nfs_test_dir_copy", sub2, true)); - ASSERT_EQ(sub1.size(), sub2.size()); - - int64_t sz1, sz2; - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file1", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir_copy/nfs_test_file1", sz2)); - ASSERT_EQ(sz1, sz2); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file2", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir_copy/nfs_test_file2", sz2)); - ASSERT_EQ(sz1, sz2); + // The kNewDstDir will be created automatically. + ASSERT_TRUE(utils::filesystem::directory_exists(kNewDstDir)); + + std::vector new_dst_filenames; + ASSERT_TRUE(utils::filesystem::get_subfiles(kNewDstDir, new_dst_filenames, true)); + std::sort(new_dst_filenames.begin(), new_dst_filenames.end()); + ASSERT_EQ(kSrcFilenames.size(), new_dst_filenames.size()); + + int i = 0; + for (const auto &new_dst_filename : new_dst_filenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + new_dst_filename, dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(src_file_sizes[i], file_size); + std::string file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(new_dst_filename, file_md5)); + ASSERT_EQ(src_file_md5s[i], file_md5); + i++; + } } nfs->stop(); diff --git a/src/perf_counter/perf_counter.h b/src/perf_counter/perf_counter.h index dad8009e41..1425bc228c 100644 --- a/src/perf_counter/perf_counter.h +++ b/src/perf_counter/perf_counter.h @@ -33,6 +33,7 @@ #include #include "utils/autoref_ptr.h" +#include "utils/fmt_utils.h" typedef enum dsn_perf_counter_type_t { COUNTER_TYPE_NUMBER, @@ -42,6 +43,7 @@ typedef enum dsn_perf_counter_type_t { COUNTER_TYPE_COUNT, COUNTER_TYPE_INVALID } dsn_perf_counter_type_t; +USER_DEFINED_ENUM_FORMATTER(dsn_perf_counter_type_t) typedef enum dsn_perf_counter_percentile_type_t { COUNTER_PERCENTILE_50, diff --git a/src/perf_counter/test/CMakeLists.txt b/src/perf_counter/test/CMakeLists.txt index a02a6923b5..434d97dc11 100644 --- a/src/perf_counter/test/CMakeLists.txt +++ b/src/perf_counter/test/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime) +set(MY_PROJ_LIBS gtest dsn_runtime rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/perf_counter/test/perf_counter_test.cpp b/src/perf_counter/test/perf_counter_test.cpp index f83679bd45..0758ef8071 100644 --- a/src/perf_counter/test/perf_counter_test.cpp +++ b/src/perf_counter/test/perf_counter_test.cpp @@ -38,6 +38,7 @@ // IWYU pragma: no_include #include #include +#include #include #include #include diff --git a/src/redis_protocol/proxy_ut/redis_proxy_test.cpp b/src/redis_protocol/proxy_ut/redis_proxy_test.cpp index 2706ae772f..400c41b4ce 100644 --- a/src/redis_protocol/proxy_ut/redis_proxy_test.cpp +++ b/src/redis_protocol/proxy_ut/redis_proxy_test.cpp @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt index 7e27ef9b77..d4609d9e58 100644 --- a/src/replica/CMakeLists.txt +++ b/src/replica/CMakeLists.txt @@ -75,7 +75,7 @@ set(MY_PROJ_LIBS PocoFoundation PocoNetSSL PocoJSON - ) + rocksdb) set(MY_BOOST_LIBS Boost::filesystem Boost::regex) diff --git a/src/replica/backup/cold_backup_context.cpp b/src/replica/backup/cold_backup_context.cpp index f9063a8966..845b96fe60 100644 --- a/src/replica/backup/cold_backup_context.cpp +++ b/src/replica/backup/cold_backup_context.cpp @@ -17,7 +17,6 @@ #include "cold_backup_context.h" -#include #include #include #include @@ -1029,7 +1028,7 @@ void cold_backup_context::on_upload_file_complete(const std::string &local_filen } else { CHECK_GT(total, 0.0); update_progress(static_cast(complete_size / total * 1000)); - LOG_INFO("{}: the progress of upload checkpoint is {}", name, _progress); + LOG_INFO("{}: the progress of upload checkpoint is {}", name, _progress.load()); } if (is_ready_for_upload()) { std::vector upload_files; diff --git a/src/replica/backup/test/CMakeLists.txt b/src/replica/backup/test/CMakeLists.txt index 21ec10f505..e3dc0ec05b 100644 --- a/src/replica/backup/test/CMakeLists.txt +++ b/src/replica/backup/test/CMakeLists.txt @@ -30,7 +30,7 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_utils hashtable gtest -) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/backup/test/config-test.ini b/src/replica/backup/test/config-test.ini index b24a8ff146..ad6433590a 100644 --- a/src/replica/backup/test/config-test.ini +++ b/src/replica/backup/test/config-test.ini @@ -51,6 +51,7 @@ partitioned = true name = replica_long [replication] +disk_min_available_space_ratio = 10 cluster_name = master-cluster [duplication-group] diff --git a/src/replica/backup/test/run.sh b/src/replica/backup/test/run.sh index 996196944e..73aa02be9a 100755 --- a/src/replica/backup/test/run.sh +++ b/src/replica/backup/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -23,6 +23,24 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi ./dsn_replica_backup_test diff --git a/src/replica/bulk_load/replica_bulk_loader.cpp b/src/replica/bulk_load/replica_bulk_loader.cpp index 60f3e5a043..643816d423 100644 --- a/src/replica/bulk_load/replica_bulk_loader.cpp +++ b/src/replica/bulk_load/replica_bulk_loader.cpp @@ -16,6 +16,8 @@ // under the License. #include +#include +#include #include #include #include @@ -44,9 +46,11 @@ #include "utils/autoref_ptr.h" #include "utils/blob.h" #include "utils/chrono_literals.h" +#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" +#include "utils/ports.h" #include "utils/string_view.h" #include "utils/thread_access_checker.h" @@ -497,7 +501,8 @@ void replica_bulk_loader::download_sst_file(const std::string &remote_dir, // We are not sure if the file was cached by system. And we couldn't // afford the io overhead which is cased by reading file in verify_file(), // so if file exist we just verify file size - if (utils::filesystem::verify_file_size(file_name, f_meta.size)) { + if (utils::filesystem::verify_file_size( + file_name, utils::FileDataType::kSensitive, f_meta.size)) { // local file exist and is verified ec = ERR_OK; f_size = f_meta.size; @@ -520,7 +525,8 @@ void replica_bulk_loader::download_sst_file(const std::string &remote_dir, if (ec == ERR_OK && !verified) { if (!f_meta.md5.empty() && f_md5 != f_meta.md5) { ec = ERR_CORRUPTION; - } else if (!utils::filesystem::verify_file_size(file_name, f_meta.size)) { + } else if (!utils::filesystem::verify_file_size( + file_name, utils::FileDataType::kSensitive, f_meta.size)) { ec = ERR_CORRUPTION; } } @@ -559,10 +565,10 @@ void replica_bulk_loader::download_sst_file(const std::string &remote_dir, error_code replica_bulk_loader::parse_bulk_load_metadata(const std::string &fname) { std::string buf; - error_code ec = utils::filesystem::read_file(fname, buf); - if (ec != ERR_OK) { - LOG_ERROR_PREFIX("read file {} failed, error = {}", fname, ec); - return ec; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &buf); + if (dsn_unlikely(!s.ok())) { + LOG_ERROR_PREFIX("read file {} failed, error = {}", fname, s.ToString()); + return ERR_FILE_OPERATION_FAILED; } blob bb = blob::create_from_bytes(std::move(buf)); diff --git a/src/replica/bulk_load/test/CMakeLists.txt b/src/replica/bulk_load/test/CMakeLists.txt index 77081196c5..a15861f61c 100644 --- a/src/replica/bulk_load/test/CMakeLists.txt +++ b/src/replica/bulk_load/test/CMakeLists.txt @@ -27,7 +27,8 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_runtime hashtable gtest -) + test_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/bulk_load/test/config-test.ini b/src/replica/bulk_load/test/config-test.ini index d9d8fd9daa..7bdc530aa8 100644 --- a/src/replica/bulk_load/test/config-test.ini +++ b/src/replica/bulk_load/test/config-test.ini @@ -91,6 +91,9 @@ rpc_call_channel = RPC_CHANNEL_TCP rpc_message_header_format = dsn rpc_timeout_milliseconds = 5000 +[replication] +disk_min_available_space_ratio = 10 + [block_service.local_service] type = local_service args = diff --git a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp index 6b94754e85..9eb7322a61 100644 --- a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp +++ b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp @@ -17,10 +17,14 @@ #include "replica/bulk_load/replica_bulk_loader.h" +#include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include +#include +#include +#include #include // IWYU pragma: keep #include #include @@ -33,10 +37,11 @@ #include "replica/test/replica_test_base.h" #include "runtime/rpc/rpc_address.h" #include "runtime/task/task_tracker.h" +#include "test_util/test_util.h" #include "utils/blob.h" #include "utils/fail_point.h" #include "utils/filesystem.h" -#include "utils/fmt_logging.h" +#include "utils/test_macros.h" namespace dsn { namespace replication { @@ -248,44 +253,21 @@ class replica_bulk_loader_test : public replica_test_base _replica->set_primary_partition_configuration(config); } - void create_local_file(const std::string &file_name) + void create_local_metadata_file() { - std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, file_name); - utils::filesystem::create_file(whole_name); - std::ofstream test_file; - test_file.open(whole_name); - test_file << "write some data.\n"; - test_file.close(); + NO_FATALS(pegasus::create_local_test_file( + utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME), &_file_meta)); - _file_meta.name = whole_name; - utils::filesystem::md5sum(whole_name, _file_meta.md5); - utils::filesystem::file_size(whole_name, _file_meta.size); - } - - error_code create_local_metadata_file() - { - create_local_file(FILE_NAME); _metadata.files.emplace_back(_file_meta); _metadata.file_total_size = _file_meta.size; - std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); - utils::filesystem::create_file(whole_name); - std::ofstream os(whole_name.c_str(), - (std::ofstream::out | std::ios::binary | std::ofstream::trunc)); - if (!os.is_open()) { - LOG_ERROR("open file {} failed", whole_name); - return ERR_FILE_OPERATION_FAILED; - } - blob bb = json::json_forwarder::encode(_metadata); - os.write((const char *)bb.data(), (std::streamsize)bb.length()); - if (os.bad()) { - LOG_ERROR("write file {} failed", whole_name); - return ERR_FILE_OPERATION_FAILED; - } - os.close(); - - return ERR_OK; + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(bb.data(), bb.length()), + whole_name, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << fmt::format( + "write file {} failed, err = {}", whole_name, s.ToString()); } bool validate_metadata() @@ -550,21 +532,21 @@ TEST_F(replica_bulk_loader_test, bulk_load_metadata_corrupt) { // create file can not parse as bulk_load_metadata structure utils::filesystem::create_directory(LOCAL_DIR); - create_local_file(METADATA); + NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, METADATA), + &_file_meta)); std::string metadata_file_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); error_code ec = test_parse_bulk_load_metadata(metadata_file_name); - ASSERT_EQ(ec, ERR_CORRUPTION); + ASSERT_EQ(ERR_CORRUPTION, ec); utils::filesystem::remove_path(LOCAL_DIR); } TEST_F(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) { utils::filesystem::create_directory(LOCAL_DIR); - error_code ec = create_local_metadata_file(); - ASSERT_EQ(ec, ERR_OK); + NO_FATALS(create_local_metadata_file()); std::string metadata_file_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); - ec = test_parse_bulk_load_metadata(metadata_file_name); + auto ec = test_parse_bulk_load_metadata(metadata_file_name); ASSERT_EQ(ec, ERR_OK); ASSERT_TRUE(validate_metadata()); utils::filesystem::remove_path(LOCAL_DIR); diff --git a/src/replica/bulk_load/test/run.sh b/src/replica/bulk_load/test/run.sh index 91ba506453..7b060300d2 100755 --- a/src/replica/bulk_load/test/run.sh +++ b/src/replica/bulk_load/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -23,6 +23,24 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi ./dsn_replica_bulk_load_test diff --git a/src/replica/duplication/load_from_private_log.cpp b/src/replica/duplication/load_from_private_log.cpp index aa458b22a4..d3d88a075d 100644 --- a/src/replica/duplication/load_from_private_log.cpp +++ b/src/replica/duplication/load_from_private_log.cpp @@ -16,6 +16,7 @@ // under the License. #include +#include #include #include "common/duplication_common.h" @@ -57,11 +58,11 @@ bool load_from_private_log::will_fail_fast() const // we try to list all files and select a new one to start (find_log_file_to_start). bool load_from_private_log::switch_to_next_log_file() { - auto file_map = _private_log->get_log_file_map(); - auto next_file_it = file_map.find(_current->index() + 1); + const auto &file_map = _private_log->get_log_file_map(); + const auto &next_file_it = file_map.find(_current->index() + 1); if (next_file_it != file_map.end()) { log_file_ptr file; - error_s es = log_utils::open_read(next_file_it->second->path(), file); + const auto &es = log_utils::open_read(next_file_it->second->path(), file); if (!es.is_ok()) { LOG_ERROR_PREFIX("{}", es); _current = nullptr; @@ -123,11 +124,11 @@ void load_from_private_log::run() void load_from_private_log::find_log_file_to_start() { // `file_map` has already excluded the useless log files during replica init. - auto file_map = _private_log->get_log_file_map(); + const auto &file_map = _private_log->get_log_file_map(); // Reopen the files. Because the internal file handle of `file_map` // is cleared once WAL replay finished. They are unable to read. - std::map new_file_map; + mutation_log::log_file_map_by_index new_file_map; for (const auto &pr : file_map) { log_file_ptr file; error_s es = log_utils::open_read(pr.second->path(), file); @@ -141,7 +142,8 @@ void load_from_private_log::find_log_file_to_start() find_log_file_to_start(std::move(new_file_map)); } -void load_from_private_log::find_log_file_to_start(std::map log_file_map) +void load_from_private_log::find_log_file_to_start( + const mutation_log::log_file_map_by_index &log_file_map) { _current = nullptr; if (dsn_unlikely(log_file_map.empty())) { diff --git a/src/replica/duplication/load_from_private_log.h b/src/replica/duplication/load_from_private_log.h index 523002b54a..56651aabfa 100644 --- a/src/replica/duplication/load_from_private_log.h +++ b/src/replica/duplication/load_from_private_log.h @@ -21,7 +21,6 @@ #include #include #include -#include #include "common/replication_other_types.h" #include "mutation_batch.h" @@ -61,7 +60,7 @@ class load_from_private_log final : public replica_base, /// Find the log file that contains `_start_decree`. void find_log_file_to_start(); - void find_log_file_to_start(std::map log_files); + void find_log_file_to_start(const mutation_log::log_file_map_by_index &log_files); void replay_log_block(); diff --git a/src/replica/duplication/mutation_batch.cpp b/src/replica/duplication/mutation_batch.cpp index 786c4d61e0..852a8ff127 100644 --- a/src/replica/duplication/mutation_batch.cpp +++ b/src/replica/duplication/mutation_batch.cpp @@ -16,9 +16,7 @@ // under the License. #include -#include #include -#include #include #include #include diff --git a/src/replica/duplication/replica_follower.cpp b/src/replica/duplication/replica_follower.cpp index f9c2a996d1..f5a29b94dd 100644 --- a/src/replica/duplication/replica_follower.cpp +++ b/src/replica/duplication/replica_follower.cpp @@ -20,7 +20,6 @@ #include "replica_follower.h" #include -#include #include #include #include diff --git a/src/replica/duplication/test/CMakeLists.txt b/src/replica/duplication/test/CMakeLists.txt index 09619343ec..faab20f7e6 100644 --- a/src/replica/duplication/test/CMakeLists.txt +++ b/src/replica/duplication/test/CMakeLists.txt @@ -30,7 +30,7 @@ set(MY_PROJ_LIBS dsn_meta_server zookeeper hashtable gtest -) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/duplication/test/config-test.ini b/src/replica/duplication/test/config-test.ini index 5eb6b16342..a06a16552a 100644 --- a/src/replica/duplication/test/config-test.ini +++ b/src/replica/duplication/test/config-test.ini @@ -50,6 +50,9 @@ partitioned = true [threadpool.THREAD_POOL_REPLICATION_LONG] name = replica_long +[replication] +disk_min_available_space_ratio = 10 + [duplication-group] master-cluster = 1 slave-cluster = 2 diff --git a/src/replica/duplication/test/duplication_test_base.h b/src/replica/duplication/test/duplication_test_base.h index fb0c7ea4f9..4152a9a370 100644 --- a/src/replica/duplication/test/duplication_test_base.h +++ b/src/replica/duplication/test/duplication_test_base.h @@ -68,9 +68,9 @@ class duplication_test_base : public replica_test_base return duplicator; } - std::map open_log_file_map(const std::string &log_dir) + mutation_log::log_file_map_by_index open_log_file_map(const std::string &log_dir) { - std::map log_file_map; + mutation_log::log_file_map_by_index log_file_map; error_s err = log_utils::open_log_file_map(log_dir, log_file_map); EXPECT_EQ(err, error_s::ok()); return log_file_map; diff --git a/src/replica/duplication/test/load_from_private_log_test.cpp b/src/replica/duplication/test/load_from_private_log_test.cpp index 342669f63c..b215be60f0 100644 --- a/src/replica/duplication/test/load_from_private_log_test.cpp +++ b/src/replica/duplication/test/load_from_private_log_test.cpp @@ -15,16 +15,19 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include +// IWYU pragma: no_include #include +// IWYU pragma: no_include // IWYU pragma: no_include +// IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include #include -#include +#include "aio/aio_task.h" +#include "aio/file_io.h" #include "common/gpid.h" #include "common/replication.codes.h" #include "common/replication_other_types.h" @@ -44,6 +47,7 @@ #include "runtime/task/task_tracker.h" #include "utils/autoref_ptr.h" #include "utils/chrono_literals.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/errors.h" #include "utils/fail_point.h" @@ -58,7 +62,6 @@ #include #include #include -#include #include #include #include @@ -303,42 +306,43 @@ TEST_F(load_from_private_log_test, start_duplication_100000_4MB) // Ensure replica_duplicator can correctly handle real-world log file TEST_F(load_from_private_log_test, handle_real_private_log) { + std::vector log_files({"log.1.0.handle_real_private_log", + "log.1.0.handle_real_private_log2", + "log.1.0.all_loaded_are_write_empties"}); + struct test_data { - std::string fname; int puts; int total; gpid id; } tests[] = { // PUT, PUT, PUT, EMPTY, PUT, EMPTY, EMPTY - {"log.1.0.handle_real_private_log", 4, 6, gpid(1, 4)}, + {4, 6, gpid(1, 4)}, // EMPTY, PUT, EMPTY - {"log.1.0.handle_real_private_log2", 1, 2, gpid(1, 4)}, + {1, 2, gpid(1, 4)}, // EMPTY, EMPTY, EMPTY - {"log.1.0.all_loaded_are_write_empties", 0, 2, gpid(1, 5)}, + {0, 2, gpid(1, 5)}, }; - for (auto tt : tests) { + ASSERT_EQ(log_files.size(), sizeof(tests) / sizeof(test_data)); + for (int i = 0; i < log_files.size(); i++) { // reset replica to specified gpid duplicator.reset(nullptr); - _replica = create_mock_replica(stub.get(), tt.id.get_app_id(), tt.id.get_partition_index()); + _replica = create_mock_replica( + stub.get(), tests[i].id.get_app_id(), tests[i].id.get_partition_index()); // Update '_log_dir' to the corresponding replica created above. _log_dir = _replica->dir(); ASSERT_TRUE(utils::filesystem::path_exists(_log_dir)) << _log_dir; // Copy the log file to '_log_dir' - boost::filesystem::path file(tt.fname); - ASSERT_TRUE(dsn::utils::filesystem::file_exists(tt.fname)) << tt.fname; - boost::system::error_code ec; - boost::filesystem::copy_file( - file, _log_dir + "/log.1.0", boost::filesystem::copy_option::overwrite_if_exists, ec); - ASSERT_TRUE(!ec) << ec.value() << ", " << ec.category().name() << ", " << ec.message(); + auto s = dsn::utils::copy_file(log_files[i], _log_dir + "/log.1.0"); + ASSERT_TRUE(s.ok()) << s.ToString(); // Start to verify. - load_and_wait_all_entries_loaded(tt.puts, tt.total, tt.id, 1, 0); + load_and_wait_all_entries_loaded(tests[i].puts, tests[i].total, tests[i].id, 1, 0); } } @@ -451,12 +455,27 @@ TEST_F(load_fail_mode_test, fail_skip_real_corrupted_file) { { // inject some bad data in the middle of the first file std::string log_path = _log_dir + "/log.1.0"; - auto file_size = boost::filesystem::file_size(log_path); - int fd = open(log_path.c_str(), O_WRONLY); + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + log_path, dsn::utils::FileDataType::kSensitive, file_size)); + auto wfile = file::open(log_path, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + const char buf[] = "xxxxxx"; - auto written_size = pwrite(fd, buf, sizeof(buf), file_size / 2); - ASSERT_EQ(written_size, sizeof(buf)); - close(fd); + auto buff_len = sizeof(buf); + auto t = ::dsn::file::write(wfile, + buf, + buff_len, + file_size / 2, + LPC_AIO_IMMEDIATE_CALLBACK, + nullptr, + [=](::dsn::error_code err, size_t n) { + CHECK_EQ(ERR_OK, err); + CHECK_EQ(buff_len, n); + }); + t->wait(); + ASSERT_EQ(ERR_OK, ::dsn::file::flush(wfile)); + ASSERT_EQ(ERR_OK, ::dsn::file::close(wfile)); } duplicator->update_fail_mode(duplication_fail_mode::FAIL_SKIP); diff --git a/src/replica/duplication/test/run.sh b/src/replica/duplication/test/run.sh index c72b65788f..cf9d4b11ef 100755 --- a/src/replica/duplication/test/run.sh +++ b/src/replica/duplication/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -23,6 +23,24 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi ./dsn_replica_dup_test diff --git a/src/replica/log_file.cpp b/src/replica/log_file.cpp index 7362aa7c41..973213b3a4 100644 --- a/src/replica/log_file.cpp +++ b/src/replica/log_file.cpp @@ -26,7 +26,6 @@ #include "log_file.h" -#include #include #include #include @@ -99,7 +98,7 @@ log_file::~log_file() { close(); } return nullptr; } - disk_file *hfile = file::open(path, O_RDONLY | O_BINARY, 0); + disk_file *hfile = file::open(path, file::FileOpenType::kReadOnly); if (!hfile) { err = ERR_FILE_OPERATION_FAILED; LOG_WARNING("open log file {} failed", path); @@ -155,7 +154,7 @@ log_file::~log_file() { close(); } return nullptr; } - disk_file *hfile = file::open(path, O_RDWR | O_CREAT | O_BINARY, 0666); + disk_file *hfile = file::open(path, file::FileOpenType::kWriteOnly); if (!hfile) { LOG_WARNING("create log {} failed", path); return nullptr; @@ -166,15 +165,15 @@ log_file::~log_file() { close(); } log_file::log_file( const char *path, disk_file *handle, int index, int64_t start_offset, bool is_read) - : _is_read(is_read) + : _crc32(0), + _start_offset(start_offset), + _end_offset(start_offset), + _handle(handle), + _is_read(is_read), + _path(path), + _index(index), + _last_write_time(0) { - _start_offset = start_offset; - _end_offset = start_offset; - _handle = handle; - _path = path; - _index = index; - _crc32 = 0; - _last_write_time = 0; memset(&_header, 0, sizeof(_header)); if (is_read) { @@ -268,6 +267,7 @@ aio_task_ptr log_file::commit_log_block(log_block &block, log_appender pending(offset, block); return commit_log_blocks(pending, evt, tracker, std::move(callback), hash); } + aio_task_ptr log_file::commit_log_blocks(log_appender &pending, dsn::task_code evt, dsn::task_tracker *tracker, @@ -333,7 +333,7 @@ aio_task_ptr log_file::commit_log_blocks(log_appender &pending, hash); } - if (utils::FLAGS_enable_latency_tracer) { + if (dsn_unlikely(utils::FLAGS_enable_latency_tracer)) { tsk->_tracer->set_parent_point_name("commit_pending_mutations"); tsk->_tracer->set_description("log"); for (const auto &mutation : pending.mutations()) { @@ -357,7 +357,7 @@ void log_file::reset_stream(size_t offset /*default = 0*/) } } -decree log_file::previous_log_max_decree(const dsn::gpid &pid) +decree log_file::previous_log_max_decree(const dsn::gpid &pid) const { auto it = _previous_log_max_decrees.find(pid); return it == _previous_log_max_decrees.end() ? 0 : it->second.max_decree; diff --git a/src/replica/log_file.h b/src/replica/log_file.h index fb1fe3b6d4..fcaff2187a 100644 --- a/src/replica/log_file.h +++ b/src/replica/log_file.h @@ -65,9 +65,9 @@ struct log_file_header // a structure to record replica's log info struct replica_log_info { - int64_t max_decree; + decree max_decree; int64_t valid_start_offset; // valid start offset in global space - replica_log_info(int64_t d, int64_t o) + replica_log_info(decree d, int64_t o) { max_decree = d; valid_start_offset = o; @@ -184,11 +184,14 @@ class log_file : public ref_counter // file path const std::string &path() const { return _path; } // previous decrees - const replica_log_info_map &previous_log_max_decrees() { return _previous_log_max_decrees; } + const replica_log_info_map &previous_log_max_decrees() const + { + return _previous_log_max_decrees; + } // previous decree for speicified gpid - decree previous_log_max_decree(const gpid &pid); + decree previous_log_max_decree(const gpid &pid) const; // file header - log_file_header &header() { return _header; } + const log_file_header &header() const { return _header; } // read file header from reader, return byte count consumed int read_file_header(binary_reader &reader); @@ -213,7 +216,7 @@ class log_file : public ref_counter friend class mock_log_file; uint32_t _crc32; - int64_t _start_offset; // start offset in the global space + const int64_t _start_offset; // start offset in the global space std::atomic _end_offset; // end offset in the global space: end_offset = start_offset + file_size class file_streamer; @@ -221,8 +224,8 @@ class log_file : public ref_counter std::unique_ptr _stream; disk_file *_handle; // file handle const bool _is_read; // if opened for read or write - std::string _path; // file path - int _index; // file index + const std::string _path; // file path + const int _index; // file index log_file_header _header; // file header uint64_t _last_write_time; // seconds from epoch time diff --git a/src/replica/mutation_cache.cpp b/src/replica/mutation_cache.cpp index c97c9b0451..1a7fa4d338 100644 --- a/src/replica/mutation_cache.cpp +++ b/src/replica/mutation_cache.cpp @@ -30,8 +30,6 @@ // https://github.com/include-what-you-use/include-what-you-use/issues/166 // TODO(yingchun): remove this pragma by using mapping.imp // IWYU pragma: no_include -#include - #include "consensus_types.h" #include "mutation.h" #include "utils/autoref_ptr.h" diff --git a/src/replica/mutation_log.cpp b/src/replica/mutation_log.cpp index c618c7cd47..ad745670b2 100644 --- a/src/replica/mutation_log.cpp +++ b/src/replica/mutation_log.cpp @@ -527,8 +527,8 @@ error_code mutation_log::open(replay_callback read_callback, io_failure_callback write_error_callback, const std::map &replay_condition) { - CHECK(!_is_opened, "cannot open a opened mutation_log"); - CHECK(nullptr == _current_log_file, ""); + CHECK(!_is_opened, "cannot open an opened mutation_log"); + CHECK_NULL(_current_log_file, ""); // create dir if necessary if (!dsn::utils::filesystem::path_exists(_dir)) { @@ -562,9 +562,8 @@ error_code mutation_log::open(replay_callback read_callback, err == ERR_INVALID_PARAMETERS) { LOG_WARNING("skip file {} during log init, err = {}", fpath, err); continue; - } else { - return err; } + return err; } if (_is_private) { @@ -592,8 +591,8 @@ error_code mutation_log::open(replay_callback read_callback, file_list.clear(); // filter useless log - std::map::iterator replay_begin = _log_files.begin(); - std::map::iterator replay_end = _log_files.end(); + log_file_map_by_index::iterator replay_begin = _log_files.begin(); + log_file_map_by_index::iterator replay_end = _log_files.end(); if (!replay_condition.empty()) { if (_is_private) { auto find = replay_condition.find(_private_gpid); @@ -609,7 +608,7 @@ error_code mutation_log::open(replay_callback read_callback, } else { // find the largest file which can be ignored. // after iterate, the 'mark_it' will point to the largest file which can be ignored. - std::map::reverse_iterator mark_it; + log_file_map_by_index::reverse_iterator mark_it; std::set kickout_replicas; replica_log_info_map max_decrees; // max_decrees for log file at mark_it. for (mark_it = _log_files.rbegin(); mark_it != _log_files.rend(); ++mark_it) { @@ -666,7 +665,7 @@ error_code mutation_log::open(replay_callback read_callback, } // replay with the found files - std::map replay_logs(replay_begin, replay_end); + log_file_map_by_index replay_logs(replay_begin, replay_end); int64_t end_offset = 0; err = replay( replay_logs, @@ -863,11 +862,8 @@ decree mutation_log::max_decree(gpid gpid) const CHECK_EQ(gpid, _private_gpid); return _private_log_info.max_decree; } else { - auto it = _shared_log_info_map.find(gpid); - if (it != _shared_log_info_map.end()) - return it->second.max_decree; - else - return 0; + const auto &it = _shared_log_info_map.find(gpid); + return it != _shared_log_info_map.end() ? it->second.max_decree : 0; } } @@ -923,7 +919,7 @@ int64_t mutation_log::total_size() const int64_t mutation_log::total_size_no_lock() const { - return _log_files.size() > 0 ? _global_end_offset - _global_start_offset : 0; + return _log_files.empty() ? 0 : _global_end_offset - _global_start_offset; } error_code mutation_log::reset_from(const std::string &dir, @@ -1095,7 +1091,7 @@ bool mutation_log::get_learn_state(gpid gpid, decree start, /*out*/ learn_state state.meta = temp_writer.get_buffer(); } - std::map files; + log_file_map_by_index files; { zauto_lock l(_lock); @@ -1202,13 +1198,13 @@ void mutation_log::get_parent_mutations_and_logs(gpid pid, // no memory data and no disk data return; } - std::map file_map = get_log_file_map(); + const auto &file_map = get_log_file_map(); bool skip_next = false; std::list learn_files; decree last_max_decree = 0; for (auto itr = file_map.rbegin(); itr != file_map.rend(); ++itr) { - log_file_ptr &log = itr->second; + const log_file_ptr &log = itr->second; if (log->end_offset() <= _private_log_info.valid_start_offset) break; @@ -1287,7 +1283,7 @@ int mutation_log::garbage_collection(gpid gpid, { CHECK(_is_private, "this method is only valid for private log"); - std::map files; + log_file_map_by_index files; decree max_decree = invalid_decree; int current_file_index = -1; @@ -1295,23 +1291,24 @@ int mutation_log::garbage_collection(gpid gpid, zauto_lock l(_lock); files = _log_files; max_decree = _private_log_info.max_decree; - if (_current_log_file != nullptr) + if (_current_log_file != nullptr) { current_file_index = _current_log_file->index(); + } } if (files.size() <= 1) { // nothing to do return 0; - } else { - // the last one should be the current log file - CHECK(current_file_index == -1 || files.rbegin()->first == current_file_index, - "invalid current_file_index, index = {}", - current_file_index); } + // the last one should be the current log file + CHECK(current_file_index == -1 || files.rbegin()->first == current_file_index, + "invalid current_file_index, index = {}", + current_file_index); + // find the largest file which can be deleted. // after iterate, the 'mark_it' will point to the largest file which can be deleted. - std::map::reverse_iterator mark_it; + log_file_map_by_index::reverse_iterator mark_it; int64_t already_reserved_size = 0; for (mark_it = files.rbegin(); mark_it != files.rend(); ++mark_it) { log_file_ptr log = mark_it->second; @@ -1403,315 +1400,253 @@ int mutation_log::garbage_collection(gpid gpid, return deleted; } -int mutation_log::garbage_collection(const replica_log_info_map &gc_condition, - int file_count_limit, - std::set &prevent_gc_replicas) +struct gc_summary_info { - CHECK(!_is_private, "this method is only valid for shared log"); - - std::map files; - replica_log_info_map max_decrees; - int current_log_index = -1; - int64_t total_log_size = 0; + dsn::gpid pid; + int min_file_index = 0; + dsn::replication::decree max_decree_gap = 0; + dsn::replication::decree garbage_max_decree = 0; + dsn::replication::decree slog_max_decree = 0; + std::string to_string() const { - zauto_lock l(_lock); - files = _log_files; - max_decrees = _shared_log_info_map; - if (_current_log_file != nullptr) - current_log_index = _current_log_file->index(); - total_log_size = total_size_no_lock(); + return fmt::format("gc_summary_info = [pid = {}, min_file_index = {}, max_decree_gap = {}, " + "garbage_max_decree = {}, slog_max_decree = {}]", + pid, + min_file_index, + max_decree_gap, + garbage_max_decree, + slog_max_decree); } - if (files.size() <= 1) { - // nothing to do - LOG_INFO("gc_shared: too few files to delete, file_count_limit = {}, reserved_log_count " - "= {}, reserved_log_size = {}, current_log_index = {}", - file_count_limit, - files.size(), - total_log_size, - current_log_index); - return (int)files.size(); - } else { - // the last one should be the current log file - CHECK(-1 == current_log_index || files.rbegin()->first == current_log_index, - "invalid current_log_index, index = {}", - current_log_index); + friend std::ostream &operator<<(std::ostream &os, const gc_summary_info &gc_summary) + { + return os << gc_summary.to_string(); } +}; - int reserved_log_count = files.size(); - int64_t reserved_log_size = total_log_size; - int reserved_smallest_log = files.begin()->first; - int reserved_largest_log = current_log_index; - - // find the largest file which can be deleted. - // after iterate, the 'mark_it' will point to the largest file which can be deleted. - std::map::reverse_iterator mark_it; - std::set kickout_replicas; - gpid stop_gc_replica; - int stop_gc_log_index = 0; - decree stop_gc_decree_gap = 0; - decree stop_gc_garbage_max_decree = 0; - decree stop_gc_log_max_decree = 0; - int file_count = 0; - for (mark_it = files.rbegin(); mark_it != files.rend(); ++mark_it) { - log_file_ptr log = mark_it->second; - CHECK_EQ(mark_it->first, log->index()); - file_count++; - - bool delete_ok = true; - - // skip current file - if (current_log_index == log->index()) { - delete_ok = false; - } +namespace { - if (delete_ok) { - std::set prevent_gc_replicas_for_this_log; +bool can_gc_replica_slog(const dsn::replication::replica_log_info_map &slog_max_decrees, + const dsn::replication::log_file_ptr &file, + const dsn::gpid &pid, + const dsn::replication::replica_log_info &replica_durable_info, + dsn::replication::gc_summary_info &gc_summary) +{ + const auto &garbage_max_decree = replica_durable_info.max_decree; + const auto &valid_start_offset = replica_durable_info.valid_start_offset; + + const auto &it = slog_max_decrees.find(pid); + if (it == slog_max_decrees.end()) { + // There's no log found in this file for this replica, thus all decrees of + // this replica in this file could deleted. + // + // `valid_start_offset` might be reset to 0 if initialize_on_load() returned + // `ERR_INCOMPLETE_DATA`, thus it's possible that `valid_start_offset == 0`. + CHECK(valid_start_offset == 0 || file->end_offset() <= valid_start_offset, + "valid start offset must be 0 or greater than the end of this log file"); + LOG_DEBUG("gc @ {}: max_decree for {} is missing vs {} as garbage max decree, it's " + "safe to delete this and all older logs for this replica", + pid, + file->path(), + garbage_max_decree); + return true; + } else if (file->end_offset() <= valid_start_offset) { + // This file has been invalid for this replica, since `valid_start_offset` was reset + // to a file with larger index than this file. Thus all decrees of this replica in + // this file could be deleted. + LOG_DEBUG("gc @ {}: log is invalid for {}, as valid start offset vs log end offset = " + "{} vs {}, it is therefore safe to delete this and all older logs for this " + "replica", + pid, + file->path(), + valid_start_offset, + file->end_offset()); + return true; + } else if (it->second.max_decree <= garbage_max_decree) { + // All decrees are no more than the garbage max decree. Since all decrees less than + // garbage max decree would be deleted, all decrees of this replica in this file + // could be deleted. + LOG_DEBUG("gc @ {}: max_decree for {} is {} vs {} as garbage max decree, it is " + "therefore safe to delete this and all older logs for this replica", + pid, + file->path(), + it->second.max_decree, + garbage_max_decree); + return true; + } - for (auto &kv : gc_condition) { - if (kickout_replicas.find(kv.first) != kickout_replicas.end()) { - // no need to consider this replica - continue; - } + // it->second.max_decree > garbage_max_decree + // + // Some decrees are more than garbage max decree, thus this file should not be deleted + // for now. + LOG_DEBUG("gc @ {}: max_decree for {} is {} vs {} as garbage max decree, it " + "is therefore not allowed to delete this and all older logs", + pid, + file->path(), + it->second.max_decree, + garbage_max_decree); + + auto gap = it->second.max_decree - garbage_max_decree; + if (file->index() < gc_summary.min_file_index || gap > gc_summary.max_decree_gap) { + // Find the oldest file of this round of iteration for gc of slog files, with the + // max decree gap between the garbage max decree and the oldest slog file. + gc_summary.pid = pid; + gc_summary.min_file_index = file->index(); + gc_summary.max_decree_gap = gap; + gc_summary.garbage_max_decree = garbage_max_decree; + gc_summary.slog_max_decree = it->second.max_decree; + } - gpid gpid = kv.first; - decree garbage_max_decree = kv.second.max_decree; - int64_t valid_start_offset = kv.second.valid_start_offset; + return false; +} - bool delete_ok_for_this_replica = false; - bool kickout_this_replica = false; - auto it3 = max_decrees.find(gpid); +} // anonymous namespace - // log not found for this replica, ok to delete - if (it3 == max_decrees.end()) { - // valid_start_offset may be reset to 0 if initialize_on_load() returns - // ERR_INCOMPLETE_DATA - CHECK(valid_start_offset == 0 || valid_start_offset >= log->end_offset(), - "valid start offset must be 0 or greater than the end of this log file"); +void mutation_log::garbage_collection(const replica_log_info_map &replica_durable_decrees, + std::set &prevent_gc_replicas) +{ + CHECK(!_is_private, "this method is only valid for shared log"); - LOG_DEBUG( - "gc @ {}: max_decree for {} is missing vs {} as garbage max decree, it's " - "safe to delete this and all older logs for this replica", - gpid, - log->path(), - garbage_max_decree); - delete_ok_for_this_replica = true; - kickout_this_replica = true; - } + // Fetch the snapshot of the latest states of the slog, such as the max decree it maintains + // for each partition. + log_file_map_by_index files; + replica_log_info_map slog_max_decrees; + int64_t total_log_size = 0; + { + zauto_lock l(_lock); + total_log_size = total_size_no_lock(); + if (_log_files.empty()) { + CHECK_EQ(total_log_size, 0); + LOG_INFO("gc_shared: slog file not found"); + return; + } - // log is invalid for this replica, ok to delete - else if (log->end_offset() <= valid_start_offset) { - LOG_DEBUG( - "gc @ {}: log is invalid for {}, as valid start offset vs log end offset = " - "{} vs {}, it is therefore safe to delete this and all older logs for this " - "replica", - gpid, - log->path(), - valid_start_offset, - log->end_offset()); - delete_ok_for_this_replica = true; - kickout_this_replica = true; - } + CHECK_NULL(_current_log_file, + "shared logs have been deprecated, thus could not be created"); + files = _log_files; + slog_max_decrees = _shared_log_info_map; + } - // all decrees are no more than garbage max decree, ok to delete - else if (it3->second.max_decree <= garbage_max_decree) { - LOG_DEBUG("gc @ {}: max_decree for {} is {} vs {} as garbage max decree, it is " - "therefore safe to delete this and all older logs for this replica", - gpid, - log->path(), - it3->second.max_decree, - garbage_max_decree); - delete_ok_for_this_replica = true; - kickout_this_replica = true; - } + reserved_slog_info reserved_slog = { + files.size(), total_log_size, files.begin()->first, files.rbegin()->first}; - else // it3->second.max_decree > garbage_max_decree - { - // should not delete this file - LOG_DEBUG("gc @ {}: max_decree for {} is {} vs {} as garbage max decree, it " - "is therefore not allowed to delete this and all older logs", - gpid, - log->path(), - it3->second.max_decree, - garbage_max_decree); - prevent_gc_replicas_for_this_log.insert(gpid); - decree gap = it3->second.max_decree - garbage_max_decree; - if (log->index() < stop_gc_log_index || gap > stop_gc_decree_gap) { - // record the max gap replica for the smallest log - stop_gc_replica = gpid; - stop_gc_log_index = log->index(); - stop_gc_decree_gap = gap; - stop_gc_garbage_max_decree = garbage_max_decree; - stop_gc_log_max_decree = it3->second.max_decree; - } - } + // Iterate over the slog files from the newest to the oldest in descending order(i.e. + // file index in descending order), to find the newest file that could be deleted(after + // iterating, `mark_it` would point to the newest file that could be deleted). + log_file_map_by_index::reverse_iterator mark_it; + std::set kickout_replicas; + gc_summary_info gc_summary; + for (mark_it = files.rbegin(); mark_it != files.rend(); ++mark_it) { + const auto &file = mark_it->second; + CHECK_EQ(mark_it->first, file->index()); - if (kickout_this_replica) { - // files before this file is useless for this replica, - // so from now on, this replica will not be considered anymore - kickout_replicas.insert(gpid); - } + bool can_gc_all_replicas_slog = true; - if (!delete_ok_for_this_replica) { - // can not delete this file, mark it, and continue to check other replicas - delete_ok = false; - } + for (const auto &replica_durable_info : replica_durable_decrees) { + if (kickout_replicas.find(replica_durable_info.first) != kickout_replicas.end()) { + // There's no need to consider this replica. + continue; } - // update prevent_gc_replicas - if (file_count > file_count_limit && !prevent_gc_replicas_for_this_log.empty()) { - prevent_gc_replicas.insert(prevent_gc_replicas_for_this_log.begin(), - prevent_gc_replicas_for_this_log.end()); + if (can_gc_replica_slog(slog_max_decrees, + file, + replica_durable_info.first, + replica_durable_info.second, + gc_summary)) { + // Log files before this file is useless for this replica, + // so from now on, this replica would not be considered any more. + kickout_replicas.insert(replica_durable_info.first); + continue; } + + // For now, this file could not be deleted. + can_gc_all_replicas_slog = false; + prevent_gc_replicas.insert(replica_durable_info.first); } - if (delete_ok) { - // found the largest file which can be deleted + if (can_gc_all_replicas_slog) { + // The newest file that could be deleted has been found. break; } - // update max_decrees for the next log file - max_decrees = log->previous_log_max_decrees(); + // Fetch max decrees of the next slog file. + slog_max_decrees = file->previous_log_max_decrees(); } if (mark_it == files.rend()) { - // no file to delete - if (stop_gc_decree_gap > 0) { - LOG_INFO("gc_shared: no file can be deleted, file_count_limit = {}, " - "reserved_log_count = {}, reserved_log_size = {}, " - "reserved_smallest_log = {}, reserved_largest_log = {}, " - "stop_gc_log_index = {}, stop_gc_replica_count = {}, " - "stop_gc_replica = {}, stop_gc_decree_gap = {}, " - "stop_gc_garbage_max_decree = {}, stop_gc_log_max_decree = {}", - file_count_limit, - reserved_log_count, - reserved_log_size, - reserved_smallest_log, - reserved_largest_log, - stop_gc_log_index, - prevent_gc_replicas.size(), - stop_gc_replica, - stop_gc_decree_gap, - stop_gc_garbage_max_decree, - stop_gc_log_max_decree); - } else { - LOG_INFO("gc_shared: no file can be deleted, file_count_limit = {}, " - "reserved_log_count = {}, reserved_log_size = {}, " - "reserved_smallest_log = {}, reserved_largest_log = {}", - file_count_limit, - reserved_log_count, - reserved_log_size, - reserved_smallest_log, - reserved_largest_log); - } - - return reserved_log_count; + // There's no file that could be deleted. + LOG_INFO("gc_shared: no file can be deleted: {}, {}, prevent_gc_replicas = {}", + reserved_slog, + gc_summary, + prevent_gc_replicas.size()); + return; } - // ok, let's delete files in increasing order of file index - // to avoid making a hole in the file list - int largest_log_to_delete = mark_it->second->index(); - int to_delete_log_count = 0; - int64_t to_delete_log_size = 0; - int deleted_log_count = 0; - int64_t deleted_log_size = 0; - int deleted_smallest_log = 0; - int deleted_largest_log = 0; - for (auto it = files.begin(); it != files.end() && it->second->index() <= largest_log_to_delete; + slog_deletion_info slog_deletion; + + // Delete files in ascending order of file index. Otherwise, deleting files in descending + // order would lead to a hole in the file list once a file failed to be deleted. + remove_obsolete_slog_files(mark_it->second->index(), files, reserved_slog, slog_deletion); + LOG_INFO("gc_shared: deleted some files: {}, {}, {}, prevent_gc_replicas = {}", + reserved_slog, + slog_deletion, + gc_summary, + prevent_gc_replicas.size()); +} + +void mutation_log::remove_obsolete_slog_files(const int max_file_index_to_delete, + log_file_map_by_index &files, + reserved_slog_info &reserved_slog, + slog_deletion_info &slog_deletion) +{ + for (auto it = files.begin(); + it != files.end() && it->second->index() <= max_file_index_to_delete; ++it) { - log_file_ptr log = it->second; - CHECK_EQ(it->first, log->index()); - to_delete_log_count++; - to_delete_log_size += log->end_offset() - log->start_offset(); + auto &file = it->second; + CHECK_EQ(it->first, file->index()); + slog_deletion.to_delete_file_count++; + slog_deletion.to_delete_log_size += file->end_offset() - file->start_offset(); - // close first - log->close(); + // Firstly close the log file. + file->close(); - // delete file - auto &fpath = log->path(); + // Delete the log file. + const auto &fpath = file->path(); if (!dsn::utils::filesystem::remove_path(fpath)) { LOG_ERROR("gc_shared: fail to remove {}, stop current gc cycle ...", fpath); break; } - // delete succeed + // The log file is deleted successfully. LOG_INFO("gc_shared: log file {} is removed", fpath); - deleted_log_count++; - deleted_log_size += log->end_offset() - log->start_offset(); - if (deleted_smallest_log == 0) - deleted_smallest_log = log->index(); - deleted_largest_log = log->index(); + slog_deletion.deleted_file_count++; + slog_deletion.deleted_log_size += file->end_offset() - file->start_offset(); + if (slog_deletion.deleted_min_file_index == 0) { + slog_deletion.deleted_min_file_index = file->index(); + } + slog_deletion.deleted_max_file_index = file->index(); - // erase from _log_files + // Remove the log file from _log_files. { zauto_lock l(_lock); _log_files.erase(it->first); _global_start_offset = _log_files.size() > 0 ? _log_files.begin()->second->start_offset() : 0; - reserved_log_count = _log_files.size(); - reserved_log_size = total_size_no_lock(); - if (reserved_log_count > 0) { - reserved_smallest_log = _log_files.begin()->first; - reserved_largest_log = _log_files.rbegin()->first; + reserved_slog.file_count = _log_files.size(); + reserved_slog.log_size = total_size_no_lock(); + if (reserved_slog.file_count > 0) { + reserved_slog.min_file_index = _log_files.begin()->first; + reserved_slog.max_file_index = _log_files.rbegin()->first; } else { - reserved_smallest_log = -1; - reserved_largest_log = -1; + reserved_slog.min_file_index = -1; + reserved_slog.max_file_index = -1; } } } - - if (stop_gc_decree_gap > 0) { - LOG_INFO("gc_shared: deleted some files, file_count_limit = {}, " - "reserved_log_count = {}, reserved_log_size = {}, " - "reserved_smallest_log = {}, reserved_largest_log = {}, " - "to_delete_log_count = {}, to_delete_log_size = {}, " - "deleted_log_count = {}, deleted_log_size = {}, " - "deleted_smallest_log = {}, deleted_largest_log = {}, " - "stop_gc_log_index = {}, stop_gc_replica_count = {}, " - "stop_gc_replica = {}, stop_gc_decree_gap = {}, " - "stop_gc_garbage_max_decree = {}, stop_gc_log_max_decree = {}", - file_count_limit, - reserved_log_count, - reserved_log_size, - reserved_smallest_log, - reserved_largest_log, - to_delete_log_count, - to_delete_log_size, - deleted_log_count, - deleted_log_size, - deleted_smallest_log, - deleted_largest_log, - stop_gc_log_index, - prevent_gc_replicas.size(), - stop_gc_replica, - stop_gc_decree_gap, - stop_gc_garbage_max_decree, - stop_gc_log_max_decree); - } else { - LOG_INFO("gc_shared: deleted some files, file_count_limit = {}, " - "reserved_log_count = {}, reserved_log_size = {}, " - "reserved_smallest_log = {}, reserved_largest_log = {}, " - "to_delete_log_count = {}, to_delete_log_size = {}, " - "deleted_log_count = {}, deleted_log_size = {}, " - "deleted_smallest_log = {}, deleted_largest_log = {}", - file_count_limit, - reserved_log_count, - reserved_log_size, - reserved_smallest_log, - reserved_largest_log, - to_delete_log_count, - to_delete_log_size, - deleted_log_count, - deleted_log_size, - deleted_smallest_log, - deleted_largest_log); - } - - return reserved_log_count; } -std::map mutation_log::get_log_file_map() const +mutation_log::log_file_map_by_index mutation_log::get_log_file_map() const { zauto_lock l(_lock); return _log_files; @@ -1719,3 +1654,5 @@ std::map mutation_log::get_log_file_map() const } // namespace replication } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(dsn::replication::gc_summary_info); diff --git a/src/replica/mutation_log.h b/src/replica/mutation_log.h index 6636f808af..b6223e847c 100644 --- a/src/replica/mutation_log.h +++ b/src/replica/mutation_log.h @@ -26,11 +26,13 @@ #pragma once +#include #include #include #include #include #include +#include #include #include #include @@ -50,6 +52,7 @@ #include "utils/autoref_ptr.h" #include "utils/error_code.h" #include "utils/errors.h" +#include "utils/fmt_utils.h" #include "utils/zlocks.h" namespace dsn { @@ -221,19 +224,17 @@ class mutation_log : public ref_counter int64_t reserve_max_size, int64_t reserve_max_time); - // garbage collection for shared log, returns reserved file count. - // `prevent_gc_replicas' will store replicas which prevent log files out of `file_count_limit' - // to be deleted. - // remove log files if satisfy: - // - for each replica "r": - // r is not in file.max_decree - // || file.max_decree[r] <= gc_condition[r].max_decree - // || file.end_offset[r] <= gc_condition[r].valid_start_offset - // - the current log file should not be removed - // thread safe - int garbage_collection(const replica_log_info_map &gc_condition, - int file_count_limit, - std::set &prevent_gc_replicas); + // Garbage collection for shared log. + // `prevent_gc_replicas' will store replicas which prevent log files from being deleted + // for gc. + // + // Since slog had been deprecated, no new slog files would be created. Therefore, our + // target is to remove all of the existing slog files according to the progressive durable + // decree for each replica. + // + // Thread safe. + void garbage_collection(const replica_log_info_map &replica_durable_decrees, + std::set &prevent_gc_replicas); // // when this is a private log, log files are learned by remote replicas @@ -285,8 +286,10 @@ class mutation_log : public ref_counter decree max_gced_decree(gpid gpid) const; decree max_gced_decree_no_lock(gpid gpid) const; + using log_file_map_by_index = std::map; + // thread-safe - std::map get_log_file_map() const; + log_file_map_by_index get_log_file_map() const; // check the consistence of valid_start_offset // thread safe @@ -300,6 +303,58 @@ class mutation_log : public ref_counter task_tracker *tracker() { return &_tracker; } + struct reserved_slog_info + { + size_t file_count = 0; + int64_t log_size = 0; + int min_file_index = 0; + int max_file_index = 0; + + std::string to_string() const + { + return fmt::format("reserved_slog_info = [file_count = {}, log_size = {}, " + "min_file_index = {}, max_file_index = {}]", + file_count, + log_size, + min_file_index, + max_file_index); + } + + friend std::ostream &operator<<(std::ostream &os, const reserved_slog_info &reserved_log) + { + return os << reserved_log.to_string(); + } + }; + + struct slog_deletion_info + { + int to_delete_file_count = 0; + int64_t to_delete_log_size = 0; + int deleted_file_count = 0; + int64_t deleted_log_size = 0; + int deleted_min_file_index = 0; + int deleted_max_file_index = 0; + + std::string to_string() const + { + return fmt::format("slog_deletion_info = [to_delete_file_count = {}, " + "to_delete_log_size = {}, deleted_file_count = {}, " + "deleted_log_size = {}, deleted_min_file_index = {}, " + "deleted_max_file_index = {}]", + to_delete_file_count, + to_delete_log_size, + deleted_file_count, + deleted_log_size, + deleted_min_file_index, + deleted_max_file_index); + } + + friend std::ostream &operator<<(std::ostream &os, const slog_deletion_info &log_deletion) + { + return os << log_deletion.to_string(); + } + }; + protected: // thread-safe // 'size' is data size to write; the '_global_end_offset' will be updated by 'size'. @@ -325,7 +380,7 @@ class mutation_log : public ref_counter replay_callback callback, /*out*/ int64_t &end_offset); - static error_code replay(std::map &log_files, + static error_code replay(log_file_map_by_index &log_files, replay_callback callback, /*out*/ int64_t &end_offset); @@ -345,6 +400,13 @@ class mutation_log : public ref_counter // get total size ithout lock. int64_t total_size_no_lock() const; + // Closing and remove all of slog files whose indexes are less than (i.e. older) or equal to + // `max_file_index_to_delete`. + void remove_obsolete_slog_files(const int max_file_index_to_delete, + log_file_map_by_index &files, + reserved_slog_info &reserved_log, + slog_deletion_info &log_deletion); + protected: std::string _dir; bool _is_private; @@ -373,12 +435,12 @@ class mutation_log : public ref_counter bool _switch_file_demand; // logs - int _last_file_index; // new log file index = _last_file_index + 1 - std::map _log_files; // index -> log_file_ptr - log_file_ptr _current_log_file; // current log file - int64_t _global_start_offset; // global start offset of all files. - // invalid if _log_files.size() == 0. - int64_t _global_end_offset; // global end offset currently + int _last_file_index; // new log file index = _last_file_index + 1 + log_file_map_by_index _log_files; // index -> log_file_ptr + log_file_ptr _current_log_file; // current log file + int64_t _global_start_offset; // global start offset of all files. + // invalid if _log_files.size() == 0. + int64_t _global_end_offset; // global end offset currently // replica log info // - log_info.max_decree: the max decree of mutations up to now @@ -410,21 +472,21 @@ class mutation_log_shared : public mutation_log { } - virtual ~mutation_log_shared() override + ~mutation_log_shared() override { close(); _tracker.cancel_outstanding_tasks(); } - virtual ::dsn::task_ptr append(mutation_ptr &mu, - dsn::task_code callback_code, - dsn::task_tracker *tracker, - aio_handler &&callback, - int hash = 0, - int64_t *pending_size = nullptr) override; + ::dsn::task_ptr append(mutation_ptr &mu, + dsn::task_code callback_code, + dsn::task_tracker *tracker, + aio_handler &&callback, + int hash = 0, + int64_t *pending_size = nullptr) override; - virtual void flush() override; - virtual void flush_once() override; + void flush() override; + void flush_once() override; private: // async write pending mutations into log file @@ -467,24 +529,22 @@ class mutation_log_private : public mutation_log, private replica_base _tracker.cancel_outstanding_tasks(); } - virtual ::dsn::task_ptr append(mutation_ptr &mu, - dsn::task_code callback_code, - dsn::task_tracker *tracker, - aio_handler &&callback, - int hash = 0, - int64_t *pending_size = nullptr) override; + ::dsn::task_ptr append(mutation_ptr &mu, + dsn::task_code callback_code, + dsn::task_tracker *tracker, + aio_handler &&callback, + int hash = 0, + int64_t *pending_size = nullptr) override; - virtual bool get_learn_state_in_memory(decree start_decree, - binary_writer &writer) const override; + bool get_learn_state_in_memory(decree start_decree, binary_writer &writer) const override; // get in-memory mutations, including pending and writing mutations - virtual void - get_in_memory_mutations(decree start_decree, - ballot start_ballot, - /*out*/ std::vector &mutation_list) const override; + void get_in_memory_mutations(decree start_decree, + ballot start_ballot, + /*out*/ std::vector &mutation_list) const override; - virtual void flush() override; - virtual void flush_once() override; + void flush() override; + void flush_once() override; private: // async write pending mutations into log file @@ -499,7 +559,7 @@ class mutation_log_private : public mutation_log, private replica_base std::shared_ptr &pending, decree max_commit); - virtual void init_states() override; + void init_states() override; // flush at most count times // if count <= 0, means flush until all data is on disk @@ -521,3 +581,6 @@ class mutation_log_private : public mutation_log, private replica_base } // namespace replication } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::mutation_log::reserved_slog_info); +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::mutation_log::slog_deletion_info); diff --git a/src/replica/mutation_log_replay.cpp b/src/replica/mutation_log_replay.cpp index 261ff4706a..316c12396e 100644 --- a/src/replica/mutation_log_replay.cpp +++ b/src/replica/mutation_log_replay.cpp @@ -133,7 +133,7 @@ namespace replication { replay_callback callback, /*out*/ int64_t &end_offset) { - std::map logs; + log_file_map_by_index logs; for (auto &fpath : log_files) { error_code err; log_file_ptr log = log_file::open_read(fpath.c_str(), err); @@ -154,7 +154,7 @@ namespace replication { return replay(logs, callback, end_offset); } -/*static*/ error_code mutation_log::replay(std::map &logs, +/*static*/ error_code mutation_log::replay(log_file_map_by_index &logs, replay_callback callback, /*out*/ int64_t &end_offset) { diff --git a/src/replica/mutation_log_utils.cpp b/src/replica/mutation_log_utils.cpp index 5083897494..46c32b4df9 100644 --- a/src/replica/mutation_log_utils.cpp +++ b/src/replica/mutation_log_utils.cpp @@ -64,7 +64,7 @@ namespace log_utils { } /*extern*/ -error_s check_log_files_continuity(const std::map &logs) +error_s check_log_files_continuity(const mutation_log::log_file_map_by_index &logs) { if (logs.empty()) { return error_s::ok(); diff --git a/src/replica/mutation_log_utils.h b/src/replica/mutation_log_utils.h index 4b61846696..9673546f82 100644 --- a/src/replica/mutation_log_utils.h +++ b/src/replica/mutation_log_utils.h @@ -31,6 +31,7 @@ #include #include "replica/log_file.h" +#include "replica/mutation_log.h" #include "utils/autoref_ptr.h" #include "utils/errors.h" #include "utils/string_view.h" @@ -44,7 +45,7 @@ extern error_s open_read(string_view path, /*out*/ log_file_ptr &file); extern error_s list_all_files(const std::string &dir, /*out*/ std::vector &files); inline error_s open_log_file_map(const std::vector &log_files, - /*out*/ std::map &log_file_map) + /*out*/ mutation_log::log_file_map_by_index &log_file_map) { for (const std::string &fname : log_files) { log_file_ptr lf; @@ -58,7 +59,7 @@ inline error_s open_log_file_map(const std::vector &log_files, } inline error_s open_log_file_map(const std::string &dir, - /*out*/ std::map &log_file_map) + /*out*/ mutation_log::log_file_map_by_index &log_file_map) { std::vector log_files; error_s es = list_all_files(dir, log_files); @@ -68,11 +69,11 @@ inline error_s open_log_file_map(const std::string &dir, return open_log_file_map(log_files, log_file_map) << "open_log_file_map(dir)"; } -extern error_s check_log_files_continuity(const std::map &logs); +extern error_s check_log_files_continuity(const mutation_log::log_file_map_by_index &logs); inline error_s check_log_files_continuity(const std::string &dir) { - std::map log_file_map; + mutation_log::log_file_map_by_index log_file_map; error_s es = open_log_file_map(dir, log_file_map); if (!es.is_ok()) { return es << "check_log_files_continuity(dir)"; diff --git a/src/replica/prepare_list.h b/src/replica/prepare_list.h index 96e4d653fb..1882f7b665 100644 --- a/src/replica/prepare_list.h +++ b/src/replica/prepare_list.h @@ -34,6 +34,7 @@ #include "replica/mutation.h" #include "replica/replica_base.h" #include "utils/error_code.h" +#include "utils/fmt_utils.h" namespace dsn { namespace replication { @@ -45,6 +46,7 @@ enum commit_type COMMIT_ALL_READY // commit (last_committed, ... ...] // - only valid when partition_status::PS_SECONDARY or partition_status::PS_PRIMARY }; +USER_DEFINED_ENUM_FORMATTER(commit_type) // prepare_list origins from the concept of `prepared list` in PacificA. // It stores an continuous and ordered list of mutations. diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp index be2a0177b5..b99a3c4219 100644 --- a/src/replica/replica.cpp +++ b/src/replica/replica.cpp @@ -27,12 +27,8 @@ #include "replica.h" #include -#include -#include -#include -#include -#include #include +#include #include #include "backup/replica_backup_manager.h" @@ -285,8 +281,7 @@ void replica::on_client_read(dsn::message_ex *request, bool ignore_throttling) // a small window where the state is not the latest yet if (last_committed_decree() < _primary_states.last_prepare_decree_on_new_primary) { - LOG_ERROR_PREFIX("last_committed_decree(%" PRId64 - ") < last_prepare_decree_on_new_primary(%" PRId64 ")", + LOG_ERROR_PREFIX("last_committed_decree({}) < last_prepare_decree_on_new_primary({})", last_committed_decree(), _primary_states.last_prepare_decree_on_new_primary); response_client_read(request, ERR_INVALID_STATE); diff --git a/src/replica/replica_2pc.cpp b/src/replica/replica_2pc.cpp index e0cff3868e..6138ad7e7d 100644 --- a/src/replica/replica_2pc.cpp +++ b/src/replica/replica_2pc.cpp @@ -259,11 +259,7 @@ void replica::init_prepare(mutation_ptr &mu, bool reconciliation, bool pop_all_c } mu->_tracer->set_name(fmt::format("mutation[{}]", mu->name())); - dlog(level, - "%s: mutation %s init_prepare, mutation_tid=%" PRIu64, - name(), - mu->name(), - mu->tid()); + dlog_f(level, "{}: mutation {} init_prepare, mutation_tid={}", name(), mu->name(), mu->tid()); // child should prepare mutation synchronously mu->set_is_sync_to_child(_primary_states.sync_send_write_request); diff --git a/src/replica/replica_http_service.h b/src/replica/replica_http_service.h index d02707441d..35b2352ac1 100644 --- a/src/replica/replica_http_service.h +++ b/src/replica/replica_http_service.h @@ -48,7 +48,14 @@ class replica_http_service : public http_server_base this, std::placeholders::_1, std::placeholders::_2), - "ip:port/replica/maual_compaction?app_id="); + "ip:port/replica/manual_compaction?app_id="); + } + + ~replica_http_service() + { + deregister_http_call("replica/duplication"); + deregister_http_call("replica/data_version"); + deregister_http_call("replica/manual_compaction"); } std::string path() const override { return "replica"; } diff --git a/src/replica/replica_learn.cpp b/src/replica/replica_learn.cpp index 75fc9c5894..3fc23ed73d 100644 --- a/src/replica/replica_learn.cpp +++ b/src/replica/replica_learn.cpp @@ -46,7 +46,8 @@ #include #include -#include "aio/aio_task.h" +#include // IWYU pragma: keep + #include "common/fs_manager.h" #include "common/gpid.h" #include "common/replication.codes.h" diff --git a/src/replica/replica_restore.cpp b/src/replica/replica_restore.cpp index 1a37479777..f2494482ff 100644 --- a/src/replica/replica_restore.cpp +++ b/src/replica/replica_restore.cpp @@ -17,6 +17,8 @@ #include #include +#include +#include #include #include #include @@ -49,10 +51,10 @@ #include "runtime/task/task_tracker.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" -#include "utils/utils.h" using namespace dsn::dist::block_service; @@ -93,41 +95,25 @@ bool replica::remove_useless_file_under_chkpt(const std::string &chkpt_dir, return true; } -bool replica::read_cold_backup_metadata(const std::string &file, +bool replica::read_cold_backup_metadata(const std::string &fname, cold_backup_metadata &backup_metadata) { - if (!::dsn::utils::filesystem::file_exists(file)) { + if (!::dsn::utils::filesystem::file_exists(fname)) { LOG_ERROR_PREFIX( - "checkpoint on remote storage media is damaged, coz file({}) doesn't exist", file); + "checkpoint on remote storage media is damaged, coz file({}) doesn't exist", fname); return false; } - int64_t file_sz = 0; - if (!::dsn::utils::filesystem::file_size(file, file_sz)) { - LOG_ERROR_PREFIX("get file({}) size failed", file); - return false; - } - std::shared_ptr buf = utils::make_shared_array(file_sz + 1); - std::ifstream fin(file, std::ifstream::in); - if (!fin.is_open()) { - LOG_ERROR_PREFIX("open file({}) failed", file); + std::string data; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &data); + if (!s.ok()) { + LOG_ERROR_PREFIX("read file '{}' failed, err = {}", fname, s.ToString()); return false; } - fin.read(buf.get(), file_sz); - CHECK_EQ_MSG(file_sz, - fin.gcount(), - "{}: read file({}) failed, need {}, but read {}", - name(), - file, - file_sz, - fin.gcount()); - fin.close(); - - buf.get()[fin.gcount()] = '\0'; - blob bb; - bb.assign(std::move(buf), 0, file_sz); - if (!::dsn::json::json_forwarder::decode(bb, backup_metadata)) { - LOG_ERROR_PREFIX("file({}) under checkpoint is damaged", file); + + if (!::dsn::json::json_forwarder::decode( + blob::create_from_bytes(std::move(data)), backup_metadata)) { + LOG_ERROR_PREFIX("file({}) under checkpoint is damaged", fname); return false; } return true; @@ -160,7 +146,8 @@ error_code replica::download_checkpoint(const configuration_restore_request &req const std::string file_name = utils::filesystem::path_combine(local_chkpt_dir, f_meta.name); if (download_err == ERR_OK || download_err == ERR_PATH_ALREADY_EXIST) { - if (!utils::filesystem::verify_file(file_name, f_meta.md5, f_meta.size)) { + if (!utils::filesystem::verify_file( + file_name, utils::FileDataType::kSensitive, f_meta.md5, f_meta.size)) { download_err = ERR_CORRUPTION; } else if (download_err == ERR_PATH_ALREADY_EXIST) { download_err = ERR_OK; diff --git a/src/replica/replica_stub.cpp b/src/replica/replica_stub.cpp index f828585562..c5bd5000a9 100644 --- a/src/replica/replica_stub.cpp +++ b/src/replica/replica_stub.cpp @@ -36,7 +36,7 @@ #include // IWYU pragma: no_include #include -#include +#include #include #include #include @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -132,6 +133,14 @@ DSN_DEFINE_bool(replication, verbose_commit_log_on_start, false, "whether to print verbose log when commit mutation when starting the server"); +DSN_DEFINE_bool( + replication, + crash_on_slog_error, + false, + "whether to exit the process while fail to open slog. If true, the process will exit and leave " + "the corrupted slog and replicas to be handled by the administrator. If false, the process " + "will continue, and remove the slog and move all the replicas to corresponding error " + "directories"); DSN_DEFINE_uint32(replication, max_concurrent_manual_emergency_checkpointing_count, 10, @@ -160,7 +169,13 @@ DSN_DEFINE_int32(replication, 32, "shared log maximum segment file size (MB)"); -DSN_DEFINE_int32(replication, log_shared_file_count_limit, 100, "shared log maximum file count"); +DSN_DEFINE_uint64( + replication, + log_shared_gc_flush_replicas_limit, + 64, + "The number of submitted replicas that are flushed for gc shared logs; 0 means no limit"); +DSN_TAG_VARIABLE(log_shared_gc_flush_replicas_limit, FT_MUTABLE); + DSN_DEFINE_int32( replication, mem_release_check_interval_ms, @@ -185,6 +200,9 @@ bool replica_stub::s_not_exit_on_log_failure = false; replica_stub::replica_stub(replica_state_subscriber subscriber /*= nullptr*/, bool is_long_subscriber /* = true*/) : serverlet("replica_stub"), + _last_prevent_gc_replica_count(0), + _real_log_shared_gc_flush_replicas_limit(0), + _mock_flush_replicas_for_test(0), _deny_client(false), _verbose_client_log(false), _verbose_commit_log(false), @@ -571,13 +589,7 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f LOG_INFO("primary_address = {}", _primary_address_str); set_options(opts); - std::ostringstream oss; - for (int i = 0; i < _options.meta_servers.size(); ++i) { - if (i != 0) - oss << ","; - oss << _options.meta_servers[i].to_string(); - } - LOG_INFO("meta_servers = {}", oss.str()); + LOG_INFO("meta_servers = {}", fmt::join(_options.meta_servers, ", ")); _deny_client = FLAGS_deny_client_on_start; _verbose_client_log = FLAGS_verbose_client_log_on_start; @@ -708,6 +720,9 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f if (err == ERR_OK) { LOG_INFO("replay shared log succeed, time_used = {} ms", finish_time - start_time); } else { + if (FLAGS_crash_on_slog_error) { + LOG_FATAL("replay shared log failed, err = {}, please check the error details", err); + } LOG_ERROR("replay shared log failed, err = {}, time_used = {} ms, clear all logs ...", err, finish_time - start_time); @@ -1757,139 +1772,203 @@ void replica_stub::on_gc_replica(replica_stub_ptr this_, gpid id) } } -void replica_stub::on_gc() +void replica_stub::gc_slog(const replica_gc_info_map &replica_gc_map) { - uint64_t start = dsn_now_ns(); + if (_log == nullptr) { + return; + } - struct gc_info - { - replica_ptr rep; - partition_status::type status; - mutation_log_ptr plog; - decree last_durable_decree; - int64_t init_offset_in_shared_log; - }; - - std::unordered_map rs; - { - zauto_read_lock l(_replicas_lock); - // collect info in lock to prevent the case that the replica is closed in replica::close() - for (auto &kv : _replicas) { - const replica_ptr &rep = kv.second; - gc_info &info = rs[kv.first]; - info.rep = rep; - info.status = rep->status(); - info.plog = rep->private_log(); - info.last_durable_decree = rep->last_durable_decree(); - info.init_offset_in_shared_log = rep->get_app()->init_info().init_offset_in_shared_log; + replica_log_info_map replica_durable_decrees; + for (auto &replica_gc : replica_gc_map) { + replica_log_info replica_log; + auto &rep = replica_gc.second.rep; + auto &plog = replica_gc.second.plog; + if (plog) { + // Flush private log to update `plog_max_commit_on_disk`, and just flush once + // to avoid flushing infinitely. + plog->flush_once(); + auto plog_max_commit_on_disk = plog->max_commit_on_disk(); + + replica_log.max_decree = + std::min(replica_gc.second.last_durable_decree, plog_max_commit_on_disk); + LOG_INFO("gc_shared: gc condition for {}, status = {}, garbage_max_decree = {}, " + "last_durable_decree= {}, plog_max_commit_on_disk = {}", + rep->name(), + enum_to_string(replica_gc.second.status), + replica_log.max_decree, + replica_gc.second.last_durable_decree, + plog_max_commit_on_disk); + } else { + replica_log.max_decree = replica_gc.second.last_durable_decree; + LOG_INFO("gc_shared: gc condition for {}, status = {}, garbage_max_decree = {}, " + "last_durable_decree = {}", + rep->name(), + enum_to_string(replica_gc.second.status), + replica_log.max_decree, + replica_gc.second.last_durable_decree); } + replica_log.valid_start_offset = replica_gc.second.init_offset_in_shared_log; + replica_durable_decrees[replica_gc.first] = replica_log; } - LOG_INFO("start to garbage collection, replica_count = {}", rs.size()); + // Garbage collection for shared log files. + std::set prevent_gc_replicas; + _log->garbage_collection(replica_durable_decrees, prevent_gc_replicas); - // gc shared prepare log + // Trigger checkpoint to flush memtables once some replicas were found that prevent + // slog files from being removed for gc. + flush_replicas_for_slog_gc(replica_gc_map, prevent_gc_replicas); + + auto total_size = _log->total_size(); + _counter_shared_log_size->set(total_size / (1024 * 1024)); + + // TODO(wangdan): currently we could not yet call _log.reset() as below to close slog and + // reset it to nullptr even if it was found that slog had become empty (which means there + // had not been any file for slog). + // if (total_size == 0) { + // _log.reset(); + // } // - // Now that checkpoint is very important for gc, we must be able to trigger checkpoint when - // necessary. - // that is, we should be able to trigger memtable flush when necessary. + // The reason for this point is that on_gc() is scheduled by timer to run asynchronously + // during the initialization of replica_stub. It might happen before slog.on_partition_reset() + // (building slog._shared_log_info_map), which means slog would be closed mistakenly before + // it was initialized completely. // - // How to trigger memtable flush? - // we add a parameter `is_emergency' in dsn_app_async_checkpoint() function, when set true, - // the undering storage system should flush memtable as soon as possiable. + // All of slog files would removed on v2.5; thus it is safe to remove all of slog code (which + // means even slog object would not be created) on the next version (namely 2.6), and this + // problem would also be resolved. +} + +void replica_stub::limit_flush_replicas_for_slog_gc(size_t prevent_gc_replica_count) +{ + const size_t log_shared_gc_flush_replicas_limit = FLAGS_log_shared_gc_flush_replicas_limit; + if (log_shared_gc_flush_replicas_limit == 0) { + // 0 for log_shared_gc_flush_replicas_limit means no limit. + _real_log_shared_gc_flush_replicas_limit = std::numeric_limits::max(); + return; + } + + if (_last_prevent_gc_replica_count == 0) { + // Initialize it for the 1st time. + _real_log_shared_gc_flush_replicas_limit = log_shared_gc_flush_replicas_limit; + return; + } + + CHECK_GE(_last_prevent_gc_replica_count, prevent_gc_replica_count); + size_t flushed_replicas = _last_prevent_gc_replica_count - prevent_gc_replica_count; + if (flushed_replicas == 0) { + // It's too busy to process more flush tasks. + _real_log_shared_gc_flush_replicas_limit = + std::min(2UL, log_shared_gc_flush_replicas_limit); + return; + } + + if (_real_log_shared_gc_flush_replicas_limit == 0 || + _real_log_shared_gc_flush_replicas_limit == std::numeric_limits::max()) { + // Once it was previously set with some special values, it should be reset. + _real_log_shared_gc_flush_replicas_limit = log_shared_gc_flush_replicas_limit; + return; + } + + if (flushed_replicas < _real_log_shared_gc_flush_replicas_limit) { + // Keep it unchanged. + return; + } + + // Increase it to process more flush tasks. + _real_log_shared_gc_flush_replicas_limit = + std::min(log_shared_gc_flush_replicas_limit, _real_log_shared_gc_flush_replicas_limit << 1); +} + +void replica_stub::flush_replicas_for_slog_gc(const replica_gc_info_map &replica_gc_map, + const std::set &prevent_gc_replicas) +{ + // Trigger checkpoints to flush memtables once some replicas were found that prevent slog files + // from being removed for gc. // - // When to trigger memtable flush? - // 1. Using `[replication].checkpoint_max_interval_hours' option, we can set max interval time - // of two adjacent checkpoints; If the time interval is arrived, then emergency checkpoint - // will be triggered. - // 2. Using `[replication].log_shared_file_count_limit' option, we can set max file count of - // shared log; If the limit is exceeded, then emergency checkpoint will be triggered; Instead - // of triggering all replicas to do checkpoint, we will only trigger a few of necessary - // replicas which block garbage collection of the oldest log file. + // How to trigger memtable flush ? + // A parameter `is_emergency' was added for `replica::background_async_checkpoint()` function; + // once it's set true, underlying storage engine would flush memtable as soon as possiable. // - if (_log != nullptr) { - replica_log_info_map gc_condition; - for (auto &kv : rs) { - replica_log_info ri; - replica_ptr &rep = kv.second.rep; - mutation_log_ptr &plog = kv.second.plog; - if (plog) { - // flush private log to update plog_max_commit_on_disk, - // and just flush once to avoid flushing infinitely - plog->flush_once(); - - decree plog_max_commit_on_disk = plog->max_commit_on_disk(); - ri.max_decree = std::min(kv.second.last_durable_decree, plog_max_commit_on_disk); - LOG_INFO("gc_shared: gc condition for {}, status = {}, garbage_max_decree = {}, " - "last_durable_decree= {}, plog_max_commit_on_disk = {}", - rep->name(), - enum_to_string(kv.second.status), - ri.max_decree, - kv.second.last_durable_decree, - plog_max_commit_on_disk); - } else { - ri.max_decree = kv.second.last_durable_decree; - LOG_INFO("gc_shared: gc condition for {}, status = {}, garbage_max_decree = {}, " - "last_durable_decree = {}", - rep->name(), - enum_to_string(kv.second.status), - ri.max_decree, - kv.second.last_durable_decree); - } - ri.valid_start_offset = kv.second.init_offset_in_shared_log; - gc_condition[kv.first] = ri; + // When memtable flush is triggered ? + // 1. After a fixed interval (specified by `[replication].gc_interval_ms` option), try to find + // if there are some replicas preventing slog files from being removed for gc; if any, all of + // them would be deleted "gradually" ("gradually" means the number of the replicas whose + // memtables are submitted to storage engine to be flushed would be limited). + // 2. `[replication].checkpoint_max_interval_hours' option specified the max interval between + // the two adjacent checkpoints. + + if (prevent_gc_replicas.empty()) { + return; + } + + limit_flush_replicas_for_slog_gc(prevent_gc_replicas.size()); + _last_prevent_gc_replica_count = prevent_gc_replicas.size(); + + LOG_INFO("gc_shared: trigger emergency checkpoints to flush replicas for gc shared logs: " + "log_shared_gc_flush_replicas_limit = {}/{}, prevent_gc_replicas({}) = {}", + _real_log_shared_gc_flush_replicas_limit, + FLAGS_log_shared_gc_flush_replicas_limit, + prevent_gc_replicas.size(), + fmt::join(prevent_gc_replicas, ", ")); + + size_t i = 0; + for (const auto &pid : prevent_gc_replicas) { + const auto &replica_gc = replica_gc_map.find(pid); + if (replica_gc == replica_gc_map.end()) { + continue; } - std::set prevent_gc_replicas; - int reserved_log_count = _log->garbage_collection( - gc_condition, FLAGS_log_shared_file_count_limit, prevent_gc_replicas); - if (reserved_log_count > FLAGS_log_shared_file_count_limit * 2) { - LOG_INFO( - "gc_shared: trigger emergency checkpoint by FLAGS_log_shared_file_count_limit, " - "file_count_limit = {}, reserved_log_count = {}, trigger all replicas to do " - "checkpoint", - FLAGS_log_shared_file_count_limit, - reserved_log_count); - for (auto &kv : rs) { - tasking::enqueue( - LPC_PER_REPLICA_CHECKPOINT_TIMER, - kv.second.rep->tracker(), - std::bind(&replica_stub::trigger_checkpoint, this, kv.second.rep, true), - kv.first.thread_hash(), - std::chrono::milliseconds(rand::next_u32(0, FLAGS_gc_interval_ms / 2))); - } - } else if (reserved_log_count > FLAGS_log_shared_file_count_limit) { - std::ostringstream oss; - int c = 0; - for (auto &i : prevent_gc_replicas) { - if (c != 0) - oss << ", "; - oss << i.to_string(); - c++; - } - LOG_INFO( - "gc_shared: trigger emergency checkpoint by FLAGS_log_shared_file_count_limit, " - "file_count_limit = {}, reserved_log_count = {}, prevent_gc_replica_count = " - "{}, trigger them to do checkpoint: {}", - FLAGS_log_shared_file_count_limit, - reserved_log_count, - prevent_gc_replicas.size(), - oss.str()); - for (auto &id : prevent_gc_replicas) { - auto find = rs.find(id); - if (find != rs.end()) { - tasking::enqueue( - LPC_PER_REPLICA_CHECKPOINT_TIMER, - find->second.rep->tracker(), - std::bind(&replica_stub::trigger_checkpoint, this, find->second.rep, true), - id.thread_hash(), - std::chrono::milliseconds(rand::next_u32(0, FLAGS_gc_interval_ms / 2))); - } - } + if (++i > _real_log_shared_gc_flush_replicas_limit) { + break; + } + + bool mock_flush = false; + FAIL_POINT_INJECT_NOT_RETURN_F( + "mock_flush_replicas_for_slog_gc", [&mock_flush, this, i](dsn::string_view str) { + CHECK(buf2bool(str, mock_flush), + "invalid mock_flush_replicas_for_slog_gc toggle, should be true or false: {}", + str); + _mock_flush_replicas_for_test = i; + }); + if (dsn_unlikely(mock_flush)) { + continue; } - _counter_shared_log_size->set(_log->total_size() / (1024 * 1024)); + tasking::enqueue( + LPC_PER_REPLICA_CHECKPOINT_TIMER, + replica_gc->second.rep->tracker(), + std::bind(&replica_stub::trigger_checkpoint, this, replica_gc->second.rep, true), + pid.thread_hash(), + std::chrono::milliseconds(rand::next_u32(0, FLAGS_gc_interval_ms / 2))); + } +} + +void replica_stub::on_gc() +{ + uint64_t start = dsn_now_ns(); + + replica_gc_info_map replica_gc_map; + { + zauto_read_lock l(_replicas_lock); + // A replica was removed from _replicas before it would be closed by replica::close(). + // Thus it's safe to use the replica after fetching its ref pointer from _replicas. + for (const auto &rep_pair : _replicas) { + const replica_ptr &rep = rep_pair.second; + + auto &replica_gc = replica_gc_map[rep_pair.first]; + replica_gc.rep = rep; + replica_gc.status = rep->status(); + replica_gc.plog = rep->private_log(); + replica_gc.last_durable_decree = rep->last_durable_decree(); + replica_gc.init_offset_in_shared_log = + rep->get_app()->init_info().init_offset_in_shared_log; + } } + LOG_INFO("start to garbage collection, replica_count = {}", replica_gc_map.size()); + gc_slog(replica_gc_map); + // statistic learning info uint64_t learning_count = 0; uint64_t learning_max_duration_time_ms = 0; @@ -1904,7 +1983,7 @@ void replica_stub::on_gc() uint64_t splitting_max_duration_time_ms = 0; uint64_t splitting_max_async_learn_time_ms = 0; uint64_t splitting_max_copy_file_size = 0; - for (auto &kv : rs) { + for (auto &kv : replica_gc_map) { replica_ptr &rep = kv.second.rep; if (rep->status() == partition_status::PS_POTENTIAL_SECONDARY) { learning_count++; diff --git a/src/replica/replica_stub.h b/src/replica/replica_stub.h index b6f54c4850..e71e8f864e 100644 --- a/src/replica/replica_stub.h +++ b/src/replica/replica_stub.h @@ -33,11 +33,13 @@ // #include +#include #include #include #include #include #include +#include #include #include #include @@ -71,12 +73,14 @@ #include "utils/autoref_ptr.h" #include "utils/error_code.h" #include "utils/flags.h" +#include "utils/fmt_utils.h" #include "utils/zlocks.h" namespace dsn { class command_deregister; class message_ex; class nfs_node; + namespace service { class copy_request; class copy_response; @@ -306,6 +310,7 @@ class replica_stub : public serverlet, public ref_counter NS_Connecting, NS_Connected }; + friend USER_DEFINED_ENUM_FORMATTER(replica_stub::replica_node_state); enum replica_life_cycle { @@ -357,6 +362,30 @@ class replica_stub : public serverlet, public ref_counter replica_life_cycle get_replica_life_cycle(gpid id); void on_gc_replica(replica_stub_ptr this_, gpid id); + struct replica_gc_info + { + replica_ptr rep; + partition_status::type status; + mutation_log_ptr plog; + decree last_durable_decree; + int64_t init_offset_in_shared_log; + }; + using replica_gc_info_map = std::unordered_map; + + // Try to remove obsolete files of shared log for garbage collection according to the provided + // states of all replicas. The purpose is to remove all of the files of shared log, since it + // has been deprecated, and would not be appended any more. + void gc_slog(const replica_gc_info_map &replica_gc_map); + + // The number of flushed replicas for the garbage collection of shared log at a time should be + // limited. + void limit_flush_replicas_for_slog_gc(size_t prevent_gc_replica_count); + + // Flush rocksdb data to sst files for replicas to facilitate garbage collection of more files + // of shared log. + void flush_replicas_for_slog_gc(const replica_gc_info_map &replica_gc_map, + const std::set &prevent_gc_replicas); + void response_client(gpid id, bool is_read, dsn::message_ex *request, @@ -419,6 +448,7 @@ class replica_stub : public serverlet, public ref_counter FRIEND_TEST(open_replica_test, open_replica_add_decree_and_ballot_check); FRIEND_TEST(replica_error_test, test_auto_trash_of_corruption); FRIEND_TEST(replica_test, test_clear_on_failure); + FRIEND_TEST(GcSlogFlushFeplicasTest, FlushReplicas); typedef std::unordered_map opening_replicas; typedef std::unordered_map> @@ -432,6 +462,15 @@ class replica_stub : public serverlet, public ref_counter closing_replicas _closing_replicas; closed_replicas _closed_replicas; + // The number of replicas that prevent slog files from being removed for gc at the last round. + size_t _last_prevent_gc_replica_count; + + // The real limit of flushed replicas for the garbage collection of shared log. + size_t _real_log_shared_gc_flush_replicas_limit; + + // The number of flushed replicas, mocked only for test. + size_t _mock_flush_replicas_for_test; + mutation_log_ptr _log; ::dsn::rpc_address _primary_address; char _primary_address_str[64]; diff --git a/src/replica/replication_app_base.cpp b/src/replica/replication_app_base.cpp index db92bdf17e..bc84c0c0a6 100644 --- a/src/replica/replication_app_base.cpp +++ b/src/replica/replication_app_base.cpp @@ -25,16 +25,14 @@ */ #include -#include +#include +#include #include -#include #include #include #include #include -#include "aio/aio_task.h" -#include "aio/file_io.h" #include "common/bulk_load_common.h" #include "common/duplication_common.h" #include "common/replica_envs.h" @@ -50,8 +48,6 @@ #include "runtime/rpc/serialization.h" #include "runtime/task/task_code.h" #include "runtime/task/task_spec.h" -#include "runtime/task/task_tracker.h" -#include "utils/autoref_ptr.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" #include "utils/blob.h" @@ -61,57 +57,31 @@ #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/latency_tracer.h" -#include "utils/ports.h" #include "utils/string_view.h" -#include "utils/threadpool_code.h" -#include "utils/utils.h" namespace dsn { -class disk_file; namespace replication { const std::string replica_init_info::kInitInfo = ".init-info"; -DEFINE_TASK_CODE_AIO(LPC_AIO_INFO_WRITE, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) - namespace { -error_code write_blob_to_file(const std::string &file, const blob &data) +error_code write_blob_to_file(const std::string &fname, const blob &data) { - std::string tmp_file = file + ".tmp"; - disk_file *hfile = file::open(tmp_file.c_str(), O_WRONLY | O_CREAT | O_BINARY | O_TRUNC, 0666); + std::string tmp_fname = fname + ".tmp"; + auto cleanup = defer([tmp_fname]() { utils::filesystem::remove_path(tmp_fname); }); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(data.data(), data.length()), + tmp_fname, + /* should_sync */ true); LOG_AND_RETURN_NOT_TRUE( - ERROR, hfile, ERR_FILE_OPERATION_FAILED, "open file {} failed", tmp_file); - auto cleanup = defer([tmp_file]() { utils::filesystem::remove_path(tmp_file); }); - - error_code err; - size_t sz = 0; - task_tracker tracker; - aio_task_ptr tsk = file::write(hfile, - data.data(), - data.length(), - 0, - LPC_AIO_INFO_WRITE, - &tracker, - [&err, &sz](error_code e, size_t s) { - err = e; - sz = s; - }, - 0); - CHECK_NOTNULL(tsk, "create file::write task failed"); - tracker.wait_outstanding_tasks(); - file::flush(hfile); - file::close(hfile); - LOG_AND_RETURN_NOT_OK(ERROR, err, "write file {} failed", tmp_file); - CHECK_EQ(data.length(), sz); - // TODO(yingchun): need fsync too？ + ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "write file {} failed", tmp_fname); LOG_AND_RETURN_NOT_TRUE(ERROR, - utils::filesystem::rename_path(tmp_file, file), + utils::filesystem::rename_path(tmp_fname, fname), ERR_FILE_OPERATION_FAILED, "move file from {} to {} failed", - tmp_file, - file); - + tmp_fname, + fname); return ERR_OK; } } // namespace @@ -146,38 +116,23 @@ error_code replica_init_info::store(const std::string &dir) return ERR_OK; } -error_code replica_init_info::load_json(const std::string &file) +error_code replica_init_info::load_json(const std::string &fname) { - std::ifstream is(file, std::ios::binary); - LOG_AND_RETURN_NOT_TRUE( - ERROR, is.is_open(), ERR_FILE_OPERATION_FAILED, "open file {} failed", file); - - int64_t sz = 0; + std::string data; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &data); + LOG_AND_RETURN_NOT_TRUE(ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "read file {} failed", fname); LOG_AND_RETURN_NOT_TRUE(ERROR, - utils::filesystem::file_size(std::string(file), sz), + json::json_forwarder::decode( + blob::create_from_bytes(std::move(data)), *this), ERR_FILE_OPERATION_FAILED, - "get file size of {} failed", - file); - - std::shared_ptr buffer(utils::make_shared_array(sz)); - is.read((char *)buffer.get(), sz); - LOG_AND_RETURN_NOT_TRUE( - ERROR, !is.bad(), ERR_FILE_OPERATION_FAILED, "read file {} failed", file); - is.close(); - - LOG_AND_RETURN_NOT_TRUE( - ERROR, - json::json_forwarder::decode(blob(buffer, sz), *this), - ERR_FILE_OPERATION_FAILED, - "decode json from file {} failed", - file); - + "decode json from file {} failed", + fname); return ERR_OK; } -error_code replica_init_info::store_json(const std::string &file) +error_code replica_init_info::store_json(const std::string &fname) { - return write_blob_to_file(file, json::json_forwarder::encode(*this)); + return write_blob_to_file(fname, json::json_forwarder::encode(*this)); } std::string replica_init_info::to_string() @@ -190,35 +145,21 @@ std::string replica_init_info::to_string() return oss.str(); } -error_code replica_app_info::load(const std::string &file) +error_code replica_app_info::load(const std::string &fname) { - std::ifstream is(file, std::ios::binary); - LOG_AND_RETURN_NOT_TRUE( - ERROR, is.is_open(), ERR_FILE_OPERATION_FAILED, "open file {} failed", file); - - int64_t sz = 0; - LOG_AND_RETURN_NOT_TRUE(ERROR, - utils::filesystem::file_size(std::string(file), sz), - ERR_FILE_OPERATION_FAILED, - "get file size of {} failed", - file); - - std::shared_ptr buffer(utils::make_shared_array(sz)); - is.read((char *)buffer.get(), sz); - is.close(); - - binary_reader reader(blob(buffer, sz)); - int magic; + std::string data; + auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &data); + LOG_AND_RETURN_NOT_TRUE(ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "read file {} failed", fname); + binary_reader reader(blob::create_from_bytes(std::move(data))); + int magic = 0; unmarshall(reader, magic, DSF_THRIFT_BINARY); - LOG_AND_RETURN_NOT_TRUE( - ERROR, magic == 0xdeadbeef, ERR_INVALID_DATA, "data in file {} is invalid (magic)", file); - + ERROR, magic == 0xdeadbeef, ERR_INVALID_DATA, "data in file {} is invalid (magic)", fname); unmarshall(reader, *_app, DSF_THRIFT_JSON); return ERR_OK; } -error_code replica_app_info::store(const std::string &file) +error_code replica_app_info::store(const std::string &fname) { binary_writer writer; int magic = 0xdeadbeef; @@ -238,7 +179,7 @@ error_code replica_app_info::store(const std::string &file) marshall(writer, tmp, DSF_THRIFT_JSON); } - return write_blob_to_file(file, writer.get_buffer()); + return write_blob_to_file(fname, writer.get_buffer()); } /*static*/ diff --git a/src/replica/replication_app_base.h b/src/replica/replication_app_base.h index e48ef8a5dd..5ae162a1bb 100644 --- a/src/replica/replication_app_base.h +++ b/src/replica/replication_app_base.h @@ -39,6 +39,7 @@ #include "replica/replica_base.h" #include "replica_admin_types.h" #include "utils/error_code.h" +#include "utils/fmt_utils.h" #include "utils/ports.h" namespace dsn { @@ -75,8 +76,8 @@ class replica_init_info std::string to_string(); private: - error_code load_json(const std::string &file); - error_code store_json(const std::string &file); + error_code load_json(const std::string &fname); + error_code store_json(const std::string &fname); }; class replica_app_info @@ -86,8 +87,8 @@ class replica_app_info public: replica_app_info(app_info *app) { _app = app; } - error_code load(const std::string &file); - error_code store(const std::string &file); + error_code load(const std::string &fname); + error_code store(const std::string &fname); }; /// The store engine interface of Pegasus. @@ -313,6 +314,6 @@ class replication_app_base : public replica_base explicit replication_app_base(replication::replica *replica); }; - +USER_DEFINED_ENUM_FORMATTER(replication_app_base::chkpt_apply_mode) } // namespace replication } // namespace dsn diff --git a/src/replica/split/test/config-test.ini b/src/replica/split/test/config-test.ini index ca0550f482..86279cd5a6 100644 --- a/src/replica/split/test/config-test.ini +++ b/src/replica/split/test/config-test.ini @@ -86,3 +86,6 @@ allow_inline = false rpc_call_channel = RPC_CHANNEL_TCP rpc_message_header_format = dsn rpc_timeout_milliseconds = 5000 + +[replication] +disk_min_available_space_ratio = 10 diff --git a/src/replica/split/test/run.sh b/src/replica/split/test/run.sh index 397d6f7a82..71820b0939 100755 --- a/src/replica/split/test/run.sh +++ b/src/replica/split/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -23,6 +23,24 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi ./dsn_replica_split_test diff --git a/src/replica/storage/simple_kv/CMakeLists.txt b/src/replica/storage/simple_kv/CMakeLists.txt index cb7d328fc1..667b4ab862 100644 --- a/src/replica/storage/simple_kv/CMakeLists.txt +++ b/src/replica/storage/simple_kv/CMakeLists.txt @@ -37,7 +37,7 @@ set(MY_PROJ_SRC ${SIMPLE_KV_THRIFT_SRCS}) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime hashtable) +set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime hashtable rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/storage/simple_kv/config.ini b/src/replica/storage/simple_kv/config.ini index e1b1013e07..ad9ec33715 100644 --- a/src/replica/storage/simple_kv/config.ini +++ b/src/replica/storage/simple_kv/config.ini @@ -152,6 +152,7 @@ max_replica_count = 3 stateful = true [replication] +disk_min_available_space_ratio = 10 prepare_timeout_ms_for_secondaries = 10000 prepare_timeout_ms_for_potential_secondaries = 20000 diff --git a/src/replica/storage/simple_kv/run.sh b/src/replica/storage/simple_kv/run.sh index 6f0b84f183..7669b4daf9 100755 --- a/src/replica/storage/simple_kv/run.sh +++ b/src/replica/storage/simple_kv/run.sh @@ -29,6 +29,25 @@ if [ ! -f dsn.replication.simple_kv ]; then exit 1 fi +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config.ini" ]; then + echo "./config.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config.ini + done +fi + ./clear.sh echo "running dsn.replication.simple_kv for 20 seconds ..." diff --git a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp index b6f0a00e3f..e78ae81efb 100644 --- a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp @@ -35,25 +35,34 @@ #include "simple_kv.server.impl.h" +#include #include +#include #include #include #include -#include #include -#include +#include #include #include +#include "aio/aio_task.h" +#include "aio/file_io.h" +#include "common/replication.codes.h" #include "consensus_types.h" #include "replica/storage/simple_kv/simple_kv.server.h" +#include "rocksdb/env.h" +#include "rocksdb/status.h" #include "runtime/serverlet.h" #include "simple_kv_types.h" +#include "utils/autoref_ptr.h" +#include "utils/binary_reader.h" +#include "utils/blob.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" +#include "utils/utils.h" namespace dsn { -class blob; namespace replication { class replica; @@ -176,75 +185,104 @@ void simple_kv_service_impl::recover(const std::string &name, int64_t version) { zauto_lock l(_lock); - std::ifstream is(name.c_str(), std::ios::binary); - if (!is.is_open()) - return; + std::unique_ptr rfile; + auto s = rocksdb::Env::Default()->NewSequentialFile(name, &rfile, rocksdb::EnvOptions()); + CHECK(s.ok(), "open log file '{}' failed, err = {}", name, s.ToString()); _store.clear(); - uint64_t count; - int magic; - - is.read((char *)&count, sizeof(count)); - is.read((char *)&magic, sizeof(magic)); + // Read header. + uint64_t count = 0; + int magic = 0; + rocksdb::Slice result; + static const uint64_t kHeaderSize = sizeof(count) + sizeof(magic); + char buff[kHeaderSize] = {0}; + s = rfile->Read(kHeaderSize, &result, buff); + CHECK(s.ok(), "read header failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + binary_reader reader(blob(buff, 0, kHeaderSize)); + CHECK_EQ(sizeof(count), reader.read(count)); + CHECK_EQ(sizeof(magic), reader.read(magic)); CHECK_EQ_MSG(magic, 0xdeadbeef, "invalid checkpoint"); + // Read kv pairs. for (uint64_t i = 0; i < count; i++) { - std::string key; - std::string value; - - uint32_t sz; - is.read((char *)&sz, (uint32_t)sizeof(sz)); - key.resize(sz); - - is.read((char *)&key[0], sz); - - is.read((char *)&sz, (uint32_t)sizeof(sz)); - value.resize(sz); - - is.read((char *)&value[0], sz); - + // Read key. + uint32_t sz = 0; + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read key size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr key_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, key_buffer.get()); + CHECK(s.ok(), "read key failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string key = result.ToString(); + + // Read value. + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read value size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr value_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, value_buffer.get()); + CHECK(s.ok(), "read value failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string value = result.ToString(); + + // Store the kv pair. _store[key] = value; } - is.close(); } ::dsn::error_code simple_kv_service_impl::sync_checkpoint() { - char name[256]; int64_t last_commit = _last_committed_decree.load(); - sprintf(name, "%s/checkpoint.%" PRId64, _dir_data.c_str(), last_commit); + std::string fname = fmt::format("{}/checkpoint.{}", data_dir(), last_commit); zauto_lock l(_lock); - if (last_commit == last_durable_decree()) { - CHECK(utils::filesystem::file_exists(name), "checkpoint file {} is missing!", name); + CHECK(utils::filesystem::file_exists(fname), "checkpoint file {} is missing!", fname); return ERR_OK; } - std::ofstream os(name, std::ios::binary); + auto wfile = file::open(fname, file::FileOpenType::kWriteOnly); + CHECK_NOTNULL(wfile, ""); +#define WRITE_DATA_SIZE(data, size) \ + do { \ + auto tsk = ::dsn::file::write( \ + wfile, (char *)&data, size, offset, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr); \ + tsk->wait(); \ + offset += size; \ + } while (false) + +#define WRITE_DATA(data) WRITE_DATA_SIZE(data, sizeof(data)) + + uint64_t offset = 0; uint64_t count = (uint64_t)_store.size(); - int magic = 0xdeadbeef; + WRITE_DATA(count); - os.write((const char *)&count, (uint32_t)sizeof(count)); - os.write((const char *)&magic, (uint32_t)sizeof(magic)); + int magic = 0xdeadbeef; + WRITE_DATA(magic); - for (auto it = _store.begin(); it != _store.end(); ++it) { - const std::string &k = it->first; + for (const auto &kv : _store) { + const std::string &k = kv.first; uint32_t sz = (uint32_t)k.length(); + WRITE_DATA(sz); + WRITE_DATA_SIZE(k[0], sz); - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&k[0], sz); - - const std::string &v = it->second; + const std::string &v = kv.second; sz = (uint32_t)v.length(); - - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&v[0], sz); + WRITE_DATA(sz); + WRITE_DATA_SIZE(v[0], sz); } +#undef WRITE_DATA +#undef WRITE_DATA_SIZE - os.close(); + CHECK_EQ(ERR_OK, file::flush(wfile)); + CHECK_EQ(ERR_OK, file::close(wfile)); // TODO: gc checkpoints set_last_durable_decree(last_commit); diff --git a/src/replica/storage/simple_kv/test/CMakeLists.txt b/src/replica/storage/simple_kv/test/CMakeLists.txt index 1d64070fb6..de86358d09 100644 --- a/src/replica/storage/simple_kv/test/CMakeLists.txt +++ b/src/replica/storage/simple_kv/test/CMakeLists.txt @@ -39,7 +39,8 @@ set(MY_PROJ_LIBS dsn_replica_server zookeeper hashtable gtest - ) + dsn_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/storage/simple_kv/test/case-000.ini b/src/replica/storage/simple_kv/test/case-000.ini index 76896bdd84..4cb2b55450 100644 --- a/src/replica/storage/simple_kv/test/case-000.ini +++ b/src/replica/storage/simple_kv/test/case-000.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -155,6 +158,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-001.ini b/src/replica/storage/simple_kv/test/case-001.ini index d5b5339f0c..08c42f699a 100644 --- a/src/replica/storage/simple_kv/test/case-001.ini +++ b/src/replica/storage/simple_kv/test/case-001.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -155,6 +158,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-002.ini b/src/replica/storage/simple_kv/test/case-002.ini index 6beb5a0246..e883afcc03 100644 --- a/src/replica/storage/simple_kv/test/case-002.ini +++ b/src/replica/storage/simple_kv/test/case-002.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -155,6 +158,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-003.ini b/src/replica/storage/simple_kv/test/case-003.ini index b6865db7ad..5ee053ad4d 100644 --- a/src/replica/storage/simple_kv/test/case-003.ini +++ b/src/replica/storage/simple_kv/test/case-003.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -155,6 +158,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-004.ini b/src/replica/storage/simple_kv/test/case-004.ini index 6dcbb398ce..a2eebf4ee6 100644 --- a/src/replica/storage/simple_kv/test/case-004.ini +++ b/src/replica/storage/simple_kv/test/case-004.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -155,6 +158,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-005.ini b/src/replica/storage/simple_kv/test/case-005.ini index 7446967ddb..1b75197419 100644 --- a/src/replica/storage/simple_kv/test/case-005.ini +++ b/src/replica/storage/simple_kv/test/case-005.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -155,6 +158,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-006.ini b/src/replica/storage/simple_kv/test/case-006.ini index beb1272fcd..4a8d09e289 100644 --- a/src/replica/storage/simple_kv/test/case-006.ini +++ b/src/replica/storage/simple_kv/test/case-006.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-100.ini b/src/replica/storage/simple_kv/test/case-100.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-100.ini +++ b/src/replica/storage/simple_kv/test/case-100.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-101.ini b/src/replica/storage/simple_kv/test/case-101.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-101.ini +++ b/src/replica/storage/simple_kv/test/case-101.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-102.ini b/src/replica/storage/simple_kv/test/case-102.ini index 9b0c33ea71..4ac4bcc5f8 100644 --- a/src/replica/storage/simple_kv/test/case-102.ini +++ b/src/replica/storage/simple_kv/test/case-102.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-103.ini b/src/replica/storage/simple_kv/test/case-103.ini index c57b99aa25..3d70ea9c5e 100644 --- a/src/replica/storage/simple_kv/test/case-103.ini +++ b/src/replica/storage/simple_kv/test/case-103.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-104.ini b/src/replica/storage/simple_kv/test/case-104.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-104.ini +++ b/src/replica/storage/simple_kv/test/case-104.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-105.ini b/src/replica/storage/simple_kv/test/case-105.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-105.ini +++ b/src/replica/storage/simple_kv/test/case-105.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-106.ini b/src/replica/storage/simple_kv/test/case-106.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-106.ini +++ b/src/replica/storage/simple_kv/test/case-106.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-107.ini b/src/replica/storage/simple_kv/test/case-107.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-107.ini +++ b/src/replica/storage/simple_kv/test/case-107.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-108.ini b/src/replica/storage/simple_kv/test/case-108.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-108.ini +++ b/src/replica/storage/simple_kv/test/case-108.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-109.ini b/src/replica/storage/simple_kv/test/case-109.ini index b674b1b40b..9a1d88c697 100644 --- a/src/replica/storage/simple_kv/test/case-109.ini +++ b/src/replica/storage/simple_kv/test/case-109.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-200.ini b/src/replica/storage/simple_kv/test/case-200.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-200.ini +++ b/src/replica/storage/simple_kv/test/case-200.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-201.ini b/src/replica/storage/simple_kv/test/case-201.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-201.ini +++ b/src/replica/storage/simple_kv/test/case-201.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-202-0.ini b/src/replica/storage/simple_kv/test/case-202-0.ini index 0dda324d66..dabdf01544 100644 --- a/src/replica/storage/simple_kv/test/case-202-0.ini +++ b/src/replica/storage/simple_kv/test/case-202-0.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-202-1.ini b/src/replica/storage/simple_kv/test/case-202-1.ini index 0dda324d66..dabdf01544 100644 --- a/src/replica/storage/simple_kv/test/case-202-1.ini +++ b/src/replica/storage/simple_kv/test/case-202-1.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-203-0.ini b/src/replica/storage/simple_kv/test/case-203-0.ini index 28af6cfe95..42c9c006c9 100644 --- a/src/replica/storage/simple_kv/test/case-203-0.ini +++ b/src/replica/storage/simple_kv/test/case-203-0.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-204.ini b/src/replica/storage/simple_kv/test/case-204.ini index 7dab9e62e6..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-204.ini +++ b/src/replica/storage/simple_kv/test/case-204.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-205.ini b/src/replica/storage/simple_kv/test/case-205.ini index 7dab9e62e6..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-205.ini +++ b/src/replica/storage/simple_kv/test/case-205.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-206.ini b/src/replica/storage/simple_kv/test/case-206.ini index 7dab9e62e6..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-206.ini +++ b/src/replica/storage/simple_kv/test/case-206.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-207.ini b/src/replica/storage/simple_kv/test/case-207.ini index 7dab9e62e6..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-207.ini +++ b/src/replica/storage/simple_kv/test/case-207.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-208.ini b/src/replica/storage/simple_kv/test/case-208.ini index 7dab9e62e6..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-208.ini +++ b/src/replica/storage/simple_kv/test/case-208.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-209.ini b/src/replica/storage/simple_kv/test/case-209.ini index 7dab9e62e6..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-209.ini +++ b/src/replica/storage/simple_kv/test/case-209.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-210.ini b/src/replica/storage/simple_kv/test/case-210.ini index a3897bb679..a9dd9383cc 100644 --- a/src/replica/storage/simple_kv/test/case-210.ini +++ b/src/replica/storage/simple_kv/test/case-210.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-211.ini b/src/replica/storage/simple_kv/test/case-211.ini index a3897bb679..a9dd9383cc 100644 --- a/src/replica/storage/simple_kv/test/case-211.ini +++ b/src/replica/storage/simple_kv/test/case-211.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-212.ini b/src/replica/storage/simple_kv/test/case-212.ini index 2b43d145aa..a61a428f21 100644 --- a/src/replica/storage/simple_kv/test/case-212.ini +++ b/src/replica/storage/simple_kv/test/case-212.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-213.ini b/src/replica/storage/simple_kv/test/case-213.ini index 247a8fea5b..660afcfff9 100644 --- a/src/replica/storage/simple_kv/test/case-213.ini +++ b/src/replica/storage/simple_kv/test/case-213.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-214.ini b/src/replica/storage/simple_kv/test/case-214.ini index 7dab9e62e6..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-214.ini +++ b/src/replica/storage/simple_kv/test/case-214.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-215.ini b/src/replica/storage/simple_kv/test/case-215.ini index 7dab9e62e6..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-215.ini +++ b/src/replica/storage/simple_kv/test/case-215.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-216.ini b/src/replica/storage/simple_kv/test/case-216.ini index 7dab9e62e6..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-216.ini +++ b/src/replica/storage/simple_kv/test/case-216.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-300-0.ini b/src/replica/storage/simple_kv/test/case-300-0.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-300-0.ini +++ b/src/replica/storage/simple_kv/test/case-300-0.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-300-1.ini b/src/replica/storage/simple_kv/test/case-300-1.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-300-1.ini +++ b/src/replica/storage/simple_kv/test/case-300-1.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-300-2.ini b/src/replica/storage/simple_kv/test/case-300-2.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-300-2.ini +++ b/src/replica/storage/simple_kv/test/case-300-2.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-301.ini b/src/replica/storage/simple_kv/test/case-301.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-301.ini +++ b/src/replica/storage/simple_kv/test/case-301.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-302.ini b/src/replica/storage/simple_kv/test/case-302.ini index 90681e6b97..fdfb947769 100644 --- a/src/replica/storage/simple_kv/test/case-302.ini +++ b/src/replica/storage/simple_kv/test/case-302.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-303.ini b/src/replica/storage/simple_kv/test/case-303.ini index 0e0e866f8d..db032f0a27 100644 --- a/src/replica/storage/simple_kv/test/case-303.ini +++ b/src/replica/storage/simple_kv/test/case-303.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-304.ini b/src/replica/storage/simple_kv/test/case-304.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-304.ini +++ b/src/replica/storage/simple_kv/test/case-304.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-305.ini b/src/replica/storage/simple_kv/test/case-305.ini index 0e0e866f8d..db032f0a27 100644 --- a/src/replica/storage/simple_kv/test/case-305.ini +++ b/src/replica/storage/simple_kv/test/case-305.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-306.ini b/src/replica/storage/simple_kv/test/case-306.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-306.ini +++ b/src/replica/storage/simple_kv/test/case-306.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-307.ini b/src/replica/storage/simple_kv/test/case-307.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-307.ini +++ b/src/replica/storage/simple_kv/test/case-307.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-400.ini b/src/replica/storage/simple_kv/test/case-400.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-400.ini +++ b/src/replica/storage/simple_kv/test/case-400.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-401.ini b/src/replica/storage/simple_kv/test/case-401.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-401.ini +++ b/src/replica/storage/simple_kv/test/case-401.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-402.ini b/src/replica/storage/simple_kv/test/case-402.ini index e71811be91..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-402.ini +++ b/src/replica/storage/simple_kv/test/case-402.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-600.ini b/src/replica/storage/simple_kv/test/case-600.ini index beb1272fcd..4a8d09e289 100644 --- a/src/replica/storage/simple_kv/test/case-600.ini +++ b/src/replica/storage/simple_kv/test/case-600.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-601.ini b/src/replica/storage/simple_kv/test/case-601.ini index 0401980fc0..20ee3c8bf0 100644 --- a/src/replica/storage/simple_kv/test/case-601.ini +++ b/src/replica/storage/simple_kv/test/case-601.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-602.ini b/src/replica/storage/simple_kv/test/case-602.ini index beb1272fcd..4a8d09e289 100644 --- a/src/replica/storage/simple_kv/test/case-602.ini +++ b/src/replica/storage/simple_kv/test/case-602.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -156,6 +159,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case-603.ini b/src/replica/storage/simple_kv/test/case-603.ini index 4b171e934b..563b86adf8 100644 --- a/src/replica/storage/simple_kv/test/case-603.ini +++ b/src/replica/storage/simple_kv/test/case-603.ini @@ -151,6 +151,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv @@ -158,6 +161,7 @@ partition_count = 1 max_replica_count = 3 [replication] +disk_min_available_space_ratio = 10 empty_write_disabled = true prepare_timeout_ms_for_secondaries = 1000 prepare_timeout_ms_for_potential_secondaries = 3000 diff --git a/src/replica/storage/simple_kv/test/case.cpp b/src/replica/storage/simple_kv/test/case.cpp index 79c06503f6..94962e65f7 100644 --- a/src/replica/storage/simple_kv/test/case.cpp +++ b/src/replica/storage/simple_kv/test/case.cpp @@ -42,7 +42,6 @@ #include #include #include -#include #include #include #include diff --git a/src/replica/storage/simple_kv/test/case.h b/src/replica/storage/simple_kv/test/case.h index 0fc0d9fc53..565119808b 100644 --- a/src/replica/storage/simple_kv/test/case.h +++ b/src/replica/storage/simple_kv/test/case.h @@ -46,6 +46,7 @@ #include "meta_admin_types.h" #include "runtime/rpc/rpc_address.h" #include "utils/error_code.h" +#include "utils/fmt_utils.h" #include "utils/singleton.h" #include "utils/zlocks.h" @@ -441,6 +442,7 @@ class client_case_line : public case_line dsn::replication::config_type::type _config_type; rpc_address _config_node; }; +USER_DEFINED_ENUM_FORMATTER(client_case_line::client_type) class test_case : public dsn::utils::singleton { @@ -505,3 +507,5 @@ class test_case : public dsn::utils::singleton } } } + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::test::event); diff --git a/src/replica/storage/simple_kv/test/client.cpp b/src/replica/storage/simple_kv/test/client.cpp index b576986080..905cdb080c 100644 --- a/src/replica/storage/simple_kv/test/client.cpp +++ b/src/replica/storage/simple_kv/test/client.cpp @@ -36,7 +36,6 @@ #include "client.h" #include -#include #include #include #include diff --git a/src/replica/storage/simple_kv/test/common.h b/src/replica/storage/simple_kv/test/common.h index 89634acfec..533dadc7e9 100644 --- a/src/replica/storage/simple_kv/test/common.h +++ b/src/replica/storage/simple_kv/test/common.h @@ -47,6 +47,7 @@ #include "common/replication_other_types.h" #include "metadata_types.h" #include "runtime/rpc/rpc_address.h" +#include "utils/fmt_utils.h" namespace dsn { class partition_configuration; @@ -213,3 +214,7 @@ struct parti_config } } } + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::test::parti_config); +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::test::replica_id); +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::replication::test::state_snapshot); diff --git a/src/replica/storage/simple_kv/test/config.ini b/src/replica/storage/simple_kv/test/config.ini index 892a318259..b014b44c3f 100644 --- a/src/replica/storage/simple_kv/test/config.ini +++ b/src/replica/storage/simple_kv/test/config.ini @@ -179,6 +179,7 @@ max_replica_count = 3 stateful = true [replication] +disk_min_available_space_ratio = 10 prepare_timeout_ms_for_secondaries = 10000 prepare_timeout_ms_for_potential_secondaries = 20000 diff --git a/src/replica/storage/simple_kv/test/run.sh b/src/replica/storage/simple_kv/test/run.sh index 7737b09c96..e10210fa41 100755 --- a/src/replica/storage/simple_kv/test/run.sh +++ b/src/replica/storage/simple_kv/test/run.sh @@ -29,6 +29,26 @@ bin=./dsn.rep_tests.simple_kv function run_single() { prefix=$1 + + if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./${prefix}.ini" ]; then + echo "./${prefix}.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./${prefix}.ini + done + fi + echo "${bin} ${prefix}.ini ${prefix}.act" ${bin} ${prefix}.ini ${prefix}.act ret=$? @@ -92,6 +112,14 @@ else fi if [ ! -z "${cases}" ]; then + OLD_TEST_OPTS=${TEST_OPTS} + TEST_OPTS=${OLD_TEST_OPTS},encrypt_data_at_rest=false + for id in ${cases}; do + run_case ${id} + echo + done + # TODO(yingchun): ENCRYPTION: add enable encryption test. + # TEST_OPTS=${OLD_TEST_OPTS},encrypt_data_at_rest=true for id in ${cases}; do run_case ${id} echo diff --git a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp index 4e2d1e4e78..021496194b 100644 --- a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp @@ -25,23 +25,35 @@ */ #include "simple_kv.server.impl.h" +#include #include #include #include #include -#include -#include +#include #include #include +#include "aio/aio_task.h" +#include "aio/file_io.h" #include "consensus_types.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/serverlet.h" +#include "runtime/task/task_code.h" #include "simple_kv_types.h" +#include "utils/autoref_ptr.h" +#include "utils/binary_reader.h" +#include "utils/blob.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" +#include "utils/threadpool_code.h" +#include "utils/utils.h" +// TODO(yingchun): most of the code are the same as +// src/replica/storage/simple_kv/simple_kv.server.impl.cpp, unify the code! namespace dsn { -class blob; namespace replication { class replica; @@ -54,6 +66,8 @@ namespace dsn { namespace replication { namespace test { +DEFINE_TASK_CODE(LPC_AIO_IMMEDIATE_CALLBACK, TASK_PRIORITY_COMMON, dsn::THREAD_POOL_DEFAULT) + bool simple_kv_service_impl::s_simple_kv_open_fail = false; bool simple_kv_service_impl::s_simple_kv_close_fail = false; bool simple_kv_service_impl::s_simple_kv_get_checkpoint_fail = false; @@ -174,34 +188,53 @@ void simple_kv_service_impl::recover(const std::string &name, int64_t version) { dsn::zauto_lock l(_lock); - std::ifstream is(name.c_str(), std::ios::binary); - if (!is.is_open()) - return; + std::unique_ptr rfile; + auto s = rocksdb::Env::Default()->NewSequentialFile(name, &rfile, rocksdb::EnvOptions()); + CHECK(s.ok(), "open log file '{}' failed, err = {}", name, s.ToString()); _store.clear(); - uint64_t count; - int magic; - - is.read((char *)&count, sizeof(count)); - is.read((char *)&magic, sizeof(magic)); + // Read header. + uint64_t count = 0; + int magic = 0; + rocksdb::Slice result; + static const uint64_t kHeaderSize = sizeof(count) + sizeof(magic); + char buff[kHeaderSize] = {0}; + s = rfile->Read(kHeaderSize, &result, buff); + CHECK(s.ok(), "read header failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + binary_reader reader(blob(buff, 0, kHeaderSize)); + CHECK_EQ(sizeof(count), reader.read(count)); + CHECK_EQ(sizeof(magic), reader.read(magic)); CHECK_EQ_MSG(magic, 0xdeadbeef, "invalid checkpoint"); + // Read kv pairs. for (uint64_t i = 0; i < count; i++) { - std::string key; - std::string value; - - uint32_t sz; - is.read((char *)&sz, (uint32_t)sizeof(sz)); - key.resize(sz); - - is.read((char *)&key[0], sz); - - is.read((char *)&sz, (uint32_t)sizeof(sz)); - value.resize(sz); - - is.read((char *)&value[0], sz); - + // Read key. + uint32_t sz = 0; + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read key size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr key_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, key_buffer.get()); + CHECK(s.ok(), "read key failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string key = result.ToString(); + + // Read value. + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read value size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr value_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, value_buffer.get()); + CHECK(s.ok(), "read value failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string value = result.ToString(); + + // Store the kv pair. _store[key] = value; } } @@ -218,30 +251,43 @@ ::dsn::error_code simple_kv_service_impl::sync_checkpoint() return ERR_OK; } - // TODO: should use async write instead - char name[256]; - sprintf(name, "%s/checkpoint.%" PRId64, data_dir().c_str(), last_commit); - std::ofstream os(name, std::ios::binary); + std::string fname = fmt::format("{}/checkpoint.{}", data_dir(), last_commit); + auto wfile = file::open(fname, file::FileOpenType::kWriteOnly); + CHECK_NOTNULL(wfile, ""); +#define WRITE_DATA_SIZE(data, size) \ + do { \ + auto tsk = ::dsn::file::write( \ + wfile, (char *)&data, size, offset, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr); \ + tsk->wait(); \ + offset += size; \ + } while (false) + +#define WRITE_DATA(data) WRITE_DATA_SIZE(data, sizeof(data)) + + uint64_t offset = 0; uint64_t count = (uint64_t)_store.size(); - int magic = 0xdeadbeef; + WRITE_DATA(count); - os.write((const char *)&count, (uint32_t)sizeof(count)); - os.write((const char *)&magic, (uint32_t)sizeof(magic)); + int magic = 0xdeadbeef; + WRITE_DATA(magic); - for (auto it = _store.begin(); it != _store.end(); ++it) { - const std::string &k = it->first; + for (const auto &kv : _store) { + const std::string &k = kv.first; uint32_t sz = (uint32_t)k.length(); + WRITE_DATA(sz); + WRITE_DATA_SIZE(k[0], sz); - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&k[0], sz); - - const std::string &v = it->second; + const std::string &v = kv.second; sz = (uint32_t)v.length(); - - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&v[0], sz); + WRITE_DATA(sz); + WRITE_DATA_SIZE(v[0], sz); } +#undef WRITE_DATA +#undef WRITE_DATA_SIZE + + CHECK_EQ(ERR_OK, file::flush(wfile)); + CHECK_EQ(ERR_OK, file::close(wfile)); set_last_durable_decree(last_commit); LOG_INFO("simple_kv_service_impl create checkpoint succeed, " diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt index 3b82d28e12..587826baad 100644 --- a/src/replica/test/CMakeLists.txt +++ b/src/replica/test/CMakeLists.txt @@ -47,7 +47,8 @@ set(MY_PROJ_LIBS dsn_meta_server zookeeper hashtable gtest - test_utils) + test_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/test/config-test.ini b/src/replica/test/config-test.ini index 1a1997cc9f..9d73bb5d01 100644 --- a/src/replica/test/config-test.ini +++ b/src/replica/test/config-test.ini @@ -87,6 +87,9 @@ rpc_call_channel = RPC_CHANNEL_TCP rpc_message_header_format = dsn rpc_timeout_milliseconds = 5000 +[replication] +disk_min_available_space_ratio = 10 + [block_service.local_service] type = local_service args = diff --git a/src/replica/test/mock_utils.h b/src/replica/test/mock_utils.h index 104bd2c072..48072f546a 100644 --- a/src/replica/test/mock_utils.h +++ b/src/replica/test/mock_utils.h @@ -445,7 +445,7 @@ class mock_mutation_log_shared : public mutation_log_shared dsn::task_tracker *tracker, aio_handler &&callback, int hash = 0, - int64_t *pending_size = nullptr) + int64_t *pending_size = nullptr) override { _mu_list.push_back(mu); return nullptr; diff --git a/src/replica/test/mutation_log_test.cpp b/src/replica/test/mutation_log_test.cpp index 6bca13336a..4f96d7d57e 100644 --- a/src/replica/test/mutation_log_test.cpp +++ b/src/replica/test/mutation_log_test.cpp @@ -26,26 +26,37 @@ #include "replica/mutation_log.h" +// IWYU pragma: no_include // IWYU pragma: no_include +// IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include +#include +#include +#include #include #include "aio/aio_task.h" +#include "aio/file_io.h" #include "backup_types.h" #include "common/replication.codes.h" #include "consensus_types.h" #include "replica/log_block.h" #include "replica/log_file.h" #include "replica/mutation.h" +#include "replica/replica_stub.h" #include "replica/test/mock_utils.h" #include "replica_test_base.h" +#include "rrdb/rrdb.code.definition.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" #include "utils/blob.h" +#include "utils/defer.h" +#include "utils/env.h" +#include "utils/fail_point.h" #include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/ports.h" @@ -56,43 +67,26 @@ class message_ex; using namespace ::dsn; using namespace ::dsn::replication; -static void copy_file(const char *from_file, const char *to_file, int64_t to_size = -1) -{ - int64_t from_size; - ASSERT_TRUE(dsn::utils::filesystem::file_size(from_file, from_size)); - ASSERT_LE(to_size, from_size); - FILE *from = fopen(from_file, "rb"); - ASSERT_TRUE(from != nullptr); - FILE *to = fopen(to_file, "wb"); - ASSERT_TRUE(to != nullptr); - if (to_size == -1) - to_size = from_size; - if (to_size > 0) { - std::unique_ptr buf(new char[to_size]); - auto n = fread(buf.get(), 1, to_size, from); - ASSERT_EQ(to_size, n); - n = fwrite(buf.get(), 1, to_size, to); - ASSERT_EQ(to_size, n); - } - int r = fclose(from); - ASSERT_EQ(0, r); - r = fclose(to); - ASSERT_EQ(0, r); -} - static void overwrite_file(const char *file, int offset, const void *buf, int size) { - FILE *f = fopen(file, "r+b"); - ASSERT_TRUE(f != nullptr); - int r = fseek(f, offset, SEEK_SET); - ASSERT_EQ(0, r); - size_t n = fwrite(buf, 1, size, f); - ASSERT_EQ(size, n); - r = fclose(f); - ASSERT_EQ(0, r); + auto wfile = file::open(file, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + auto t = ::dsn::file::write(wfile, + (const char *)buf, + size, + offset, + LPC_AIO_IMMEDIATE_CALLBACK, + nullptr, + [=](::dsn::error_code err, size_t n) { + CHECK_EQ(ERR_OK, err); + CHECK_EQ(size, n); + }); + t->wait(); + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); } -TEST(replication, log_file) +TEST(replication_test, log_file) { replica_log_info_map mdecrees; gpid gpid(1, 0); @@ -121,7 +115,7 @@ TEST(replication, log_file) lf->write_file_header(temp_writer, mdecrees); writer->add(temp_writer.get_buffer()); ASSERT_EQ(mdecrees, lf->previous_log_max_decrees()); - log_file_header &h = lf->header(); + const auto &h = lf->header(); ASSERT_EQ(100, h.start_global_offset); } @@ -188,7 +182,7 @@ TEST(replication, log_file) // bad file data: empty file ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.0")); - copy_file(fpath.c_str(), "log.1.0", 0); + dsn::utils::copy_file_by_size(fpath, "log.1.0", 0); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.0")); lf = log_file::open_read("log.1.0", err); ASSERT_TRUE(lf == nullptr); @@ -198,7 +192,7 @@ TEST(replication, log_file) // bad file data: incomplete log_block_header ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.1")); - copy_file(fpath.c_str(), "log.1.1", sizeof(log_block_header) - 1); + dsn::utils::copy_file_by_size(fpath, "log.1.1", sizeof(log_block_header) - 1); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.1")); lf = log_file::open_read("log.1.1", err); ASSERT_TRUE(lf == nullptr); @@ -208,7 +202,7 @@ TEST(replication, log_file) // bad file data: bad log_block_header (magic = 0xfeadbeef) ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.2")); - copy_file(fpath.c_str(), "log.1.2"); + dsn::utils::copy_file_by_size(fpath, "log.1.2"); int32_t bad_magic = 0xfeadbeef; overwrite_file("log.1.2", FIELD_OFFSET(log_block_header, magic), &bad_magic, sizeof(bad_magic)); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.2")); @@ -220,7 +214,7 @@ TEST(replication, log_file) // bad file data: bad log_block_header (crc check failed) ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.3")); - copy_file(fpath.c_str(), "log.1.3"); + dsn::utils::copy_file_by_size(fpath, "log.1.3"); int32_t bad_crc = 0; overwrite_file("log.1.3", FIELD_OFFSET(log_block_header, body_crc), &bad_crc, sizeof(bad_crc)); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.3")); @@ -232,14 +226,13 @@ TEST(replication, log_file) // bad file data: incomplete block body ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.4")); - copy_file(fpath.c_str(), "log.1.4", sizeof(log_block_header) + 1); + dsn::utils::copy_file_by_size(fpath, "log.1.4", sizeof(log_block_header) + 1); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.4")); lf = log_file::open_read("log.1.4", err); ASSERT_TRUE(lf == nullptr); ASSERT_EQ(ERR_INCOMPLETE_DATA, err); ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.4")); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.4.removed")); - ASSERT_TRUE(dsn::utils::filesystem::rename_path("log.1.4.removed", "log.1.4")); // read the file for test offset = 100; @@ -249,7 +242,7 @@ TEST(replication, log_file) ASSERT_EQ(1, lf->index()); ASSERT_EQ(100, lf->start_offset()); int64_t sz; - ASSERT_TRUE(dsn::utils::filesystem::file_size(fpath, sz)); + ASSERT_TRUE(dsn::utils::filesystem::file_size(fpath, dsn::utils::FileDataType::kSensitive, sz)); ASSERT_EQ(lf->start_offset() + sz, lf->end_offset()); // read data @@ -450,6 +443,83 @@ class mutation_log_test : public replica_test_base ASSERT_GE(log_files.size(), 1); } } + + mutation_ptr generate_slog_mutation(const gpid &pid, const decree d, const std::string &data) + { + mutation_ptr mu(new mutation()); + mu->data.header.ballot = 1; + mu->data.header.decree = d; + mu->data.header.pid = pid; + mu->data.header.last_committed_decree = d - 1; + mu->data.header.log_offset = 0; + mu->data.header.timestamp = d; + + mu->data.updates.push_back(mutation_update()); + mu->data.updates.back().code = dsn::apps::RPC_RRDB_RRDB_PUT; + mu->data.updates.back().data = blob::create_from_bytes(std::string(data)); + + mu->client_requests.push_back(nullptr); + + return mu; + } + + void generate_slog_file(const std::vector> &replica_mutations, + mutation_log_ptr &mlog, + decree &d, + std::unordered_map &valid_start_offsets, + std::pair &slog_file_start_offset) + { + for (size_t i = 0; i < replica_mutations.size(); ++i) { + const auto &pid = replica_mutations[i].first; + + for (size_t j = 0; j < replica_mutations[i].second; ++j) { + if (i == 0) { + // Record the start offset of each slog file. + slog_file_start_offset.first = pid; + slog_file_start_offset.second = mlog->get_global_offset(); + } + + const auto &it = valid_start_offsets.find(pid); + if (it == valid_start_offsets.end()) { + // Add new partition with its start offset in slog. + valid_start_offsets.emplace(pid, mlog->get_global_offset()); + mlog->set_valid_start_offset_on_open(pid, mlog->get_global_offset()); + } + + // Append a mutation. + auto mu = generate_slog_mutation(pid, d++, "test data"); + mlog->append(mu, LPC_AIO_IMMEDIATE_CALLBACK, mlog->tracker(), nullptr, 0); + } + } + + // Wait until all mutations are written into this file. + mlog->tracker()->wait_outstanding_tasks(); + } + + void generate_slog_files(const std::vector>> &files, + mutation_log_ptr &mlog, + std::unordered_map &valid_start_offsets, + std::vector> &slog_file_start_offsets) + { + valid_start_offsets.clear(); + slog_file_start_offsets.resize(files.size()); + + decree d = 1; + for (size_t i = 0; i < files.size(); ++i) { + generate_slog_file(files[i], mlog, d, valid_start_offsets, slog_file_start_offsets[i]); + if (i + 1 < files.size()) { + // Do not create a new slog file after the last file is generated. + mlog->create_new_log_file(); + // Wait until file header is written. + mlog->tracker()->wait_outstanding_tasks(); + } + } + + // Close and reset `_current_log_file` since slog has been deprecated and would not be + // used again. + mlog->_current_log_file->close(); + mlog->_current_log_file = nullptr; + } }; TEST_F(mutation_log_test, replay_single_file_1000) { test_replay_single_file(1000); } @@ -606,5 +676,182 @@ TEST_F(mutation_log_test, reset_from_while_writing) mlog->flush(); ASSERT_EQ(actual.size(), expected.size()); } + +TEST_F(mutation_log_test, gc_slog) +{ + // Remove the slog dir and create a new one. + const std::string slog_dir("./slog_test"); + ASSERT_TRUE(dsn::utils::filesystem::remove_path(slog_dir)); + ASSERT_TRUE(dsn::utils::filesystem::create_directory(slog_dir)); + + // Create and open slog object, which would be closed at the end of the scope. + mutation_log_ptr mlog = new mutation_log_shared(slog_dir, 1, false); + auto cleanup = dsn::defer([mlog]() { mlog->close(); }); + ASSERT_EQ(ERR_OK, mlog->open(nullptr, nullptr)); + + // Each line describes a sequence of mutations written to specified replicas by + // specified numbers. + // + // From these sequences the decrees for each partition could be concluded as below: + // {1, 1}: 9 ~ 15 + // {1, 2}: 16 ~ 22 + // {2, 5}: 1 ~ 8, 23 ~ 38 + // {2, 7}: 39 ~ 46 + // {5, 6}: 47 ~ 73 + const std::vector>> files = { + {{{2, 5}, 8}, {{1, 1}, 7}, {{1, 2}, 2}}, + {{{1, 2}, 5}}, + {{{2, 5}, 16}, {{2, 7}, 8}, {{5, 6}, 27}}}; + + // Each line describes a progress of durable decrees for all of replicas: decrees are + // continuously being applied and becoming durable. + const std::vector> durable_decrees = { + {{{1, 1}, 10}, {{1, 2}, 17}, {{2, 5}, 6}, {{2, 7}, 39}, {{5, 6}, 47}}, + {{{1, 1}, 15}, {{1, 2}, 18}, {{2, 5}, 7}, {{2, 7}, 40}, {{5, 6}, 57}}, + {{{1, 1}, 15}, {{1, 2}, 20}, {{2, 5}, 8}, {{2, 7}, 42}, {{5, 6}, 61}}, + {{{1, 1}, 15}, {{1, 2}, 22}, {{2, 5}, 23}, {{2, 7}, 44}, {{5, 6}, 65}}, + {{{1, 1}, 15}, {{1, 2}, 22}, {{2, 5}, 27}, {{2, 7}, 46}, {{5, 6}, 66}}, + {{{1, 1}, 15}, {{1, 2}, 22}, {{2, 5}, 32}, {{2, 7}, 46}, {{5, 6}, 67}}, + {{{1, 1}, 15}, {{1, 2}, 22}, {{2, 5}, 38}, {{2, 7}, 46}, {{5, 6}, 72}}, + {{{1, 1}, 15}, {{1, 2}, 22}, {{2, 5}, 38}, {{2, 7}, 46}, {{5, 6}, 73}}, + }; + const std::vector remaining_slog_files = {3, 3, 2, 1, 1, 1, 1, 0}; + const std::vector> expected_prevent_gc_replicas = { + {{1, 1}, {1, 2}, {2, 5}, {2, 7}, {5, 6}}, + {{1, 2}, {2, 5}, {2, 7}, {5, 6}}, + {{1, 2}, {2, 5}, {2, 7}, {5, 6}}, + {{2, 5}, {2, 7}, {5, 6}}, + {{2, 5}, {5, 6}}, + {{2, 5}, {5, 6}}, + {{5, 6}}, + {}, + }; + + // Each line describes an action, that during a round (related to the index of + // `durable_decrees`), which replica should be reset to the start offset of an + // slog file (related to the index of `files` and `slog_file_start_offsets`). + const std::unordered_map set_to_slog_file_start_offsets = { + {2, 1}, + }; + + // Create slog files and write some data into them according to test cases. + std::unordered_map valid_start_offsets; + std::vector> slog_file_start_offsets; + generate_slog_files(files, mlog, valid_start_offsets, slog_file_start_offsets); + + for (size_t i = 0; i < durable_decrees.size(); ++i) { + std::cout << "Update No." << i << " group of durable decrees" << std::endl; + + // Update the progress of durable_decrees for each partition. + replica_log_info_map replica_durable_decrees; + for (const auto &d : durable_decrees[i]) { + replica_durable_decrees.emplace( + d.first, replica_log_info(d.second, valid_start_offsets[d.first])); + } + + // Test condition for `valid_start_offset`, see `can_gc_replica_slog`. + const auto &set_to_start = set_to_slog_file_start_offsets.find(i); + if (set_to_start != set_to_slog_file_start_offsets.end()) { + const auto &start_offset = slog_file_start_offsets[set_to_start->second]; + replica_durable_decrees[start_offset.first].valid_start_offset = start_offset.second; + } + + // Run garbage collection for a round. + std::set actual_prevent_gc_replicas; + mlog->garbage_collection(replica_durable_decrees, actual_prevent_gc_replicas); + + // Check if the number of remaining slog files after garbage collection is desired. + std::vector file_list; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(slog_dir, file_list, false)); + ASSERT_EQ(remaining_slog_files[i], file_list.size()); + + // Check if the replicas that prevent garbage collection (i.e. cannot be removed by + // garbage collection) is expected. + ASSERT_EQ(expected_prevent_gc_replicas[i], actual_prevent_gc_replicas); + } +} + +using gc_slog_flush_replicas_case = std::tuple, uint64_t, size_t, size_t, size_t>; + +class GcSlogFlushFeplicasTest : public testing::TestWithParam +{ +}; + +DSN_DECLARE_uint64(log_shared_gc_flush_replicas_limit); + +TEST_P(GcSlogFlushFeplicasTest, FlushReplicas) +{ + std::set prevent_gc_replicas; + size_t last_prevent_gc_replica_count; + uint64_t limit; + size_t last_limit; + size_t expected_flush_replicas; + std::tie(prevent_gc_replicas, + last_prevent_gc_replica_count, + limit, + last_limit, + expected_flush_replicas) = GetParam(); + + replica_stub::replica_gc_info_map replica_gc_map; + for (const auto &r : prevent_gc_replicas) { + replica_gc_map.emplace(r, replica_stub::replica_gc_info()); + } + + const auto reserved_log_shared_gc_flush_replicas_limit = + FLAGS_log_shared_gc_flush_replicas_limit; + FLAGS_log_shared_gc_flush_replicas_limit = limit; + + dsn::fail::setup(); + dsn::fail::cfg("mock_flush_replicas_for_slog_gc", "void(true)"); + + replica_stub stub; + stub._last_prevent_gc_replica_count = last_prevent_gc_replica_count; + stub._real_log_shared_gc_flush_replicas_limit = last_limit; + + stub.flush_replicas_for_slog_gc(replica_gc_map, prevent_gc_replicas); + EXPECT_EQ(expected_flush_replicas, stub._mock_flush_replicas_for_test); + + dsn::fail::teardown(); + + FLAGS_log_shared_gc_flush_replicas_limit = reserved_log_shared_gc_flush_replicas_limit; +} + +const std::vector gc_slog_flush_replicas_tests = { + // Initially, there is no limit on flushed replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 0, 0, 0, 6}, + // Initially, there is no limit on flushed replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 1, 0, 5, 6}, + // Initially, limit is less than the number of replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 0, 1, 0, 1}, + // Initially, limit is less than the number of replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 0, 2, 0, 2}, + // Initially, limit is just equal to the number of replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 0, 6, 0, 6}, + // Initially, limit is more than the number of replicas. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 0, 7, 0, 6}, + // No replica has been flushed during previous round. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 6, 6, 6, 2}, + // No replica has been flushed during previous round. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 6, 1, 2, 1}, + // The previous limit is 0. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 7, 5, 0, 5}, + // The previous limit is infinite. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 7, 5, std::numeric_limits::max(), 5}, + // The number of previously flushed replicas is less than the previous limit. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 7, 5, 0, 5}, + // The number of previously flushed replicas reaches the previous limit. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 8, 6, 2, 4}, + // The number of previously flushed replicas reaches the previous limit. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 12, 6, 6, 6}, + // The number of previously flushed replicas is more than the previous limit. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 9, 3, 2, 3}, + // The number of previously flushed replicas is more than the previous limit. + {{{1, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}, 9, 5, 2, 4}, +}; + +INSTANTIATE_TEST_CASE_P(MutationLogTest, + GcSlogFlushFeplicasTest, + testing::ValuesIn(gc_slog_flush_replicas_tests)); + } // namespace replication } // namespace dsn diff --git a/src/replica/test/replica_disk_migrate_test.cpp b/src/replica/test/replica_disk_migrate_test.cpp index 024b039cef..e90cbf2fa0 100644 --- a/src/replica/test/replica_disk_migrate_test.cpp +++ b/src/replica/test/replica_disk_migrate_test.cpp @@ -18,11 +18,9 @@ */ #include -#include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include #include diff --git a/src/replica/test/replica_test.cpp b/src/replica/test/replica_test.cpp index f1ff0acc9d..40a93cc95f 100644 --- a/src/replica/test/replica_test.cpp +++ b/src/replica/test/replica_test.cpp @@ -54,7 +54,6 @@ #include "replica/test/mock_utils.h" #include "replica_test_base.h" #include "runtime/api_layer1.h" -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_message.h" diff --git a/src/replica/test/run.sh b/src/replica/test/run.sh index 6d645f3327..12152709dd 100755 --- a/src/replica/test/run.sh +++ b/src/replica/test/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # The MIT License (MIT) # # Copyright (c) 2015 Microsoft Corporation @@ -28,6 +28,25 @@ if [ -z "${REPORT_DIR}" ]; then REPORT_DIR="." fi +if [ -n ${TEST_OPTS} ]; then + if [ ! -f "./config-test.ini" ]; then + echo "./config-test.ini does not exists" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config-test.ini + done +fi + ./clear.sh output_xml="${REPORT_DIR}/dsn.replica.test.1.xml" GTEST_OUTPUT="xml:${output_xml}" ./dsn.replica.test diff --git a/src/runtime/providers.common.cpp b/src/runtime/providers.common.cpp index 52fe571f68..b3fc676680 100644 --- a/src/runtime/providers.common.cpp +++ b/src/runtime/providers.common.cpp @@ -33,7 +33,6 @@ * xxxx-xx-xx, author, fix bug about xxx */ -#include #include #include "runtime/env_provider.h" diff --git a/src/runtime/rpc/asio_net_provider.cpp b/src/runtime/rpc/asio_net_provider.cpp index 7677d071cb..95569b2045 100644 --- a/src/runtime/rpc/asio_net_provider.cpp +++ b/src/runtime/rpc/asio_net_provider.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -242,7 +241,7 @@ void asio_udp_provider::send_message(message_ex *request) [=](const boost::system::error_code &error, std::size_t bytes_transferred) { if (error) { LOG_WARNING("send udp packet to ep {}:{} failed, message = {}", - ep.address(), + ep.address().to_string(), ep.port(), error.message()); // we do not handle failure here, rpc matcher would handle timeouts diff --git a/src/runtime/rpc/asio_rpc_session.cpp b/src/runtime/rpc/asio_rpc_session.cpp index a8518cb261..5006a257f3 100644 --- a/src/runtime/rpc/asio_rpc_session.cpp +++ b/src/runtime/rpc/asio_rpc_session.cpp @@ -28,7 +28,6 @@ #include // IWYU pragma: keep // IWYU pragma: no_include -#include #include #include #include diff --git a/src/runtime/rpc/network.h b/src/runtime/rpc/network.h index 72f6108533..1dd15d11d6 100644 --- a/src/runtime/rpc/network.h +++ b/src/runtime/rpc/network.h @@ -39,6 +39,7 @@ #include "runtime/task/task_spec.h" #include "utils/autoref_ptr.h" #include "utils/error_code.h" +#include "utils/fmt_utils.h" #include "utils/join_point.h" #include "utils/link.h" #include "utils/synchronize.h" @@ -282,6 +283,8 @@ class rpc_session : public ref_counter SS_CONNECTED, SS_DISCONNECTED }; + friend USER_DEFINED_ENUM_FORMATTER(rpc_session::session_state); + mutable utils::ex_lock_nr _lock; // [ volatile session_state _connect_state; diff --git a/src/runtime/rpc/rpc_address.h b/src/runtime/rpc/rpc_address.h index 7b8cc68201..e5afd7270a 100644 --- a/src/runtime/rpc/rpc_address.h +++ b/src/runtime/rpc/rpc_address.h @@ -27,12 +27,15 @@ #pragma once #include // IWYU pragma: keep - #include #include +// IWYU pragma: no_include #include #include #include +#include + +#include "utils/fmt_utils.h" namespace apache { namespace thrift { @@ -47,6 +50,7 @@ typedef enum dsn_host_type_t { HOST_TYPE_IPV4 = 1, HOST_TYPE_GROUP = 2, } dsn_host_type_t; +USER_DEFINED_ENUM_FORMATTER(dsn_host_type_t) namespace dsn { @@ -208,6 +212,8 @@ class rpc_address } // namespace dsn +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::rpc_address); + namespace std { template <> diff --git a/src/runtime/rpc/rpc_host_port.h b/src/runtime/rpc/rpc_host_port.h index bfeba0123e..be825b22c4 100644 --- a/src/runtime/rpc/rpc_host_port.h +++ b/src/runtime/rpc/rpc_host_port.h @@ -25,11 +25,13 @@ #include #include #include +#include #include #include "runtime/rpc/rpc_address.h" #include "utils/errors.h" #include "utils/fmt_logging.h" +#include "utils/fmt_utils.h" namespace apache { namespace thrift { @@ -116,6 +118,8 @@ inline bool operator!=(const host_port &hp1, const host_port &hp2) { return !(hp } // namespace dsn +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::host_port); + namespace std { template <> struct hash<::dsn::host_port> diff --git a/src/runtime/rpc/thrift_message_parser.cpp b/src/runtime/rpc/thrift_message_parser.cpp index e67c2ad0ce..e01d342d80 100644 --- a/src/runtime/rpc/thrift_message_parser.cpp +++ b/src/runtime/rpc/thrift_message_parser.cpp @@ -27,7 +27,6 @@ #include "thrift_message_parser.h" #include -#include #include #include #include @@ -46,6 +45,7 @@ #include "utils/crc.h" #include "utils/endians.h" #include "utils/fmt_logging.h" +#include "utils/fmt_utils.h" #include "utils/string_view.h" #include "utils/strings.h" @@ -428,3 +428,5 @@ thrift_message_parser::thrift_message_parser() thrift_message_parser::~thrift_message_parser() = default; } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(apache::thrift::protocol::TMessageType); diff --git a/src/runtime/scheduler.cpp b/src/runtime/scheduler.cpp index 9ccc20afad..458dde7163 100644 --- a/src/runtime/scheduler.cpp +++ b/src/runtime/scheduler.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -276,10 +277,9 @@ void scheduler::schedule() _time_ns = ts; } - // randomize the events, and see - std::random_shuffle( - events->begin(), events->end(), [](int n) { return rand::next_u32(0, n - 1); }); - + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(events->begin(), events->end(), g); for (auto e : *events) { if (e.app_task != nullptr) { task *t = e.app_task; diff --git a/src/runtime/security/init.cpp b/src/runtime/security/init.cpp index 36d78e567c..870dd32204 100644 --- a/src/runtime/security/init.cpp +++ b/src/runtime/security/init.cpp @@ -23,11 +23,14 @@ #include "utils/errors.h" #include "utils/flags.h" #include "utils/fmt_logging.h" +#include "utils/strings.h" namespace dsn { namespace security { +DSN_DECLARE_bool(enable_auth); DSN_DECLARE_string(krb5_config); DSN_DECLARE_string(krb5_keytab); +DSN_DECLARE_string(krb5_principal); /*** * set kerberos envs(for more details: @@ -43,6 +46,13 @@ void set_krb5_env(bool is_server) error_s init_kerberos(bool is_server) { + // When FLAGS_enable_auth is set but lacks of necessary parameters to execute kinit by itself, + // then try to obtain the principal under the current Unix account for identity + // authentication automatically. + if (FLAGS_enable_auth && utils::is_empty(FLAGS_krb5_keytab) && + utils::is_empty(FLAGS_krb5_principal)) { + return run_get_principal_without_kinit(); + } // set kerberos env set_krb5_env(is_server); diff --git a/src/runtime/security/kinit_context.cpp b/src/runtime/security/kinit_context.cpp index 08ff5eea86..7b281d5d3a 100644 --- a/src/runtime/security/kinit_context.cpp +++ b/src/runtime/security/kinit_context.cpp @@ -93,6 +93,9 @@ class kinit_context : public utils::singleton public: // implementation of 'kinit -k -t ' error_s kinit(); + // If kinit has been executed outside the program, then directly obtain the principal + // information of the unix account for permission verification. + error_s get_principal_without_kinit(); const std::string &username() const { return _user_name; } private: @@ -174,6 +177,26 @@ error_s kinit_context::kinit() return error_s::ok(); } +// obtain _principal info under the current unix account for permission verification. +error_s kinit_context::get_principal_without_kinit() +{ + // get krb5_ctx + init_krb5_ctx(); + + // acquire credential cache handle + KRB5_RETURN_NOT_OK(krb5_cc_default(_krb5_context, &_ccache), + "couldn't acquire credential cache handle"); + + // get '_principal' from '_ccache' + KRB5_RETURN_NOT_OK(krb5_cc_get_principal(_krb5_context, _ccache, &_principal), + "get principal from cache failed"); + + // get '_user_name' from '_principal' + RETURN_NOT_OK(parse_username_from_principal()); + + return error_s::ok(); +} + void kinit_context::init_krb5_ctx() { static std::once_flag once; @@ -333,6 +356,11 @@ error_s kinit_context::wrap_krb5_err(krb5_error_code krb5_err, const std::string error_s run_kinit() { return kinit_context::instance().kinit(); } +error_s run_get_principal_without_kinit() +{ + return kinit_context::instance().get_principal_without_kinit(); +} + const std::string &get_username() { return kinit_context::instance().username(); } } // namespace security } // namespace dsn diff --git a/src/runtime/security/kinit_context.h b/src/runtime/security/kinit_context.h index 05a0de3993..bb97a43f22 100644 --- a/src/runtime/security/kinit_context.h +++ b/src/runtime/security/kinit_context.h @@ -24,6 +24,7 @@ namespace dsn { namespace security { extern error_s run_kinit(); +extern error_s run_get_principal_without_kinit(); extern const std::string &get_username(); } // namespace security } // namespace dsn diff --git a/src/runtime/security/negotiation_manager.cpp b/src/runtime/security/negotiation_manager.cpp index e7c05d6501..d289732dc9 100644 --- a/src/runtime/security/negotiation_manager.cpp +++ b/src/runtime/security/negotiation_manager.cpp @@ -21,7 +21,6 @@ #include "client_negotiation.h" #include "failure_detector/fd.code.definition.h" -#include "fmt/core.h" #include "http/http_server.h" #include "negotiation_utils.h" #include "runtime/rpc/network.h" diff --git a/src/runtime/service_engine.cpp b/src/runtime/service_engine.cpp index 90d42a68b7..ee0c742d81 100644 --- a/src/runtime/service_engine.cpp +++ b/src/runtime/service_engine.cpp @@ -27,7 +27,6 @@ #include "service_engine.h" #include -#include #include #include #include diff --git a/src/runtime/task/task.cpp b/src/runtime/task/task.cpp index 6e08bd714d..7d83948f3b 100644 --- a/src/runtime/task/task.cpp +++ b/src/runtime/task/task.cpp @@ -120,8 +120,8 @@ task::task(dsn::task_code code, int hash, service_node *node) } else { auto p = get_current_node(); CHECK_NOTNULL(p, - "tasks without explicit service node " - "can only be created inside threads which is attached to specific node"); + "tasks without explicit service node can only be created " + "inside threads which is attached to specific node"); _node = p; } diff --git a/src/runtime/task/task_code.h b/src/runtime/task/task_code.h index 2dafdc51f3..2dd03bf701 100644 --- a/src/runtime/task/task_code.h +++ b/src/runtime/task/task_code.h @@ -31,6 +31,7 @@ #include #include "utils/enum_helper.h" +#include "utils/fmt_utils.h" #include "utils/ports.h" #include "utils/threadpool_code.h" @@ -53,6 +54,7 @@ typedef enum dsn_task_type_t { TASK_TYPE_COUNT, TASK_TYPE_INVALID } dsn_task_type_t; +USER_DEFINED_ENUM_FORMATTER(dsn_task_type_t) ENUM_BEGIN(dsn_task_type_t, TASK_TYPE_INVALID) ENUM_REG(TASK_TYPE_RPC_REQUEST) @@ -115,9 +117,12 @@ class task_code const char *to_string() const; - constexpr bool operator==(const task_code &r) { return _internal_code == r._internal_code; } + constexpr bool operator==(const task_code &r) const + { + return _internal_code == r._internal_code; + } - constexpr bool operator!=(const task_code &r) { return !(*this == r); } + constexpr bool operator!=(const task_code &r) const { return !(*this == r); } constexpr operator int() const { return _internal_code; } @@ -202,3 +207,5 @@ DEFINE_TASK_CODE(TASK_CODE_INVALID, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) DEFINE_TASK_CODE(TASK_CODE_EXEC_INLINED, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::task_code); diff --git a/src/runtime/task/task_engine.cpp b/src/runtime/task/task_engine.cpp index c729a3b611..11ac1b03fd 100644 --- a/src/runtime/task/task_engine.cpp +++ b/src/runtime/task/task_engine.cpp @@ -28,7 +28,6 @@ // IWYU pragma: no_include #include -#include #include #include diff --git a/src/runtime/task/task_spec.h b/src/runtime/task/task_spec.h index 5de2b56e66..ea3b18ea55 100644 --- a/src/runtime/task/task_spec.h +++ b/src/runtime/task/task_spec.h @@ -48,6 +48,7 @@ #include "utils/enum_helper.h" #include "utils/exp_delay.h" #include "utils/extensible_object.h" +#include "utils/fmt_utils.h" #include "utils/join_point.h" #include "utils/threadpool_code.h" @@ -85,6 +86,7 @@ typedef enum grpc_mode_t { GRPC_COUNT, GRPC_INVALID } grpc_mode_t; +USER_DEFINED_ENUM_FORMATTER(grpc_mode_t) ENUM_BEGIN(grpc_mode_t, GRPC_INVALID) ENUM_REG(GRPC_TO_LEADER) @@ -99,6 +101,7 @@ typedef enum throttling_mode_t { TM_COUNT, TM_INVALID } throttling_mode_t; +USER_DEFINED_ENUM_FORMATTER(throttling_mode_t) ENUM_BEGIN(throttling_mode_t, TM_INVALID) ENUM_REG(TM_NONE) @@ -128,11 +131,13 @@ ENUM_END(dsn_msg_serialize_format) DEFINE_CUSTOMIZED_ID_TYPE(network_header_format) DEFINE_CUSTOMIZED_ID(network_header_format, NET_HDR_INVALID) DEFINE_CUSTOMIZED_ID(network_header_format, NET_HDR_DSN) +USER_DEFINED_ENUM_FORMATTER(network_header_format) // define network channel types for RPC DEFINE_CUSTOMIZED_ID_TYPE(rpc_channel) DEFINE_CUSTOMIZED_ID(rpc_channel, RPC_CHANNEL_TCP) DEFINE_CUSTOMIZED_ID(rpc_channel, RPC_CHANNEL_UDP) +USER_DEFINED_ENUM_FORMATTER(rpc_channel) class aio_task; class message_ex; diff --git a/src/runtime/test/CMakeLists.txt b/src/runtime/test/CMakeLists.txt index c0146eb73b..01f17e8c50 100644 --- a/src/runtime/test/CMakeLists.txt +++ b/src/runtime/test/CMakeLists.txt @@ -33,6 +33,7 @@ set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio dsn_meta_server + rocksdb ) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/runtime/test/client_negotiation_test.cpp b/src/runtime/test/client_negotiation_test.cpp index 00eae954be..74dab238ac 100644 --- a/src/runtime/test/client_negotiation_test.cpp +++ b/src/runtime/test/client_negotiation_test.cpp @@ -22,7 +22,6 @@ #include #include -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_holder.h" diff --git a/src/runtime/test/pipeline_test.cpp b/src/runtime/test/pipeline_test.cpp index 22ae223f66..bdc1e5a868 100644 --- a/src/runtime/test/pipeline_test.cpp +++ b/src/runtime/test/pipeline_test.cpp @@ -27,6 +27,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include "common/replication.codes.h" diff --git a/src/runtime/test/server_negotiation_test.cpp b/src/runtime/test/server_negotiation_test.cpp index 907ed0ca8a..0ed7471642 100644 --- a/src/runtime/test/server_negotiation_test.cpp +++ b/src/runtime/test/server_negotiation_test.cpp @@ -23,7 +23,6 @@ #include #include -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_holder.h" diff --git a/src/runtime/test/task_test.cpp b/src/runtime/test/task_test.cpp index ba4f3d8875..5716c8ce41 100644 --- a/src/runtime/test/task_test.cpp +++ b/src/runtime/test/task_test.cpp @@ -17,7 +17,6 @@ #include "runtime/task/task.h" -#include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -66,7 +65,7 @@ class task_test : public ::testing::Test static void test_signal_finished_task() { - disk_file *fp = file::open("config-test.ini", O_RDONLY | O_BINARY, 0); + disk_file *fp = file::open("config-test.ini", file::FileOpenType::kReadOnly); // this aio task is enqueued into read-queue of disk_engine char buffer[128]; @@ -80,6 +79,7 @@ class task_test : public ::testing::Test // signal a finished task won't cause failure t->signal_waiters(); // signal_waiters may return false t->signal_waiters(); + ASSERT_EQ(ERR_OK, file::close(fp)); } }; diff --git a/src/server/available_detector.cpp b/src/server/available_detector.cpp index 73fb81b26c..ca90170fdf 100644 --- a/src/server/available_detector.cpp +++ b/src/server/available_detector.cpp @@ -19,6 +19,7 @@ #include "available_detector.h" +#include // IWYU pragma: no_include #include #include @@ -30,6 +31,8 @@ #include #include +#include // IWYU pragma: keep + #include "base/pegasus_key_schema.h" #include "client/replication_ddl_client.h" #include "common/common.h" diff --git a/src/server/compaction_filter_rule.h b/src/server/compaction_filter_rule.h index 47e780e808..14c157731a 100644 --- a/src/server/compaction_filter_rule.h +++ b/src/server/compaction_filter_rule.h @@ -19,15 +19,17 @@ #pragma once -#include #include #include +#include + #include "base/pegasus_value_schema.h" #include "common/json_helper.h" #include "utils/blob.h" #include "utils/enum_helper.h" #include "utils/factory_store.h" +#include "utils/fmt_utils.h" #include "utils/string_view.h" namespace pegasus { @@ -86,6 +88,7 @@ enum string_match_type SMT_MATCH_POSTFIX, SMT_INVALID, }; +USER_DEFINED_ENUM_FORMATTER(string_match_type) ENUM_BEGIN(string_match_type, SMT_INVALID) ENUM_REG(SMT_MATCH_ANYWHERE) ENUM_REG(SMT_MATCH_PREFIX) diff --git a/src/server/config.ini b/src/server/config.ini index 2d2b7d19d6..543fa316f6 100644 --- a/src/server/config.ini +++ b/src/server/config.ini @@ -278,7 +278,7 @@ stateful = true plog_force_flush = false log_shared_file_size_mb = 128 - log_shared_file_count_limit = 100 + log_shared_gc_flush_replicas_limit = 64 log_shared_batch_buffer_kb = 0 log_shared_force_flush = false log_shared_pending_size_throttling_threshold_kb = 0 diff --git a/src/server/info_collector.h b/src/server/info_collector.h index cf882c1177..258a3d6318 100644 --- a/src/server/info_collector.h +++ b/src/server/info_collector.h @@ -19,10 +19,11 @@ #pragma once -// IWYU pragma: no_include #include #include #include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include #include diff --git a/src/server/pegasus_event_listener.cpp b/src/server/pegasus_event_listener.cpp index e5414d8b79..24faf954e7 100644 --- a/src/server/pegasus_event_listener.cpp +++ b/src/server/pegasus_event_listener.cpp @@ -20,10 +20,9 @@ #include "pegasus_event_listener.h" #include -#include #include #include -#include +#include #include #include "common/gpid.h" diff --git a/src/server/pegasus_mutation_duplicator.cpp b/src/server/pegasus_mutation_duplicator.cpp index e9295af356..87c1adbf3e 100644 --- a/src/server/pegasus_mutation_duplicator.cpp +++ b/src/server/pegasus_mutation_duplicator.cpp @@ -20,13 +20,11 @@ #include "pegasus_mutation_duplicator.h" #include -#include #include #include #include #include #include -#include #include #include #include diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index e6e9d48edc..9bc325ba9d 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include @@ -38,11 +38,14 @@ #include // IWYU pragma: keep #include #include +#include #include #include #include #include +#include +#include "base/idl_utils.h" // IWYU pragma: keep #include "base/pegasus_key_schema.h" #include "base/pegasus_utils.h" #include "base/pegasus_value_schema.h" @@ -1632,12 +1635,12 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) rocksdb::DBOptions loaded_db_opt; std::vector loaded_cf_descs; rocksdb::ColumnFamilyOptions loaded_data_cf_opts; + rocksdb::ConfigOptions config_options; // Set `ignore_unknown_options` true for forward compatibility. - auto status = rocksdb::LoadLatestOptions(rdb_path, - rocksdb::Env::Default(), - &loaded_db_opt, - &loaded_cf_descs, - /*ignore_unknown_options=*/true); + config_options.ignore_unknown_options = true; + config_options.env = rocksdb::Env::Default(); + auto status = + rocksdb::LoadLatestOptions(config_options, rdb_path, &loaded_db_opt, &loaded_cf_descs); if (!status.ok()) { // Here we ignore an invalid argument error related to `pegasus_data_version` and // `pegasus_data` options, which were used in old version rocksdbs (before 2.1.0). @@ -1663,7 +1666,7 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) // We don't use `loaded_data_cf_opts` directly because pointer-typed options will // only be initialized with default values when calling 'LoadLatestOptions', see // 'rocksdb/utilities/options_util.h'. - reset_usage_scenario_options(loaded_data_cf_opts, &_table_data_cf_opts); + reset_rocksdb_options(loaded_data_cf_opts, &_table_data_cf_opts); _db_opts.allow_ingest_behind = parse_allow_ingest_behind(envs); } } else { @@ -1675,11 +1678,14 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) std::vector column_families( {{DATA_COLUMN_FAMILY_NAME, _table_data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}}); - auto s = rocksdb::CheckOptionsCompatibility(rdb_path, - rocksdb::Env::Default(), - _db_opts, - column_families, - /*ignore_unknown_options=*/true); + rocksdb::ConfigOptions config_options; + config_options.ignore_unknown_options = true; + config_options.ignore_unsupported_options = true; + config_options.sanity_level = + rocksdb::ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible; + config_options.env = rocksdb::Env::Default(); + auto s = + rocksdb::CheckOptionsCompatibility(config_options, rdb_path, _db_opts, column_families); if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) { LOG_ERROR_PREFIX("rocksdb::CheckOptionsCompatibility failed, error = {}", s.ToString()); return dsn::ERR_LOCAL_APP_FAILURE; @@ -2037,7 +2043,8 @@ ::dsn::error_code pegasus_server_impl::async_checkpoint(bool flush_memtable) } int64_t checkpoint_decree = 0; - ::dsn::error_code err = copy_checkpoint_to_dir_unsafe(tmp_dir.c_str(), &checkpoint_decree); + ::dsn::error_code err = + copy_checkpoint_to_dir_unsafe(tmp_dir.c_str(), &checkpoint_decree, flush_memtable); if (err != ::dsn::ERR_OK) { LOG_ERROR_PREFIX("copy_checkpoint_to_dir_unsafe failed with err = {}", err.to_string()); return ::dsn::ERR_LOCAL_APP_FAILURE; @@ -2156,7 +2163,7 @@ ::dsn::error_code pegasus_server_impl::copy_checkpoint_to_dir_unsafe(const char {{DATA_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}, {META_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}}); status = rocksdb::DB::OpenForReadOnly( - rocksdb::DBOptions(), checkpoint_dir, column_families, &handles_opened, &snapshot_db); + _db_opts, checkpoint_dir, column_families, &handles_opened, &snapshot_db); if (!status.ok()) { LOG_ERROR_PREFIX( "OpenForReadOnly from {} failed, error = {}", checkpoint_dir, status.ToString()); @@ -2306,7 +2313,7 @@ bool pegasus_server_impl::validate_filter(::dsn::apps::filter_type::type filter_ } } default: - CHECK(false, "unsupported filter type: %d", filter_type); + CHECK(false, "unsupported filter type: {}", filter_type); } return false; } @@ -2625,6 +2632,67 @@ pegasus_server_impl::get_restore_dir_from_env(const std::map &envs) +{ + if (envs.empty()) { + return; + } + + std::unordered_map new_options; + for (const auto &option : ROCKSDB_DYNAMIC_OPTIONS) { + const auto &find = envs.find(option); + if (find == envs.end()) { + continue; + } + + std::vector args; + // split_args example: Parse "write_buffer_size" from "rocksdb.write_buffer_size" + dsn::utils::split_args(option.c_str(), args, '.'); + CHECK_EQ(args.size(), 2); + new_options[args[1]] = find->second; + } + + // doing set option + if (!new_options.empty() && set_options(new_options)) { + LOG_INFO("Set rocksdb dynamic options success"); + } +} + +void pegasus_server_impl::set_rocksdb_options_before_creating( + const std::map &envs) +{ + if (envs.empty()) { + return; + } + + for (const auto &option : pegasus::ROCKSDB_STATIC_OPTIONS) { + const auto &find = envs.find(option); + if (find == envs.end()) { + continue; + } + + const auto &setter = cf_opts_setters.find(option); + CHECK_TRUE(setter != cf_opts_setters.end()); + if (setter->second(find->second, _data_cf_opts)) { + LOG_INFO_PREFIX("Set {} \"{}\" succeed", find->first, find->second); + } + } + + for (const auto &option : pegasus::ROCKSDB_DYNAMIC_OPTIONS) { + const auto &find = envs.find(option); + if (find == envs.end()) { + continue; + } + + const auto &setter = cf_opts_setters.find(option); + CHECK_TRUE(setter != cf_opts_setters.end()); + if (setter->second(find->second, _data_cf_opts)) { + LOG_INFO_PREFIX("Set {} \"{}\" succeed", find->first, find->second); + } + } +} + void pegasus_server_impl::update_app_envs(const std::map &envs) { update_usage_scenario(envs); @@ -2637,6 +2705,7 @@ void pegasus_server_impl::update_app_envs(const std::map &envs) { envs[ROCKSDB_ENV_USAGE_SCENARIO_KEY] = _usage_scenario; + // write_buffer_size involves random values (refer to pegasus_server_impl::set_usage_scenario), + // so it can only be taken from _data_cf_opts + envs[ROCKSDB_WRITE_BUFFER_SIZE] = std::to_string(_data_cf_opts.write_buffer_size); + + // Get Data ColumnFamilyOptions directly from _data_cf + rocksdb::ColumnFamilyDescriptor desc; + CHECK_TRUE(_data_cf->GetDescriptor(&desc).ok()); + for (const auto &option : pegasus::ROCKSDB_STATIC_OPTIONS) { + auto getter = cf_opts_getters.find(option); + CHECK_TRUE(getter != cf_opts_getters.end()); + std::string option_val; + getter->second(desc.options, option_val); + envs[option] = option_val; + } + for (const auto &option : pegasus::ROCKSDB_DYNAMIC_OPTIONS) { + if (option.compare(ROCKSDB_WRITE_BUFFER_SIZE) == 0) { + continue; + } + auto getter = cf_opts_getters.find(option); + CHECK_TRUE(getter != cf_opts_getters.end()); + std::string option_val; + getter->second(desc.options, option_val); + envs[option] = option_val; + } } void pegasus_server_impl::update_usage_scenario(const std::map &envs) @@ -2962,7 +3056,7 @@ std::string pegasus_server_impl::compression_type_to_str(rocksdb::CompressionTyp case rocksdb::kZSTD: return "zstd"; default: - LOG_ERROR_PREFIX("Unsupported compression type: {}.", type); + LOG_ERROR_PREFIX("Unsupported compression type: {}.", static_cast(type)); return ""; } } @@ -3038,6 +3132,23 @@ bool pegasus_server_impl::set_usage_scenario(const std::string &usage_scenario) } } +void pegasus_server_impl::reset_rocksdb_options(const rocksdb::ColumnFamilyOptions &base_opts, + rocksdb::ColumnFamilyOptions *target_opts) +{ + LOG_INFO_PREFIX("Reset rocksdb envs options"); + // Reset rocksdb option includes two aspects: + // 1. Set usage_scenario related rocksdb options + // 2. Rocksdb option set in app envs, consists of ROCKSDB_DYNAMIC_OPTIONS and + // ROCKSDB_STATIC_OPTIONS + + // aspect 1: + reset_usage_scenario_options(base_opts, target_opts); + + // aspect 2: + target_opts->num_levels = base_opts.num_levels; + target_opts->write_buffer_size = base_opts.write_buffer_size; +} + void pegasus_server_impl::reset_usage_scenario_options( const rocksdb::ColumnFamilyOptions &base_opts, rocksdb::ColumnFamilyOptions *target_opts) { @@ -3179,7 +3290,7 @@ ::dsn::error_code pegasus_server_impl::check_column_families(const std::string & *missing_meta_cf = true; *missing_data_cf = true; std::vector column_families; - auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path, &column_families); + auto s = rocksdb::DB::ListColumnFamilies(_db_opts, path, &column_families); if (!s.ok()) { LOG_ERROR_PREFIX("rocksdb::DB::ListColumnFamilies failed, error = {}", s.ToString()); if (s.IsCorruption() && diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index d156acdce6..02c6421c7a 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -20,8 +20,10 @@ #pragma once #include +#include #include #include +#include #include #include #include @@ -69,6 +71,7 @@ class WriteBufferManager; namespace dsn { class blob; class message_ex; + namespace replication { class detect_hotkey_request; class detect_hotkey_response; @@ -328,6 +331,10 @@ class pegasus_server_impl : public pegasus_read_service void update_user_specified_compaction(const std::map &envs); + void update_rocksdb_dynamic_options(const std::map &envs); + + void set_rocksdb_options_before_creating(const std::map &envs); + void update_throttling_controller(const std::map &envs); bool parse_allow_ingest_behind(const std::map &envs); @@ -359,6 +366,9 @@ class pegasus_server_impl : public pegasus_read_service void reset_usage_scenario_options(const rocksdb::ColumnFamilyOptions &base_opts, rocksdb::ColumnFamilyOptions *target_opts); + void reset_rocksdb_options(const rocksdb::ColumnFamilyOptions &base_opts, + rocksdb::ColumnFamilyOptions *target_opts); + // return true if successfully set bool set_options(const std::unordered_map &new_options); @@ -468,6 +478,7 @@ class pegasus_server_impl : public pegasus_read_service // Dynamically calculate the value of current data_cf option according to the conf module file // and usage scenario rocksdb::ColumnFamilyOptions _table_data_cf_opts; + rocksdb::BlockBasedTableOptions _tbl_opts; rocksdb::ColumnFamilyOptions _meta_cf_opts; rocksdb::ReadOptions _data_cf_rd_opts; std::string _usage_scenario; diff --git a/src/server/pegasus_server_impl_init.cpp b/src/server/pegasus_server_impl_init.cpp index 896f8ab945..528b2bd8e9 100644 --- a/src/server/pegasus_server_impl_init.cpp +++ b/src/server/pegasus_server_impl_init.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -137,7 +138,7 @@ DSN_DEFINE_bool(pegasus.server, DSN_DEFINE_bool(pegasus.server, rocksdb_disable_table_block_cache, false, - "rocksdb tbl_opts.no_block_cache"); + "rocksdb _tbl_opts.no_block_cache"); DSN_DEFINE_bool(pegasus.server, rocksdb_enable_write_buffer_manager, false, @@ -422,6 +423,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) _rng_rd_opts.rocksdb_iteration_threshold_time_ms = FLAGS_rocksdb_iteration_threshold_time_ms; // init rocksdb::DBOptions + _db_opts.env = rocksdb::Env::Default(); _db_opts.create_if_missing = true; // atomic flush data CF and meta CF, aim to keep consistency of 'last flushed decree' in meta CF // and data in data CF. @@ -466,12 +468,11 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) CHECK(parse_compression_types("none", _meta_cf_opts.compression_per_level), "parse rocksdb_compression_type failed."); - rocksdb::BlockBasedTableOptions tbl_opts; - tbl_opts.read_amp_bytes_per_bit = FLAGS_read_amp_bytes_per_bit; + _tbl_opts.read_amp_bytes_per_bit = FLAGS_read_amp_bytes_per_bit; if (FLAGS_rocksdb_disable_table_block_cache) { - tbl_opts.no_block_cache = true; - tbl_opts.block_restart_interval = 4; + _tbl_opts.no_block_cache = true; + _tbl_opts.block_restart_interval = 4; } else { // If block cache is enabled, all replicas on this server will share the same block cache // object. It's convenient to control the total memory used by this server, and the LRU @@ -484,7 +485,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) }); // every replica has the same block cache - tbl_opts.block_cache = _s_block_cache; + _tbl_opts.block_cache = _s_block_cache; } // FLAGS_rocksdb_limiter_max_write_megabytes_per_sec <= 0 means close the rate limit. @@ -520,7 +521,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) FLAGS_rocksdb_total_size_across_write_buffer); _s_write_buffer_manager = std::make_shared( static_cast(FLAGS_rocksdb_total_size_across_write_buffer), - tbl_opts.block_cache); + _tbl_opts.block_cache); }); _db_opts.write_buffer_manager = _s_write_buffer_manager; } @@ -541,33 +542,33 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) CHECK(index_type_item != INDEX_TYPE_STRING_MAP.end(), "[pegasus.server]rocksdb_index_type should be one among binary_search, " "hash_search, two_level_index_search or binary_search_with_first_key."); - tbl_opts.index_type = index_type_item->second; + _tbl_opts.index_type = index_type_item->second; LOG_INFO_PREFIX("rocksdb_index_type = {}", FLAGS_rocksdb_index_type); - tbl_opts.partition_filters = FLAGS_rocksdb_partition_filters; + _tbl_opts.partition_filters = FLAGS_rocksdb_partition_filters; // TODO(yingchun): clean up these useless log ? - LOG_INFO_PREFIX("rocksdb_partition_filters = {}", tbl_opts.partition_filters); + LOG_INFO_PREFIX("rocksdb_partition_filters = {}", _tbl_opts.partition_filters); - tbl_opts.metadata_block_size = FLAGS_rocksdb_metadata_block_size; - LOG_INFO_PREFIX("rocksdb_metadata_block_size = {}", tbl_opts.metadata_block_size); + _tbl_opts.metadata_block_size = FLAGS_rocksdb_metadata_block_size; + LOG_INFO_PREFIX("rocksdb_metadata_block_size = {}", _tbl_opts.metadata_block_size); - tbl_opts.cache_index_and_filter_blocks = FLAGS_rocksdb_cache_index_and_filter_blocks; + _tbl_opts.cache_index_and_filter_blocks = FLAGS_rocksdb_cache_index_and_filter_blocks; LOG_INFO_PREFIX("rocksdb_cache_index_and_filter_blocks = {}", - tbl_opts.cache_index_and_filter_blocks); + _tbl_opts.cache_index_and_filter_blocks); - tbl_opts.pin_top_level_index_and_filter = FLAGS_rocksdb_pin_top_level_index_and_filter; + _tbl_opts.pin_top_level_index_and_filter = FLAGS_rocksdb_pin_top_level_index_and_filter; LOG_INFO_PREFIX("rocksdb_pin_top_level_index_and_filter = {}", - tbl_opts.pin_top_level_index_and_filter); + _tbl_opts.pin_top_level_index_and_filter); - tbl_opts.cache_index_and_filter_blocks_with_high_priority = + _tbl_opts.cache_index_and_filter_blocks_with_high_priority = FLAGS_rocksdb_cache_index_and_filter_blocks_with_high_priority; LOG_INFO_PREFIX("rocksdb_cache_index_and_filter_blocks_with_high_priority = {}", - tbl_opts.cache_index_and_filter_blocks_with_high_priority); + _tbl_opts.cache_index_and_filter_blocks_with_high_priority); - tbl_opts.pin_l0_filter_and_index_blocks_in_cache = + _tbl_opts.pin_l0_filter_and_index_blocks_in_cache = FLAGS_rocksdb_pin_l0_filter_and_index_blocks_in_cache; LOG_INFO_PREFIX("rocksdb_pin_l0_filter_and_index_blocks_in_cache = {}", - tbl_opts.pin_l0_filter_and_index_blocks_in_cache); + _tbl_opts.pin_l0_filter_and_index_blocks_in_cache); // Bloom filter configurations. if (!FLAGS_rocksdb_disable_bloom_filter) { @@ -584,8 +585,8 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) // 50 | 0.225453 | ~0.00003 // Recommend using no more than three decimal digits after the decimal point, as in 6.667. // More details: https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter - tbl_opts.format_version = FLAGS_rocksdb_format_version; - tbl_opts.filter_policy.reset( + _tbl_opts.format_version = FLAGS_rocksdb_format_version; + _tbl_opts.filter_policy.reset( rocksdb::NewBloomFilterPolicy(FLAGS_rocksdb_bloom_filter_bits_per_key, false)); if (dsn::utils::equals(FLAGS_rocksdb_filter_type, "prefix")) { @@ -596,8 +597,8 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) } } - _data_cf_opts.table_factory.reset(NewBlockBasedTableFactory(tbl_opts)); - _meta_cf_opts.table_factory.reset(NewBlockBasedTableFactory(tbl_opts)); + _data_cf_opts.table_factory.reset(NewBlockBasedTableFactory(_tbl_opts)); + _meta_cf_opts.table_factory.reset(NewBlockBasedTableFactory(_tbl_opts)); _key_ttl_compaction_filter_factory = std::make_shared(); _data_cf_opts.compaction_filter_factory = _key_ttl_compaction_filter_factory; diff --git a/src/server/pegasus_server_write.cpp b/src/server/pegasus_server_write.cpp index 0147050a65..1c6399cc27 100644 --- a/src/server/pegasus_server_write.cpp +++ b/src/server/pegasus_server_write.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include #include #include #include diff --git a/src/server/pegasus_write_service.cpp b/src/server/pegasus_write_service.cpp index 73f6cc8d6f..6ee7109fe0 100644 --- a/src/server/pegasus_write_service.cpp +++ b/src/server/pegasus_write_service.cpp @@ -18,10 +18,8 @@ */ #include -#include #include #include -#include #include #include "base/pegasus_rpc_types.h" diff --git a/src/server/pegasus_write_service_impl.h b/src/server/pegasus_write_service_impl.h index b75b379a3f..69b71c6ba4 100644 --- a/src/server/pegasus_write_service_impl.h +++ b/src/server/pegasus_write_service_impl.h @@ -21,6 +21,7 @@ #include +#include "base/idl_utils.h" #include "base/pegasus_key_schema.h" #include "logging_utils.h" #include "meta_store.h" @@ -28,6 +29,7 @@ #include "pegasus_write_service.h" #include "rocksdb_wrapper.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/string_conv.h" #include "utils/strings.h" @@ -75,7 +77,8 @@ inline dsn::error_code get_external_files_path(const std::string &bulk_load_dir, for (const auto &f_meta : metadata.files) { const auto &file_name = dsn::utils::filesystem::path_combine(bulk_load_dir, f_meta.name); if (verify_before_ingest && - !dsn::utils::filesystem::verify_file(file_name, f_meta.md5, f_meta.size)) { + !dsn::utils::filesystem::verify_file( + file_name, dsn::utils::FileDataType::kSensitive, f_meta.md5, f_meta.size)) { break; } files_path.emplace_back(file_name); @@ -270,8 +273,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base if (!is_check_type_supported(update.check_type)) { LOG_ERROR_PREFIX("invalid argument for check_and_set: decree = {}, error = {}", decree, - "check type {} not supported", - update.check_type); + fmt::format("check type {} not supported", update.check_type)); resp.error = rocksdb::Status::kInvalidArgument; // we should write empty record to update rocksdb's last flushed decree return empty_put(decree); @@ -391,8 +393,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base if (!is_check_type_supported(update.check_type)) { LOG_ERROR_PREFIX("invalid argument for check_and_mutate: decree = {}, error = {}", decree, - "check type {} not supported", - update.check_type); + fmt::format("check type {} not supported", update.check_type)); resp.error = rocksdb::Status::kInvalidArgument; // we should write empty record to update rocksdb's last flushed decree return empty_put(decree); diff --git a/src/server/test/capacity_unit_calculator_test.cpp b/src/server/test/capacity_unit_calculator_test.cpp index cdfe1bddd4..93db546e14 100644 --- a/src/server/test/capacity_unit_calculator_test.cpp +++ b/src/server/test/capacity_unit_calculator_test.cpp @@ -17,13 +17,14 @@ * under the License. */ -#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include +#include +#include #include #include #include diff --git a/src/server/test/config.ini b/src/server/test/config.ini index a1125b265a..1ec547264a 100644 --- a/src/server/test/config.ini +++ b/src/server/test/config.ini @@ -144,6 +144,7 @@ max_replica_count = 3 stateful = true [replication] +disk_min_available_space_ratio = 10 data_dirs_black_list_file = /home/mi/.pegasus_data_dirs_black_list cluster_name = onebox @@ -186,7 +187,7 @@ log_private_reserve_max_size_mb = 0 log_private_reserve_max_time_seconds = 0 log_shared_file_size_mb = 32 -log_shared_file_count_limit = 32 +log_shared_gc_flush_replicas_limit = 64 log_shared_batch_buffer_kb = 0 log_shared_force_flush = false diff --git a/src/server/test/hashkey_transform_test.cpp b/src/server/test/hashkey_transform_test.cpp index 5ece1c7fb6..52f4d5a6be 100644 --- a/src/server/test/hashkey_transform_test.cpp +++ b/src/server/test/hashkey_transform_test.cpp @@ -19,6 +19,7 @@ #include "server/hashkey_transform.h" +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/server/test/hotkey_collector_test.cpp b/src/server/test/hotkey_collector_test.cpp index 7b784a6d2a..be98def07d 100644 --- a/src/server/test/hotkey_collector_test.cpp +++ b/src/server/test/hotkey_collector_test.cpp @@ -17,6 +17,7 @@ #include "server/hotkey_collector.h" +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/server/test/pegasus_compression_options_test.cpp b/src/server/test/pegasus_compression_options_test.cpp index 9fa38daf66..b33eeeac14 100644 --- a/src/server/test/pegasus_compression_options_test.cpp +++ b/src/server/test/pegasus_compression_options_test.cpp @@ -20,6 +20,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 1363d8054b..718446821d 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -21,6 +21,7 @@ #include #include #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -29,7 +30,9 @@ #include #include #include +#include #include +#include #include "pegasus_const.h" #include "pegasus_server_test_base.h" @@ -95,24 +98,68 @@ class pegasus_server_impl_test : public pegasus_server_test_base ASSERT_EQ(before_count + test.expect_perf_counter_incr, after_count); } } + + void test_open_db_with_rocksdb_envs(bool is_restart) + { + struct create_test + { + std::string env_key; + std::string env_value; + std::string expect_value; + } tests[] = { + {"rocksdb.num_levels", "5", "5"}, {"rocksdb.write_buffer_size", "33554432", "33554432"}, + }; + + std::map all_test_envs; + { + // Make sure all rocksdb options of ROCKSDB_DYNAMIC_OPTIONS and ROCKSDB_STATIC_OPTIONS + // are tested. + for (const auto &test : tests) { + all_test_envs[test.env_key] = test.env_value; + } + for (const auto &option : pegasus::ROCKSDB_DYNAMIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + for (const auto &option : pegasus::ROCKSDB_STATIC_OPTIONS) { + ASSERT_TRUE(all_test_envs.find(option) != all_test_envs.end()); + } + } + + ASSERT_EQ(dsn::ERR_OK, start(all_test_envs)); + if (is_restart) { + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); + ASSERT_EQ(dsn::ERR_OK, start()); + } + + std::map query_envs; + _server->query_app_envs(query_envs); + for (const auto &test : tests) { + const auto &iter = query_envs.find(test.env_key); + if (iter != query_envs.end()) { + ASSERT_EQ(iter->second, test.expect_value); + } else { + ASSERT_TRUE(false) << fmt::format("query_app_envs not supported {}", test.env_key); + } + } + } }; TEST_F(pegasus_server_impl_test, test_table_level_slow_query) { - start(); + ASSERT_EQ(dsn::ERR_OK, start()); test_table_level_slow_query(); } TEST_F(pegasus_server_impl_test, default_data_version) { - start(); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_EQ(_server->_pegasus_data_version, 1); } TEST_F(pegasus_server_impl_test, test_open_db_with_latest_options) { // open a new db with no app env. - start(); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_NORMAL, _server->_usage_scenario); // set bulk_load scenario for the db. ASSERT_TRUE(_server->set_usage_scenario(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD)); @@ -121,8 +168,8 @@ TEST_F(pegasus_server_impl_test, test_open_db_with_latest_options) ASSERT_EQ(1000000000, opts.level0_file_num_compaction_trigger); ASSERT_EQ(true, opts.disable_auto_compactions); // reopen the db. - _server->stop(false); - start(); + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario); ASSERT_EQ(opts.level0_file_num_compaction_trigger, _server->_db->GetOptions().level0_file_num_compaction_trigger); @@ -133,22 +180,34 @@ TEST_F(pegasus_server_impl_test, test_open_db_with_app_envs) { std::map envs; envs[ROCKSDB_ENV_USAGE_SCENARIO_KEY] = ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD; - start(envs); + ASSERT_EQ(dsn::ERR_OK, start(envs)); ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario); } +TEST_F(pegasus_server_impl_test, test_open_db_with_rocksdb_envs) +{ + // Hint: Verify the set_rocksdb_options_before_creating function by boolean is_restart=false. + test_open_db_with_rocksdb_envs(false); +} + +TEST_F(pegasus_server_impl_test, test_restart_db_with_rocksdb_envs) +{ + // Hint: Verify the reset_rocksdb_options function by boolean is_restart=true. + test_open_db_with_rocksdb_envs(true); +} + TEST_F(pegasus_server_impl_test, test_stop_db_twice) { - start(); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_TRUE(_server->_is_open); ASSERT_TRUE(_server->_db != nullptr); - _server->stop(false); + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); ASSERT_FALSE(_server->_is_open); ASSERT_TRUE(_server->_db == nullptr); // stop again - _server->stop(false); + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); ASSERT_FALSE(_server->_is_open); ASSERT_TRUE(_server->_db == nullptr); } diff --git a/src/server/test/pegasus_server_write_test.cpp b/src/server/test/pegasus_server_write_test.cpp index 0e8c571147..3a9cfc2b1d 100644 --- a/src/server/test/pegasus_server_write_test.cpp +++ b/src/server/test/pegasus_server_write_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/server/test/pegasus_write_service_impl_test.cpp b/src/server/test/pegasus_write_service_impl_test.cpp index f466f9f3ac..0d32540fda 100644 --- a/src/server/test/pegasus_write_service_impl_test.cpp +++ b/src/server/test/pegasus_write_service_impl_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/server/test/pegasus_write_service_test.cpp b/src/server/test/pegasus_write_service_test.cpp index 8971076bb0..f277551bc6 100644 --- a/src/server/test/pegasus_write_service_test.cpp +++ b/src/server/test/pegasus_write_service_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/server/test/rocksdb_wrapper_test.cpp b/src/server/test/rocksdb_wrapper_test.cpp index 24d88e6564..86542c64fe 100644 --- a/src/server/test/rocksdb_wrapper_test.cpp +++ b/src/server/test/rocksdb_wrapper_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/server/test/run.sh b/src/server/test/run.sh index 76baf02a0b..d8da4db1a6 100755 --- a/src/server/test/run.sh +++ b/src/server/test/run.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -23,6 +23,25 @@ exit_if_fail() { fi } +if [ -n ${TEST_OPTS} ]; then + if [ ! -f ./config.ini ]; then + echo "./config.ini does not exists !" + exit 1 + fi + + OPTS=`echo ${TEST_OPTS} | xargs` + config_kvs=(${OPTS//,/ }) + for config_kv in ${config_kvs[@]}; do + config_kv=`echo $config_kv | xargs` + kv=(${config_kv//=/ }) + if [ ! ${#kv[*]} -eq 2 ]; then + echo "Invalid config kv !" + exit 1 + fi + sed -i '/^\s*'"${kv[0]}"'/c '"${kv[0]}"' = '"${kv[1]}" ./config.ini + done +fi + ./pegasus_unit_test exit_if_fail $? "run unit test failed" diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index 30d8a22a77..585a64f9c0 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -48,6 +49,7 @@ #include "perf_counter/perf_counter_utils.h" #include "remote_cmd/remote_command.h" #include "tools/mutation_log_tool.h" +#include "utils/fmt_utils.h" #include "utils/string_view.h" #include "utils/strings.h" #include "utils/synchronize.h" @@ -73,6 +75,8 @@ enum scan_data_operator SCAN_GEN_GEO, SCAN_AND_MULTI_SET }; +USER_DEFINED_ENUM_FORMATTER(scan_data_operator) + class top_container { public: diff --git a/src/shell/commands/bulk_load.cpp b/src/shell/commands/bulk_load.cpp index eba70633e8..6f6f16d73b 100644 --- a/src/shell/commands/bulk_load.cpp +++ b/src/shell/commands/bulk_load.cpp @@ -22,10 +22,10 @@ #include // IWYU pragma: no_include #include -#include #include #include #include +#include #include #include #include diff --git a/src/shell/commands/data_operations.cpp b/src/shell/commands/data_operations.cpp index 2a92aea9e7..f13b2fec99 100644 --- a/src/shell/commands/data_operations.cpp +++ b/src/shell/commands/data_operations.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include diff --git a/src/shell/commands/detect_hotkey.cpp b/src/shell/commands/detect_hotkey.cpp index 5eab8b6f9c..b413c63b77 100644 --- a/src/shell/commands/detect_hotkey.cpp +++ b/src/shell/commands/detect_hotkey.cpp @@ -16,10 +16,8 @@ // under the License. #include -#include #include #include -#include #include #include #include diff --git a/src/shell/commands/table_management.cpp b/src/shell/commands/table_management.cpp index b083f9ffbf..818ca8a715 100644 --- a/src/shell/commands/table_management.cpp +++ b/src/shell/commands/table_management.cpp @@ -20,7 +20,6 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include #include diff --git a/src/test/function_test/CMakeLists.txt b/src/test/function_test/CMakeLists.txt index d642cfb093..b3453846b3 100644 --- a/src/test/function_test/CMakeLists.txt +++ b/src/test/function_test/CMakeLists.txt @@ -16,11 +16,11 @@ # under the License. add_subdirectory(utils) -add_subdirectory(backup_restore_test) -add_subdirectory(base_api_test) -add_subdirectory(bulk_load_test) -add_subdirectory(detect_hotspot_test) -add_subdirectory(partition_split_test) -add_subdirectory(recovery_test) -add_subdirectory(restore_test) -add_subdirectory(throttle_test) +add_subdirectory(backup_restore) +add_subdirectory(base_api) +add_subdirectory(bulk_load) +add_subdirectory(detect_hotspot) +add_subdirectory(partition_split) +add_subdirectory(recovery) +add_subdirectory(restore) +add_subdirectory(throttle) diff --git a/src/test/function_test/backup_restore_test/CMakeLists.txt b/src/test/function_test/backup_restore/CMakeLists.txt similarity index 100% rename from src/test/function_test/backup_restore_test/CMakeLists.txt rename to src/test/function_test/backup_restore/CMakeLists.txt diff --git a/src/test/function_test/backup_restore_test/main.cpp b/src/test/function_test/backup_restore/main.cpp similarity index 100% rename from src/test/function_test/backup_restore_test/main.cpp rename to src/test/function_test/backup_restore/main.cpp diff --git a/src/test/function_test/backup_restore_test/test_backup_and_restore.cpp b/src/test/function_test/backup_restore/test_backup_and_restore.cpp similarity index 100% rename from src/test/function_test/backup_restore_test/test_backup_and_restore.cpp rename to src/test/function_test/backup_restore/test_backup_and_restore.cpp diff --git a/src/test/function_test/base_api_test/CMakeLists.txt b/src/test/function_test/base_api/CMakeLists.txt similarity index 100% rename from src/test/function_test/base_api_test/CMakeLists.txt rename to src/test/function_test/base_api/CMakeLists.txt diff --git a/src/test/function_test/base_api_test/integration_test.cpp b/src/test/function_test/base_api/integration_test.cpp similarity index 100% rename from src/test/function_test/base_api_test/integration_test.cpp rename to src/test/function_test/base_api/integration_test.cpp diff --git a/src/test/function_test/base_api_test/main.cpp b/src/test/function_test/base_api/main.cpp similarity index 100% rename from src/test/function_test/base_api_test/main.cpp rename to src/test/function_test/base_api/main.cpp diff --git a/src/test/function_test/base_api_test/test_basic.cpp b/src/test/function_test/base_api/test_basic.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_basic.cpp rename to src/test/function_test/base_api/test_basic.cpp diff --git a/src/test/function_test/base_api_test/test_batch_get.cpp b/src/test/function_test/base_api/test_batch_get.cpp similarity index 99% rename from src/test/function_test/base_api_test/test_batch_get.cpp rename to src/test/function_test/base_api/test_batch_get.cpp index 622fc2a425..d0bf4c1e52 100644 --- a/src/test/function_test/base_api_test/test_batch_get.cpp +++ b/src/test/function_test/base_api/test_batch_get.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include diff --git a/src/test/function_test/base_api_test/test_check_and_mutate.cpp b/src/test/function_test/base_api/test_check_and_mutate.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_check_and_mutate.cpp rename to src/test/function_test/base_api/test_check_and_mutate.cpp diff --git a/src/test/function_test/base_api_test/test_check_and_set.cpp b/src/test/function_test/base_api/test_check_and_set.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_check_and_set.cpp rename to src/test/function_test/base_api/test_check_and_set.cpp diff --git a/src/test/function_test/base_api_test/test_copy.cpp b/src/test/function_test/base_api/test_copy.cpp similarity index 96% rename from src/test/function_test/base_api_test/test_copy.cpp rename to src/test/function_test/base_api/test_copy.cpp index 910a07fff6..d2155f12f0 100644 --- a/src/test/function_test/base_api_test/test_copy.cpp +++ b/src/test/function_test/base_api/test_copy.cpp @@ -98,9 +98,9 @@ class copy_data_test : public test_util ASSERT_EQ(dsn::ERR_OK, ddl_client_->create_app( destination_app_name, "pegasus", default_partitions, 3, {}, false)); - srouce_client_ = + source_client_ = pegasus_client_factory::get_client(cluster_name_.c_str(), source_app_name.c_str()); - ASSERT_NE(nullptr, srouce_client_); + ASSERT_NE(nullptr, source_client_); destination_client_ = pegasus_client_factory::get_client(cluster_name_.c_str(), destination_app_name.c_str()); ASSERT_NE(nullptr, destination_client_); @@ -132,7 +132,7 @@ class copy_data_test : public test_util while (expect_data_[empty_hash_key].size() < 1000) { sort_key = random_string(); value = random_string(); - ASSERT_EQ(PERR_OK, srouce_client_->set(empty_hash_key, sort_key, value)) + ASSERT_EQ(PERR_OK, source_client_->set(empty_hash_key, sort_key, value)) << "hash_key=" << hash_key << ", sort_key=" << sort_key; expect_data_[empty_hash_key][sort_key] = value; } @@ -142,7 +142,7 @@ class copy_data_test : public test_util while (expect_data_[hash_key].size() < 10) { sort_key = random_string(); value = random_string(); - ASSERT_EQ(PERR_OK, srouce_client_->set(hash_key, sort_key, value)) + ASSERT_EQ(PERR_OK, source_client_->set(hash_key, sort_key, value)) << "hash_key=" << hash_key << ", sort_key=" << sort_key; expect_data_[hash_key][sort_key] = value; } @@ -163,7 +163,7 @@ class copy_data_test : public test_util char buffer_[256]; map> expect_data_; - pegasus_client *srouce_client_; + pegasus_client *source_client_; pegasus_client *destination_client_; }; const char copy_data_test::CCH[] = @@ -176,7 +176,7 @@ TEST_F(copy_data_test, EMPTY_HASH_KEY_COPY) pegasus_client::scan_options options; options.return_expire_ts = true; vector raw_scanners; - ASSERT_EQ(PERR_OK, srouce_client_->get_unordered_scanners(INT_MAX, options, raw_scanners)); + ASSERT_EQ(PERR_OK, source_client_->get_unordered_scanners(INT_MAX, options, raw_scanners)); LOG_INFO("open source app scanner succeed, partition_count = {}", raw_scanners.size()); diff --git a/src/test/function_test/base_api_test/test_incr.cpp b/src/test/function_test/base_api/test_incr.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_incr.cpp rename to src/test/function_test/base_api/test_incr.cpp diff --git a/src/test/function_test/base_api_test/test_range_read.cpp b/src/test/function_test/base_api/test_range_read.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_range_read.cpp rename to src/test/function_test/base_api/test_range_read.cpp diff --git a/src/test/function_test/base_api_test/test_recall.cpp b/src/test/function_test/base_api/test_recall.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_recall.cpp rename to src/test/function_test/base_api/test_recall.cpp diff --git a/src/test/function_test/base_api_test/test_scan.cpp b/src/test/function_test/base_api/test_scan.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_scan.cpp rename to src/test/function_test/base_api/test_scan.cpp diff --git a/src/test/function_test/base_api_test/test_ttl.cpp b/src/test/function_test/base_api/test_ttl.cpp similarity index 100% rename from src/test/function_test/base_api_test/test_ttl.cpp rename to src/test/function_test/base_api/test_ttl.cpp diff --git a/src/test/function_test/bulk_load_test/CMakeLists.txt b/src/test/function_test/bulk_load/CMakeLists.txt similarity index 98% rename from src/test/function_test/bulk_load_test/CMakeLists.txt rename to src/test/function_test/bulk_load/CMakeLists.txt index 92a677ddf8..ea27eed37b 100644 --- a/src/test/function_test/bulk_load_test/CMakeLists.txt +++ b/src/test/function_test/bulk_load/CMakeLists.txt @@ -37,7 +37,8 @@ set(MY_PROJ_LIBS gssapi_krb5 krb5 function_test_utils - ) + test_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/test/function_test/bulk_load_test/main.cpp b/src/test/function_test/bulk_load/main.cpp similarity index 100% rename from src/test/function_test/bulk_load_test/main.cpp rename to src/test/function_test/bulk_load/main.cpp diff --git a/src/test/function_test/bulk_load/test_bulk_load.cpp b/src/test/function_test/bulk_load/test_bulk_load.cpp new file mode 100644 index 0000000000..28e52858f3 --- /dev/null +++ b/src/test/function_test/bulk_load/test_bulk_load.cpp @@ -0,0 +1,348 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +// IWYU pragma: no_include +// IWYU pragma: no_include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "base/pegasus_const.h" +#include "block_service/local/local_service.h" +#include "bulk_load_types.h" +#include "client/partition_resolver.h" +#include "client/replication_ddl_client.h" +#include "common/json_helper.h" +#include "include/pegasus/client.h" +#include "include/pegasus/error.h" +#include "meta/meta_bulk_load_service.h" +#include "meta_admin_types.h" +#include "test/function_test/utils/test_util.h" +#include "utils/blob.h" +#include "utils/enum_helper.h" +#include "utils/error_code.h" +#include "utils/errors.h" +#include "utils/filesystem.h" +#include "utils/test_macros.h" + +using namespace ::dsn; +using namespace ::dsn::replication; +using namespace pegasus; +using std::map; +using std::string; + +/// +/// Files: +/// `pegasus-bulk-load-function-test-files` folder stores sst files and metadata files used for +/// bulk load function tests +/// - `mock_bulk_load_info` sub-directory stores stores wrong bulk_load_info +/// - `bulk_load_root` sub-directory stores right data +/// - Please do not rename any files or directories under this folder +/// +/// The app to test bulk load functionality: +/// - partition count should be 8 +/// +/// Data: +/// hashkey: hash${i} sortkey: sort${i} value: newValue i=[0, 1000] +/// hashkey: hashkey${j} sortkey: sortkey${j} value: newValue j=[0, 1000] +/// +class bulk_load_test : public test_util +{ +protected: + bulk_load_test() : test_util(map({{"rocksdb.allow_ingest_behind", "true"}})) + { + TRICKY_CODE_TO_AVOID_LINK_ERROR; + bulk_load_local_app_root_ = + fmt::format("{}/{}/{}", kLocalBulkLoadRoot, kCluster, app_name_); + } + + void SetUp() override + { + test_util::SetUp(); + NO_FATALS(copy_bulk_load_files()); + } + + void TearDown() override + { + ASSERT_EQ(ERR_OK, ddl_client_->drop_app(app_name_, 0)); + NO_FATALS(run_cmd_from_project_root("rm -rf " + kLocalBulkLoadRoot)); + } + + // Generate the 'bulk_load_info' file according to 'bli' to path 'bulk_load_info_path'. + void generate_bulk_load_info(const bulk_load_info &bli, const std::string &bulk_load_info_path) + { + auto value = dsn::json::json_forwarder::encode(bli); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(value.data(), value.length()), + bulk_load_info_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + } + + // Generate the '.bulk_load_info.meta' file according to the 'bulk_load_info' file + // in path 'bulk_load_info_path'. + void generate_bulk_load_info_meta(const std::string &bulk_load_info_path) + { + dist::block_service::file_metadata fm; + ASSERT_TRUE(utils::filesystem::file_size(bulk_load_info_path, fm.size)); + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(bulk_load_info_path, fm.md5)); + std::string value = nlohmann::json(fm).dump(); + auto bulk_load_info_meta_path = + fmt::format("{}/{}/{}/.bulk_load_info.meta", kLocalBulkLoadRoot, kCluster, app_name_); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(value), + bulk_load_info_meta_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + } + + void copy_bulk_load_files() + { + // TODO(yingchun): remove the 'mock_bulk_load_info' file, because we can generate it. + // Prepare bulk load files. + // The source data has 8 partitions. + ASSERT_EQ(8, partition_count_); + NO_FATALS(run_cmd_from_project_root("mkdir -p " + kLocalBulkLoadRoot)); + NO_FATALS(run_cmd_from_project_root( + fmt::format("cp -r {}/{} {}", kSourceFilesRoot, kBulkLoad, kLocalServiceRoot))); + + // Generate 'bulk_load_info'. + auto bulk_load_info_path = + fmt::format("{}/{}/{}/bulk_load_info", kLocalBulkLoadRoot, kCluster, app_name_); + NO_FATALS(generate_bulk_load_info(bulk_load_info(app_id_, app_name_, partition_count_), + bulk_load_info_path)); + + // Generate '.bulk_load_info.meta'. + NO_FATALS(generate_bulk_load_info_meta(bulk_load_info_path)); + } + + error_code start_bulk_load(bool ingest_behind = false) + { + return ddl_client_ + ->start_bulk_load(app_name_, kCluster, kProvider, kBulkLoad, ingest_behind) + .get_value() + .err; + } + + void remove_file(const string &file_path) + { + NO_FATALS(run_cmd_from_project_root("rm " + file_path)); + } + + void update_allow_ingest_behind(const string &allow_ingest_behind) + { + const auto ret = ddl_client_->set_app_envs( + app_name_, {ROCKSDB_ALLOW_INGEST_BEHIND}, {allow_ingest_behind}); + ASSERT_EQ(ERR_OK, ret.get_value().err); + std::cout << "sleep 31s to wait app_envs update" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(31)); + } + + bulk_load_status::type wait_bulk_load_finish(int64_t remain_seconds) + { + int64_t sleep_time = 5; + auto err = ERR_OK; + + auto last_status = bulk_load_status::BLS_INVALID; + // when bulk load end, err will be ERR_INVALID_STATE + while (remain_seconds > 0 && err == ERR_OK) { + sleep_time = std::min(sleep_time, remain_seconds); + remain_seconds -= sleep_time; + std::cout << "sleep " << sleep_time << "s to query bulk status" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(sleep_time)); + + auto resp = ddl_client_->query_bulk_load(app_name_).get_value(); + err = resp.err; + if (err == ERR_OK) { + last_status = resp.app_status; + } + } + return last_status; + } + + void verify_bulk_load_data() + { + NO_FATALS(verify_data(kBulkLoadHashKeyPrefix1, kBulkLoadSortKeyPrefix1)); + NO_FATALS(verify_data(kBulkLoadHashKeyPrefix2, kBulkLoadSortKeyPrefix2)); + } + + void verify_data(const string &hashkey_prefix, const string &sortkey_prefix) + { + for (int i = 0; i < kBulkLoadItemCount; ++i) { + string hash_key = hashkey_prefix + std::to_string(i); + for (int j = 0; j < kBulkLoadItemCount; ++j) { + string sort_key = sortkey_prefix + std::to_string(j); + string actual_value; + ASSERT_EQ(PERR_OK, client_->get(hash_key, sort_key, actual_value)) + << hash_key << "," << sort_key; + ASSERT_EQ(kBulkLoadValue, actual_value) << hash_key << "," << sort_key; + } + } + } + + enum class operation + { + GET, + SET, + DEL, + NO_VALUE + }; + void operate_data(operation op, const string &value, int count) + { + for (int i = 0; i < count; ++i) { + auto hash_key = fmt::format("{}{}", kBulkLoadHashKeyPrefix2, i); + auto sort_key = fmt::format("{}{}", kBulkLoadSortKeyPrefix2, i); + switch (op) { + case operation::GET: { + string actual_value; + ASSERT_EQ(PERR_OK, client_->get(hash_key, sort_key, actual_value)); + ASSERT_EQ(value, actual_value); + } break; + case operation::DEL: { + ASSERT_EQ(PERR_OK, client_->del(hash_key, sort_key)); + } break; + case operation::SET: { + ASSERT_EQ(PERR_OK, client_->set(hash_key, sort_key, value)); + } break; + case operation::NO_VALUE: { + string actual_value; + ASSERT_EQ(PERR_NOT_FOUND, client_->get(hash_key, sort_key, actual_value)); + } break; + default: + ASSERT_TRUE(false); + break; + } + } + } + + void check_bulk_load(bool ingest_behind, + const std::string &value_before_bulk_load, + const std::string &value_after_bulk_load) + { + // Write some data before bulk load. + NO_FATALS(operate_data(operation::SET, value_before_bulk_load, 10)); + NO_FATALS(operate_data(operation::GET, value_before_bulk_load, 10)); + + // Start bulk load and wait until it complete. + ASSERT_EQ(ERR_OK, start_bulk_load(ingest_behind)); + ASSERT_EQ(bulk_load_status::BLS_SUCCEED, wait_bulk_load_finish(300)); + + std::cout << "Start to verify data..." << std::endl; + if (ingest_behind) { + // Values have NOT been overwritten by the bulk load data. + NO_FATALS(operate_data(operation::GET, value_before_bulk_load, 10)); + NO_FATALS(verify_data(kBulkLoadHashKeyPrefix1, kBulkLoadSortKeyPrefix1)); + } else { + // Values have been overwritten by the bulk load data. + NO_FATALS(operate_data(operation::GET, kBulkLoadValue, 10)); + NO_FATALS(verify_bulk_load_data()); + } + + // Write new data succeed after bulk load. + NO_FATALS(operate_data(operation::SET, value_after_bulk_load, 20)); + NO_FATALS(operate_data(operation::GET, value_after_bulk_load, 20)); + + // Delete data succeed after bulk load. + NO_FATALS(operate_data(operation::DEL, "", 15)); + NO_FATALS(operate_data(operation::NO_VALUE, "", 15)); + } + +protected: + string bulk_load_local_app_root_; + const string kSourceFilesRoot = + "src/test/function_test/bulk_load/pegasus-bulk-load-function-test-files"; + const string kLocalServiceRoot = "onebox/block_service/local_service"; + const string kLocalBulkLoadRoot = "onebox/block_service/local_service/bulk_load_root"; + const string kBulkLoad = "bulk_load_root"; + const string kCluster = "cluster"; + const string kProvider = "local_service"; + + const int32_t kBulkLoadItemCount = 1000; + const string kBulkLoadHashKeyPrefix1 = "hashkey"; + const string kBulkLoadSortKeyPrefix1 = "sortkey"; + const string kBulkLoadValue = "newValue"; + + // Real time write operations will use this prefix as well. + const string kBulkLoadHashKeyPrefix2 = "hash"; + const string kBulkLoadSortKeyPrefix2 = "sort"; +}; + +// Test bulk load failed because the 'bulk_load_info' file is missing +TEST_F(bulk_load_test, missing_bulk_load_info) +{ + NO_FATALS(remove_file(bulk_load_local_app_root_ + "/bulk_load_info")); + ASSERT_EQ(ERR_OBJECT_NOT_FOUND, start_bulk_load()); +} + +// Test bulk load failed because the 'bulk_load_info' file is inconsistent with the actual app info. +TEST_F(bulk_load_test, inconsistent_bulk_load_info) +{ + // Only 'app_id' and 'partition_count' will be checked in Pegasus server, so just inject these + // kind of inconsistencies. + bulk_load_info tests[] = {{app_id_ + 1, app_name_, partition_count_}, + {app_id_, app_name_, partition_count_ * 2}}; + for (const auto &test : tests) { + // Generate inconsistent 'bulk_load_info'. + auto bulk_load_info_path = + fmt::format("{}/{}/{}/bulk_load_info", kLocalBulkLoadRoot, kCluster, app_name_); + NO_FATALS(generate_bulk_load_info(test, bulk_load_info_path)); + + // Generate '.bulk_load_info.meta'. + NO_FATALS(generate_bulk_load_info_meta(bulk_load_info_path)); + + ASSERT_EQ(ERR_INCONSISTENT_STATE, start_bulk_load()) << test.app_id << "," << test.app_name + << "," << test.partition_count; + } +} + +// Test bulk load failed because partition[0]'s 'bulk_load_metadata' file is missing. +TEST_F(bulk_load_test, missing_p0_bulk_load_metadata) +{ + NO_FATALS(remove_file(bulk_load_local_app_root_ + "/0/bulk_load_metadata")); + ASSERT_EQ(ERR_OK, start_bulk_load()); + ASSERT_EQ(bulk_load_status::BLS_FAILED, wait_bulk_load_finish(300)); +} + +// Test bulk load failed because the allow_ingest_behind config is inconsistent. +TEST_F(bulk_load_test, allow_ingest_behind_inconsistent) +{ + NO_FATALS(update_allow_ingest_behind("false")); + ASSERT_EQ(ERR_INCONSISTENT_STATE, start_bulk_load(true)); +} + +// Test normal bulk load, old data will be overwritten by bulk load data. +TEST_F(bulk_load_test, normal) { check_bulk_load(false, "oldValue", "valueAfterBulkLoad"); } + +// Test normal bulk load with allow_ingest_behind=true, old data will NOT be overwritten by bulk +// load data. +TEST_F(bulk_load_test, allow_ingest_behind) +{ + NO_FATALS(update_allow_ingest_behind("true")); + check_bulk_load(true, "oldValue", "valueAfterBulkLoad"); +} diff --git a/src/test/function_test/bulk_load_test/test_bulk_load.cpp b/src/test/function_test/bulk_load_test/test_bulk_load.cpp deleted file mode 100644 index 83b2dd6ecc..0000000000 --- a/src/test/function_test/bulk_load_test/test_bulk_load.cpp +++ /dev/null @@ -1,317 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// IWYU pragma: no_include -// IWYU pragma: no_include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "base/pegasus_const.h" -#include "bulk_load_types.h" -#include "client/replication_ddl_client.h" -#include "include/pegasus/client.h" -#include "include/pegasus/error.h" -#include "meta_admin_types.h" -#include "metadata_types.h" -#include "test/function_test/utils/test_util.h" -#include "utils/error_code.h" -#include "utils/errors.h" -#include "utils/filesystem.h" -#include "utils/utils.h" - -using namespace ::dsn; -using namespace ::dsn::replication; -using namespace pegasus; -using std::map; -using std::string; - -/// -/// Files: -/// `pegasus-bulk-load-function-test-files` folder stores sst files and metadata files used for -/// bulk load function tests -/// - `mock_bulk_load_info` sub-directory stores stores wrong bulk_load_info -/// - `bulk_load_root` sub-directory stores right data -/// - Please do not rename any files or directories under this folder -/// -/// The app who is executing bulk load: -/// - app_name is `temp`, app_id is 2, partition_count is 8 -/// -/// Data: -/// hashkey: hashi sortkey: sorti value: newValue i=[0, 1000] -/// hashkey: hashkeyj sortkey: sortkeyj value: newValue j=[0, 1000] -/// -class bulk_load_test : public test_util -{ -protected: - bulk_load_test() : test_util(map({{"rocksdb.allow_ingest_behind", "true"}})) - { - TRICKY_CODE_TO_AVOID_LINK_ERROR; - bulk_load_local_root_ = - utils::filesystem::path_combine("onebox/block_service/local_service/", LOCAL_ROOT); - } - - void SetUp() override - { - test_util::SetUp(); - ASSERT_NO_FATAL_FAILURE(copy_bulk_load_files()); - } - - void TearDown() override - { - ASSERT_EQ(ERR_OK, ddl_client_->drop_app(app_name_, 0)); - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root("rm -rf onebox/block_service")); - } - - void copy_bulk_load_files() - { - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root("mkdir -p onebox/block_service")); - ASSERT_NO_FATAL_FAILURE( - run_cmd_from_project_root("mkdir -p onebox/block_service/local_service")); - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root( - "cp -r src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files/" + - LOCAL_ROOT + " onebox/block_service/local_service")); - string cmd = "echo '{\"app_id\":" + std::to_string(app_id_) + - ",\"app_name\":\"temp\",\"partition_count\":8}' > " - "onebox/block_service/local_service/bulk_load_root/cluster/temp/" - "bulk_load_info"; - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root(cmd)); - } - - error_code start_bulk_load(bool ingest_behind = false) - { - auto err_resp = - ddl_client_->start_bulk_load(app_name_, CLUSTER, PROVIDER, LOCAL_ROOT, ingest_behind); - return err_resp.get_value().err; - } - - void remove_file(const string &file_path) - { - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root("rm " + file_path)); - } - - void replace_bulk_load_info() - { - string cmd = "cp -R " - "src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files/" - "mock_bulk_load_info/. " + - bulk_load_local_root_ + "/" + CLUSTER + "/" + app_name_ + "/"; - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root(cmd)); - } - - void update_allow_ingest_behind(const string &allow_ingest_behind) - { - // update app envs - std::vector keys; - keys.emplace_back(ROCKSDB_ALLOW_INGEST_BEHIND); - std::vector values; - values.emplace_back(allow_ingest_behind); - ASSERT_EQ(ERR_OK, ddl_client_->set_app_envs(app_name_, keys, values).get_value().err); - std::cout << "sleep 31s to wait app_envs update" << std::endl; - std::this_thread::sleep_for(std::chrono::seconds(31)); - } - - bulk_load_status::type wait_bulk_load_finish(int64_t seconds) - { - int64_t sleep_time = 5; - error_code err = ERR_OK; - - bulk_load_status::type last_status = bulk_load_status::BLS_INVALID; - // when bulk load end, err will be ERR_INVALID_STATE - while (seconds > 0 && err == ERR_OK) { - sleep_time = sleep_time > seconds ? seconds : sleep_time; - seconds -= sleep_time; - std::cout << "sleep " << sleep_time << "s to query bulk status" << std::endl; - std::this_thread::sleep_for(std::chrono::seconds(sleep_time)); - - auto resp = ddl_client_->query_bulk_load(app_name_).get_value(); - err = resp.err; - if (err == ERR_OK) { - last_status = resp.app_status; - } - } - return last_status; - } - - void verify_bulk_load_data() - { - ASSERT_NO_FATAL_FAILURE(verify_data("hashkey", "sortkey")); - ASSERT_NO_FATAL_FAILURE(verify_data(HASHKEY_PREFIX, SORTKEY_PREFIX)); - } - - void verify_data(const string &hashkey_prefix, const string &sortkey_prefix) - { - const string &expected_value = VALUE; - for (int i = 0; i < COUNT; ++i) { - string hash_key = hashkey_prefix + std::to_string(i); - for (int j = 0; j < COUNT; ++j) { - string sort_key = sortkey_prefix + std::to_string(j); - string act_value; - ASSERT_EQ(PERR_OK, client_->get(hash_key, sort_key, act_value)) << hash_key << "," - << sort_key; - ASSERT_EQ(expected_value, act_value) << hash_key << "," << sort_key; - } - } - } - - enum operation - { - GET, - SET, - DEL, - NO_VALUE - }; - void operate_data(bulk_load_test::operation op, const string &value, int count) - { - for (int i = 0; i < count; ++i) { - string hash_key = HASHKEY_PREFIX + std::to_string(i); - string sort_key = SORTKEY_PREFIX + std::to_string(i); - switch (op) { - case bulk_load_test::operation::GET: { - string act_value; - ASSERT_EQ(PERR_OK, client_->get(hash_key, sort_key, act_value)); - ASSERT_EQ(value, act_value); - } break; - case bulk_load_test::operation::DEL: { - ASSERT_EQ(PERR_OK, client_->del(hash_key, sort_key)); - } break; - case bulk_load_test::operation::SET: { - ASSERT_EQ(PERR_OK, client_->set(hash_key, sort_key, value)); - } break; - case bulk_load_test::operation::NO_VALUE: { - string act_value; - ASSERT_EQ(PERR_NOT_FOUND, client_->get(hash_key, sort_key, act_value)); - } break; - default: - ASSERT_TRUE(false); - break; - } - } - } - -protected: - string bulk_load_local_root_; - - const string LOCAL_ROOT = "bulk_load_root"; - const string CLUSTER = "cluster"; - const string PROVIDER = "local_service"; - - const string HASHKEY_PREFIX = "hash"; - const string SORTKEY_PREFIX = "sort"; - const string VALUE = "newValue"; - const int32_t COUNT = 1000; -}; - -/// -/// case1: lack of `bulk_load_info` file -/// case2: `bulk_load_info` file inconsistent with app_info -/// -TEST_F(bulk_load_test, bulk_load_test_failed) -{ - // bulk load failed because `bulk_load_info` file is missing - ASSERT_NO_FATAL_FAILURE( - remove_file(bulk_load_local_root_ + "/" + CLUSTER + "/" + app_name_ + "/bulk_load_info")); - ASSERT_EQ(ERR_OBJECT_NOT_FOUND, start_bulk_load()); - - // bulk load failed because `bulk_load_info` file inconsistent with current app_info - ASSERT_NO_FATAL_FAILURE(replace_bulk_load_info()); - ASSERT_EQ(ERR_INCONSISTENT_STATE, start_bulk_load()); -} - -/// -/// case1: lack of `bulk_load_metadata` file -/// case2: bulk load succeed with data verfied -/// case3: bulk load data consistent: -/// - old data will be overrided by bulk load data -/// - get/set/del succeed after bulk load -/// -TEST_F(bulk_load_test, bulk_load_tests) -{ - // bulk load failed because partition[0] `bulk_load_metadata` file is missing - ASSERT_NO_FATAL_FAILURE(remove_file(bulk_load_local_root_ + "/" + CLUSTER + "/" + app_name_ + - "/0/bulk_load_metadata")); - ASSERT_EQ(ERR_OK, start_bulk_load()); - // bulk load will get FAILED - ASSERT_EQ(bulk_load_status::BLS_FAILED, wait_bulk_load_finish(300)); - - // recover complete files - ASSERT_NO_FATAL_FAILURE(copy_bulk_load_files()); - - // write old data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "oldValue", 10)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "oldValue", 10)); - - ASSERT_EQ(ERR_OK, start_bulk_load()); - ASSERT_EQ(bulk_load_status::BLS_SUCCEED, wait_bulk_load_finish(300)); - std::cout << "Start to verify data..." << std::endl; - ASSERT_NO_FATAL_FAILURE(verify_bulk_load_data()); - - // value overide by bulk_loaded_data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, VALUE, 10)); - - // write data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "valueAfterBulkLoad", 20)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "valueAfterBulkLoad", 20)); - - // del data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::DEL, "", 15)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::NO_VALUE, "", 15)); -} - -/// -/// case1: inconsistent ingest_behind -/// case2: bulk load(ingest_behind) succeed with data verfied -/// case3: bulk load data consistent: -/// - bulk load data will be overrided by old data -/// - get/set/del succeed after bulk load -/// -TEST_F(bulk_load_test, bulk_load_ingest_behind_tests) -{ - ASSERT_NO_FATAL_FAILURE(update_allow_ingest_behind("false")); - - // app envs allow_ingest_behind = false, request ingest_behind = true - ASSERT_EQ(ERR_INCONSISTENT_STATE, start_bulk_load(true)); - - ASSERT_NO_FATAL_FAILURE(update_allow_ingest_behind("true")); - - // write old data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "oldValue", 10)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "oldValue", 10)); - - ASSERT_EQ(ERR_OK, start_bulk_load(true)); - ASSERT_EQ(bulk_load_status::BLS_SUCCEED, wait_bulk_load_finish(300)); - - std::cout << "Start to verify data..." << std::endl; - // value overide by bulk_loaded_data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "oldValue", 10)); - ASSERT_NO_FATAL_FAILURE(verify_data("hashkey", "sortkey")); - - // write data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "valueAfterBulkLoad", 20)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "valueAfterBulkLoad", 20)); - - // del data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::DEL, "", 15)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::NO_VALUE, "", 15)); -} diff --git a/src/test/function_test/detect_hotspot_test/CMakeLists.txt b/src/test/function_test/detect_hotspot/CMakeLists.txt similarity index 100% rename from src/test/function_test/detect_hotspot_test/CMakeLists.txt rename to src/test/function_test/detect_hotspot/CMakeLists.txt diff --git a/src/test/function_test/detect_hotspot_test/main.cpp b/src/test/function_test/detect_hotspot/main.cpp similarity index 100% rename from src/test/function_test/detect_hotspot_test/main.cpp rename to src/test/function_test/detect_hotspot/main.cpp diff --git a/src/test/function_test/detect_hotspot_test/test_detect_hotspot.cpp b/src/test/function_test/detect_hotspot/test_detect_hotspot.cpp similarity index 100% rename from src/test/function_test/detect_hotspot_test/test_detect_hotspot.cpp rename to src/test/function_test/detect_hotspot/test_detect_hotspot.cpp diff --git a/src/test/function_test/partition_split_test/CMakeLists.txt b/src/test/function_test/partition_split/CMakeLists.txt similarity index 100% rename from src/test/function_test/partition_split_test/CMakeLists.txt rename to src/test/function_test/partition_split/CMakeLists.txt diff --git a/src/test/function_test/partition_split_test/main.cpp b/src/test/function_test/partition_split/main.cpp similarity index 100% rename from src/test/function_test/partition_split_test/main.cpp rename to src/test/function_test/partition_split/main.cpp diff --git a/src/test/function_test/partition_split_test/test_split.cpp b/src/test/function_test/partition_split/test_split.cpp similarity index 100% rename from src/test/function_test/partition_split_test/test_split.cpp rename to src/test/function_test/partition_split/test_split.cpp diff --git a/src/test/function_test/recovery_test/CMakeLists.txt b/src/test/function_test/recovery/CMakeLists.txt similarity index 100% rename from src/test/function_test/recovery_test/CMakeLists.txt rename to src/test/function_test/recovery/CMakeLists.txt diff --git a/src/test/function_test/recovery_test/main.cpp b/src/test/function_test/recovery/main.cpp similarity index 100% rename from src/test/function_test/recovery_test/main.cpp rename to src/test/function_test/recovery/main.cpp diff --git a/src/test/function_test/recovery_test/test_recovery.cpp b/src/test/function_test/recovery/test_recovery.cpp similarity index 100% rename from src/test/function_test/recovery_test/test_recovery.cpp rename to src/test/function_test/recovery/test_recovery.cpp diff --git a/src/test/function_test/restore_test/CMakeLists.txt b/src/test/function_test/restore/CMakeLists.txt similarity index 100% rename from src/test/function_test/restore_test/CMakeLists.txt rename to src/test/function_test/restore/CMakeLists.txt diff --git a/src/test/function_test/restore_test/main.cpp b/src/test/function_test/restore/main.cpp similarity index 100% rename from src/test/function_test/restore_test/main.cpp rename to src/test/function_test/restore/main.cpp diff --git a/src/test/function_test/restore_test/test_restore.cpp b/src/test/function_test/restore/test_restore.cpp similarity index 100% rename from src/test/function_test/restore_test/test_restore.cpp rename to src/test/function_test/restore/test_restore.cpp diff --git a/src/test/function_test/run.sh b/src/test/function_test/run.sh index 9bd561386b..947cc35f8f 100755 --- a/src/test/function_test/run.sh +++ b/src/test/function_test/run.sh @@ -26,7 +26,7 @@ fi if [ -n ${TEST_OPTS} ]; then if [ ! -f ./config.ini ]; then - echo "./config.ini does not exists !" + echo "./config.ini does not exists" exit 1 fi diff --git a/src/test/function_test/throttle_test/CMakeLists.txt b/src/test/function_test/throttle/CMakeLists.txt similarity index 100% rename from src/test/function_test/throttle_test/CMakeLists.txt rename to src/test/function_test/throttle/CMakeLists.txt diff --git a/src/test/function_test/throttle_test/main.cpp b/src/test/function_test/throttle/main.cpp similarity index 100% rename from src/test/function_test/throttle_test/main.cpp rename to src/test/function_test/throttle/main.cpp diff --git a/src/test/function_test/throttle_test/test_throttle.cpp b/src/test/function_test/throttle/test_throttle.cpp similarity index 99% rename from src/test/function_test/throttle_test/test_throttle.cpp rename to src/test/function_test/throttle/test_throttle.cpp index a29c9c5fe1..23583a7bfe 100644 --- a/src/test/function_test/throttle_test/test_throttle.cpp +++ b/src/test/function_test/throttle/test_throttle.cpp @@ -131,7 +131,7 @@ struct throttle_test_recorder // read/write throttle function test // the details of records are saved in -// `./src/builder/test/function_test/throttle_test/throttle_test_result.txt` +// `./src/builder/test/function_test/throttle/throttle_test_result.txt` class throttle_test : public test_util { public: diff --git a/src/test_util/CMakeLists.txt b/src/test_util/CMakeLists.txt index 267c825e40..b1e7ac2dff 100644 --- a/src/test_util/CMakeLists.txt +++ b/src/test_util/CMakeLists.txt @@ -22,6 +22,6 @@ set(MY_PROJ_NAME test_utils) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest) +set(MY_PROJ_LIBS gtest rocksdb) dsn_add_static_library() diff --git a/src/test_util/test_util.cpp b/src/test_util/test_util.cpp index 5ab185503d..935d50631a 100644 --- a/src/test_util/test_util.cpp +++ b/src/test_util/test_util.cpp @@ -18,7 +18,6 @@ #include "test_util.h" #include -#include #include #include #include @@ -26,12 +25,33 @@ #include "gtest/gtest-message.h" #include "gtest/gtest-test-part.h" #include "gtest/gtest.h" +#include "metadata_types.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/api_layer1.h" #include "utils/defer.h" +#include "utils/env.h" +#include "utils/error_code.h" +#include "utils/filesystem.h" #include "utils/fmt_logging.h" namespace pegasus { +void create_local_test_file(const std::string &full_name, dsn::replication::file_meta *fm) +{ + ASSERT_NE(fm, nullptr); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice("write some data."), + full_name, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + fm->name = full_name; + ASSERT_EQ(dsn::ERR_OK, dsn::utils::filesystem::md5sum(full_name, fm->md5)); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + full_name, dsn::utils::FileDataType::kSensitive, fm->size)); +} + void AssertEventually(const std::function &f, int timeout_sec, WaitBackoff backoff) { // TODO(yingchun): should use mono time diff --git a/src/test_util/test_util.h b/src/test_util/test_util.h index ec7bce6e2e..23b1624be7 100644 --- a/src/test_util/test_util.h +++ b/src/test_util/test_util.h @@ -19,12 +19,54 @@ #pragma once +#include +#include +#include #include +#include +#include +#include "fmt/core.h" +#include "runtime/api_layer1.h" +#include "utils/flags.h" #include "utils/test_macros.h" +namespace dsn { +namespace replication { +class file_meta; +} // namespace replication +} // namespace dsn + +DSN_DECLARE_bool(encrypt_data_at_rest); + namespace pegasus { +// A base parameterized test class for testing enable/disable encryption at rest. +class encrypt_data_test_base : public testing::TestWithParam +{ +public: + encrypt_data_test_base() { FLAGS_encrypt_data_at_rest = GetParam(); } +}; + +class stop_watch +{ +public: + stop_watch() { _start_ms = dsn_now_ms(); } + void stop_and_output(const std::string &msg) + { + auto duration_ms = + std::chrono::duration_cast>( + std::chrono::milliseconds(static_cast(dsn_now_ms() - _start_ms))) + .count(); + fmt::print(stdout, "{}, cost {} ms\n", msg, duration_ms); + } + +private: + uint64_t _start_ms = 0; +}; + +void create_local_test_file(const std::string &full_name, dsn::replication::file_meta *fm); + #define ASSERT_EVENTUALLY(expr) \ do { \ AssertEventually(expr); \ @@ -39,7 +81,7 @@ namespace pegasus { #define ASSERT_IN_TIME_WITH_FIXED_INTERVAL(expr, sec) \ do { \ - AssertEventually(expr, sec, WaitBackoff::NONE); \ + AssertEventually(expr, sec, ::pegasus::WaitBackoff::NONE); \ NO_PENDING_FATALS(); \ } while (0) diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index e17fef9001..1127a868ef 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -31,7 +31,7 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) -set(MY_PROJ_LIBS dsn_http crypto) +set(MY_PROJ_LIBS dsn_http crypto rocksdb) # Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/utils/alloc.h b/src/utils/alloc.h index 195f740f6b..542f4efb51 100644 --- a/src/utils/alloc.h +++ b/src/utils/alloc.h @@ -17,6 +17,8 @@ #pragma once +#include + #include "utils/ports.h" // The check for the definition of CACHELINE_SIZE has to be put after including "utils/ports.h", @@ -24,7 +26,7 @@ #ifdef CACHELINE_SIZE #include -#include +#include // IWYU pragma: keep #include #include #include diff --git a/src/utils/api_utilities.h b/src/utils/api_utilities.h index b32ff52220..4e1d664ed0 100644 --- a/src/utils/api_utilities.h +++ b/src/utils/api_utilities.h @@ -34,6 +34,7 @@ #include #include "ports.h" +#include "utils/fmt_utils.h" /*! @defgroup logging Logging Service @@ -57,6 +58,8 @@ typedef enum dsn_log_level_t { LOG_LEVEL_INVALID } dsn_log_level_t; +USER_DEFINED_ENUM_FORMATTER(dsn_log_level_t) + // logs with level smaller than this start_level will not be logged extern dsn_log_level_t dsn_log_start_level; extern dsn_log_level_t dsn_log_get_start_level(); diff --git a/src/utils/autoref_ptr.h b/src/utils/autoref_ptr.h index a08ebc6ee4..a3a7dfd2b2 100644 --- a/src/utils/autoref_ptr.h +++ b/src/utils/autoref_ptr.h @@ -159,6 +159,8 @@ class ref_ptr void swap(ref_ptr &r) noexcept { std::swap(_obj, r._obj); } + void reset(T *obj = nullptr) { *this = obj; } + T *get() const { return _obj; } operator T *() const { return _obj; } diff --git a/src/utils/command_manager.cpp b/src/utils/command_manager.cpp index 723f8f3447..38d84054b5 100644 --- a/src/utils/command_manager.cpp +++ b/src/utils/command_manager.cpp @@ -27,7 +27,6 @@ #include "utils/command_manager.h" #include -#include #include #include #include // IWYU pragma: keep diff --git a/src/utils/config_api.cpp b/src/utils/config_api.cpp index 6319f39dab..537ffed659 100644 --- a/src/utils/config_api.cpp +++ b/src/utils/config_api.cpp @@ -25,6 +25,9 @@ */ #include "utils/config_api.h" + +#include + #include "utils/configuration.h" dsn::configuration g_config; diff --git a/src/utils/configuration.cpp b/src/utils/configuration.cpp index 9aea53448e..c69a556d2c 100644 --- a/src/utils/configuration.cpp +++ b/src/utils/configuration.cpp @@ -33,14 +33,16 @@ * xxxx-xx-xx, author, fix bug about xxx */ -#include +#include +#include +#include #include #include #include #include #include "utils/configuration.h" -#include "utils/filesystem.h" +#include "utils/env.h" #include "utils/strings.h" namespace dsn { @@ -67,33 +69,17 @@ bool configuration::load(const char *file_name, const char *arguments) { _file_name = std::string(file_name); - FILE *fd = ::fopen(file_name, "rb"); - if (fd == nullptr) { - std::string cdir; - dsn::utils::filesystem::get_current_directory(cdir); - printf("ERROR: cannot open file %s in %s, err = %s\n", - file_name, - cdir.c_str(), - strerror(errno)); - return false; - } - ::fseek(fd, 0, SEEK_END); - int len = ftell(fd); - if (len == -1 || len == 0) { - printf("ERROR: cannot get length of %s, err = %s\n", file_name, strerror(errno)); - ::fclose(fd); + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), _file_name, &_file_data); + if (!s.ok()) { + fmt::print(stderr, "ERROR: read file '{}' failed, err = {}\n", _file_name, s.ToString()); return false; } - _file_data.resize(len + 1); - ::fseek(fd, 0, SEEK_SET); - auto sz = ::fread((char *)_file_data.c_str(), len, 1, fd); - ::fclose(fd); - if (sz != 1) { - printf("ERROR: cannot read correct data of %s, err = %s\n", file_name, strerror(errno)); + if (_file_data.empty()) { + fmt::print(stderr, "ERROR: file '{}' is empty\n", _file_name); return false; } - _file_data[len] = '\n'; // replace data with arguments if (arguments != nullptr) { diff --git a/src/utils/env.cpp b/src/utils/env.cpp new file mode 100644 index 0000000000..7e49d999a5 --- /dev/null +++ b/src/utils/env.cpp @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "env.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "utils/defer.h" +#include "utils/filesystem.h" +#include "utils/flags.h" +#include "utils/fmt_logging.h" +#include "utils/utils.h" + +DSN_DEFINE_bool(pegasus.server, + encrypt_data_at_rest, + false, + "Whether the sensitive files should be encrypted on the file system."); + +DSN_DEFINE_string(pegasus.server, + server_key_for_testing, + "server_key_for_testing", + "The encrypted server key to use in the filesystem. NOTE: only for testing."); + +DSN_DEFINE_string(pegasus.server, + encryption_method, + "AES128CTR", + "The encryption method to use in the filesystem. Now " + "supports AES128CTR, AES192CTR, AES256CTR and SM4CTR."); + +DSN_DEFINE_bool(replication, + enable_direct_io, + false, + "Whether to enable direct I/O when download files"); +DSN_TAG_VARIABLE(enable_direct_io, FT_MUTABLE); + +namespace dsn { +namespace utils { + +rocksdb::Env *NewEncryptedEnv() +{ + // Create an encryption provider. + std::shared_ptr provider; + auto provider_id = + fmt::format("AES:{},{}", FLAGS_server_key_for_testing, FLAGS_encryption_method); + auto s = rocksdb::EncryptionProvider::CreateFromString( + rocksdb::ConfigOptions(), provider_id, &provider); + CHECK(s.ok(), "Failed to create encryption provider: {}", s.ToString()); + + // Create an encrypted env. + return NewEncryptedEnv(rocksdb::Env::Default(), provider); +} + +rocksdb::Env *PegasusEnv(FileDataType type) +{ + // Return an encrypted env only when the file is sensitive and FLAGS_encrypt_data_at_rest + // is enabled at the same time. + if (FLAGS_encrypt_data_at_rest && type == FileDataType::kSensitive) { + static rocksdb::Env *env = NewEncryptedEnv(); + return env; + } + + // Otherwise, return a common non-encrypted env. + static rocksdb::Env *env = rocksdb::Env::Default(); + return env; +} + +namespace { +rocksdb::Status do_copy_file(const std::string &src_fname, + dsn::utils::FileDataType src_type, + const std::string &dst_fname, + dsn::utils::FileDataType dst_type, + int64_t remain_size, + uint64_t *total_size) +{ + rocksdb::EnvOptions src_env_options; + src_env_options.use_direct_reads = FLAGS_enable_direct_io; + std::unique_ptr src_file; + auto s = + dsn::utils::PegasusEnv(src_type)->NewSequentialFile(src_fname, &src_file, src_env_options); + LOG_AND_RETURN_NOT_RDB_OK(WARNING, s, "failed to open file {} for reading", src_fname); + + // Limit the size of the file to be copied. + int64_t src_file_size; + CHECK_TRUE(dsn::utils::filesystem::file_size(src_fname, src_type, src_file_size)); + if (remain_size == -1) { + // Copy the whole file if 'remain_size' is -1. + remain_size = src_file_size; + } else { + remain_size = std::min(remain_size, src_file_size); + } + + rocksdb::EnvOptions dst_env_options; + dst_env_options.use_direct_writes = FLAGS_enable_direct_io; + std::unique_ptr dst_file; + s = dsn::utils::PegasusEnv(dst_type)->NewWritableFile(dst_fname, &dst_file, dst_env_options); + LOG_AND_RETURN_NOT_RDB_OK(WARNING, s, "failed to open file {} for writing", dst_fname); + + // Read at most 4MB once. + // TODO(yingchun): make it configurable. + const uint64_t kCopyBlockSize = 4 << 20; + auto buffer = dsn::utils::make_shared_array(kCopyBlockSize); + uint64_t offset = 0; + do { + int bytes_per_copy = std::min(remain_size, static_cast(kCopyBlockSize)); + // Reach the EOF. + if (bytes_per_copy <= 0) { + break; + } + + rocksdb::Slice result; + LOG_AND_RETURN_NOT_RDB_OK(WARNING, + src_file->Read(bytes_per_copy, &result, buffer.get()), + "failed to read file {}", + src_fname); + CHECK(!result.empty(), + "read file {} at offset {} with size {} failed", + src_fname, + offset, + bytes_per_copy); + LOG_AND_RETURN_NOT_RDB_OK( + WARNING, dst_file->Append(result), "failed to write file {}", dst_fname); + + offset += result.size(); + remain_size -= result.size(); + + // Reach the EOF. + if (result.size() < bytes_per_copy) { + break; + } + } while (true); + LOG_AND_RETURN_NOT_RDB_OK(WARNING, dst_file->Fsync(), "failed to fsync file {}", dst_fname); + + if (total_size != nullptr) { + *total_size = offset; + } + + LOG_INFO("copy file from {} to {}, total size {}", src_fname, dst_fname, offset); + return rocksdb::Status::OK(); +} +} // anonymous namespace + +rocksdb::Status +copy_file(const std::string &src_fname, const std::string &dst_fname, uint64_t *total_size) +{ + // TODO(yingchun): Consider to use hard link, i.e. rocksdb::Env()::LinkFile(). + return do_copy_file( + src_fname, FileDataType::kSensitive, dst_fname, FileDataType::kSensitive, -1, total_size); +} + +rocksdb::Status +encrypt_file(const std::string &src_fname, const std::string &dst_fname, uint64_t *total_size) +{ + return do_copy_file(src_fname, + FileDataType::kNonSensitive, + dst_fname, + FileDataType::kSensitive, + -1, + total_size); +} + +rocksdb::Status encrypt_file(const std::string &fname, uint64_t *total_size) +{ + // TODO(yingchun): add timestamp to the tmp encrypted file name. + std::string tmp_fname = fname + ".encrypted.tmp"; + auto cleanup = dsn::defer([tmp_fname]() { utils::filesystem::remove_path(tmp_fname); }); + LOG_AND_RETURN_NOT_RDB_OK( + WARNING, encrypt_file(fname, tmp_fname, total_size), "failed to encrypt file {}", fname); + if (!::dsn::utils::filesystem::rename_path(tmp_fname, fname)) { + LOG_WARNING("rename file from {} to {} failed", tmp_fname, fname); + return rocksdb::Status::IOError("rename file failed"); + } + return rocksdb::Status::OK(); +} + +rocksdb::Status +copy_file_by_size(const std::string &src_fname, const std::string &dst_fname, int64_t limit_size) +{ + return do_copy_file(src_fname, + FileDataType::kSensitive, + dst_fname, + FileDataType::kSensitive, + limit_size, + nullptr); +} + +} // namespace utils +} // namespace dsn diff --git a/src/utils/env.h b/src/utils/env.h new file mode 100644 index 0000000000..cdaf16aaa5 --- /dev/null +++ b/src/utils/env.h @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace dsn { +namespace utils { + +// Indicate whether the file is sensitive or not. +// Only the sensitive file will be encrypted if FLAGS_encrypt_data_at_rest +// is enabled at the same time. +enum class FileDataType +{ + kSensitive = 0, + kNonSensitive = 1 +}; + +static const size_t kEncryptionHeaderkSize = rocksdb::kDefaultPageSize; + +// Get the rocksdb::Env instance for the given file type. +rocksdb::Env *PegasusEnv(FileDataType type); + +// Encrypt the original non-encrypted 'src_fname' to 'dst_fname'. +// The 'total_size' is the total size of the file content, exclude the file encryption header +// (typically 4KB). +rocksdb::Status encrypt_file(const std::string &src_fname, + const std::string &dst_fname, + uint64_t *total_size = nullptr); + +// Similar to the above, but encrypt the file in the same path. +rocksdb::Status encrypt_file(const std::string &fname, uint64_t *total_size = nullptr); + +// Copy the original 'src_fname' to 'dst_fname'. +// Both 'src_fname' and 'dst_fname' are sensitive files. +rocksdb::Status copy_file(const std::string &src_fname, + const std::string &dst_fname, + uint64_t *total_size = nullptr); + +// Similar to the above, but copy the file by a limited size. +// Both 'src_fname' and 'dst_fname' are sensitive files, 'limit_size' is the max size of the +// file to copy, and -1 means no limit. +rocksdb::Status copy_file_by_size(const std::string &src_fname, + const std::string &dst_fname, + int64_t limit_size = -1); +} // namespace utils +} // namespace dsn diff --git a/src/utils/error_code.h b/src/utils/error_code.h index 25feabd48a..04df97947a 100644 --- a/src/utils/error_code.h +++ b/src/utils/error_code.h @@ -30,6 +30,7 @@ #include #include +#include "utils/fmt_utils.h" #include "utils/ports.h" namespace apache { @@ -53,9 +54,12 @@ class error_code const char *to_string() const; - constexpr bool operator==(const error_code &r) { return _internal_code == r._internal_code; } + constexpr bool operator==(const error_code &r) const + { + return _internal_code == r._internal_code; + } - constexpr bool operator!=(const error_code &r) { return !(*this == r); } + constexpr bool operator!=(const error_code &r) const { return !(*this == r); } constexpr operator int() const { return _internal_code; } @@ -176,4 +180,8 @@ DEFINE_ERR_CODE(ERR_RANGER_POLICIES_NO_NEED_UPDATE) DEFINE_ERR_CODE(ERR_RDB_CORRUPTION) DEFINE_ERR_CODE(ERR_DISK_IO_ERROR) + +DEFINE_ERR_CODE(ERR_CURL_FAILED) } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::error_code); diff --git a/src/utils/errors.h b/src/utils/errors.h index 8cc47a74b8..ce36befca9 100644 --- a/src/utils/errors.h +++ b/src/utils/errors.h @@ -31,6 +31,7 @@ #include "utils/api_utilities.h" #include "utils/error_code.h" #include "utils/fmt_logging.h" +#include "utils/fmt_utils.h" #include "utils/ports.h" #include "utils/smart_pointers.h" #include "utils/string_view.h" @@ -96,6 +97,8 @@ class error_s return true; } + explicit operator bool() const noexcept { return is_ok(); } + std::string description() const { if (!_info) { @@ -219,12 +222,14 @@ class error_with } // namespace dsn +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::error_s); + #define FMT_ERR(ec, msg, args...) error_s::make(ec, fmt::format(msg, ##args)) #define RETURN_NOT_OK(s) \ do { \ const ::dsn::error_s &_s = (s); \ - if (dsn_unlikely(!_s.is_ok())) { \ + if (dsn_unlikely(!_s)) { \ return _s; \ } \ } while (false); diff --git a/src/utils/fail_point_impl.h b/src/utils/fail_point_impl.h index b8b96fd9c2..22212e41d9 100644 --- a/src/utils/fail_point_impl.h +++ b/src/utils/fail_point_impl.h @@ -30,13 +30,15 @@ #pragma once -#include "utils/fail_point.h" -#include "utils/api_utilities.h" -#include "utils/ports.h" #include #include #include +#include "utils/api_utilities.h" +#include "utils/fail_point.h" +#include "utils/fmt_utils.h" +#include "utils/ports.h" + namespace dsn { namespace fail { @@ -97,6 +99,7 @@ struct fail_point int _freq{100}; int _max_cnt{-1}; // TODO(wutao1): not thread-safe }; +USER_DEFINED_ENUM_FORMATTER(fail_point::task_type) struct fail_point_registry { diff --git a/src/utils/filesystem.cpp b/src/utils/filesystem.cpp index 1954f29b5a..211aefbecb 100644 --- a/src/utils/filesystem.cpp +++ b/src/utils/filesystem.cpp @@ -33,31 +33,31 @@ * xxxx-xx-xx, author, fix bug about xxx */ -// IWYU pragma: no_include -#include // IWYU pragma: keep #include #include #include -#include #include #include #include #include +#include +#include +#include #include #include #include +// IWYU pragma: no_include #include -#include -#include +#include #include "utils/defer.h" +#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/ports.h" #include "utils/safe_strerror_posix.h" #include "utils/string_view.h" -#include "utils/strings.h" #define getcwd_ getcwd #define rmdir_ rmdir @@ -94,8 +94,7 @@ static inline int get_stat_internal(const std::string &npath, struct stat_ &st) return err; } -// TODO(yingchun): remove the return value because it's always 0. -int get_normalized_path(const std::string &path, std::string &npath) +void get_normalized_path(const std::string &path, std::string &npath) { char sep; size_t i; @@ -105,7 +104,7 @@ int get_normalized_path(const std::string &path, std::string &npath) if (path.empty()) { npath = ""; - return 0; + return; } len = path.length(); @@ -131,8 +130,6 @@ int get_normalized_path(const std::string &path, std::string &npath) CHECK_NE_MSG(tls_path_buffer[0], _FS_NULL, "Normalized path cannot be empty!"); npath = tls_path_buffer; - - return 0; } static __thread struct @@ -210,44 +207,31 @@ bool path_exists(const std::string &path) return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + get_normalized_path(path, npath); return dsn::utils::filesystem::path_exists_internal(npath, FTW_NS); } bool directory_exists(const std::string &path) { - std::string npath; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); return dsn::utils::filesystem::path_exists_internal(npath, FTW_D); } bool file_exists(const std::string &path) { - std::string npath; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); return dsn::utils::filesystem::path_exists_internal(npath, FTW_F); } @@ -257,23 +241,18 @@ static bool get_subpaths(const std::string &path, bool recursive, int typeflags) { - std::string npath; - bool ret; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); if (!dsn::utils::filesystem::path_exists_internal(npath, FTW_D)) { return false; } + bool ret; switch (typeflags) { case FTW_F: ret = dsn::utils::filesystem::file_tree_walk( @@ -351,17 +330,12 @@ static bool remove_directory(const std::string &npath) bool remove_path(const std::string &path) { - std::string npath; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); if (dsn::utils::filesystem::path_exists_internal(npath, FTW_F)) { bool ret = (::remove(npath.c_str()) == 0); @@ -389,22 +363,17 @@ bool rename_path(const std::string &path1, const std::string &path2) return ret; } -bool file_size(const std::string &path, int64_t &sz) +bool deprecated_file_size(const std::string &path, int64_t &sz) { - struct stat_ st; - std::string npath; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); - err = dsn::utils::filesystem::get_stat_internal(npath, st); + struct stat_ st; + int err = dsn::utils::filesystem::get_stat_internal(npath, st); if (err != 0) { return false; } @@ -414,7 +383,23 @@ bool file_size(const std::string &path, int64_t &sz) } sz = st.st_size; + return true; +} + +bool file_size(const std::string &path, int64_t &sz) +{ + return file_size(path, dsn::utils::FileDataType::kNonSensitive, sz); +} +bool file_size(const std::string &path, FileDataType type, int64_t &sz) +{ + uint64_t file_size = 0; + auto s = dsn::utils::PegasusEnv(type)->GetFileSize(path, &file_size); + if (!s.ok()) { + LOG_ERROR("GetFileSize failed, file '{}', err = {}", path, s.ToString()); + return false; + } + sz = file_size; return true; } @@ -442,18 +427,14 @@ bool create_directory(const std::string &path) std::string npath; std::string cpath; size_t len; - int err; if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + get_normalized_path(path, npath); - err = dsn::utils::filesystem::create_directory_component(npath); + int err = dsn::utils::filesystem::create_directory_component(npath); if (err == 0) { return true; } else if (err != ENOENT) { @@ -497,53 +478,25 @@ bool create_directory(const std::string &path) bool create_file(const std::string &path) { - size_t pos; std::string npath; - int fd; - int mode; - int err; - - if (path.empty()) { - return false; - } - - if (_FS_ISSEP(path.back())) { - return false; - } - - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } - - if (dsn::utils::filesystem::path_exists_internal(npath, FTW_F)) { - return true; - } + get_normalized_path(path, npath); - if (dsn::utils::filesystem::path_exists_internal(npath, FTW_D)) { - return false; - } - - pos = npath.find_last_of("\\/"); + auto pos = npath.find_last_of("\\/"); if ((pos != std::string::npos) && (pos > 0)) { auto ppath = npath.substr(0, pos); if (!dsn::utils::filesystem::create_directory(ppath)) { + LOG_WARNING("fail to create directory {}", ppath); return false; } } - mode = 0775; - fd = ::creat(npath.c_str(), mode); - if (fd == -1) { - err = errno; - LOG_WARNING("create_file {} failed, err = {}", path, safe_strerror(err)); + std::unique_ptr wfile; + auto s = rocksdb::Env::Default()->ReopenWritableFile(path, &wfile, rocksdb::EnvOptions()); + if (dsn_unlikely(!s.ok())) { + LOG_WARNING("fail to create file {}, err={}", path, s.ToString()); return false; } - if (::close_(fd) != 0) { - LOG_WARNING("create_file {}, failed to close the file handle.", path); - } - return true; } @@ -608,23 +561,20 @@ std::string get_file_name(const std::string &path) std::string path_combine(const std::string &path1, const std::string &path2) { - int err; - std::string path3; std::string npath; - if (path1.empty()) { - err = dsn::utils::filesystem::get_normalized_path(path2, npath); + get_normalized_path(path2, npath); } else if (path2.empty()) { - err = dsn::utils::filesystem::get_normalized_path(path1, npath); + get_normalized_path(path1, npath); } else { - path3 = path1; + std::string path3 = path1; path3.append(1, _FS_SLASH); path3.append(path2); - err = dsn::utils::filesystem::get_normalized_path(path3, npath); + get_normalized_path(path3, npath); } - return ((err == 0) ? npath : ""); + return npath; } bool get_current_directory(std::string &path) @@ -641,20 +591,15 @@ bool get_current_directory(std::string &path) bool last_write_time(const std::string &path, time_t &tm) { - struct stat_ st; - std::string npath; - int err; - if (path.empty()) { return false; } - err = get_normalized_path(path, npath); - if (err != 0) { - return false; - } + std::string npath; + get_normalized_path(path, npath); - err = dsn::utils::filesystem::get_stat_internal(npath, st); + struct stat_ st; + int err = dsn::utils::filesystem::get_stat_internal(npath, st); if (err != 0) { return false; } @@ -694,7 +639,7 @@ bool get_disk_space_info(const std::string &path, disk_space_info &info) FAIL_POINT_INJECT_F("filesystem_get_disk_space_info", [&info](string_view str) { info.capacity = 100 * 1024 * 1024; if (str.find("insufficient") != string_view::npos) { - info.available = 5 * 1024 * 1024; + info.available = 512 * 1024; } else { info.available = 50 * 1024 * 1024; } @@ -725,6 +670,54 @@ bool link_file(const std::string &src, const std::string &target) } error_code md5sum(const std::string &file_path, /*out*/ std::string &result) +{ + result.clear(); + if (!::dsn::utils::filesystem::file_exists(file_path)) { + LOG_ERROR("md5sum error: file {} not exist", file_path); + return ERR_OBJECT_NOT_FOUND; + } + + std::unique_ptr sfile; + auto s = rocksdb::Env::Default()->NewSequentialFile(file_path, &sfile, rocksdb::EnvOptions()); + if (!sfile) { + LOG_ERROR("md5sum error: open file {} failed, err={}", file_path, s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } + + const int64_t kBufferSize = 4096; + char buf[kBufferSize]; + unsigned char out[MD5_DIGEST_LENGTH] = {0}; + MD5_CTX c; + CHECK_EQ(1, MD5_Init(&c)); + while (true) { + rocksdb::Slice res; + s = sfile->Read(kBufferSize, &res, buf); + if (!s.ok()) { + MD5_Final(out, &c); + LOG_ERROR("md5sum error: read file {} failed, err={}", file_path, s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } + if (res.empty()) { + break; + } + CHECK_EQ(1, MD5_Update(&c, buf, res.size())); + if (res.size() < kBufferSize) { + break; + } + } + CHECK_EQ(1, MD5_Final(out, &c)); + + char str[MD5_DIGEST_LENGTH * 2 + 1]; + str[MD5_DIGEST_LENGTH * 2] = 0; + for (int n = 0; n < MD5_DIGEST_LENGTH; n++) { + sprintf(str + n + n, "%02x", out[n]); + } + result.assign(str); + + return ERR_OK; +} + +error_code deprecated_md5sum(const std::string &file_path, /*out*/ std::string &result) { result.clear(); // if file not exist, we return ERR_OBJECT_NOT_FOUND @@ -790,37 +783,8 @@ std::pair is_directory_empty(const std::string &dirname) return res; } -error_code read_file(const std::string &fname, std::string &buf) -{ - if (!file_exists(fname)) { - LOG_ERROR("file({}) doesn't exist", fname); - return ERR_FILE_OPERATION_FAILED; - } - - int64_t file_sz = 0; - if (!file_size(fname, file_sz)) { - LOG_ERROR("get file({}) size failed", fname); - return ERR_FILE_OPERATION_FAILED; - } - - buf.resize(file_sz); - std::ifstream fin(fname, std::ifstream::in); - if (!fin.is_open()) { - LOG_ERROR("open file({}) failed", fname); - return ERR_FILE_OPERATION_FAILED; - } - fin.read(&buf[0], file_sz); - CHECK_EQ_MSG(file_sz, - fin.gcount(), - "read file({}) failed, file_size = {} but read size = {}", - fname, - file_sz, - fin.gcount()); - fin.close(); - return ERR_OK; -} - bool verify_file(const std::string &fname, + FileDataType type, const std::string &expected_md5, const int64_t &expected_fsize) { @@ -829,7 +793,7 @@ bool verify_file(const std::string &fname, return false; } int64_t f_size = 0; - if (!file_size(fname, f_size)) { + if (!file_size(fname, type, f_size)) { LOG_ERROR("verify file({}) failed, becaused failed to get file size", fname); return false; } @@ -850,14 +814,14 @@ bool verify_file(const std::string &fname, return true; } -bool verify_file_size(const std::string &fname, const int64_t &expected_fsize) +bool verify_file_size(const std::string &fname, FileDataType type, const int64_t &expected_fsize) { if (!file_exists(fname)) { LOG_ERROR("file({}) is not existed", fname); return false; } int64_t f_size = 0; - if (!file_size(fname, f_size)) { + if (!file_size(fname, type, f_size)) { LOG_ERROR("verify file({}) size failed, becaused failed to get file size", fname); return false; } @@ -871,22 +835,6 @@ bool verify_file_size(const std::string &fname, const int64_t &expected_fsize) return true; } -bool verify_data_md5(const std::string &fname, - const char *data, - const size_t data_size, - const std::string &expected_md5) -{ - std::string md5 = string_md5(data, data_size); - if (md5 != expected_md5) { - LOG_ERROR("verify data({}) failed, because data damaged, size: md5: {} VS {}", - fname, - md5, - expected_md5); - return false; - } - return true; -} - bool create_directory(const std::string &path, std::string &absolute_path, std::string &err_msg) { FAIL_POINT_INJECT_F("filesystem_create_directory", [path](string_view str) { @@ -908,20 +856,6 @@ bool create_directory(const std::string &path, std::string &absolute_path, std:: return true; } -bool write_file(const std::string &fname, std::string &buf) -{ - if (!file_exists(fname)) { - LOG_ERROR("file({}) doesn't exist", fname); - return false; - } - - std::ofstream fstream; - fstream.open(fname.c_str()); - fstream << buf; - fstream.close(); - return true; -} - bool check_dir_rw(const std::string &path, std::string &err_msg) { FAIL_POINT_INJECT_F("filesystem_check_dir_rw", [path](string_view str) { @@ -932,23 +866,28 @@ bool check_dir_rw(const std::string &path, std::string &err_msg) path.find(broken_disk_dir) == std::string::npos; }); - std::string fname = "read_write_test_file"; - std::string fpath = path_combine(path, fname); - if (!create_file(fpath)) { - err_msg = fmt::format("Fail to create test file {}.", fpath); + static const std::string kTestValue = "test_value"; + static const std::string kFname = "read_write_test_file"; + std::string fpath = path_combine(path, kFname); + auto cleanup = defer([&fpath]() { remove_path(fpath); }); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(kTestValue), + fpath, + /* should_sync */ true); + if (dsn_unlikely(!s.ok())) { + err_msg = fmt::format("fail to write file {}, err={}", fpath, s.ToString()); return false; } - auto cleanup = defer([&fpath]() { remove_path(fpath); }); - std::string value = "test_value"; - if (!write_file(fpath, value)) { - err_msg = fmt::format("Fail to write file {}.", fpath); + std::string read_data; + s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fpath, &read_data); + if (dsn_unlikely(!s.ok())) { + err_msg = fmt::format("fail to read file {}, err={}", fpath, s.ToString()); return false; } - std::string buf; - if (read_file(fpath, buf) != ERR_OK || buf != value) { - err_msg = fmt::format("Fail to read file {} or get wrong value({}).", fpath, buf); + if (dsn_unlikely(read_data != kTestValue)) { + err_msg = fmt::format("get wrong value '{}' from file {}", read_data, fpath); return false; } diff --git a/src/utils/filesystem.h b/src/utils/filesystem.h index 4229f551d2..2142d1be1a 100644 --- a/src/utils/filesystem.h +++ b/src/utils/filesystem.h @@ -61,9 +61,13 @@ namespace dsn { namespace utils { +enum class FileDataType; + namespace filesystem { -int get_normalized_path(const std::string &path, std::string &npath); +// TODO(yingchun): Consider using rocksdb APIs to rewrite the following functions. + +void get_normalized_path(const std::string &path, std::string &npath); bool get_absolute_path(const std::string &path1, std::string &path2); @@ -96,7 +100,15 @@ bool remove_path(const std::string &path); // this will always remove target path if exist bool rename_path(const std::string &path1, const std::string &path2); +// Get the file size. The encryption header is considered as part of the file if it is an encrypted +// file. +// TODO(yingchun): refactor to use uint64_t. bool file_size(const std::string &path, int64_t &sz); +// The legacy file_size(), just for testing. +bool deprecated_file_size(const std::string &path, int64_t &sz); +// Get the file size. The encryption header is not considered as part of the file if it is an +// encrypted file and 'type' is specified as FileDataType::kSensitive. +bool file_size(const std::string &path, FileDataType type, int64_t &sz); bool create_directory(const std::string &path); @@ -126,6 +138,7 @@ bool get_disk_space_info(const std::string &path, disk_space_info &info); bool link_file(const std::string &src, const std::string &target); error_code md5sum(const std::string &file_path, /*out*/ std::string &result); +error_code deprecated_md5sum(const std::string &file_path, /*out*/ std::string &result); // return value: // - : @@ -133,27 +146,19 @@ error_code md5sum(const std::string &file_path, /*out*/ std::string &result); // B is represent wheter the directory is empty, true means empty, otherwise false std::pair is_directory_empty(const std::string &dirname); -error_code read_file(const std::string &fname, /*out*/ std::string &buf); - // compare file metadata calculated by fname with expected md5 and file_size bool verify_file(const std::string &fname, + FileDataType type, const std::string &expected_md5, const int64_t &expected_fsize); -bool verify_file_size(const std::string &fname, const int64_t &expected_fsize); - -bool verify_data_md5(const std::string &fname, - const char *data, - const size_t data_size, - const std::string &expected_md5); +bool verify_file_size(const std::string &fname, FileDataType type, const int64_t &expected_fsize); // create driectory and get absolute path bool create_directory(const std::string &path, /*out*/ std::string &absolute_path, /*out*/ std::string &err_msg); -bool write_file(const std::string &fname, std::string &buf); - // check if directory is readable and writable // call `create_directory` before to make `path` exist bool check_dir_rw(const std::string &path, /*out*/ std::string &err_msg); diff --git a/src/utils/flags.h b/src/utils/flags.h index 304238b7bb..355fa14ded 100644 --- a/src/utils/flags.h +++ b/src/utils/flags.h @@ -22,6 +22,7 @@ // IWYU pragma: no_include #include #include +#include #include "enum_helper.h" #include "errors.h" diff --git a/src/utils/fmt_logging.h b/src/utils/fmt_logging.h index 1f6e961207..0a6ac64d99 100644 --- a/src/utils/fmt_logging.h +++ b/src/utils/fmt_logging.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include "utils/api_utilities.h" @@ -57,14 +58,15 @@ #define CHECK_EXPRESSION(expression, evaluation, ...) \ do { \ if (dsn_unlikely(!(evaluation))) { \ - dlog_f(LOG_LEVEL_FATAL, "assertion expression: " #expression); \ - dlog_f(LOG_LEVEL_FATAL, __VA_ARGS__); \ - dsn_coredump(); \ + std::string assertion_info("assertion expression: [" #expression "] "); \ + assertion_info += fmt::format(__VA_ARGS__); \ + LOG_FATAL(assertion_info); \ } \ } while (false) #define CHECK(x, ...) CHECK_EXPRESSION(x, x, __VA_ARGS__) -#define CHECK_NOTNULL(p, ...) CHECK(p != nullptr, __VA_ARGS__) +#define CHECK_NOTNULL(p, ...) CHECK((p) != nullptr, __VA_ARGS__) +#define CHECK_NULL(p, ...) CHECK((p) == nullptr, __VA_ARGS__) // Macros for writing log message prefixed by log_prefix(). #define LOG_DEBUG_PREFIX(...) LOG_DEBUG("[{}] {}", log_prefix(), fmt::format(__VA_ARGS__)) @@ -272,6 +274,16 @@ inline const char *null_str_printer(const char *s) { return s == nullptr ? "(nul LOG_AND_RETURN_NOT_TRUE(level, _err == ::dsn::ERR_OK, _err, __VA_ARGS__); \ } while (0) +// Return the given rocksdb::Status 's' if it is not OK. +#define LOG_AND_RETURN_NOT_RDB_OK(level, s, ...) \ + do { \ + const auto &_s = (s); \ + if (dsn_unlikely(!_s.ok())) { \ + LOG_##level("{}: {}", _s.ToString(), fmt::format(__VA_ARGS__)); \ + return _s; \ + } \ + } while (0) + #ifndef NDEBUG #define DCHECK CHECK #define DCHECK_NOTNULL CHECK_NOTNULL diff --git a/src/utils/fmt_utils.h b/src/utils/fmt_utils.h new file mode 100644 index 0000000000..9624bb881a --- /dev/null +++ b/src/utils/fmt_utils.h @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#define USER_DEFINED_STRUCTURE_FORMATTER(type) \ + template <> \ + struct fmt::formatter : ostream_formatter \ + { \ + } + +#define USER_DEFINED_ENUM_FORMATTER(type) \ + inline auto format_as(type e)->int { return e; } diff --git a/src/utils/je_ctl.cpp b/src/utils/je_ctl.cpp index fa6a7168d4..38f5bb1be0 100644 --- a/src/utils/je_ctl.cpp +++ b/src/utils/je_ctl.cpp @@ -75,7 +75,7 @@ const char *je_stats_type_to_opts(je_stats_type type) size_t je_stats_type_to_default_buf_sz(je_stats_type type) { static const size_t buf_sz_map[] = { - 2 * 1024, 4 * 1024, 1024 * 1024, 2 * 1024 * 1024, + 2 * 1024, 4 * 1024, 8 * 1024 * 1024, 8 * 1024 * 1024, }; RETURN_ARRAY_ELEM_BY_ENUM_TYPE(type, buf_sz_map); diff --git a/src/utils/long_adder_bench/CMakeLists.txt b/src/utils/long_adder_bench/CMakeLists.txt index f63efc8a96..480b048079 100644 --- a/src/utils/long_adder_bench/CMakeLists.txt +++ b/src/utils/long_adder_bench/CMakeLists.txt @@ -27,7 +27,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime dsn_utils) +set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/utils/long_adder_bench/long_adder_bench.cpp b/src/utils/long_adder_bench/long_adder_bench.cpp index 93ec649c7b..0c2a12deed 100644 --- a/src/utils/long_adder_bench/long_adder_bench.cpp +++ b/src/utils/long_adder_bench/long_adder_bench.cpp @@ -20,12 +20,11 @@ #include #include #include -#include #include #include #include -#include "runtime/api_layer1.h" +#include "test_util/test_util.h" #include "utils/long_adder.h" #include "utils/ports.h" #include "utils/process_utils.h" @@ -133,7 +132,7 @@ void run_bench(int64_t num_operations, int64_t num_threads, const char *name) std::vector threads; - uint64_t start = dsn_now_ns(); + pegasus::stop_watch sw; for (int64_t i = 0; i < num_threads; i++) { threads.emplace_back([num_operations, &adder]() { for (int64_t i = 0; i < num_operations; ++i) { @@ -144,19 +143,11 @@ void run_bench(int64_t num_operations, int64_t num_threads, const char *name) for (auto &t : threads) { t.join(); } - uint64_t end = dsn_now_ns(); - - auto duration_ns = static_cast(end - start); - std::chrono::nanoseconds nano(duration_ns); - auto duration_s = std::chrono::duration_cast>(nano).count(); - - fmt::print(stdout, - "Running {} operations of {} with {} threads took {} seconds, result = {}.\n", - num_operations, - name, - num_threads, - duration_s, - adder.value()); + sw.stop_and_output(fmt::format("Running {} operations of {} with {} threads, result = {}", + num_operations, + name, + num_threads, + adder.value())); } int main(int argc, char **argv) diff --git a/src/utils/metrics.cpp b/src/utils/metrics.cpp index 2a15fb5aa9..b5800d3cd1 100644 --- a/src/utils/metrics.cpp +++ b/src/utils/metrics.cpp @@ -23,6 +23,7 @@ #include #include +#include "http/http_method.h" #include "runtime/api_layer1.h" #include "utils/flags.h" #include "utils/rand.h" @@ -275,7 +276,7 @@ const dsn::metric_filters::metric_fields_type kBriefMetricFields = get_brief_met void metrics_http_service::get_metrics_handler(const http_request &req, http_response &resp) { - if (req.method != http_method::HTTP_METHOD_GET) { + if (req.method != http_method::GET) { resp.body = encode_error_as_json("please use 'GET' method while querying for metrics"); resp.status_code = http_status_code::bad_request; return; @@ -617,8 +618,8 @@ void metric_timer::on_timer(const boost::system::error_code &ec) } while (0) if (dsn_unlikely(!!ec)) { - CHECK_EQ_MSG(ec, - boost::system::errc::operation_canceled, + CHECK_EQ_MSG(static_cast(boost::system::errc::operation_canceled), + ec.value(), "failed to exec on_timer with an error that cannot be handled: {}", ec.message()); diff --git a/src/utils/output_utils.cpp b/src/utils/output_utils.cpp index 6b30172217..dfaa799633 100644 --- a/src/utils/output_utils.cpp +++ b/src/utils/output_utils.cpp @@ -17,6 +17,7 @@ #include "utils/output_utils.h" +#include // IWYU pragma: no_include #include diff --git a/src/utils/output_utils.h b/src/utils/output_utils.h index c66d55f734..201cee938b 100644 --- a/src/utils/output_utils.h +++ b/src/utils/output_utils.h @@ -20,6 +20,7 @@ // IWYU pragma: no_include #include #include +#include #include // IWYU pragma: keep #include // IWYU pragma: no_include diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index f982670186..16c4e95e14 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -29,7 +29,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include @@ -38,10 +37,14 @@ #include "runtime/api_layer1.h" #include "runtime/task/task_spec.h" #include "utils/command_manager.h" +#include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" +#include "utils/ports.h" #include "utils/process_utils.h" +#include "utils/string_conv.h" +#include "utils/string_view.h" #include "utils/strings.h" #include "utils/time_utils.h" @@ -93,6 +96,28 @@ static void print_header(FILE *fp, dsn_log_level_t log_level) log_prefixed_message_func().c_str()); } +namespace { + +inline void process_fatal_log(dsn_log_level_t log_level) +{ + if (dsn_likely(log_level < LOG_LEVEL_FATAL)) { + return; + } + + bool coredump = true; + FAIL_POINT_INJECT_NOT_RETURN_F("coredump_for_fatal_log", [&coredump](dsn::string_view str) { + CHECK(buf2bool(str, coredump), + "invalid coredump toggle for fatal log, should be true or false: {}", + str); + }); + + if (dsn_likely(coredump)) { + dsn_coredump(); + } +} + +} // anonymous namespace + screen_logger::screen_logger(bool short_header) { _short_header = short_header; } screen_logger::~screen_logger(void) {} @@ -112,6 +137,8 @@ void screen_logger::dsn_logv(const char *file, } vprintf(fmt, args); printf("\n"); + + process_fatal_log(log_level); } void screen_logger::flush() { ::fflush(stdout); } @@ -257,6 +284,8 @@ void simple_logger::dsn_logv(const char *file, printf("\n"); } + process_fatal_log(log_level); + if (++_lines >= 200000) { create_log_file(); } @@ -287,6 +316,8 @@ void simple_logger::dsn_log(const char *file, printf("%s\n", str); } + process_fatal_log(log_level); + if (++_lines >= 200000) { create_log_file(); } diff --git a/src/utils/simple_logger.h b/src/utils/simple_logger.h index 74a44d2dde..71535b26fc 100644 --- a/src/utils/simple_logger.h +++ b/src/utils/simple_logger.h @@ -46,18 +46,20 @@ class screen_logger : public logging_provider explicit screen_logger(bool short_header); ~screen_logger() override; - virtual void dsn_logv(const char *file, - const char *function, - const int line, - dsn_log_level_t log_level, - const char *fmt, - va_list args); + void dsn_logv(const char *file, + const char *function, + const int line, + dsn_log_level_t log_level, + const char *fmt, + va_list args) override; - virtual void dsn_log(const char *file, - const char *function, - const int line, - dsn_log_level_t log_level, - const char *str){}; + void dsn_log(const char *file, + const char *function, + const int line, + dsn_log_level_t log_level, + const char *str) override + { + } virtual void flush(); diff --git a/src/utils/string_view.h b/src/utils/string_view.h index ee3664c576..ab867448e2 100644 --- a/src/utils/string_view.h +++ b/src/utils/string_view.h @@ -49,6 +49,7 @@ #include #include "ports.h" +#include "utils/fmt_utils.h" namespace dsn { @@ -428,3 +429,5 @@ inline bool operator!=(string_view x, string_view y) noexcept { return !(x == y) std::ostream &operator<<(std::ostream &o, string_view piece); } // namespace dsn + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::string_view); diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index fb284b0e86..d77464c360 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -33,7 +33,8 @@ set(MY_PROJ_LIBS dsn_http dsn_runtime dsn_utils gtest - ) + test_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/utils/test/autoref_ptr_test.cpp b/src/utils/test/autoref_ptr_test.cpp index 6a8fe8d9d7..4ff6a15294 100644 --- a/src/utils/test/autoref_ptr_test.cpp +++ b/src/utils/test/autoref_ptr_test.cpp @@ -8,6 +8,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include diff --git a/src/utils/test/env.cpp b/src/utils/test/env.cpp index 5a05f1879b..49e389729f 100644 --- a/src/utils/test/env.cpp +++ b/src/utils/test/env.cpp @@ -33,17 +33,31 @@ * xxxx-xx-xx, author, fix bug about xxx */ +#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include +#include #include +#include +#include "test_util/test_util.h" +#include "utils/env.h" +#include "utils/error_code.h" +#include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/rand.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + using namespace ::dsn; -TEST(core, env) +TEST(env_test, rand) { uint64_t xs[] = {0, std::numeric_limits::max() - 1, 0xdeadbeef}; @@ -55,3 +69,271 @@ TEST(core, env) EXPECT_TRUE(r == x || r == (x + 1)); } } + +TEST(env_test, get_env) +{ + FLAGS_encrypt_data_at_rest = false; + auto *env_no_enc1 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive); + auto *env_no_enc2 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); + ASSERT_EQ(env_no_enc1, env_no_enc2); + + FLAGS_encrypt_data_at_rest = true; + auto *env_no_enc3 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive); + ASSERT_EQ(env_no_enc1, env_no_enc3); + + auto *env_enc1 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); + ASSERT_NE(env_no_enc1, env_enc1); +} + +class env_file_test : public pegasus::encrypt_data_test_base +{ +public: + env_file_test() : pegasus::encrypt_data_test_base() + { + // The size of an actual encrypted file should plus kEncryptionHeaderkSize bytes if consider + // it as kNonSensitive. + if (FLAGS_encrypt_data_at_rest) { + _extra_size = dsn::utils::kEncryptionHeaderkSize; + } + } + uint64_t _extra_size = 0; +}; + +INSTANTIATE_TEST_CASE_P(, env_file_test, ::testing::Values(false, true)); + +TEST_P(env_file_test, encrypt_file_2_files) +{ + const std::string kFileName = "encrypt_file_2_files"; + const std::string kEncryptedFileName = kFileName + ".encrypted"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare a non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + + // Check encrypt_file(src_fname, dst_fname, total_size). + // Loop twice to check overwrite. + for (int i = 0; i < 2; ++i) { + uint64_t encrypt_file_size; + s = dsn::utils::encrypt_file(kFileName, kEncryptedFileName, &encrypt_file_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_EQ(kFileContentSize, encrypt_file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kEncryptedFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kEncryptedFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + kEncryptedFileName, + &data); + ASSERT_EQ(kFileContent, data); + } +} + +TEST_P(env_file_test, encrypt_file_1_file) +{ + const std::string kFileName = "encrypt_file_1_file"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare a non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + + // Check encrypt_file(fname, total_size). + uint64_t encrypt_file_size; + s = dsn::utils::encrypt_file(kFileName, &encrypt_file_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_EQ(kFileContentSize, encrypt_file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), kFileName, &data); + ASSERT_EQ(kFileContent, data); +} + +TEST_P(env_file_test, copy_file) +{ + const std::string kFileName = "copy_file"; + const std::string kCopyFileName = kFileName + ".copy"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare an encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + + // Check copy_file(src_fname, dst_fname, total_size). + // Loop twice to check overwrite. + for (int i = 0; i < 2; ++i) { + uint64_t copy_file_size; + s = dsn::utils::copy_file(kFileName, kCopyFileName, ©_file_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_EQ(kFileContentSize, copy_file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), kCopyFileName, &data); + ASSERT_EQ(kFileContent, data); + } +} + +TEST_P(env_file_test, copy_file_by_size) +{ + const std::string kFileName = "copy_file_by_size"; + std::string kCopyFileName = kFileName + ".copy"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare an encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + + // Check copy_file_by_size(src_fname, dst_fname, limit_size). + struct test_case + { + int64_t limit_size; + int64_t expect_size; + } tests[] = {{-1, kFileContentSize}, + {0, 0}, + {10, 10}, + {kFileContentSize, kFileContentSize}, + {kFileContentSize + 10, kFileContentSize}}; + for (const auto &test : tests) { + s = dsn::utils::copy_file_by_size(kFileName, kCopyFileName, test.limit_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + + int64_t actual_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kSensitive, actual_size)); + ASSERT_EQ(test.expect_size, actual_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(test.expect_size + _extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), kCopyFileName, &data); + ASSERT_EQ(std::string(test.expect_size, 'a'), data); + } +} + +TEST_P(env_file_test, copy_non_encrypt_file) +{ + const std::string kFileName = "copy_non_encrypt_file"; + std::string kCopyFileName = kFileName + ".copy"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare a non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + + // Check copy_file() on non-sensitive file. + s = dsn::utils::copy_file(kFileName, kCopyFileName); + if (FLAGS_encrypt_data_at_rest) { + // copy_file() consider the source file as encrypted, so it will fail. + ASSERT_TRUE(s.IsCorruption()) << s.ToString(); + ASSERT_TRUE(s.ToString().find( + fmt::format("Corruption: Invalid encryption header in {}", kFileName)) == 0) + << s.ToString(); + } else { + // Although copy_file() consider the source file as non-encrypted, but it will succeed if + // FLAGS_encrypt_data_at_rest is disabled. + ASSERT_TRUE(s.ok()) << s.ToString(); + int64_t copy_file_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kNonSensitive, copy_file_size)); + ASSERT_EQ(kFileContentSize, copy_file_size); + } + + // Check copy_file_by_size() on non-sensitive file. + s = dsn::utils::copy_file_by_size(kFileName, kCopyFileName); + if (FLAGS_encrypt_data_at_rest) { + // copy_file_by_size() consider the source file as encrypted, so it will fail. + ASSERT_TRUE(s.IsCorruption()) << s.ToString(); + ASSERT_TRUE(s.ToString().find( + fmt::format("Corruption: Invalid encryption header in {}", kFileName)) == 0) + << s.ToString(); + } else { + // Although copy_file_by_size() consider the source file as non-encrypted, but it will + // succeed if FLAGS_encrypt_data_at_rest is disabled. + ASSERT_TRUE(s.ok()) << s.ToString(); + int64_t copy_file_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kCopyFileName, dsn::utils::FileDataType::kNonSensitive, copy_file_size)); + ASSERT_EQ(kFileContentSize, copy_file_size); + } +} diff --git a/src/utils/test/fail_point_test.cpp b/src/utils/test/fail_point_test.cpp index 74d576065a..b7668b03c9 100644 --- a/src/utils/test/fail_point_test.cpp +++ b/src/utils/test/fail_point_test.cpp @@ -31,6 +31,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include "utils/fail_point.h" #include "utils/fail_point_impl.h" diff --git a/src/utils/test/file_system_test.cpp b/src/utils/test/file_system_test.cpp index f6da3e2b38..f497a908b3 100644 --- a/src/utils/test/file_system_test.cpp +++ b/src/utils/test/file_system_test.cpp @@ -15,30 +15,165 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include +#include #include +#include "utils/env.h" +#include "utils/error_code.h" #include "utils/filesystem.h" +#include "utils/flags.h" + +DSN_DECLARE_bool(encrypt_data_at_rest); namespace dsn { namespace utils { namespace filesystem { -TEST(verify_file, verify_file_test) +TEST(filesystem_test, compare_with_legacy_file_size) +{ + const std::string kFileName = "file_size_test"; + std::set test_file_sizes({0, 100}); + for (const auto &test_file_size : test_file_sizes) { + const std::string kFileContent(test_file_size, 'a'); + + // Prepare a non-encrypted test file. + auto s = rocksdb::WriteStringToFile( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // The file size should be the same as the legacy file size. + int64_t actual_file_size; + ASSERT_TRUE(file_size(kFileName, actual_file_size)); + ASSERT_EQ(test_file_size, actual_file_size); + ASSERT_TRUE(deprecated_file_size(kFileName, actual_file_size)); + ASSERT_EQ(test_file_size, actual_file_size); + } +} + +TEST(filesystem_test_p, non_encrypted_file_size) +{ + FLAGS_encrypt_data_at_rest = false; + const std::string kFileName = "non_encrypted_file_size"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare the non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + int64_t actual_file_size; + // Check file_size(path, sz) + ASSERT_TRUE(file_size(kFileName, actual_file_size)); + ASSERT_EQ(kFileContentSize, actual_file_size); + // Check file_size(path, type, sz) + ASSERT_TRUE(file_size(kFileName, FileDataType::kNonSensitive, actual_file_size)); + ASSERT_EQ(kFileContentSize, actual_file_size); + // Check file_size(path, type, sz) with kSensitive type. + // It's able to get the correct file size because FLAGS_encrypt_data_at_rest is disabled. + ASSERT_TRUE(file_size(kFileName, FileDataType::kSensitive, actual_file_size)); + ASSERT_EQ(kFileContentSize, actual_file_size); +} + +TEST(filesystem_test_p, encrypted_file_size) +{ + FLAGS_encrypt_data_at_rest = true; + const std::string kFileName = "encrypted_file_size"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare the non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + int64_t actual_file_size; + // Check file_size(path, sz), the encryption header size is counted. + ASSERT_TRUE(file_size(kFileName, actual_file_size)); + ASSERT_EQ(kFileContentSize + kEncryptionHeaderkSize, actual_file_size); + // Check file_size(path, type, sz) with correct type. + ASSERT_TRUE(file_size(kFileName, FileDataType::kSensitive, actual_file_size)); + ASSERT_EQ(kFileContentSize, actual_file_size); + // Check file_size(path, type, sz) with kNonSensitive type, the encryption header size is + // counted. + ASSERT_TRUE(file_size(kFileName, FileDataType::kNonSensitive, actual_file_size)); + ASSERT_EQ(kFileContentSize + kEncryptionHeaderkSize, actual_file_size); +} + +// The old filesystem API doesn't support sensitive files, so skip testing +// FLAGS_encrypt_data_at_rest=true. +TEST(filesystem_test, check_new_md5sum) { + FLAGS_encrypt_data_at_rest = false; + + struct file_info + { + int64_t size; + } tests[]{{4095}, {4096}, {4097}}; + + for (const auto &test : tests) { + std::string fname = "test_file"; + // deprecated_md5sum doesn't support kSensitive files, so use kNonSensitive here. + auto s = rocksdb::WriteStringToFile( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(std::string(test.size, 'a')), + fname, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + // Check the file size. + int64_t file_fsize; + ASSERT_TRUE(file_size(fname, FileDataType::kNonSensitive, file_fsize)); + ASSERT_EQ(test.size, file_fsize); + + // Get the md5sum. + std::string md5sum1; + ASSERT_EQ(ERR_OK, md5sum(fname, md5sum1)); + ASSERT_FALSE(md5sum1.empty()); + + // Check the md5sum is repeatable. + std::string md5sum2; + ASSERT_EQ(ERR_OK, md5sum(fname, md5sum2)); + ASSERT_EQ(md5sum1, md5sum2); + + // Check the md5sum is the same to deprecated_md5sum. + ASSERT_EQ(ERR_OK, deprecated_md5sum(fname, md5sum2)); + ASSERT_EQ(md5sum1, md5sum2); + + utils::filesystem::remove_path(fname); + } +} + +TEST(filesystem_test, verify_file_test) +{ + FLAGS_encrypt_data_at_rest = false; + const std::string &fname = "test_file"; std::string expected_md5; int64_t expected_fsize; create_file(fname); md5sum(fname, expected_md5); - file_size(fname, expected_fsize); + ASSERT_TRUE(file_size(fname, FileDataType::kNonSensitive, expected_fsize)); - ASSERT_TRUE(verify_file(fname, expected_md5, expected_fsize)); - ASSERT_FALSE(verify_file(fname, "wrong_md5", 10086)); - ASSERT_FALSE(verify_file("file_not_exists", "wrong_md5", 10086)); + ASSERT_TRUE(verify_file(fname, FileDataType::kNonSensitive, expected_md5, expected_fsize)); + ASSERT_FALSE(verify_file(fname, FileDataType::kNonSensitive, "wrong_md5", 10086)); + ASSERT_FALSE(verify_file("file_not_exists", FileDataType::kNonSensitive, "wrong_md5", 10086)); remove_path(fname); } diff --git a/src/utils/test/file_utils.cpp b/src/utils/test/file_utils.cpp index 004e228b4b..c4cadeb959 100644 --- a/src/utils/test/file_utils.cpp +++ b/src/utils/test/file_utils.cpp @@ -25,876 +25,327 @@ */ // IWYU pragma: no_include +// IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include #include #include #include #include +#include "test_util/test_util.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" -static void file_utils_test_setup() +class file_utils : public pegasus::encrypt_data_test_base { - std::string path; - bool ret; - - path = "./file_utils_temp.txt"; - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_FALSE(ret); - - path = "./file_utils_temp"; - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_FALSE(ret); -} +public: + void file_utils_test_setup() + { + std::string path = "./file_utils_temp.txt"; + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + ASSERT_FALSE(dsn::utils::filesystem::file_exists(path)); + + path = "./file_utils_temp"; + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + ASSERT_FALSE(dsn::utils::filesystem::directory_exists(path)); + } -static void file_utils_test_get_process_image_path() -{ - std::string path; - std::string imagepath; - dsn::error_code ret; - // int pid; + void file_utils_test_get_process_image_path() + { + std::string imagepath; + ASSERT_TRUE(dsn::utils::filesystem::get_current_directory(imagepath)); + imagepath = dsn::utils::filesystem::path_combine(imagepath, "dsn_utils_tests"); - if (!dsn::utils::filesystem::get_current_directory(imagepath)) { - EXPECT_TRUE(false); + std::string path; + ASSERT_EQ(dsn::ERR_OK, dsn::utils::filesystem::get_current_process_image_path(path)); } - imagepath = dsn::utils::filesystem::path_combine(imagepath, "dsn_utils_tests"); - ret = dsn::utils::filesystem::get_current_process_image_path(path); - EXPECT_TRUE(ret == dsn::ERR_OK); - // TODO: not always true when running dir is not where the test resides - // EXPECT_TRUE(path == imagepath); // e: vs E: -} + void file_utils_test_get_normalized_path() + { + struct same_normalized_paths + { + std::string path; + } same_normalized_path_tests[] = {{"\\\\?\\"}, + {"c:\\"}, + {"c:"}, + {"\\\\?\\c:\\"}, + {"\\\\?\\c:"}, + {"c:\\a"}, + {"c:\\\\a"}, + {"c:\\\\a\\"}, + {"c:\\\\a\\\\"}, + {"\\\\?\\c:\\a"}, + {"\\\\?\\c:\\\\a"}, + {"\\\\?\\c:\\\\a\\"}, + {"\\\\?\\c:\\\\a\\\\"}, + {"\\"}, + {"\\\\"}, + {"\\\\\\"}, + {"\\\\a"}, + {"\\\\\\a"}, + {"\\\\a\\"}, + {"\\\\\\a\\"}, + {"\\\\\\a\\\\"}, + {"/"}, + {"c:/a"}, + {"."}, + {"./a"}, + {"./a/b"}, + {".."}, + {"../a"}, + {"../a/b"}}; + for (const auto &test : same_normalized_path_tests) { + std::string npath; + dsn::utils::filesystem::get_normalized_path(test.path, npath); + ASSERT_EQ(test.path, npath); + } + + struct normalized_paths + { + std::string path; + std::string normalized_path; + } normalized_path_tests[] = { + {"//", "/"}, + {"//?/", "/?"}, + {"//a", "/a"}, + {"//a/", "/a"}, + {"//a//", "/a"}, + {"c:/", "c:"}, + {"c://", "c:"}, + {"c:/a/", "c:/a"}, + {"c://a/", "c:/a"}, + {"c://a//", "c:/a"}, + {"/////////////////////////////////////////////////////////////////", "/"}, + {"/////////////////////////////////////////////////////////////////a/////////////////" + "b///" + "////////", + "/a/b"}, + {"./", "."}, + {".//a", "./a"}, + {"./a/", "./a"}, + {"./a/b/", "./a/b"}, + {".///a////b///", "./a/b"}, + {"../", ".."}, + {"..//a", "../a"}, + {"../a/", "../a"}, + {"../a/b/", "../a/b"}, + {"..///a////b///", "../a/b"}}; + for (const auto &test : normalized_path_tests) { + std::string npath; + dsn::utils::filesystem::get_normalized_path(test.path, npath); + ASSERT_EQ(test.normalized_path, npath) << test.path; + } + } -static void file_utils_test_get_normalized_path() -{ - int ret; - std::string path; - std::string npath; - - path = "\\\\?\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "c:\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "c:"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\?\\c:\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\?\\c:"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "c:\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "c:\\\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "c:\\\\a\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "c:\\\\a\\\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\?\\c:\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\?\\c:\\\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\?\\c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\?\\c:\\\\a\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\?\\c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\?\\c:\\\\a\\\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\?\\c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "\\\\\\a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\a\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\\\a\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "\\\\\\a\\\\"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "//"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\"); -#else - EXPECT_TRUE(npath == "/"); -#endif - - path = "//?/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\?\\"); -#else - EXPECT_TRUE(npath == "/?"); -#endif - - path = "//a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == "/a"); -#endif - - path = "//a/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == "/a"); -#endif - - path = "//a//"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a"); -#else - EXPECT_TRUE(npath == "/a"); -#endif - - path = "c:/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\"); -#else - EXPECT_TRUE(npath == "c:"); -#endif - - path = "c://"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\"); -#else - EXPECT_TRUE(npath == "c:"); -#endif - - path = "c:/a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "c:/a/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == "c:/a"); -#endif - - path = "c://a/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == "c:/a"); -#endif - - path = "c://a//"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "c:\\a"); -#else - EXPECT_TRUE(npath == "c:/a"); -#endif - - path = "/////////////////////////////////////////////////////////////////"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\"); -#else - EXPECT_TRUE(npath == "/"); -#endif - - path = "/////////////////////////////////////////////////////////////////a/////////////////b///" - "////////"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "\\\\a\\b"); -#else - EXPECT_TRUE(npath == "/a/b"); -#endif - - path = "."; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "./"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == "."); - - path = "./a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = ".//a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a"); -#else - EXPECT_TRUE(npath == "./a"); -#endif - - path = "./a/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a"); -#else - EXPECT_TRUE(npath == "./a"); -#endif - - path = "./a/b"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a\\b"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "./a/b/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a\\b"); -#else - EXPECT_TRUE(npath == "./a/b"); -#endif - - path = ".///a////b///"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == ".\\a\\b"); -#else - EXPECT_TRUE(npath == "./a/b"); -#endif - - path = ".."; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == path); - - path = "../"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); - EXPECT_TRUE(npath == ".."); - - path = "../a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "..//a"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a"); -#else - EXPECT_TRUE(npath == "../a"); -#endif - - path = "../a/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a"); -#else - EXPECT_TRUE(npath == "../a"); -#endif - - path = "../a/b"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a\\b"); -#else - EXPECT_TRUE(npath == path); -#endif - - path = "../a/b/"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a\\b"); -#else - EXPECT_TRUE(npath == "../a/b"); -#endif - - path = "..///a////b///"; - ret = dsn::utils::filesystem::get_normalized_path(path, npath); - EXPECT_TRUE(ret == 0); -#ifdef _WIN32 - EXPECT_TRUE(npath == "..\\a\\b"); -#else - EXPECT_TRUE(npath == "../a/b"); -#endif -} + void file_utils_test_get_current_directory() + { + std::string path; + ASSERT_TRUE(dsn::utils::filesystem::get_current_directory(path)); + ASSERT_TRUE(!path.empty()); + } -static void file_utils_test_get_current_directory() -{ - std::string path; - bool ret; + void file_utils_test_path_combine() + { + struct combine_paths + { + std::string path1; + std::string path2; + std::string combined_path; + } tests[] = {{"", "", ""}, + {"c:", "Windows\\explorer.exe", "c:/Windows\\explorer.exe"}, + {"c:", "\\Windows\\explorer.exe", "c:/Windows\\explorer.exe"}, + {"c:\\", "\\Windows\\explorer.exe", "c:\\/Windows\\explorer.exe"}, + {"/bin", "ls", "/bin/ls"}, + {"/bin/", "ls", "/bin/ls"}, + {"/bin", "/ls", "/bin/ls"}, + {"/bin/", "/ls", "/bin/ls"}}; + for (const auto &test : tests) { + std::string path = dsn::utils::filesystem::path_combine(test.path1, test.path2); + ASSERT_EQ(test.combined_path, path) << test.path1 << " + " << test.path2; + } + } - path = ""; - ret = dsn::utils::filesystem::get_current_directory(path); - EXPECT_TRUE(ret); - EXPECT_TRUE(!path.empty()); -} + void file_utils_test_get_file_name() + { + struct combine_paths + { + std::string path; + std::string file_name; + } tests[] = {{"", ""}, + {"c:", "c:"}, + {"c:\\", ""}, + {"c:1.txt", "c:1.txt"}, + {"c:\\1.txt", "1.txt"}, + {"c:\\Windows\\1.txt", "1.txt"}, + {"/bin/", ""}, + {"/bin/ls", "ls"}}; + for (const auto &test : tests) { + std::string file_name = dsn::utils::filesystem::get_file_name(test.path); + ASSERT_EQ(test.file_name, file_name) << test.path; + } + } -static void file_utils_test_path_combine() -{ - std::string path; - std::string path1; - std::string path2; - - path1 = ""; - path2 = ""; - path = dsn::utils::filesystem::path_combine(path1, path2); - EXPECT_TRUE(path == ""); - - path1 = "c:"; - path2 = "Windows\\explorer.exe"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "c:Windows\\explorer.exe"); -#else - EXPECT_TRUE(path == "c:/Windows\\explorer.exe"); -#endif - - path1 = "c:"; - path2 = "\\Windows\\explorer.exe"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "c:\\Windows\\explorer.exe"); -#else - EXPECT_TRUE(path == "c:/Windows\\explorer.exe"); -#endif - - path1 = "c:\\"; - path2 = "\\Windows\\explorer.exe"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "c:\\Windows\\explorer.exe"); -#else - EXPECT_TRUE(path == "c:\\/Windows\\explorer.exe"); -#endif - - path1 = "/bin"; - path2 = "ls"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "\\bin\\ls"); -#else - EXPECT_TRUE(path == "/bin/ls"); -#endif - - path1 = "/bin/"; - path2 = "ls"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "\\bin\\ls"); -#else - EXPECT_TRUE(path == "/bin/ls"); -#endif - - path1 = "/bin"; - path2 = "/ls"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "\\bin\\ls"); -#else - EXPECT_TRUE(path == "/bin/ls"); -#endif - - path1 = "/bin/"; - path2 = "/ls"; - path = dsn::utils::filesystem::path_combine(path1, path2); -#ifdef _WIN32 - EXPECT_TRUE(path == "\\bin\\ls"); -#else - EXPECT_TRUE(path == "/bin/ls"); -#endif -} + void file_utils_test_create() + { + std::string path = "./file_utils_temp.txt"; + ASSERT_TRUE(dsn::utils::filesystem::create_file(path)); + ASSERT_TRUE(dsn::utils::filesystem::file_exists(path)); + + time_t current_time = ::time(nullptr); + ASSERT_NE(current_time, 1); + + auto s = rocksdb::WriteStringToFile( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice("Hello world!"), + path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + time_t last_write_time; + ASSERT_TRUE(dsn::utils::filesystem::last_write_time(path, last_write_time)); + ASSERT_NE(last_write_time, -1); + ASSERT_GE(last_write_time, current_time); + + path = "./file_utils_temp"; + ASSERT_TRUE(dsn::utils::filesystem::create_directory(path)); + ASSERT_TRUE(dsn::utils::filesystem::directory_exists(path)); + + path = "./file_utils_temp/a/b/c/d//"; + ASSERT_TRUE(dsn::utils::filesystem::create_directory(path)); + ASSERT_TRUE(dsn::utils::filesystem::directory_exists(path)); + + struct create_files + { + std::string filename; + } tests[] = {{"./file_utils_temp/a/1.txt"}, + {"./file_utils_temp/a/2.txt"}, + {"./file_utils_temp/b/c/d/1.txt"}}; + for (const auto &test : tests) { + ASSERT_TRUE(dsn::utils::filesystem::create_file(test.filename)) << test.filename; + ASSERT_TRUE(dsn::utils::filesystem::file_exists(test.filename)) << test.filename; + } + } -static void file_utils_test_get_file_name() -{ - std::string path1; - std::string path2; - - path1 = ""; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == ""); - - path1 = "c:"; - path2 = dsn::utils::filesystem::get_file_name(path1); -#ifdef _WIN32 - EXPECT_TRUE(path2 == ""); -#else - EXPECT_TRUE(path2 == "c:"); -#endif - - path1 = "c:\\"; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == ""); - - path1 = "c:1.txt"; - path2 = dsn::utils::filesystem::get_file_name(path1); -#ifdef _WIN32 - EXPECT_TRUE(path2 == "1.txt"); -#else - EXPECT_TRUE(path2 == "c:1.txt"); -#endif - - path1 = "c:\\1.txt"; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == "1.txt"); - - path1 = "c:\\Windows\\1.txt"; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == "1.txt"); - - path1 = "/bin/"; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == ""); - - path1 = "/bin/ls"; - path2 = dsn::utils::filesystem::get_file_name(path1); - EXPECT_TRUE(path2 == "ls"); -} + void file_utils_test_file_size() + { + std::string path = "./file_utils_temp.txt"; + int64_t sz; + ASSERT_TRUE( + dsn::utils::filesystem::file_size(path, dsn::utils::FileDataType::kNonSensitive, sz)); + ASSERT_EQ(12, sz); + + path = "./file_utils_temp2.txt"; + ASSERT_FALSE( + dsn::utils::filesystem::file_size(path, dsn::utils::FileDataType::kNonSensitive, sz)); + } -static void file_utils_test_create() -{ - std::string path; - bool ret; - - path = "./file_utils_temp.txt"; - ret = dsn::utils::filesystem::create_file(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_TRUE(ret); - - time_t current_time = ::time(nullptr); - EXPECT_TRUE(current_time != 1); - - std::ofstream myfile(path.c_str(), std::ios::out | std::ios::app | std::ios::binary); - EXPECT_TRUE(myfile.is_open()); - myfile << "Hello world!"; - myfile.close(); - - time_t last_write_time; - ret = dsn::utils::filesystem::last_write_time(path, last_write_time); - EXPECT_TRUE(ret); - EXPECT_TRUE((last_write_time != -1) && (last_write_time >= current_time)); - - path = "./file_utils_temp"; - ret = dsn::utils::filesystem::create_directory(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/a/b/c/d//"; - ret = dsn::utils::filesystem::create_directory(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/a/1.txt"; - ret = dsn::utils::filesystem::create_file(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/a/1.txt"; - ret = dsn::utils::filesystem::create_file(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/a/2.txt"; - ret = dsn::utils::filesystem::create_file(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/b/c/d/1.txt"; - ret = dsn::utils::filesystem::create_file(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_TRUE(ret); -} + void file_utils_test_path_exists() + { + std::string path = "/"; + ASSERT_TRUE(dsn::utils::filesystem::path_exists(path)); + ASSERT_TRUE(dsn::utils::filesystem::directory_exists(path)); + ASSERT_FALSE(dsn::utils::filesystem::file_exists(path)); -static void file_utils_test_file_size() -{ - std::string path; - int64_t sz; - bool ret; - - path = "./file_utils_temp.txt"; - ret = dsn::utils::filesystem::file_size(path, sz); - EXPECT_TRUE(ret); - EXPECT_TRUE(sz == 12); - - path = "./file_utils_temp2.txt"; - ret = dsn::utils::filesystem::file_size(path, sz); - EXPECT_FALSE(ret); -} + path = "./not_exists_not_exists"; + ASSERT_FALSE(dsn::utils::filesystem::path_exists(path)); -static void file_utils_test_path_exists() -{ - std::string path; - bool ret; - - path = "c:\\"; - ret = dsn::utils::filesystem::path_exists(path); -#ifdef _WIN32 - EXPECT_TRUE(ret); -#else - EXPECT_FALSE(ret); -#endif - - path = "c:\\"; - ret = dsn::utils::filesystem::directory_exists(path); -#ifdef _WIN32 - EXPECT_TRUE(ret); -#else - EXPECT_FALSE(ret); -#endif - - path = "c:\\"; - ret = dsn::utils::filesystem::file_exists(path); -#ifdef _WIN32 - EXPECT_FALSE(ret); -#else - EXPECT_FALSE(ret); -#endif - - path = "/"; - ret = dsn::utils::filesystem::path_exists(path); - EXPECT_TRUE(ret); - - path = "/"; - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_TRUE(ret); - - path = "/"; - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_FALSE(ret); - - path = "./not_exists_not_exists"; - ret = dsn::utils::filesystem::path_exists(path); - EXPECT_FALSE(ret); - - path = "c:\\Windows\\System32\\notepad.exe"; - ret = dsn::utils::filesystem::path_exists(path); -#ifdef _WIN32 - EXPECT_TRUE(ret); -#else - EXPECT_FALSE(ret); -#endif - - path = "c:\\Windows\\System32\\notepad.exe"; - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_FALSE(ret); - - path = "c:\\Windows\\System32\\notepad.exe"; - ret = dsn::utils::filesystem::file_exists(path); -#ifdef _WIN32 - EXPECT_TRUE(ret); -#else - EXPECT_FALSE(ret); -#endif - - path = "/bin/ls"; - ret = dsn::utils::filesystem::path_exists(path); -#ifdef _WIN32 - EXPECT_FALSE(ret); -#else - EXPECT_TRUE(ret); -#endif - - path = "/bin/ls"; - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_FALSE(ret); - - path = "/bin/ls"; - ret = dsn::utils::filesystem::file_exists(path); -#ifdef _WIN32 - EXPECT_FALSE(ret); -#else - EXPECT_TRUE(ret); -#endif -} + path = "/bin/ls"; + ASSERT_TRUE(dsn::utils::filesystem::path_exists(path)); + ASSERT_FALSE(dsn::utils::filesystem::directory_exists(path)); + ASSERT_TRUE(dsn::utils::filesystem::file_exists(path)); + } -static void file_utils_test_get_paths() -{ - std::string path; - bool ret; - std::vector file_list; - - path = "."; - ret = dsn::utils::filesystem::get_subfiles(path, file_list, false); - EXPECT_TRUE(ret); -#ifdef _WIN32 - EXPECT_TRUE(file_list.size() >= 3); -#else - EXPECT_TRUE(file_list.size() >= 2); -#endif - file_list.clear(); - - path = "."; - ret = dsn::utils::filesystem::get_subfiles(path, file_list, true); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() >= 3); - file_list.clear(); - - path = "../../"; - ret = dsn::utils::filesystem::get_subfiles(path, file_list, true); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() >= 3); - file_list.clear(); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::get_subfiles(path, file_list, true); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 3); - file_list.clear(); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::get_subdirectories(path, file_list, true); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 7); - file_list.clear(); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::get_subdirectories(path, file_list, false); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 2); - file_list.clear(); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::get_subpaths(path, file_list, true); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 10); - file_list.clear(); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::get_subpaths(path, file_list, false); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 2); - file_list.clear(); - - path = "./file_utils_temp/a/"; - ret = dsn::utils::filesystem::get_subfiles(path, file_list, false); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 2); - file_list.clear(); - - path = "./file_utils_temp/a/"; - ret = dsn::utils::filesystem::get_subpaths(path, file_list, false); - EXPECT_TRUE(ret); - EXPECT_TRUE(file_list.size() == 3); - file_list.clear(); -} + void file_utils_test_get_paths() + { + std::string path = "."; + std::vector file_list; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(path, file_list, false)); + ASSERT_GE(file_list.size(), 2); + file_list.clear(); + + path = "."; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(path, file_list, true)); + ASSERT_GE(file_list.size(), 3); + file_list.clear(); + + path = "../../"; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(path, file_list, true)); + ASSERT_GE(file_list.size(), 3); + file_list.clear(); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(path, file_list, true)); + ASSERT_EQ(file_list.size(), 3); + file_list.clear(); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subdirectories(path, file_list, true)); + ASSERT_EQ(file_list.size(), 7); + file_list.clear(); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subdirectories(path, file_list, false)); + ASSERT_EQ(file_list.size(), 2); + file_list.clear(); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subpaths(path, file_list, true)); + ASSERT_EQ(file_list.size(), 10); + file_list.clear(); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subpaths(path, file_list, false)); + ASSERT_EQ(file_list.size(), 2); + file_list.clear(); + + path = "./file_utils_temp/a/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(path, file_list, false)); + ASSERT_EQ(file_list.size(), 2); + file_list.clear(); + + path = "./file_utils_temp/a/"; + ASSERT_TRUE(dsn::utils::filesystem::get_subpaths(path, file_list, false)); + ASSERT_EQ(file_list.size(), 3); + file_list.clear(); + } -static void file_utils_test_rename() -{ - std::string path; - std::string path2; - bool ret; - - path = "./file_utils_temp/b/c/d/1.txt"; - path2 = "./file_utils_temp/b/c/d/2.txt"; - ret = dsn::utils::filesystem::rename_path(path, path2); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_FALSE(ret); - ret = dsn::utils::filesystem::file_exists(path2); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::rename_path(path, path2); - EXPECT_FALSE(ret); -} + void file_utils_test_rename() + { + std::string path = "./file_utils_temp/b/c/d/1.txt"; + std::string path2 = "./file_utils_temp/b/c/d/2.txt"; + ASSERT_TRUE(dsn::utils::filesystem::rename_path(path, path2)); + ASSERT_FALSE(dsn::utils::filesystem::file_exists(path)); + ASSERT_TRUE(dsn::utils::filesystem::file_exists(path2)); + ASSERT_FALSE(dsn::utils::filesystem::rename_path(path, path2)); + } -static void file_utils_test_remove() -{ - std::string path; - std::vector file_list; - bool ret; - - path = "./file_utils_temp.txt"; - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::file_exists(path); - EXPECT_FALSE(ret); - - path = "./file_utils_temp/a/2.txt"; - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - - path = "./file_utils_temp/"; - ret = dsn::utils::filesystem::remove_path(path); - EXPECT_TRUE(ret); - ret = dsn::utils::filesystem::directory_exists(path); - EXPECT_FALSE(ret); -} + void file_utils_test_remove() + { + std::string path = "./file_utils_temp.txt"; + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + ASSERT_FALSE(dsn::utils::filesystem::file_exists(path)); + + path = "./file_utils_temp/a/2.txt"; + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + + path = "./file_utils_temp/"; + ASSERT_TRUE(dsn::utils::filesystem::remove_path(path)); + ASSERT_FALSE(dsn::utils::filesystem::directory_exists(path)); + } + + void file_utils_test_cleanup() {} +}; -static void file_utils_test_cleanup() {} +INSTANTIATE_TEST_CASE_P(, file_utils, ::testing::Values(false)); -TEST(core, file_utils) +TEST_P(file_utils, basic) { file_utils_test_setup(); file_utils_test_get_process_image_path(); diff --git a/src/utils/test/fmt_logging_test.cpp b/src/utils/test/fmt_logging_test.cpp index fc0f18f383..a8c5f7d13a 100644 --- a/src/utils/test/fmt_logging_test.cpp +++ b/src/utils/test/fmt_logging_test.cpp @@ -25,11 +25,10 @@ */ #include -#include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include +#include #include "common/gpid.h" #include "common/replication.codes.h" diff --git a/src/utils/test/je_ctl_test.cpp b/src/utils/test/je_ctl_test.cpp index 06e64be17c..b0e0db576e 100644 --- a/src/utils/test/je_ctl_test.cpp +++ b/src/utils/test/je_ctl_test.cpp @@ -17,10 +17,11 @@ #ifdef DSN_USE_JEMALLOC -#include "utils/je_ctl.h" - #include +#include "utils/je_ctl.h" +#include "utils/test_macros.h" + namespace dsn { namespace { @@ -53,10 +54,10 @@ void check_configs_marks(const std::string &stats) void check_arena_marks(const std::string &stats) { // Marks for merged arenas. - ASSERT_NE(stats.find("Merged arenas stats:"), std::string::npos); + ASSERT_NE(stats.find("Merged arenas stats:"), std::string::npos) << stats; // Marks for each arena. - ASSERT_NE(stats.find("arenas[0]:"), std::string::npos); + ASSERT_NE(stats.find("arenas[0]:"), std::string::npos) << stats; } } // anonymous namespace @@ -64,41 +65,41 @@ void check_arena_marks(const std::string &stats) TEST(je_ctl_test, dump_summary_stats) { std::string stats; - je_dump_stats(je_stats_type::SUMMARY_STATS, stats); + NO_FATALS(je_dump_stats(je_stats_type::SUMMARY_STATS, stats)); - check_base_stats_marks_with_end(stats); + NO_FATALS(check_base_stats_marks_with_end(stats)); } TEST(je_ctl_test, dump_configs) { std::string stats; - je_dump_stats(je_stats_type::CONFIGS, stats); + NO_FATALS(je_dump_stats(je_stats_type::CONFIGS, stats)); - check_base_stats_marks_with_end(stats); - check_configs_marks(stats); + NO_FATALS(check_base_stats_marks_with_end(stats)); + NO_FATALS(check_configs_marks(stats)); } TEST(je_ctl_test, dump_brief_arena_stats) { std::string stats; - je_dump_stats(je_stats_type::BRIEF_ARENA_STATS, stats); + NO_FATALS(je_dump_stats(je_stats_type::BRIEF_ARENA_STATS, stats)); // Since there may be many arenas, "End" mark is not required to be checked here. - check_base_stats_marks(stats); - check_arena_marks(stats); + NO_FATALS(check_base_stats_marks(stats)); + NO_FATALS(check_arena_marks(stats)); } TEST(je_ctl_test, dump_detailed_stats) { std::string stats; - je_dump_stats(je_stats_type::DETAILED_STATS, stats); + NO_FATALS(je_dump_stats(je_stats_type::DETAILED_STATS, stats)); // Since there may be many arenas, "End" mark is not required to be checked here. - check_base_stats_marks(stats); + NO_FATALS(check_base_stats_marks(stats)); // Detailed stats will contain all information, therefore everything should be checked. - check_configs_marks(stats); - check_arena_marks(stats); + NO_FATALS(check_configs_marks(stats)); + NO_FATALS(check_arena_marks(stats)); ASSERT_NE(stats.find("bins:"), std::string::npos); ASSERT_NE(stats.find("extents:"), std::string::npos); } diff --git a/src/utils/test/join_point_test.cpp b/src/utils/test/join_point_test.cpp index 6751fe94ae..a23740579a 100644 --- a/src/utils/test/join_point_test.cpp +++ b/src/utils/test/join_point_test.cpp @@ -29,7 +29,6 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include namespace dsn { diff --git a/src/utils/test/lock.std.cpp b/src/utils/test/lock.std.cpp index c1530e4bd8..c7027a8861 100644 --- a/src/utils/test/lock.std.cpp +++ b/src/utils/test/lock.std.cpp @@ -27,6 +27,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include "runtime/rpc/rpc_address.h" diff --git a/src/utils/test/logging.cpp b/src/utils/test/logging.cpp index f526dc5a28..eb3170d2a9 100644 --- a/src/utils/test/logging.cpp +++ b/src/utils/test/logging.cpp @@ -38,6 +38,7 @@ #include #include "utils/api_utilities.h" +#include "utils/fail_point.h" #include "utils/fmt_logging.h" TEST(core, logging) @@ -63,7 +64,7 @@ TEST(core, logging_big_log) big_str.c_str()); } -TEST(core, dlog_f) +TEST(core, dlog) { struct test_case { @@ -76,7 +77,16 @@ TEST(core, dlog_f) {dsn_log_level_t::LOG_LEVEL_ERROR, "\\x00%d\\x00\\x01%n/nm"}, {dsn_log_level_t::LOG_LEVEL_FATAL, "\\x00%d\\x00\\x01%n/nm"}}; + dsn::fail::setup(); + dsn::fail::cfg("coredump_for_fatal_log", "void(false)"); + for (auto test : tests) { - dlog_f(test.level, "sortkey = {}", test.str); + // Test logging_provider::dsn_log + dlog_f(test.level, "dlog_f: sortkey = {}", test.str); + + // Test logging_provider::dsn_logv + dlog(test.level, "dlog: sortkey = %s", test.str.c_str()); } + + dsn::fail::teardown(); } diff --git a/src/utils/test/main.cpp b/src/utils/test/main.cpp index f611528ef4..2be6302e1a 100644 --- a/src/utils/test/main.cpp +++ b/src/utils/test/main.cpp @@ -16,6 +16,7 @@ // under the License. #include +#include #include "utils/flags.h" #include "utils/logging_provider.h" diff --git a/src/utils/test/memutil_test.cpp b/src/utils/test/memutil_test.cpp index 4ddf62eaa1..87c4da8dc3 100644 --- a/src/utils/test/memutil_test.cpp +++ b/src/utils/test/memutil_test.cpp @@ -19,6 +19,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include TEST(MemUtilTest, memmatch) { diff --git a/src/utils/test/metrics_test.cpp b/src/utils/test/metrics_test.cpp index b2118f03ec..8d65c65dae 100644 --- a/src/utils/test/metrics_test.cpp +++ b/src/utils/test/metrics_test.cpp @@ -2513,7 +2513,7 @@ void test_http_get_metrics(const std::string &request_string, ASSERT_TRUE(req_res.is_ok()); const auto &req = req_res.get_value(); - std::cout << "method: " << req.method << std::endl; + std::cout << "method: " << enum_to_string(req.method) << std::endl; http_response resp; test_get_metrics_handler(req, resp); diff --git a/src/utils/test/nth_element_bench/CMakeLists.txt b/src/utils/test/nth_element_bench/CMakeLists.txt index 217d9c4363..2bd530690c 100644 --- a/src/utils/test/nth_element_bench/CMakeLists.txt +++ b/src/utils/test/nth_element_bench/CMakeLists.txt @@ -27,7 +27,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime dsn_utils) +set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/utils/test/nth_element_test.cpp b/src/utils/test/nth_element_test.cpp index 854cc6fd24..e3771fd538 100644 --- a/src/utils/test/nth_element_test.cpp +++ b/src/utils/test/nth_element_test.cpp @@ -17,6 +17,7 @@ #include "utils/nth_element.h" +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/utils/test/nth_element_utils.h b/src/utils/test/nth_element_utils.h index e8fdc9320f..85e618a318 100644 --- a/src/utils/test/nth_element_utils.h +++ b/src/utils/test/nth_element_utils.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -93,7 +94,10 @@ class nth_element_case_generator auto delta = _rand(_range_size); value += delta; } - std::random_shuffle(array.begin(), array.end()); + + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(array.begin(), array.end(), g); } private: diff --git a/src/utils/test/sema.cpp b/src/utils/test/sema.cpp index 0982ee8837..40b14824b2 100644 --- a/src/utils/test/sema.cpp +++ b/src/utils/test/sema.cpp @@ -39,6 +39,7 @@ // IWYU pragma: no_include #include #include +#include #include TEST(core, Semaphore) diff --git a/src/utils/test/smart_pointers_test.cpp b/src/utils/test/smart_pointers_test.cpp index a2113b4b36..7a6bba05f0 100644 --- a/src/utils/test/smart_pointers_test.cpp +++ b/src/utils/test/smart_pointers_test.cpp @@ -17,6 +17,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include diff --git a/src/utils/threadpool_code.h b/src/utils/threadpool_code.h index 2ff31d739f..383c1d5247 100644 --- a/src/utils/threadpool_code.h +++ b/src/utils/threadpool_code.h @@ -29,6 +29,7 @@ #include #include "ports.h" +#include "utils/fmt_utils.h" namespace dsn { class threadpool_code @@ -66,3 +67,5 @@ class threadpool_code DEFINE_THREAD_POOL_CODE(THREAD_POOL_INVALID) DEFINE_THREAD_POOL_CODE(THREAD_POOL_DEFAULT) } + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::threadpool_code); diff --git a/src/utils/utils.h b/src/utils/utils.h index f624ba48f2..fd40cb136f 100644 --- a/src/utils/utils.h +++ b/src/utils/utils.h @@ -28,7 +28,7 @@ #include #include -#include +#include // IWYU pragma: keep #include #include #include diff --git a/src/zookeeper/lock_struct.h b/src/zookeeper/lock_struct.h index dead1153ea..eca2696ad5 100644 --- a/src/zookeeper/lock_struct.h +++ b/src/zookeeper/lock_struct.h @@ -42,6 +42,7 @@ #include "runtime/task/future_types.h" #include "utils/autoref_ptr.h" #include "utils/distributed_lock_service.h" +#include "utils/fmt_utils.h" #include "utils/thread_access_checker.h" namespace dsn { @@ -57,6 +58,7 @@ enum lock_state unlocking, state_count }; +USER_DEFINED_ENUM_FORMATTER(lock_state) struct zoolock_pair { diff --git a/src/zookeeper/test/CMakeLists.txt b/src/zookeeper/test/CMakeLists.txt index ed2f742ac7..6179f8d1ec 100644 --- a/src/zookeeper/test/CMakeLists.txt +++ b/src/zookeeper/test/CMakeLists.txt @@ -41,7 +41,7 @@ set(MY_PROJ_LIBS gtest ssl crypto - ) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/zookeeper/zookeeper_session.h b/src/zookeeper/zookeeper_session.h index e7d3e86258..41323fccf7 100644 --- a/src/zookeeper/zookeeper_session.h +++ b/src/zookeeper/zookeeper_session.h @@ -44,6 +44,7 @@ #include "runtime/service_app.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" +#include "utils/fmt_utils.h" #include "utils/synchronize.h" struct String_vector; @@ -205,3 +206,5 @@ class zookeeper_session }; } } + +USER_DEFINED_STRUCTURE_FORMATTER(::dsn::dist::zookeeper_session); diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index a0984a2cad..f005f321aa 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -21,9 +21,9 @@ project(pegasus_thirdparties) cmake_minimum_required(VERSION 3.11.0) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - # require at least gcc 5.4.0 - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.4.0) - message(FATAL_ERROR "GCC version must be at least 5.4.0!") + # require at least gcc 7.0.0 + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0.0) + message(FATAL_ERROR "GCC version must be at least 7.0.0!") endif () endif () @@ -159,35 +159,6 @@ ExternalProject_Add(thrift DEPENDS boost ) -if (NOT APPLE) - # kerberos - ExternalProject_Add(krb5 - URL ${OSS_URL_PREFIX}/krb5-1.16.1.tar.gz - http://web.mit.edu/kerberos/dist/krb5/1.16/krb5-1.16.1.tar.gz - URL_MD5 848e9b80d6aaaa798e3f3df24b83c407 - CONFIGURE_COMMAND cd src && ./configure --prefix=${TP_OUTPUT} - BUILD_COMMAND cd src && make - INSTALL_COMMAND cd src && make install - BUILD_IN_SOURCE 1 - ) - - # cyrus-sasl - ExternalProject_Add(cyrus-sasl - URL ${OSS_URL_PREFIX}/cyrus-sasl-2.1.27.tar.gz - http://www.cyrusimap.org/releases/cyrus-sasl-2.1.27.tar.gz - URL_MD5 a33820c66e0622222c5aefafa1581083 - CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} - --enable-gssapi=${TP_OUTPUT} - --enable-scram=no - --enable-digest=no - --enable-cram=no - --enable-otp=no - BUILD_COMMAND make - INSTALL_COMMAND make install - BUILD_IN_SOURCE 1 - ) -endif() - check_cxx_compiler_flag(-Wformat-overflow COMPILER_SUPPORTS_FORMAT_OVERFLOW) if (COMPILER_SUPPORTS_FORMAT_OVERFLOW) set(ZOOKEEPER_CFLAGS -Wno-error=format-overflow) @@ -212,9 +183,6 @@ ExternalProject_Add(zookeeper INSTALL_COMMAND "" BUILD_IN_SOURCE 1 ) -if (NOT APPLE) - add_dependencies(zookeeper cyrus-sasl krb5) -endif () ExternalProject_Add(libevent URL ${OSS_URL_PREFIX}/libevent-release-2.1.8-stable.tar.gz @@ -276,11 +244,9 @@ ExternalProject_Add(fds BUILD_IN_SOURCE 1 ) -# fmtlib >=6.x requires c++14 support, do not update this library for now ExternalProject_Add(fmt - URL ${OSS_URL_PREFIX}/fmt-5.3.0.tar.gz - https://github.com/fmtlib/fmt/archive/refs/tags/5.3.0.tar.gz - URL_MD5 1015bf3ff2a140dfe03de50ee2469401 + URL https://github.com/fmtlib/fmt/archive/refs/tags/10.1.1.tar.gz + URL_MD5 0d41a16f1b3878d44e6fd7ff1f6cc45a CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DCMAKE_BUILD_TYPE=release -DFMT_TEST=false @@ -310,11 +276,7 @@ ExternalProject_Add(civetweb ExternalProject_Get_property(civetweb SOURCE_DIR) set(civetweb_SRC ${SOURCE_DIR}) -ExternalProject_Add(curl - URL ${OSS_URL_PREFIX}/curl-7.47.0.tar.gz - http://curl.haxx.se/download/curl-7.47.0.tar.gz - URL_MD5 5109d1232d208dfd712c0272b8360393 - CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} +set(CURL_OPTIONS --disable-dict --disable-file --disable-ftp @@ -335,6 +297,22 @@ ExternalProject_Add(curl --without-libssh2 --without-ssl --without-libidn + --without-zstd + ) +if (APPLE) + set(CURL_OPTIONS + ${CURL_OPTIONS} + --without-nghttp2 + --without-libidn2 + --without-brotli + ) +endif () +ExternalProject_Add(curl + URL ${OSS_URL_PREFIX}/curl-8.4.0.tar.gz + http://curl.haxx.se/download/curl-8.4.0.tar.gz + URL_MD5 533e8a3b1228d5945a6a512537bea4c7 + CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} + ${CURL_OPTIONS} BUILD_IN_SOURCE 1 ) @@ -392,15 +370,15 @@ ExternalProject_Add(jemalloc BUILD_IN_SOURCE 1 ) -option(ROCKSDB_PORTABLE "build a portable binary" OFF) +option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0) -# The patch name '0879c240' means the patch of rocksdb: -# https://github.com/facebook/rocksdb/commit/0879c240404b00142ba4718f36cd3f2bd537192d +# This is the commit "Update the status badge to pegasus-kv/rocksdb's own (#18) " on +# branch v8.5.3-pegasus-encrypt of https://github.com/pegasus-kv/rocksdb.git. +# The v8.5.3-pegasus-encrypt branch is based on the v8.5.3 tag of facebook/rocksdb and add +# the encryption feature. ExternalProject_Add(rocksdb - URL ${OSS_URL_PREFIX}/rocksdb-6.6.4.tar.gz - https://github.com/facebook/rocksdb/archive/refs/tags/v6.6.4.tar.gz - URL_MD5 7f7fcca3e96b7d83ef332804c90070c8 - PATCH_COMMAND patch -p1 < ${TP_DIR}/rocksdb_fix_atomic_flush_0879c240.patch + URL https://github.com/pegasus-kv/rocksdb/archive/e72106597e6b7924485cadc2433b66029f1aaf17.zip + URL_MD5 6f6daef703586ce788643bbb8a984fff DEPENDS jemalloc CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DFAIL_ON_WARNINGS=OFF @@ -413,6 +391,7 @@ ExternalProject_Add(rocksdb -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF -DUSE_RTTI=ON + -DWITH_OPENSSL=ON # enable encryption -DCMAKE_BUILD_TYPE=Release -DWITH_JEMALLOC=${USE_JEMALLOC} -DJEMALLOC_ROOT_DIR=${TP_OUTPUT}