diff --git a/.github/workflows/standardization_lint.yaml b/.github/workflows/standardization_lint.yaml index ef94b89951..66bcd16276 100644 --- a/.github/workflows/standardization_lint.yaml +++ b/.github/workflows/standardization_lint.yaml @@ -50,18 +50,6 @@ jobs: - uses: actions/checkout@v3 - uses: gaurav-nelson/github-action-markdown-link-check@1.0.13 - related_issue: - name: Check issue - runs-on: ubuntu-latest - steps: - - uses: neofinancial/ticket-check-action@v1.3.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - titleFormat: '%title%' - quiet: true - bodyRegex: '#(\d+)' - bodyURLRegex: 'http(s?):\/\/(github.com)(\/apache)(\/incubator-pegasus)(\/issues)\/\d+' - dockerfile_linter: name: Lint Dockerfile runs-on: ubuntu-latest diff --git a/.licenserc.yaml b/.licenserc.yaml index 53d316d23b..2cea94f325 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -20,80 +20,104 @@ header: copyright-owner: Apache Software Foundation paths-ignore: + # Configuration files that could not be added with copyright info (otherwise would lead to error). + - '.rat-excludes' + - '**/*.csv' + - '**/*.json' # All the type of licenses of this project should be added to LICENSE. + - 'DISCLAIMER-WIP' - 'LICENSE' - 'NOTICE' - - '.github/pull_request_template.md' + # Template files of issues and pull requests for Github. - '.github/ISSUE_TEMPLATE/bug_report.md' - '.github/ISSUE_TEMPLATE/feature-request.md' - '.github/ISSUE_TEMPLATE/general_question.md' - - '.rat-excludes' - - 'DISCLAIMER-WIP' + - '.github/pull_request_template.md' + # Image files for docs. + - '**/*.jpg' + - '**/*.png' + # Files in pdf format. + - '**/*.pdf' + # Special files for golang. + - '**/go.sum' + # TODO(wangdan): Generated files for go client, could generate dynamically? + - 'go-client/idl/base/GoUnusedProtection__.go' + - 'go-client/idl/base/dsn_err_string.go' + - 'go-client/idl/base/rocskdb_err_string.go' + # Special files for nodejs. + - '**/.npmigonre' + # Special files for python. - 'python-client/requirement.txt' - - '.devcontainer/devcontainer.json' + # Text files used for tests and could not be added with copyright info (otherwise would lead to error). + - 'src/aio/test/copy_source.txt' + - 'src/runtime/test/command.txt' + - 'src/failure_detector/test/gtest.filter' + - 'src/meta/test/meta_state/gtest.filter' + - 'src/meta/test/suite1' + - 'src/meta/test/suite2' + - 'src/nfs/test/nfs_test_file1' + - 'src/nfs/test/nfs_test_file2' + - 'src/runtime/test/gtest.filter' + # Used for tests and should be empty, or ignore all comment lines (otherwise would lead to error). + - 'src/utils/test/config-empty.ini' + # Binary files used for tests and could not be added with copyright info (otherwise would lead to error). + - 'src/replica/duplication/test/log.1.0.handle_real_private_log' + - 'src/replica/duplication/test/log.1.0.handle_real_private_log2' + - 'src/replica/duplication/test/log.1.0.all_loaded_are_write_empties' + # Used for patches for thirdparties. + - 'thirdparty/fix_fds_for_macos.patch' + - 'thirdparty/fix_jemalloc_for_m1_on_macos.patch' + - 'thirdparty/fix_libevent_for_macos.patch' + - 'thirdparty/fix_prometheus-cpp_limits.patch' + - 'thirdparty/fix_s2_for_aarch64.patch' + - 'thirdparty/fix_thrift_for_cpp11.patch' + - 'thirdparty/rocksdb_fix_atomic_flush_0879c240.patch' # TODO(yingchun): shell/* files are import from thirdparties, we can move them to thirdparty later. + # Copyright (c) 2016, Adi Shavit - 'src/shell/argh.h' + # Copyright (c) 2010-2016, Salvatore Sanfilippo, etc. - 'src/shell/linenoise/linenoise.c' + # Copyright (c) 2010-2014, Salvatore Sanfilippo, etc. - 'src/shell/linenoise/linenoise.h' + # Copyright (c) 2006-2015, Salvatore Sanfilippo, etc. - 'src/shell/sds/sds.c' - 'src/shell/sds/sds.h' - 'src/shell/sds/sdsalloc.h' - - '**/go.sum' - - '**/*.csv' - - '**/*.json' - - 'go-client/idl/base/dsn_err_string.go' - - 'go-client/idl/base/rocskdb_err_string.go' - - 'go-client/idl/base/GoUnusedProtection__.go' - - '**/.npmigonre' # Copyright (c) Facebook, Inc - 'src/utils/TokenBucket.h' - 'src/utils/test/TokenBucketTest.cpp' - 'src/utils/test/TokenBucketTest.h' + # https://github.com/preshing/modern-cpp-threading/blob/master/LICENSE + - 'src/utils/hpc_locks/autoreseteventcondvar.h' # https://github.com/preshing/cpp11-on-multicore/blob/master/LICENSE - 'src/utils/hpc_locks/autoresetevent.h' - - 'src/utils/hpc_locks/autoreseteventcondvar.h' - 'src/utils/hpc_locks/benaphore.h' - 'src/utils/hpc_locks/bitfield.h' - 'src/utils/hpc_locks/readme.txt' - 'src/utils/hpc_locks/rwlock.h' - 'src/utils/hpc_locks/sema.h' - # Copyright (c) xxxx The Chromium Authors + # Copyright (c) 2011 The Chromium Authors - 'src/utils/safe_strerror_posix.h' + # Copyright (c) 2012 The Chromium Authors - 'src/runtime/build_config.h' - 'src/utils/test/autoref_ptr_test.cpp' + # Copyright (c) 2006-2009 The Chromium Authors - 'src/utils/safe_strerror_posix.cpp' # Copyright 2017 The Abseil Authors - 'src/utils/absl/base/internal/invoke.h' - 'src/utils/absl/utility/utility.h' + - 'src/utils/memutil.h' - 'src/utils/smart_pointers.h' + - 'src/utils/string_view.cpp' - 'src/utils/string_view.h' - 'src/utils/test/memutil_test.cpp' - - 'src/utils/test/string_view_test.cpp' - 'src/utils/test/smart_pointers_test.cpp' - - 'src/utils/memutil.h' - - 'src/utils/string_view.cpp' + - 'src/utils/test/string_view_test.cpp' # Copyright (c) 2010-2011, Rob Jansen - 'cmake_modules/FindRT.cmake' - 'cmake_modules/FindDL.cmake' # Copyright (c) 2017 Guillaume Papin - 'scripts/run-clang-format.py' - # need manual fix - - 'src/failure_detector/test/gtest.filter' - - 'src/meta/test/meta_state/gtest.filter' - - 'src/meta/test/suite1' - - 'src/meta/test/suite2' - - 'src/nfs/test/nfs_test_file1' - - 'src/nfs/test/nfs_test_file2' - - 'src/runtime/test/gtest.filter' - - 'thirdparty/fix_fds_for_macos.patch' - - 'thirdparty/fix_jemalloc_for_m1_on_macos.patch' - - 'thirdparty/fix_libevent_for_macos.patch' - - 'thirdparty/fix_prometheus-cpp_limits.patch' - - 'thirdparty/fix_s2_for_aarch64.patch' - - 'thirdparty/fix_thrift_for_cpp11.patch' - - 'thirdparty/rocksdb_fix_atomic_flush_0879c240.patch' - # should be empty, or ignore all comment lines - - 'src/utils/test/config-empty.ini' # The MIT License (MIT), Copyright (c) 2015 Microsoft Corporation - 'cmake_modules/BaseFunctions.cmake' - 'docs/rdsn-README.md' @@ -105,7 +129,6 @@ header: - 'idl/replica_admin.thrift' - 'scripts/compile_thrift.py' - 'scripts/learn_stat.py' - - 'src/common/api_common.h' - 'src/runtime/api_layer1.h' - 'src/runtime/api_task.h' - 'src/utils/api_utilities.h' @@ -113,7 +136,6 @@ header: - 'src/common/json_helper.h' - 'src/runtime/rpc/rpc_stream.h' - 'src/runtime/rpc/serialization.h' - - 'src/common/serialization_helper/dsn.layer2_types.h' - 'src/common/serialization_helper/dsn_types.h' - 'src/common/serialization_helper/thrift_helper.h' - 'src/runtime/serverlet.h' @@ -131,7 +153,6 @@ header: - 'src/client/partition_resolver.h' - 'src/replica/replica_base.h' - 'src/common/replica_envs.h' - - 'src/replica/replica_test_utils.h' - 'src/common/replication.codes.h' - 'src/replica/replication_app_base.h' - 'src/client/replication_ddl_client.h' @@ -187,7 +208,6 @@ header: - 'src/utils/configuration.h' - 'src/utils/crc.h' - 'src/utils/customizable_id.h' - - 'src/utils/dlib.h' - 'src/utils/enum_helper.h' - 'src/utils/error_code.h' - 'src/utils/errors.h' @@ -220,7 +240,6 @@ header: - 'src/aio/test/aio.cpp' - 'src/aio/test/clear.sh' - 'src/aio/test/config.ini' - - 'src/aio/test/copy_source.txt' - 'src/aio/test/run.sh' - 'src/block_service/test/config-test.ini' - 'src/client/CMakeLists.txt' @@ -363,7 +382,6 @@ header: - 'src/replica/replica_learn.cpp' - 'src/replica/replica_stub.cpp' - 'src/replica/replica_stub.h' - - 'src/replica/replica_test_utils.cpp' - 'src/replica/replication_app_base.cpp' - 'src/replica/replication_service_app.cpp' - 'src/replica/split/test/config-test.ini' @@ -514,7 +532,6 @@ header: - 'src/replica/test/run.sh' - 'src/runtime/CMakeLists.txt' - 'src/runtime/core_main.cpp' - - 'src/runtime/dsn.layer2_types.cpp' - 'src/runtime/env.sim.cpp' - 'src/runtime/env.sim.h' - 'src/runtime/fault_injector.cpp' @@ -569,7 +586,6 @@ header: - 'src/runtime/test/address_test.cpp' - 'src/runtime/test/async_call.cpp' - 'src/runtime/test/clear.sh' - - 'src/runtime/test/command.txt' - 'src/runtime/test/config-test-corrupt-message.ini' - 'src/runtime/test/config-test-sim.ini' - 'src/runtime/test/config-test.ini' diff --git a/LICENSE b/LICENSE index f6862595d8..06453d0c0f 100644 --- a/LICENSE +++ b/LICENSE @@ -231,7 +231,8 @@ limitations under the License. -------------------------------------------------------------------------------- -src/shell/linenoise/* - BSD-2-Clause License +src/shell/linenoise/linenoise.h - BSD-2-Clause License +src/shell/linenoise/LICENSE Copyright (c) 2010-2014, Salvatore Sanfilippo Copyright (c) 2010-2013, Pieter Noordhuis @@ -263,6 +264,38 @@ src/shell/linenoise/* - BSD-2-Clause License -------------------------------------------------------------------------------- +src/shell/linenoise/linenoise.c - BSD-2-Clause License + + Copyright (c) 2010-2016, Salvatore Sanfilippo + Copyright (c) 2010-2013, Pieter Noordhuis + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + src/shell/sds/* - BSD-2-Clause License Copyright (c) 2006-2015, Salvatore Sanfilippo @@ -327,12 +360,12 @@ src/shell/argh.h - BSD-3-Clause License -------------------------------------------------------------------------------- -src/utils/smart_pointers.h - Apache 2.0 License -src/utils/string_view.h -src/utils/absl/base/internal/invoke.h +src/utils/absl/base/internal/invoke.h - Apache 2.0 License src/utils/absl/utility/utility.h src/utils/memutil.h +src/utils/smart_pointers.h src/utils/string_view.cpp +src/utils/string_view.h src/utils/test/memutil_test.cpp src/utils/test/smart_pointers_test.cpp src/utils/test/string_view_test.cpp @@ -406,11 +439,12 @@ limitations under the License. -------------------------------------------------------------------------------- src/utils/hpc_locks/autoreseteventcondvar.h - zlib License -src/utils/hpc_locks/rwlock.h src/utils/hpc_locks/autoresetevent.h -src/utils/hpc_locks/sema.h -src/utils/hpc_locks/bitfield.h src/utils/hpc_locks/benaphore.h +src/utils/hpc_locks/bitfield.h +src/utils/hpc_locks/readme.txt +src/utils/hpc_locks/rwlock.h +src/utils/hpc_locks/sema.h Copyright (c) 2015 Jeff Preshing diff --git a/NOTICE b/NOTICE index 6b69a1ad10..4bef57620b 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Pegasus -Copyright 2022 The Apache Software Foundation +Copyright 2023 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/docker/pegasus-build-env/ubuntu1804/Dockerfile b/docker/pegasus-build-env/ubuntu1804/Dockerfile index 667b64120e..fb4c9a47ac 100644 --- a/docker/pegasus-build-env/ubuntu1804/Dockerfile +++ b/docker/pegasus-build-env/ubuntu1804/Dockerfile @@ -63,7 +63,7 @@ RUN add-apt-repository ppa:git-core/ppa -y; \ apt-get install pkg-config -y --no-install-recommends; \ rm -rf /var/lib/apt/lists/* -RUN pip3 install --upgrade pip && pip3 install --no-cache-dir cmake +RUN pip3 install --no-cache-dir --upgrade pip && pip3 install --no-cache-dir cmake RUN wget --progress=dot:giga https://github.com/apache/thrift/archive/refs/tags/0.11.0.tar.gz -P /opt/thrift && \ cd /opt/thrift && tar xzf 0.11.0.tar.gz && cd thrift-0.11.0 && ./bootstrap.sh && \ diff --git a/java-client/pom.xml b/java-client/pom.xml index 405ce3f7a0..f1d327d49d 100644 --- a/java-client/pom.xml +++ b/java-client/pom.xml @@ -29,7 +29,7 @@ org.apache.pegasus pegasus-client - 2.4-SNAPSHOT + 2.5.0-SNAPSHOT jar Pegasus Java Client diff --git a/java-client/scripts/ci-test.sh b/java-client/scripts/ci-test.sh index c1b1f2346b..935ee96369 100755 --- a/java-client/scripts/ci-test.sh +++ b/java-client/scripts/ci-test.sh @@ -59,8 +59,8 @@ cd ../ pushd scripts -echo "bash recompile_thrift.sh" -bash recompile_thrift.sh +echo "run recompile_thrift.sh" +./recompile_thrift.sh popd mvn spotless:apply diff --git a/nodejs-client/package.json b/nodejs-client/package.json index c413ca3926..eba987f600 100644 --- a/nodejs-client/package.json +++ b/nodejs-client/package.json @@ -1,6 +1,6 @@ { "name": "pegasus-nodejs-client", - "version": "1.0.14", + "version": "2.5.0", "description": "offical pegasus nodejs client", "main": "index.js", "scripts": { diff --git a/python-client/pypegasus/__init__.py b/python-client/pypegasus/__init__.py index 151cbb8a4b..9a8e0626c5 100644 --- a/python-client/pypegasus/__init__.py +++ b/python-client/pypegasus/__init__.py @@ -18,4 +18,4 @@ # specific language governing permissions and limitations # under the License. -__version__ = '0.0.1.2' +__version__ = '2.5.0' diff --git a/scala-client/build.sbt b/scala-client/build.sbt index 8aa5fa2a79..38313ad51b 100644 --- a/scala-client/build.sbt +++ b/scala-client/build.sbt @@ -17,7 +17,7 @@ * under the License. */ -version := "2.2.0-3-release" +version := "2.5.0-SNAPSHOT" organization := "org.apache" @@ -54,6 +54,6 @@ credentials += Credentials( libraryDependencies ++= Seq( "com.google.guava" % "guava" % "21.0", - "org.apache.pegasus" % "pegasus-client" % "2.4-SNAPSHOT", + "org.apache.pegasus" % "pegasus-client" % "2.5.0-SNAPSHOT", "org.scalatest" %% "scalatest" % "3.0.3" % Test ) diff --git a/scala-client/scripts/ci-test.sh b/scala-client/scripts/ci-test.sh index 9b721c6edd..dddb168ef3 100755 --- a/scala-client/scripts/ci-test.sh +++ b/scala-client/scripts/ci-test.sh @@ -43,6 +43,10 @@ sbt scalafmtSbtCheck scalafmtCheck test:scalafmtCheck git clone https://github.com/apache/incubator-pegasus.git cd incubator-pegasus/java-client git checkout master +cd scripts +echo "run recompile_thrift.sh" +./recompile_thrift.sh +cd .. mvn clean package -DskipTests -Dcheckstyle.skip=true mvn clean install -DskipTests -Dcheckstyle.skip=true cd .. diff --git a/scripts/check_license.py b/scripts/check_license.py new file mode 100755 index 0000000000..8151979bd5 --- /dev/null +++ b/scripts/check_license.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import pprint + +PRJ_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +YML_PATH = os.path.join(PRJ_PATH, '.licenserc.yaml') + +IGNORED_STARTS_WITH = ['.git/', '.idea/'] +IGNORED_ENDS_WITH = ['.swp', '.npmigonre', 'go.sum', '.csv', '.json', '.pdf', '.jpg', '.png'] +IGNORED_NAMES = {'.licenserc.yaml', 'LICENSE', 'tags'} + +COPYRIGHT_MARKERS = [ + "Copyright (c) 2016, Adi Shavit", + "Copyright (c) 2010-2016, Salvatore Sanfilippo", + "Copyright (c) 2010-2014, Salvatore Sanfilippo", + "Copyright (c) 2006-2015, Salvatore Sanfilippo", + "Copyright (c) Facebook, Inc", + "https://github.com/preshing/modern-cpp-threading", + "https://github.com/preshing/cpp11-on-multicore", + "Copyright (c) 2011 The Chromium Authors", + "Copyright (c) 2012 The Chromium Authors", + "Copyright (c) 2006-2009 The Chromium Authors", + "Copyright 2017 The Abseil Authors", + "Copyright (c) 2010-2011, Rob Jansen", + "Copyright (c) 2017 Guillaume Papin", + "Copyright (c) 2015 Microsoft Corporation", +] +IGNORED_COPYRIGHT_MARKERS = ["http://www.apache.org/licenses/LICENSE-2.0"] + +NO_COPYRIGHT_MARKER_KEY = "NO_COPYRIGHT_MARKER" +IGNORED_COPYRIGHT_MARKER_KEY = "IGNORED_COPYRIGHT_MARKER" + + +def mark_file(path): + with open(path) as f: + try: + for line in f: + for marker in IGNORED_COPYRIGHT_MARKERS: + if marker in line: + return IGNORED_COPYRIGHT_MARKER_KEY + + for marker in COPYRIGHT_MARKERS: + if marker in line: + return marker + except UnicodeDecodeError: + # Ignore UnicodeDecodeError, since some files might be binary. + pass + + # No marker was found, thus marked with no copyright. + return NO_COPYRIGHT_MARKER_KEY + + +def is_path_ignored(path): + for header in IGNORED_STARTS_WITH: + if path.startswith(header): + return True + + for trailer in IGNORED_ENDS_WITH: + if path.endswith(trailer): + return True + + return False + + +def is_name_ignored(name): + return name in IGNORED_NAMES + + +def classify_files(): + """ + Scan all the files of the project, mark the ones that have copyright info. + """ + marked_files = {} + + for abs_dir, sub_dirs, file_names in os.walk(PRJ_PATH): + rel_dir = os.path.relpath(abs_dir, PRJ_PATH) + if rel_dir == '.': + # Drop the possible prefixed './' for the relative paths. + rel_dir = '' + + for name in file_names: + # Some kinds of files should be ignored. + if is_name_ignored(name): + continue + + rel_path = os.path.join(rel_dir, name) + + # Some kinds of dirs/files should be ignored. + if is_path_ignored(rel_path): + continue + + path = os.path.join(abs_dir, name) + marker = mark_file(path) + + # Some kinds of copyright could be ignored, such as Apache LICENSE-2.0. + if marker == IGNORED_COPYRIGHT_MARKER_KEY: + continue + + if marker not in marked_files: + marked_files[marker] = set() + marked_files[marker].add(rel_path) + + return marked_files + + +def parse_yml(): + """ + Scan all the files in .licenserc.yaml, mark the ones that have copyright info. + """ + marked_files = {} + + with open(YML_PATH) as f: + # The files without copyright info are marked with the specific key. + current_marker = NO_COPYRIGHT_MARKER_KEY + for line in f: + for marker in COPYRIGHT_MARKERS: + if marker in line: + # Files in following lines would belong to this copyright. + current_marker = marker + break + else: + begin_idx = line.find("'") + if begin_idx < 0: + # There's no file in this line, thus copyright would be reset. + current_marker = NO_COPYRIGHT_MARKER_KEY + continue + + begin_idx += 1 + end_idx = line.find("'", begin_idx) + if end_idx < 0: + raise ValueError("Invalid file path line in {yml_path}".format(yml_path=YML_PATH)) + + path = line[begin_idx:end_idx] + + # Some kinds of dirs/files should be ignored. + if is_name_ignored(os.path.basename(path)): + continue + if is_path_ignored(path): + continue + + if current_marker not in marked_files: + marked_files[current_marker] = set() + marked_files[current_marker].add(path) + + return marked_files + + +def check_diff(): + """ + Check if .licenserc.yaml is consistent with all real files of the project. + """ + yml_marked_files = parse_yml() + marked_files = classify_files() + for yml_marker, yml_files in yml_marked_files.items(): + if yml_marker not in marked_files: + print( + "marker {yml_marker} in {yml_path} not found in any file of the project".format(yml_marker=yml_marker, + yml_path=YML_PATH)) + continue + + files = marked_files[yml_marker] + yml_plus = yml_files - files + yml_minus = files - yml_files + if not yml_plus and not yml_minus: + # .licenserc.yaml is consistent with the project. + print( + "No diff found for marker '{yml_marker}' in {yml_path}".format(yml_marker=yml_marker, + yml_path=YML_PATH)) + del marked_files[yml_marker] + continue + + print("Diff found for marker '{yml_marker}' in {yml_path}:".format(yml_marker=yml_marker, yml_path=YML_PATH)) + if yml_plus: + # Files in .licenserc.yaml, but not in the project. + print("{plus}: {yml_plus}".format(plus='+' * len(yml_plus), yml_marker=yml_marker, yml_plus=yml_plus)) + if yml_minus: + # Files in the project, but not in .licenserc.yaml. + print("{minus}: {yml_minus}".format(minus='-' * len(yml_minus), yml_minus=yml_minus)) + + del marked_files[yml_marker] + + if not marked_files: + return + + print("markers in some files of the project not found in {yml_path}:".format(yml_path=YML_PATH)) + pprint.pprint(marked_files) + + +def main(): + check_diff() + + +if __name__ == '__main__': + main() diff --git a/src/aio/test/aio.cpp b/src/aio/test/aio.cpp index fa6b0114ae..2fadded6db 100644 --- a/src/aio/test/aio.cpp +++ b/src/aio/test/aio.cpp @@ -81,8 +81,7 @@ class aio_test : public pegasus::encrypt_data_test_base const std::string kTestFileName = "aio_test.txt"; }; -// TODO(yingchun): ENCRYPTION: add enable encryption test. -INSTANTIATE_TEST_CASE_P(, aio_test, ::testing::Values(false)); +INSTANTIATE_TEST_CASE_P(, aio_test, ::testing::Values(false, true)); TEST_P(aio_test, basic) { diff --git a/src/base/pegasus_utils.h b/src/base/pegasus_utils.h index b521209681..20c65b926e 100644 --- a/src/base/pegasus_utils.h +++ b/src/base/pegasus_utils.h @@ -28,8 +28,11 @@ #include #include +#include "utils/flags.h" #include "utils/string_view.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + namespace dsn { class rpc_address; } // namespace dsn @@ -40,6 +43,7 @@ namespace utils { // it's seconds since 2016.01.01-00:00:00 GMT const uint32_t epoch_begin = 1451606400; inline uint32_t epoch_now() { return time(nullptr) - epoch_begin; } +const static std::string kRedactedString = ""; // extract "host" from rpc_address void addr2host(const ::dsn::rpc_address &addr, char *str, int len); @@ -97,6 +101,15 @@ std::string c_escape_string(const T &src, bool always_escape = false) return s; } +template +std::string c_escape_sensitive_string(const T &src, bool always_escape = false) +{ + if (FLAGS_encrypt_data_at_rest) { + return kRedactedString; + } + return c_escape_string(src, always_escape); +} + // ---------------------------------------------------------------------- // c_unescape_string() // Copies 'src' to 'dest', unescaping '0xFF'-style escape sequences to @@ -106,6 +119,16 @@ std::string c_escape_string(const T &src, bool always_escape = false) // ---------------------------------------------------------------------- int c_unescape_string(const std::string &src, std::string &dest); +template +const std::string &redact_sensitive_string(const T &src) +{ + if (FLAGS_encrypt_data_at_rest) { + return kRedactedString; + } else { + return src; + } +} + inline dsn::string_view to_string_view(rocksdb::Slice s) { return {s.data(), s.size()}; } inline rocksdb::Slice to_rocksdb_slice(dsn::string_view s) { return {s.data(), s.size()}; } diff --git a/src/base/test/redact_sensitive_string_test.cpp b/src/base/test/redact_sensitive_string_test.cpp new file mode 100644 index 0000000000..d0bbcd7c61 --- /dev/null +++ b/src/base/test/redact_sensitive_string_test.cpp @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// IWYU pragma: no_include +// IWYU pragma: no_include +#include +#include + +#include "base/pegasus_utils.h" + +const std::string test_string = "pegasus"; + +TEST(pegasus_utils, redact_sensitive_string) +{ + FLAGS_encrypt_data_at_rest = true; + auto result_string = pegasus::utils::redact_sensitive_string(test_string); + ASSERT_EQ(pegasus::utils::kRedactedString, result_string); +} + +TEST(pegasus_utils, redact_sensitive_string_with_encrypt) +{ + FLAGS_encrypt_data_at_rest = false; + auto result_string = pegasus::utils::redact_sensitive_string(test_string); + ASSERT_EQ("pegasus", result_string); +} + +TEST(pegasus_utils, c_escape_sensitive_string_with_no_encrypt_and_escape) +{ + FLAGS_encrypt_data_at_rest = false; + auto result_string = pegasus::utils::c_escape_sensitive_string(test_string, false); + ASSERT_EQ("pegasus", result_string); +} + +TEST(pegasus_utils, c_escape_sensitive_string_with_no_encrypt) +{ + FLAGS_encrypt_data_at_rest = false; + auto result_string = pegasus::utils::c_escape_sensitive_string(test_string, true); + ASSERT_EQ("\\x70\\x65\\x67\\x61\\x73\\x75\\x73", result_string); +} + +TEST(pegasus_utils, c_escape_sensitive_string_with_encrypt) +{ + FLAGS_encrypt_data_at_rest = true; + auto result_string = pegasus::utils::c_escape_sensitive_string(test_string, false); + ASSERT_EQ(pegasus::utils::kRedactedString, result_string); +} + +TEST(pegasus_utils, c_escape_sensitive_string_with_encrypt_and_escape) +{ + FLAGS_encrypt_data_at_rest = true; + auto result_string = pegasus::utils::c_escape_sensitive_string(test_string, true); + ASSERT_EQ(pegasus::utils::kRedactedString, result_string); +} diff --git a/src/block_service/fds/fds_service.cpp b/src/block_service/fds/fds_service.cpp index bed4a4fdaa..8be5709ab5 100644 --- a/src/block_service/fds/fds_service.cpp +++ b/src/block_service/fds/fds_service.cpp @@ -39,6 +39,7 @@ #include "utils/TokenBucket.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -606,7 +607,8 @@ dsn::task_ptr fds_file_object::upload(const upload_request &req, const std::string &local_file = req.input_local_name; // get file size int64_t file_sz = 0; - dsn::utils::filesystem::file_size(local_file, file_sz); + dsn::utils::filesystem::file_size( + local_file, dsn::utils::FileDataType::kSensitive, file_sz); upload_response resp; // TODO: we can cache the whole file in buffer, then upload the buffer rather than the @@ -671,6 +673,7 @@ dsn::task_ptr fds_file_object::download(const download_request &req, t->set_tracker(tracker); download_response resp; + // TODO(yingchun): use rocksdb API to implement this. std::shared_ptr handle(new std::ofstream( req.output_local_name, std::ios::binary | std::ios::out | std::ios::trunc)); if (!handle->is_open()) { diff --git a/src/block_service/hdfs/hdfs_service.cpp b/src/block_service/hdfs/hdfs_service.cpp index 5909f40de0..e50a8efe25 100644 --- a/src/block_service/hdfs/hdfs_service.cpp +++ b/src/block_service/hdfs/hdfs_service.cpp @@ -385,8 +385,8 @@ dsn::task_ptr hdfs_file_object::upload(const upload_request &req, rocksdb::EnvOptions env_options; env_options.use_direct_reads = FLAGS_enable_direct_io; std::unique_ptr rfile; - auto s = rocksdb::Env::Default()->NewSequentialFile( - req.input_local_name, &rfile, env_options); + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(req.input_local_name, &rfile, env_options); if (!s.ok()) { LOG_ERROR( "open local file '{}' failed, err = {}", req.input_local_name, s.ToString()); @@ -552,7 +552,8 @@ dsn::task_ptr hdfs_file_object::download(const download_request &req, rocksdb::EnvOptions env_options; env_options.use_direct_writes = FLAGS_enable_direct_io; std::unique_ptr wfile; - auto s = rocksdb::Env::Default()->NewWritableFile(target_file, &wfile, env_options); + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewWritableFile(target_file, &wfile, env_options); if (!s.ok()) { LOG_ERROR("create local file '{}' failed, err = {}", target_file, s.ToString()); break; diff --git a/src/block_service/local/local_service.cpp b/src/block_service/local/local_service.cpp index 9175961340..ccccea7d82 100644 --- a/src/block_service/local/local_service.cpp +++ b/src/block_service/local/local_service.cpp @@ -269,7 +269,8 @@ error_code local_file_object::load_metadata() std::string metadata_path = local_service::get_metafile(file_name()); std::string data; - auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), metadata_path, &data); + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), metadata_path, &data); if (!s.ok()) { LOG_WARNING("read file '{}' failed, err = {}", metadata_path, s.ToString()); return ERR_FS_INTERNAL; @@ -294,10 +295,11 @@ error_code local_file_object::store_metadata() meta.size = _size; std::string data = nlohmann::json(meta).dump(); std::string metadata_path = local_service::get_metafile(file_name()); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice(data), - metadata_path, - /* should_sync */ true); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(data), + metadata_path, + /* should_sync */ true); if (!s.ok()) { LOG_WARNING("store to metadata file {} failed, err={}", metadata_path, s.ToString()); return ERR_FS_INTERNAL; @@ -341,7 +343,7 @@ dsn::task_ptr local_file_object::write(const write_request &req, do { auto s = rocksdb::WriteStringToFile( - rocksdb::Env::Default(), + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), rocksdb::Slice(req.buffer.data(), req.buffer.length()), file_name(), /* should_sync */ true); @@ -418,7 +420,8 @@ dsn::task_ptr local_file_object::read(const read_request &req, rocksdb::EnvOptions env_options; env_options.use_direct_reads = FLAGS_enable_direct_io; std::unique_ptr sfile; - auto s = rocksdb::Env::Default()->NewSequentialFile(file_name(), &sfile, env_options); + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(file_name(), &sfile, env_options); if (!s.ok()) { LOG_WARNING("open file '{}' failed, err = {}", file_name(), s.ToString()); resp.err = ERR_FS_INTERNAL; diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt index 19dd06808b..0a6d66e406 100644 --- a/src/block_service/test/CMakeLists.txt +++ b/src/block_service/test/CMakeLists.txt @@ -28,6 +28,7 @@ set(MY_PROJ_LIBS dsn.block_service.fds dsn.block_service.hdfs dsn_runtime + dsn_utils galaxy-fds-sdk-cpp PocoNet PocoFoundation diff --git a/src/block_service/test/block_service_manager_test.cpp b/src/block_service/test/block_service_manager_test.cpp index ef93ed1962..a059062b25 100644 --- a/src/block_service/test/block_service_manager_test.cpp +++ b/src/block_service/test/block_service_manager_test.cpp @@ -72,8 +72,7 @@ class block_service_manager_test : public pegasus::encrypt_data_test_base std::string FILE_NAME = "test_file"; }; -// TODO(yingchun): ENCRYPTION: add enable encryption test. -INSTANTIATE_TEST_CASE_P(, block_service_manager_test, ::testing::Values(false)); +INSTANTIATE_TEST_CASE_P(, block_service_manager_test, ::testing::Values(false, true)); TEST_P(block_service_manager_test, remote_file_not_exist) { diff --git a/src/block_service/test/fds_service_test.cpp b/src/block_service/test/fds_service_test.cpp index cb14aa5cd2..d06f3718a1 100644 --- a/src/block_service/test/fds_service_test.cpp +++ b/src/block_service/test/fds_service_test.cpp @@ -147,6 +147,7 @@ void FDSClientTest::TearDown() {} DEFINE_TASK_CODE(lpc_btest, TASK_PRIORITY_HIGH, dsn::THREAD_POOL_DEFAULT) +// TODO(yingchun): add encryption test when FDSClient supports encryption. TEST_F(FDSClientTest, test_basic_operation) { const char *files[] = {"/fdstest/fdstest1/test1/test1", diff --git a/src/block_service/test/hdfs_service_test.cpp b/src/block_service/test/hdfs_service_test.cpp index 0eaec29e88..948ef990f2 100644 --- a/src/block_service/test/hdfs_service_test.cpp +++ b/src/block_service/test/hdfs_service_test.cpp @@ -90,7 +90,8 @@ void HDFSClientTest::generate_test_file(const std::string &filename) { int lines = FLAGS_num_test_file_lines; std::unique_ptr wfile; - auto s = rocksdb::Env::Default()->NewWritableFile(filename, &wfile, rocksdb::EnvOptions()); + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewWritableFile(filename, &wfile, rocksdb::EnvOptions()); ASSERT_TRUE(s.ok()) << s.ToString(); for (int i = 0; i < lines; ++i) { rocksdb::Slice data(fmt::format("{:04}d_this_is_a_simple_test_file\n", i)); @@ -128,8 +129,7 @@ void HDFSClientTest::write_test_files_async(const std::string &local_test_path, } } -// TODO(yingchun): ENCRYPTION: add enable encryption test. -INSTANTIATE_TEST_CASE_P(, HDFSClientTest, ::testing::Values(false)); +INSTANTIATE_TEST_CASE_P(, HDFSClientTest, ::testing::Values(false, true)); TEST_P(HDFSClientTest, test_hdfs_read_write) { diff --git a/src/block_service/test/local_service_test.cpp b/src/block_service/test/local_service_test.cpp index e4e6aeb2ee..a852f68ca6 100644 --- a/src/block_service/test/local_service_test.cpp +++ b/src/block_service/test/local_service_test.cpp @@ -36,6 +36,7 @@ #include "block_service/local/local_service.h" #include "test_util/test_util.h" +#include "utils/env.h" #include "utils/error_code.h" namespace dsn { @@ -47,8 +48,7 @@ class local_service_test : public pegasus::encrypt_data_test_base { }; -// TODO(yingchun): ENCRYPTION: add enable encryption test. -INSTANTIATE_TEST_CASE_P(, local_service_test, ::testing::Values(false)); +INSTANTIATE_TEST_CASE_P(, local_service_test, ::testing::Values(false, true)); TEST_P(local_service_test, store_metadata) { @@ -60,7 +60,8 @@ TEST_P(local_service_test, store_metadata) ASSERT_TRUE(boost::filesystem::exists(meta_file_path)); std::string data; - auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), meta_file_path, &data); + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), meta_file_path, &data); ASSERT_TRUE(s.ok()) << s.ToString(); nlohmann::json j = nlohmann::json::parse(data); @@ -76,10 +77,11 @@ TEST_P(local_service_test, load_metadata) { nlohmann::json j({{"md5", "abcde"}, {"size", 5}}); std::string data = j.dump(); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice(data), - meta_file_path, - /* should_sync */ true); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(data), + meta_file_path, + /* should_sync */ true); ASSERT_TRUE(s.ok()) << s.ToString(); ASSERT_EQ(ERR_OK, file.load_metadata()); @@ -88,10 +90,11 @@ TEST_P(local_service_test, load_metadata) } { - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice("invalid json string"), - meta_file_path, - /* should_sync */ true); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice("invalid json string"), + meta_file_path, + /* should_sync */ true); ASSERT_TRUE(s.ok()) << s.ToString(); local_file_object file2("a.txt"); @@ -101,10 +104,11 @@ TEST_P(local_service_test, load_metadata) { nlohmann::json j({{"md5", "abcde"}, {"no such key", "illegal"}}); std::string data = j.dump(); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice(data), - meta_file_path, - /* should_sync */ true); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(data), + meta_file_path, + /* should_sync */ true); ASSERT_TRUE(s.ok()) << s.ToString(); local_file_object file2("a.txt"); diff --git a/src/client_lib/mutation.cpp b/src/client_lib/mutation.cpp index e27eb0cf26..ba0431c3af 100644 --- a/src/client_lib/mutation.cpp +++ b/src/client_lib/mutation.cpp @@ -24,7 +24,7 @@ #include #include "../base/pegasus_utils.h" -#include "utils/string_view.h" +#include "utils/enum_helper.h" using namespace ::dsn; diff --git a/src/common/fs_manager.h b/src/common/fs_manager.h index 65fb0243ac..a8985f22ef 100644 --- a/src/common/fs_manager.h +++ b/src/common/fs_manager.h @@ -172,10 +172,10 @@ class fs_manager friend class replica_disk_migrator; friend class replica_disk_test_base; friend class open_replica_test; - FRIEND_TEST(fs_manager, find_best_dir_for_new_replica); - FRIEND_TEST(fs_manager, get_dir_node); + FRIEND_TEST(fs_manager_test, find_best_dir_for_new_replica); + FRIEND_TEST(fs_manager_test, get_dir_node); FRIEND_TEST(open_replica_test, open_replica_add_decree_and_ballot_check); - FRIEND_TEST(replica_error_test, test_auto_trash_of_corruption); + FRIEND_TEST(replica_test, test_auto_trash_of_corruption); }; } // replication } // dsn diff --git a/src/common/test/fs_manager_test.cpp b/src/common/test/fs_manager_test.cpp index 542b122f1c..6027c649d6 100644 --- a/src/common/test/fs_manager_test.cpp +++ b/src/common/test/fs_manager_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include @@ -33,6 +34,7 @@ #include "common/gpid.h" #include "common/replication_other_types.h" #include "metadata_types.h" +#include "test_util/test_util.h" #include "utils/fail_point.h" #include "utils/filesystem.h" @@ -47,7 +49,13 @@ TEST(dir_node, replica_dir) ASSERT_EQ("path/1.0.test", dn.replica_dir("test", gpid(1, 0))); } -TEST(fs_manager, initialize) +class fs_manager_test : public pegasus::encrypt_data_test_base +{ +}; + +INSTANTIATE_TEST_CASE_P(, fs_manager_test, ::testing::Values(false, true)); + +TEST_P(fs_manager_test, initialize) { fail::setup(); struct broken_disk_test @@ -69,7 +77,7 @@ TEST(fs_manager, initialize) fail::teardown(); } -TEST(fs_manager, dir_update_disk_status) +TEST_P(fs_manager_test, dir_update_disk_status) { struct update_disk_status { @@ -94,7 +102,7 @@ TEST(fs_manager, dir_update_disk_status) } } -TEST(fs_manager, get_dir_node) +TEST_P(fs_manager_test, get_dir_node) { fs_manager fm; fm.initialize({"./data1"}, {"data1"}); @@ -115,7 +123,7 @@ TEST(fs_manager, get_dir_node) ASSERT_EQ(nullptr, fm.get_dir_node(base_dir + "/data2/replica1")); } -TEST(fs_manager, find_replica_dir) +TEST_P(fs_manager_test, find_replica_dir) { fs_manager fm; fm.initialize({"./data1", "./data2", "./data3"}, {"data1", "data2", "data3"}); @@ -137,7 +145,7 @@ TEST(fs_manager, find_replica_dir) ASSERT_EQ(dn, dn1); } -TEST(fs_manager, create_replica_dir_if_necessary) +TEST_P(fs_manager_test, create_replica_dir_if_necessary) { fs_manager fm; @@ -154,7 +162,7 @@ TEST(fs_manager, create_replica_dir_if_necessary) ASSERT_EQ("data1", dn->tag); } -TEST(fs_manager, create_child_replica_dir) +TEST_P(fs_manager_test, create_child_replica_dir) { fs_manager fm; fm.initialize({"./data1", "./data2", "./data3"}, {"data1", "data2", "data3"}); @@ -174,7 +182,7 @@ TEST(fs_manager, create_child_replica_dir) ASSERT_EQ(dir, child_dir); } -TEST(fs_manager, find_best_dir_for_new_replica) +TEST_P(fs_manager_test, find_best_dir_for_new_replica) { // dn1 | 1.0, 1.1 +1.6 // dn2 | 1.2, 1.3 +1.7 2.0 diff --git a/src/geo/bench/bench.cpp b/src/geo/bench/bench.cpp index 0ac170b02e..18324e7673 100644 --- a/src/geo/bench/bench.cpp +++ b/src/geo/bench/bench.cpp @@ -34,6 +34,7 @@ #include "geo/lib/geo_client.h" #include "geo/lib/latlng_codec.h" +#include "utils/env.h" #include "utils/errors.h" #include "utils/fmt_logging.h" #include "utils/string_conv.h" @@ -110,7 +111,7 @@ int main(int argc, char **argv) RESULT_COUNT }; auto statistics = rocksdb::CreateDBStatistics(); - rocksdb::Env *env = rocksdb::Env::Default(); + rocksdb::Env *env = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); uint64_t start = env->NowNanos(); std::atomic count(test_count); dsn::utils::notify_event get_completed; diff --git a/src/geo/lib/geo_client.cpp b/src/geo/lib/geo_client.cpp index 0d4927695e..31953fa81d 100644 --- a/src/geo/lib/geo_client.cpp +++ b/src/geo/lib/geo_client.cpp @@ -138,8 +138,8 @@ int geo_client::set(const std::string &hash_key, auto async_set_callback = [&](int ec_, pegasus_client::internal_info &&info_) { if (ec_ != PERR_OK) { LOG_ERROR("set data failed. hash_key={}, sort_key={}, error={}", - hash_key, - sort_key, + utils::redact_sensitive_string(hash_key), + utils::redact_sensitive_string(sort_key), get_error_string(ec_)); ret = ec_; } @@ -186,8 +186,8 @@ void geo_client::async_set(const std::string &hash_key, if (ec_ != PERR_OK) { LOG_ERROR("set {} data failed. hash_key={}, sort_key={}, error={}", data_type_ == DataType::common ? "common" : "geo", - hash_key, - sort_key, + utils::redact_sensitive_string(hash_key), + utils::redact_sensitive_string(sort_key), get_error_string(ec_)); *ret = ec_; } @@ -238,8 +238,8 @@ int geo_client::get(const std::string &hash_key, lng_degrees = lng_degrees_; } else { LOG_WARNING("get data failed. hash_key={}, sort_key={}, error={}", - hash_key, - sort_key, + utils::redact_sensitive_string(hash_key), + utils::redact_sensitive_string(sort_key), get_error_string(ec_)); } ret = ec_; @@ -269,8 +269,8 @@ void geo_client::async_get(const std::string &hash_key, S2LatLng latlng; if (!_codec.decode_from_value(value_, latlng)) { LOG_ERROR("decode_from_value failed. hash_key={}, sort_key={}, value={}", - hash_key, - sort_key, + utils::redact_sensitive_string(hash_key), + utils::redact_sensitive_string(sort_key), value_); cb(PERR_GEO_DECODE_VALUE_ERROR, id, 0, 0); return; @@ -290,8 +290,8 @@ int geo_client::del(const std::string &hash_key, auto async_del_callback = [&](int ec_, pegasus_client::internal_info &&info_) { if (ec_ != PERR_OK) { LOG_ERROR("del data failed. hash_key={}, sort_key={}, error={}", - hash_key, - sort_key, + utils::redact_sensitive_string(hash_key), + utils::redact_sensitive_string(sort_key), get_error_string(ec_)); ret = ec_; } @@ -358,8 +358,8 @@ void geo_client::async_del(const std::string &hash_key, if (ec__ != PERR_OK) { LOG_ERROR("del {} data failed. hash_key={}, sort_key={}, error={}", data_type_ == DataType::common ? "common" : "geo", - hash_key, - sort_key, + utils::redact_sensitive_string(hash_key), + utils::redact_sensitive_string(sort_key), get_error_string(ec_)); *ret = ec__; } @@ -391,8 +391,8 @@ int geo_client::set_geo_data(const std::string &hash_key, if (ec_ != PERR_OK) { ret = ec_; LOG_ERROR("set geo data failed. hash_key={}, sort_key={}, error={}", - hash_key, - sort_key, + utils::redact_sensitive_string(hash_key), + utils::redact_sensitive_string(sort_key), get_error_string(ec_)); } set_completed.notify(); @@ -521,8 +521,8 @@ void geo_client::async_search_radial(const std::string &hash_key, ](int ec_, std::string &&value_, pegasus_client::internal_info &&) mutable { if (ec_ != PERR_OK) { LOG_ERROR("get failed. hash_key={}, sort_key={}, error={}", - hash_key, - sort_key, + utils::redact_sensitive_string(hash_key), + utils::redact_sensitive_string(sort_key), get_error_string(ec_)); cb(ec_, {}); return; @@ -531,8 +531,8 @@ void geo_client::async_search_radial(const std::string &hash_key, S2LatLng latlng; if (!_codec.decode_from_value(value_, latlng)) { LOG_ERROR("decode_from_value failed. hash_key={}, sort_key={}, value={}", - hash_key, - sort_key, + utils::redact_sensitive_string(hash_key), + utils::redact_sensitive_string(sort_key), value_); cb(ec_, {}); return; @@ -726,8 +726,8 @@ bool geo_client::generate_geo_keys(const std::string &hash_key, S2LatLng latlng; if (!_codec.decode_from_value(value, latlng)) { LOG_ERROR("decode_from_value failed. hash_key={}, sort_key={}, value={}", - hash_key, - sort_key, + utils::redact_sensitive_string(hash_key), + utils::redact_sensitive_string(sort_key), value); return false; } @@ -919,7 +919,8 @@ void geo_client::do_scan(pegasus_client::pegasus_scanner_wrapper scanner_wrapper if (distance <= S2Earth::ToMeters(cap_ptr->radius())) { std::string origin_hash_key, origin_sort_key; if (!restore_origin_keys(geo_sort_key, origin_hash_key, origin_sort_key)) { - LOG_ERROR("restore_origin_keys failed. geo_sort_key={}", geo_sort_key); + LOG_ERROR("restore_origin_keys failed. geo_sort_key={}", + utils::redact_sensitive_string(geo_sort_key)); cb(); return; } @@ -955,10 +956,10 @@ int geo_client::distance(const std::string &hash_key1, LOG_ERROR( "get distance failed. hash_key1={}, sort_key1={}, hash_key2={}, sort_key2={}, " "error={}", - hash_key1, - sort_key1, - hash_key2, - sort_key2, + utils::redact_sensitive_string(hash_key1), + utils::redact_sensitive_string(sort_key1), + utils::redact_sensitive_string(hash_key2), + utils::redact_sensitive_string(sort_key2), get_error_string(ec_)); ret = ec_; } @@ -988,10 +989,10 @@ void geo_client::async_distance(const std::string &hash_key1, if (ec_ != PERR_OK) { LOG_ERROR("get data failed. hash_key1={}, sort_key1={}, hash_key2={}, sort_key2={}, " "error={}", - hash_key1, - sort_key1, - hash_key2, - sort_key2, + utils::redact_sensitive_string(hash_key1), + utils::redact_sensitive_string(sort_key1), + utils::redact_sensitive_string(hash_key2), + utils::redact_sensitive_string(sort_key2), get_error_string(ec_)); *ret = ec_; } diff --git a/src/geo/test/geo_test.cpp b/src/geo/test/geo_test.cpp index f0777a9b03..e9df7a422a 100644 --- a/src/geo/test/geo_test.cpp +++ b/src/geo/test/geo_test.cpp @@ -18,6 +18,7 @@ */ #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -55,6 +56,10 @@ namespace geo { DSN_DECLARE_int32(min_level); +// TODO(yingchun): it doesn't make sense to derive from pegasus::encrypt_data_test_base to test +// encryption or non-encryption senarios, because the Pegasus cluster has been started with a +// fixed value of FLAGS_encrypt_data_at_rest. +// We can test the senarios after clearing and restarting the cluster. class geo_client_test : public ::testing::Test { public: diff --git a/src/include/pegasus/version.h b/src/include/pegasus/version.h index 35aa370055..37995f9f9f 100644 --- a/src/include/pegasus/version.h +++ b/src/include/pegasus/version.h @@ -18,4 +18,4 @@ */ #pragma once -#define PEGASUS_VERSION "2.4.0-SNAPSHOT" +#define PEGASUS_VERSION "2.5.0-SNAPSHOT" diff --git a/src/meta/meta_state_service_simple.cpp b/src/meta/meta_state_service_simple.cpp index 776cfba342..aa54612032 100644 --- a/src/meta/meta_state_service_simple.cpp +++ b/src/meta/meta_state_service_simple.cpp @@ -41,6 +41,7 @@ #include "runtime/task/task.h" #include "utils/autoref_ptr.h" #include "utils/binary_reader.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/strings.h" @@ -243,8 +244,8 @@ error_code meta_state_service_simple::initialize(const std::vector std::string log_path = dsn::utils::filesystem::path_combine(work_dir, "meta_state_service.log"); if (utils::filesystem::file_exists(log_path)) { std::unique_ptr log_file; - auto s = - rocksdb::Env::Default()->NewSequentialFile(log_path, &log_file, rocksdb::EnvOptions()); + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(log_path, &log_file, rocksdb::EnvOptions()); CHECK(s.ok(), "open log file '{}' failed, err = {}", log_path, s.ToString()); while (true) { diff --git a/src/meta/test/meta_backup_test.cpp b/src/meta/test/meta_backup_test.cpp index 212da5aea0..3692e0724f 100644 --- a/src/meta/test/meta_backup_test.cpp +++ b/src/meta/test/meta_backup_test.cpp @@ -38,6 +38,7 @@ #include "meta_test_base.h" #include "runtime/api_layer1.h" #include "runtime/rpc/rpc_address.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/fail_point.h" #include "utils/filesystem.h" @@ -108,7 +109,8 @@ class backup_service_test : public meta_test_base cold_backup::get_app_metadata_file(backup_root, app->app_name, app_id, backup_id); int64_t metadata_file_size = 0; - if (!dsn::utils::filesystem::file_size(metadata_file, metadata_file_size)) { + if (!dsn::utils::filesystem::file_size( + metadata_file, dsn::utils::FileDataType::kSensitive, metadata_file_size)) { return false; } return metadata_file_size > 0; diff --git a/src/meta/test/meta_state/meta_state_service.cpp b/src/meta/test/meta_state/meta_state_service.cpp index 4959670b29..2bc59246dc 100644 --- a/src/meta/test/meta_state/meta_state_service.cpp +++ b/src/meta/test/meta_state/meta_state_service.cpp @@ -293,8 +293,7 @@ class meta_state_service_test : public pegasus::encrypt_data_test_base { }; -// TODO(yingchun): ENCRYPTION: add enable encryption test. -INSTANTIATE_TEST_CASE_P(, meta_state_service_test, ::testing::Values(false)); +INSTANTIATE_TEST_CASE_P(, meta_state_service_test, ::testing::Values(false, true)); TEST_P(meta_state_service_test, simple) { diff --git a/src/nfs/test/CMakeLists.txt b/src/nfs/test/CMakeLists.txt index 64c7967ace..d3f26cc006 100644 --- a/src/nfs/test/CMakeLists.txt +++ b/src/nfs/test/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio rocksdb) +set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio rocksdb test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/nfs/test/main.cpp b/src/nfs/test/main.cpp index 0788129928..60564bf6b7 100644 --- a/src/nfs/test/main.cpp +++ b/src/nfs/test/main.cpp @@ -28,6 +28,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include @@ -48,8 +49,11 @@ #include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/threadpool_code.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + using namespace dsn; DEFINE_TASK_CODE_AIO(LPC_AIO_TEST_NFS, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) @@ -63,8 +67,7 @@ class nfs_test : public pegasus::encrypt_data_test_base { }; -// TODO(yingchun): ENCRYPTION: add enable encryption test. -INSTANTIATE_TEST_CASE_P(, nfs_test, ::testing::Values(false)); +INSTANTIATE_TEST_CASE_P(, nfs_test, ::testing::Values(false, true)); TEST_P(nfs_test, basic) { @@ -82,6 +85,13 @@ TEST_P(nfs_test, basic) // Prepare the source files information. std::vector kSrcFilenames({"nfs_test_file1", "nfs_test_file2"}); + if (FLAGS_encrypt_data_at_rest) { + for (auto &src_filename : kSrcFilenames) { + auto s = dsn::utils::encrypt_file(src_filename, src_filename + ".encrypted"); + ASSERT_TRUE(s.ok()) << s.ToString(); + src_filename += ".encrypted"; + } + } std::vector src_file_sizes; std::vector src_file_md5s; for (const auto &src_filename : kSrcFilenames) { diff --git a/src/replica/backup/replica_backup_server.cpp b/src/replica/backup/replica_backup_server.cpp index 3ed7c878c5..23de8972b3 100644 --- a/src/replica/backup/replica_backup_server.cpp +++ b/src/replica/backup/replica_backup_server.cpp @@ -51,6 +51,12 @@ replica_backup_server::replica_backup_server(const replica_stub *rs) : _stub(rs) }); } +replica_backup_server::~replica_backup_server() +{ + dsn_rpc_unregiser_handler(RPC_COLD_BACKUP); + dsn_rpc_unregiser_handler(RPC_CLEAR_COLD_BACKUP); +} + void replica_backup_server::on_cold_backup(backup_rpc rpc) { const backup_request &request = rpc.request(); diff --git a/src/replica/backup/replica_backup_server.h b/src/replica/backup/replica_backup_server.h index c2786fcbda..7d30f78cc1 100644 --- a/src/replica/backup/replica_backup_server.h +++ b/src/replica/backup/replica_backup_server.h @@ -30,6 +30,7 @@ class replica_backup_server { public: explicit replica_backup_server(const replica_stub *rs); + ~replica_backup_server(); private: void on_cold_backup(backup_rpc rpc); diff --git a/src/replica/backup/test/replica_backup_manager_test.cpp b/src/replica/backup/test/replica_backup_manager_test.cpp index f01ecfc196..ec3fb957f3 100644 --- a/src/replica/backup/test/replica_backup_manager_test.cpp +++ b/src/replica/backup/test/replica_backup_manager_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -39,7 +40,9 @@ class replica_backup_manager_test : public replica_test_base } }; -TEST_F(replica_backup_manager_test, clear_cold_backup) +INSTANTIATE_TEST_CASE_P(, replica_backup_manager_test, ::testing::Values(false, true)); + +TEST_P(replica_backup_manager_test, clear_cold_backup) { std::string policy_name = "test_policy"; diff --git a/src/replica/bulk_load/replica_bulk_loader.cpp b/src/replica/bulk_load/replica_bulk_loader.cpp index 643816d423..fcf5c0bf84 100644 --- a/src/replica/bulk_load/replica_bulk_loader.cpp +++ b/src/replica/bulk_load/replica_bulk_loader.cpp @@ -565,7 +565,8 @@ void replica_bulk_loader::download_sst_file(const std::string &remote_dir, error_code replica_bulk_loader::parse_bulk_load_metadata(const std::string &fname) { std::string buf; - auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &buf); + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fname, &buf); if (dsn_unlikely(!s.ok())) { LOG_ERROR_PREFIX("read file {} failed, error = {}", fname, s.ToString()); return ERR_FILE_OPERATION_FAILED; diff --git a/src/replica/bulk_load/test/CMakeLists.txt b/src/replica/bulk_load/test/CMakeLists.txt index a15861f61c..86ca7e8b3b 100644 --- a/src/replica/bulk_load/test/CMakeLists.txt +++ b/src/replica/bulk_load/test/CMakeLists.txt @@ -30,7 +30,7 @@ set(MY_PROJ_LIBS dsn_meta_server test_utils rocksdb) -set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) +set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex rocksdb test_utils) set(MY_BINPLACES config-test.ini diff --git a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp index 9eb7322a61..3a27ee3ba1 100644 --- a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp +++ b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp @@ -39,6 +39,7 @@ #include "runtime/task/task_tracker.h" #include "test_util/test_util.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/test_macros.h" @@ -257,15 +258,15 @@ class replica_bulk_loader_test : public replica_test_base { NO_FATALS(pegasus::create_local_test_file( utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME), &_file_meta)); - _metadata.files.emplace_back(_file_meta); _metadata.file_total_size = _file_meta.size; std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); blob bb = json::json_forwarder::encode(_metadata); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice(bb.data(), bb.length()), - whole_name, - /* should_sync */ true); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(bb.data(), bb.length()), + whole_name, + /* should_sync */ true); ASSERT_TRUE(s.ok()) << fmt::format( "write file {} failed, err = {}", whole_name, s.ToString()); } @@ -434,14 +435,16 @@ class replica_bulk_loader_test : public replica_test_base std::string FILE_NAME = "test_sst_file"; }; +INSTANTIATE_TEST_CASE_P(, replica_bulk_loader_test, ::testing::Values(false, true)); + // on_bulk_load unit tests -TEST_F(replica_bulk_loader_test, on_bulk_load_not_primary) +TEST_P(replica_bulk_loader_test, on_bulk_load_not_primary) { create_bulk_load_request(bulk_load_status::BLS_DOWNLOADING); ASSERT_EQ(test_on_bulk_load(), ERR_INVALID_STATE); } -TEST_F(replica_bulk_loader_test, on_bulk_load_ballot_change) +TEST_P(replica_bulk_loader_test, on_bulk_load_ballot_change) { create_bulk_load_request(bulk_load_status::BLS_DOWNLOADING, BALLOT + 1); mock_primary_states(); @@ -449,7 +452,7 @@ TEST_F(replica_bulk_loader_test, on_bulk_load_ballot_change) } // on_group_bulk_load unit tests -TEST_F(replica_bulk_loader_test, on_group_bulk_load_test) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_test) { struct test_struct { @@ -476,7 +479,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_test) } // start_downloading unit tests -TEST_F(replica_bulk_loader_test, start_downloading_test) +TEST_P(replica_bulk_loader_test, start_downloading_test) { // Test cases: // - stub concurrent downloading count excceed @@ -503,7 +506,7 @@ TEST_F(replica_bulk_loader_test, start_downloading_test) } // start_downloading unit tests -TEST_F(replica_bulk_loader_test, rollback_to_downloading_test) +TEST_P(replica_bulk_loader_test, rollback_to_downloading_test) { fail::cfg("replica_bulk_loader_download_files", "return()"); struct test_struct @@ -523,12 +526,12 @@ TEST_F(replica_bulk_loader_test, rollback_to_downloading_test) } // parse_bulk_load_metadata unit tests -TEST_F(replica_bulk_loader_test, bulk_load_metadata_not_exist) +TEST_P(replica_bulk_loader_test, bulk_load_metadata_not_exist) { ASSERT_EQ(test_parse_bulk_load_metadata("path_not_exist"), ERR_FILE_OPERATION_FAILED); } -TEST_F(replica_bulk_loader_test, bulk_load_metadata_corrupt) +TEST_P(replica_bulk_loader_test, bulk_load_metadata_corrupt) { // create file can not parse as bulk_load_metadata structure utils::filesystem::create_directory(LOCAL_DIR); @@ -540,7 +543,7 @@ TEST_F(replica_bulk_loader_test, bulk_load_metadata_corrupt) utils::filesystem::remove_path(LOCAL_DIR); } -TEST_F(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) +TEST_P(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) { utils::filesystem::create_directory(LOCAL_DIR); NO_FATALS(create_local_metadata_file()); @@ -553,7 +556,7 @@ TEST_F(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) } // finish download test -TEST_F(replica_bulk_loader_test, finish_download_test) +TEST_P(replica_bulk_loader_test, finish_download_test) { mock_downloading_progress(100, 50, 50); stub->set_bulk_load_downloading_count(3); @@ -564,7 +567,7 @@ TEST_F(replica_bulk_loader_test, finish_download_test) } // start ingestion test -TEST_F(replica_bulk_loader_test, start_ingestion_test) +TEST_P(replica_bulk_loader_test, start_ingestion_test) { mock_group_progress(bulk_load_status::BLS_DOWNLOADED); test_start_ingestion(); @@ -572,7 +575,7 @@ TEST_F(replica_bulk_loader_test, start_ingestion_test) } // handle_bulk_load_finish unit tests -TEST_F(replica_bulk_loader_test, bulk_load_finish_test) +TEST_P(replica_bulk_loader_test, bulk_load_finish_test) { // Test cases // - bulk load succeed @@ -655,7 +658,7 @@ TEST_F(replica_bulk_loader_test, bulk_load_finish_test) } // pause_bulk_load unit tests -TEST_F(replica_bulk_loader_test, pause_bulk_load_test) +TEST_P(replica_bulk_loader_test, pause_bulk_load_test) { const int32_t stub_downloading_count = 3; // Test cases: @@ -686,7 +689,7 @@ TEST_F(replica_bulk_loader_test, pause_bulk_load_test) } // report_group_download_progress unit tests -TEST_F(replica_bulk_loader_test, report_group_download_progress_test) +TEST_P(replica_bulk_loader_test, report_group_download_progress_test) { struct test_struct { @@ -711,7 +714,7 @@ TEST_F(replica_bulk_loader_test, report_group_download_progress_test) } // report_group_ingestion_status unit tests -TEST_F(replica_bulk_loader_test, report_group_ingestion_status_test) +TEST_P(replica_bulk_loader_test, report_group_ingestion_status_test) { struct ingestion_struct @@ -785,27 +788,27 @@ TEST_F(replica_bulk_loader_test, report_group_ingestion_status_test) } // report_group_context_clean_flag unit tests -TEST_F(replica_bulk_loader_test, report_group_cleanup_flag_in_unhealthy_state) +TEST_P(replica_bulk_loader_test, report_group_cleanup_flag_in_unhealthy_state) { // _primary_states.membership.secondaries is empty mock_replica_config(partition_status::PS_PRIMARY); ASSERT_FALSE(test_report_group_cleaned_up()); } -TEST_F(replica_bulk_loader_test, report_group_cleanup_flag_not_cleaned_up) +TEST_P(replica_bulk_loader_test, report_group_cleanup_flag_not_cleaned_up) { mock_group_cleanup_flag(bulk_load_status::BLS_SUCCEED, true, false); ASSERT_FALSE(test_report_group_cleaned_up()); } -TEST_F(replica_bulk_loader_test, report_group_cleanup_flag_all_cleaned_up) +TEST_P(replica_bulk_loader_test, report_group_cleanup_flag_all_cleaned_up) { mock_group_cleanup_flag(bulk_load_status::BLS_INVALID, true, true); ASSERT_TRUE(test_report_group_cleaned_up()); } // report_group_is_paused unit tests -TEST_F(replica_bulk_loader_test, report_group_is_paused_test) +TEST_P(replica_bulk_loader_test, report_group_is_paused_test) { struct test_struct { @@ -819,21 +822,21 @@ TEST_F(replica_bulk_loader_test, report_group_is_paused_test) } // on_group_bulk_load_reply unit tests -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_downloading_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_downloading_error) { mock_group_progress(bulk_load_status::BLS_DOWNLOADING, 30, 30, 60); test_on_group_bulk_load_reply(bulk_load_status::BLS_DOWNLOADING, BALLOT, ERR_BUSY); ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_downloaded_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_downloaded_error) { mock_group_progress(bulk_load_status::BLS_DOWNLOADED); test_on_group_bulk_load_reply(bulk_load_status::BLS_DOWNLOADED, BALLOT, ERR_INVALID_STATE); ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_ingestion_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_ingestion_error) { mock_group_ingestion_states(ingestion_status::IS_RUNNING, ingestion_status::IS_SUCCEED); test_on_group_bulk_load_reply( @@ -841,7 +844,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_ingestion_error) ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_succeed_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_succeed_error) { mock_group_cleanup_flag(bulk_load_status::BLS_SUCCEED); test_on_group_bulk_load_reply( @@ -849,14 +852,14 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_succeed_error) ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_failed_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_failed_error) { mock_group_ingestion_states(ingestion_status::IS_RUNNING, ingestion_status::IS_SUCCEED); test_on_group_bulk_load_reply(bulk_load_status::BLS_FAILED, BALLOT, ERR_OK, ERR_TIMEOUT); ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_pausing_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_pausing_error) { mock_group_progress(bulk_load_status::BLS_PAUSED, 100, 50, 10); test_on_group_bulk_load_reply( @@ -864,7 +867,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_pausing_error) ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_rpc_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_rpc_error) { mock_group_cleanup_flag(bulk_load_status::BLS_INVALID, true, false); test_on_group_bulk_load_reply(bulk_load_status::BLS_CANCELED, BALLOT, ERR_OBJECT_NOT_FOUND); @@ -872,7 +875,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_rpc_error) } // validate_status unit test -TEST_F(replica_bulk_loader_test, validate_status_test) +TEST_P(replica_bulk_loader_test, validate_status_test) { struct validate_struct { diff --git a/src/replica/disk_cleaner.cpp b/src/replica/disk_cleaner.cpp index 9488c32a69..cc9ffeb661 100644 --- a/src/replica/disk_cleaner.cpp +++ b/src/replica/disk_cleaner.cpp @@ -19,11 +19,13 @@ #include "disk_cleaner.h" +#include #include #include #include #include #include +#include #include "common/fs_manager.h" #include "metadata_types.h" @@ -32,6 +34,9 @@ #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" +#include "utils/macros.h" +#include "utils/string_conv.h" +#include "utils/string_view.h" namespace dsn { namespace replication { @@ -70,6 +75,108 @@ const std::string kFolderSuffixBak = ".bak"; const std::string kFolderSuffixOri = ".ori"; const std::string kFolderSuffixTmp = ".tmp"; +namespace { + +// TODO(wangdan): we could study later whether ctime (i.e. `st_ctime` within `struct stat`, +// the time of last status change) could be used instead of mtime (i.e. `st_ctime` within +// `struct stat`, the last write time), since ctime of the new directory would be updated +// to the current time once rename() is called, while mtime would not be updated. +bool get_expiration_timestamp_by_last_write_time(const std::string &path, + uint64_t delay_seconds, + uint64_t &expiration_timestamp_s) +{ + time_t last_write_time_s; + if (!dsn::utils::filesystem::last_write_time(path, last_write_time_s)) { + LOG_WARNING("gc_disk: failed to get last write time of {}", path); + return false; + } + + expiration_timestamp_s = static_cast(last_write_time_s) + delay_seconds; + return true; +} + +// Unix timestamp in microseconds for 2010-01-01 00:00:00 GMT+0000. +// This timestamp could be used as the minimum, since it's far earlier than the time when +// Pegasus was born. +#define MIN_TIMESTAMP_US 1262304000000000 +#define MIN_TIMESTAMP_US_LENGTH (sizeof(STRINGIFY(MIN_TIMESTAMP_US)) - 1) + +// Parse timestamp from the directory name. +// +// There are only 2 kinds of directory names that could include timestamp: one is the faulty +// replicas whose name has suffix ".err"; another is the dropped replicas whose name has +// suffix ".gar". The examples for both kinds of directory names: +// 1.1.pegasus.1698843209235962.err +// 1.2.pegasus.1698843214240709.gar +// +// Specify the size of suffix by `suffix_size`. For both kinds of names (.err and .gar), +// `suffix_size` is 4. +// +// The timestamp is the number just before the suffix, between the 2 dots. For example, in +// 1.1.pegasus.1698843209235962.err, 1698843209235962 is the timestamp in microseconds, +// generated by dsn_now_us(). +// +// `timestamp_us` is parsed result while returning true; otherwise, it would never be assigned. +bool parse_timestamp_us(const std::string &name, size_t suffix_size, uint64_t ×tamp_us) +{ + CHECK_GE(name.size(), suffix_size); + + if (suffix_size == name.size()) { + return false; + } + + const size_t end_idx = name.size() - suffix_size; + auto begin_idx = name.find_last_of('.', end_idx - 1); + if (begin_idx == std::string::npos || ++begin_idx >= end_idx) { + return false; + } + + const auto length = end_idx - begin_idx; + if (length < MIN_TIMESTAMP_US_LENGTH) { + return false; + } + + // std::isdigit() is not an addressable standard library function, thus it can't be used + // directly as an algorithm predicate. + // + // See following docs for details. + // https://stackoverflow.com/questions/75868796/differences-between-isdigit-and-stdisdigit + // https://en.cppreference.com/w/cpp/string/byte/isdigit + const auto begin_itr = name.cbegin() + begin_idx; + if (!std::all_of( + begin_itr, begin_itr + length, [](unsigned char c) { return std::isdigit(c); })) { + return false; + } + + const auto ok = + dsn::buf2uint64(dsn::string_view(name.data() + begin_idx, length), timestamp_us); + return ok ? timestamp_us > MIN_TIMESTAMP_US : false; +} + +bool get_expiration_timestamp(const std::string &name, + const std::string &path, + size_t suffix_size, + uint64_t delay_seconds, + uint64_t &expiration_timestamp_s) +{ + uint64_t timestamp_us = 0; + if (!parse_timestamp_us(name, suffix_size, timestamp_us)) { + // Once the timestamp could not be extracted from the directory name, the last write time + // would be used as the base time to compute the expiration time. + LOG_WARNING("gc_disk: failed to parse timestamp from {}, turn to " + "the last write time for {}", + name, + path); + return get_expiration_timestamp_by_last_write_time( + path, delay_seconds, expiration_timestamp_s); + } + + expiration_timestamp_s = timestamp_us / 1000000 + delay_seconds; + return true; +} + +} // anonymous namespace + error_s disk_remove_useless_dirs(const std::vector> &dir_nodes, /*output*/ disk_cleaning_report &report) { @@ -87,59 +194,85 @@ error_s disk_remove_useless_dirs(const std::vector> &d } sub_list.insert(sub_list.end(), tmp_list.begin(), tmp_list.end()); } - for (auto &fpath : sub_list) { - auto name = dsn::utils::filesystem::get_file_name(fpath); - if (!is_data_dir_removable(name)) { - continue; - } - std::string folder_suffix = name.substr(name.length() - 4); - - time_t mt; - if (!dsn::utils::filesystem::last_write_time(fpath, mt)) { - LOG_WARNING("gc_disk: failed to get last write time of {}", fpath); - continue; - } - auto last_write_time = (uint64_t)mt; - uint64_t current_time_ms = dsn_now_ms(); - uint64_t remove_interval_seconds = current_time_ms / 1000; + for (const auto &path : sub_list) { + uint64_t expiration_timestamp_s = 0; - // don't delete ".bak" directory because it is backed by administrator. - if (folder_suffix == kFolderSuffixErr) { + // Note: don't delete ".bak" directory since it could be did by administrator. + const auto name = dsn::utils::filesystem::get_file_name(path); + if (boost::algorithm::ends_with(name, kFolderSuffixErr)) { report.error_replica_count++; - remove_interval_seconds = FLAGS_gc_disk_error_replica_interval_seconds; - } else if (folder_suffix == kFolderSuffixGar) { + if (!get_expiration_timestamp(name, + path, + kFolderSuffixErr.size(), + FLAGS_gc_disk_error_replica_interval_seconds, + expiration_timestamp_s)) { + continue; + } + } else if (boost::algorithm::ends_with(name, kFolderSuffixGar)) { report.garbage_replica_count++; - remove_interval_seconds = FLAGS_gc_disk_garbage_replica_interval_seconds; - } else if (folder_suffix == kFolderSuffixTmp) { + if (!get_expiration_timestamp(name, + path, + kFolderSuffixGar.size(), + FLAGS_gc_disk_garbage_replica_interval_seconds, + expiration_timestamp_s)) { + continue; + } + } else if (boost::algorithm::ends_with(name, kFolderSuffixTmp)) { report.disk_migrate_tmp_count++; - remove_interval_seconds = FLAGS_gc_disk_migration_tmp_replica_interval_seconds; - } else if (folder_suffix == kFolderSuffixOri) { + if (!get_expiration_timestamp_by_last_write_time( + path, + FLAGS_gc_disk_migration_tmp_replica_interval_seconds, + expiration_timestamp_s)) { + continue; + } + } else if (boost::algorithm::ends_with(name, kFolderSuffixOri)) { report.disk_migrate_origin_count++; - remove_interval_seconds = FLAGS_gc_disk_migration_origin_replica_interval_seconds; + if (!get_expiration_timestamp_by_last_write_time( + path, + FLAGS_gc_disk_migration_origin_replica_interval_seconds, + expiration_timestamp_s)) { + continue; + } + } else { + continue; } - if (last_write_time + remove_interval_seconds <= current_time_ms / 1000) { - if (!dsn::utils::filesystem::remove_path(fpath)) { - LOG_WARNING("gc_disk: failed to delete directory '{}', time_used_ms = {}", - fpath, - dsn_now_ms() - current_time_ms); - } else { + const auto current_time_ms = dsn_now_ms(); + if (expiration_timestamp_s <= current_time_ms / 1000) { + if (dsn::utils::filesystem::remove_path(path)) { LOG_WARNING("gc_disk: replica_dir_op succeed to delete directory '{}'" ", time_used_ms = {}", - fpath, + path, dsn_now_ms() - current_time_ms); report.remove_dir_count++; + } else { + LOG_WARNING("gc_disk: failed to delete directory '{}', time_used_ms = {}", + path, + dsn_now_ms() - current_time_ms); } } else { LOG_INFO("gc_disk: reserve directory '{}', wait_seconds = {}", - fpath, - last_write_time + remove_interval_seconds - current_time_ms / 1000); + path, + expiration_timestamp_s - current_time_ms / 1000); } } return error_s::ok(); } +bool is_data_dir_removable(const std::string &dir) +{ + return boost::algorithm::ends_with(dir, kFolderSuffixErr) || + boost::algorithm::ends_with(dir, kFolderSuffixGar) || + boost::algorithm::ends_with(dir, kFolderSuffixTmp) || + boost::algorithm::ends_with(dir, kFolderSuffixOri); +} + +bool is_data_dir_invalid(const std::string &dir) +{ + return is_data_dir_removable(dir) || boost::algorithm::ends_with(dir, kFolderSuffixBak); +} + void move_to_err_path(const std::string &path, const std::string &log_prefix) { const std::string new_path = fmt::format("{}.{}{}", path, dsn_now_us(), kFolderSuffixErr); @@ -150,5 +283,6 @@ void move_to_err_path(const std::string &path, const std::string &log_prefix) new_path); LOG_WARNING("{}: succeed to move directory from '{}' to '{}'", log_prefix, path, new_path); } + } // namespace replication } // namespace dsn diff --git a/src/replica/disk_cleaner.h b/src/replica/disk_cleaner.h index 7961d084cf..9e2edb7523 100644 --- a/src/replica/disk_cleaner.h +++ b/src/replica/disk_cleaner.h @@ -55,26 +55,13 @@ struct disk_cleaning_report extern error_s disk_remove_useless_dirs(const std::vector> &dir_nodes, /*output*/ disk_cleaning_report &report); -inline bool is_data_dir_removable(const std::string &dir) -{ - if (dir.length() < 4) { - return false; - } - const std::string folder_suffix = dir.substr(dir.length() - 4); - return (folder_suffix == kFolderSuffixErr || folder_suffix == kFolderSuffixGar || - folder_suffix == kFolderSuffixTmp || folder_suffix == kFolderSuffixOri); -} +bool is_data_dir_removable(const std::string &dir); -// Note: ".bak" is invalid but not allow delete, because it can be backed by administrator. -inline bool is_data_dir_invalid(const std::string &dir) -{ - if (dir.length() < 4) { - return false; - } - const std::string folder_suffix = dir.substr(dir.length() - 4); - return is_data_dir_removable(dir) || folder_suffix == kFolderSuffixBak; -} +// Note: ".bak" is invalid but not allowed to be deleted, because it could be did by +// administrator on purpose. +bool is_data_dir_invalid(const std::string &dir); void move_to_err_path(const std::string &path, const std::string &log_prefix); + } // namespace replication } // namespace dsn diff --git a/src/replica/duplication/duplication_sync_timer.cpp b/src/replica/duplication/duplication_sync_timer.cpp index d04d014fe6..9e315c111a 100644 --- a/src/replica/duplication/duplication_sync_timer.cpp +++ b/src/replica/duplication/duplication_sync_timer.cpp @@ -35,12 +35,21 @@ #include "runtime/task/async_calls.h" #include "runtime/task/task_code.h" #include "utils/autoref_ptr.h" +#include "utils/chrono_literals.h" #include "utils/error_code.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/threadpool_code.h" +DSN_DEFINE_uint64( + replication, + duplication_sync_period_second, + 10, + "The period seconds of duplication to sync data from local cluster to remote cluster"); + namespace dsn { namespace replication { +using namespace literals::chrono_literals; DEFINE_TASK_CODE(LPC_DUPLICATION_SYNC_TIMER, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) @@ -169,14 +178,14 @@ void duplication_sync_timer::close() void duplication_sync_timer::start() { - LOG_INFO("run duplication sync periodically in {}s", DUPLICATION_SYNC_PERIOD_SECOND); + LOG_INFO("run duplication sync periodically in {}s", FLAGS_duplication_sync_period_second); _timer_task = tasking::enqueue_timer(LPC_DUPLICATION_SYNC_TIMER, &_stub->_tracker, [this]() { run(); }, - DUPLICATION_SYNC_PERIOD_SECOND * 1_s, + FLAGS_duplication_sync_period_second * 1_s, 0, - DUPLICATION_SYNC_PERIOD_SECOND * 1_s); + FLAGS_duplication_sync_period_second * 1_s); } std::multimap diff --git a/src/replica/duplication/duplication_sync_timer.h b/src/replica/duplication/duplication_sync_timer.h index 4a112b1d58..ff1b6c0660 100644 --- a/src/replica/duplication/duplication_sync_timer.h +++ b/src/replica/duplication/duplication_sync_timer.h @@ -26,7 +26,6 @@ #include "duplication_types.h" #include "replica/replica.h" #include "runtime/task/task.h" -#include "utils/chrono_literals.h" #include "utils/zlocks.h" namespace dsn { @@ -35,10 +34,6 @@ class error_code; namespace replication { class replica_stub; -using namespace literals::chrono_literals; - -constexpr int DUPLICATION_SYNC_PERIOD_SECOND = 10; - // Per-server(replica_stub)-instance. class duplication_sync_timer { diff --git a/src/replica/duplication/test/CMakeLists.txt b/src/replica/duplication/test/CMakeLists.txt index faab20f7e6..b0abc3bd9f 100644 --- a/src/replica/duplication/test/CMakeLists.txt +++ b/src/replica/duplication/test/CMakeLists.txt @@ -30,6 +30,7 @@ set(MY_PROJ_LIBS dsn_meta_server zookeeper hashtable gtest + test_utils rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/duplication/test/dup_replica_http_service_test.cpp b/src/replica/duplication/test/dup_replica_http_service_test.cpp index 17fe0f1c7f..80b2fb87c3 100644 --- a/src/replica/duplication/test/dup_replica_http_service_test.cpp +++ b/src/replica/duplication/test/dup_replica_http_service_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -39,7 +40,9 @@ class dup_replica_http_service_test : public duplication_test_base { }; -TEST_F(dup_replica_http_service_test, query_duplication_handler) +INSTANTIATE_TEST_CASE_P(, dup_replica_http_service_test, ::testing::Values(false, true)); + +TEST_P(dup_replica_http_service_test, query_duplication_handler) { auto pri = stub->add_primary_replica(1, 1); diff --git a/src/replica/duplication/test/duplication_sync_timer_test.cpp b/src/replica/duplication/test/duplication_sync_timer_test.cpp index d12c7f50dd..c680ddcbea 100644 --- a/src/replica/duplication/test/duplication_sync_timer_test.cpp +++ b/src/replica/duplication/test/duplication_sync_timer_test.cpp @@ -17,6 +17,7 @@ #include "replica/duplication/duplication_sync_timer.h" +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -376,19 +377,21 @@ class duplication_sync_timer_test : public duplication_test_base std::unique_ptr dup_sync; }; -TEST_F(duplication_sync_timer_test, duplication_sync) { test_duplication_sync(); } +INSTANTIATE_TEST_CASE_P(, duplication_sync_timer_test, ::testing::Values(false, true)); -TEST_F(duplication_sync_timer_test, update_duplication_map) { test_update_duplication_map(); } +TEST_P(duplication_sync_timer_test, duplication_sync) { test_duplication_sync(); } -TEST_F(duplication_sync_timer_test, update_on_non_primary) { test_update_on_non_primary(); } +TEST_P(duplication_sync_timer_test, update_duplication_map) { test_update_duplication_map(); } -TEST_F(duplication_sync_timer_test, update_confirmed_points) { test_update_confirmed_points(); } +TEST_P(duplication_sync_timer_test, update_on_non_primary) { test_update_on_non_primary(); } -TEST_F(duplication_sync_timer_test, on_duplication_sync_reply) { test_on_duplication_sync_reply(); } +TEST_P(duplication_sync_timer_test, update_confirmed_points) { test_update_confirmed_points(); } -TEST_F(duplication_sync_timer_test, replica_status_transition) { test_replica_status_transition(); } +TEST_P(duplication_sync_timer_test, on_duplication_sync_reply) { test_on_duplication_sync_reply(); } -TEST_F(duplication_sync_timer_test, receive_illegal_duplication_status) +TEST_P(duplication_sync_timer_test, replica_status_transition) { test_replica_status_transition(); } + +TEST_P(duplication_sync_timer_test, receive_illegal_duplication_status) { test_receive_illegal_duplication_status(); } diff --git a/src/replica/duplication/test/load_from_private_log_test.cpp b/src/replica/duplication/test/load_from_private_log_test.cpp index b215be60f0..e44bb74a3d 100644 --- a/src/replica/duplication/test/load_from_private_log_test.cpp +++ b/src/replica/duplication/test/load_from_private_log_test.cpp @@ -71,6 +71,7 @@ #include "duplication_test_base.h" #include "replica/duplication/load_from_private_log.h" #include "replica/mutation_log_utils.h" +#include "test_util/test_util.h" namespace dsn { namespace replication { @@ -276,39 +277,48 @@ class load_from_private_log_test : public duplication_test_base std::unique_ptr duplicator; }; -TEST_F(load_from_private_log_test, find_log_file_to_start) { test_find_log_file_to_start(); } +INSTANTIATE_TEST_CASE_P(, load_from_private_log_test, ::testing::Values(false, true)); -TEST_F(load_from_private_log_test, start_duplication_10000_4MB) +TEST_P(load_from_private_log_test, find_log_file_to_start) { test_find_log_file_to_start(); } + +TEST_P(load_from_private_log_test, start_duplication_10000_4MB) { test_start_duplication(10000, 4); } -TEST_F(load_from_private_log_test, start_duplication_50000_4MB) +TEST_P(load_from_private_log_test, start_duplication_50000_4MB) { test_start_duplication(50000, 4); } -TEST_F(load_from_private_log_test, start_duplication_10000_1MB) +TEST_P(load_from_private_log_test, start_duplication_10000_1MB) { test_start_duplication(10000, 1); } -TEST_F(load_from_private_log_test, start_duplication_50000_1MB) +TEST_P(load_from_private_log_test, start_duplication_50000_1MB) { test_start_duplication(50000, 1); } -TEST_F(load_from_private_log_test, start_duplication_100000_4MB) +TEST_P(load_from_private_log_test, start_duplication_100000_4MB) { test_start_duplication(100000, 4); } // Ensure replica_duplicator can correctly handle real-world log file -TEST_F(load_from_private_log_test, handle_real_private_log) +TEST_P(load_from_private_log_test, handle_real_private_log) { std::vector log_files({"log.1.0.handle_real_private_log", "log.1.0.handle_real_private_log2", "log.1.0.all_loaded_are_write_empties"}); + if (FLAGS_encrypt_data_at_rest) { + for (int i = 0; i < log_files.size(); i++) { + auto s = dsn::utils::encrypt_file(log_files[i], log_files[i] + ".encrypted"); + ASSERT_TRUE(s.ok()) << s.ToString(); + log_files[i] += ".encrypted"; + } + } struct test_data { @@ -346,9 +356,9 @@ TEST_F(load_from_private_log_test, handle_real_private_log) } } -TEST_F(load_from_private_log_test, restart_duplication) { test_restart_duplication(); } +TEST_P(load_from_private_log_test, restart_duplication) { test_restart_duplication(); } -TEST_F(load_from_private_log_test, ignore_useless) +TEST_P(load_from_private_log_test, ignore_useless) { utils::filesystem::remove_path(_log_dir); @@ -414,7 +424,9 @@ class load_fail_mode_test : public load_from_private_log_test std::unique_ptr end_stage; }; -TEST_F(load_fail_mode_test, fail_skip) +INSTANTIATE_TEST_CASE_P(, load_fail_mode_test, ::testing::Values(false, true)); + +TEST_P(load_fail_mode_test, fail_skip) { duplicator->update_fail_mode(duplication_fail_mode::FAIL_SKIP); ASSERT_EQ(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); @@ -432,7 +444,7 @@ TEST_F(load_fail_mode_test, fail_skip) ASSERT_GT(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); } -TEST_F(load_fail_mode_test, fail_slow) +TEST_P(load_fail_mode_test, fail_slow) { duplicator->update_fail_mode(duplication_fail_mode::FAIL_SLOW); ASSERT_EQ(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); @@ -451,7 +463,7 @@ TEST_F(load_fail_mode_test, fail_slow) ASSERT_EQ(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); } -TEST_F(load_fail_mode_test, fail_skip_real_corrupted_file) +TEST_P(load_fail_mode_test, fail_skip_real_corrupted_file) { { // inject some bad data in the middle of the first file std::string log_path = _log_dir + "/log.1.0"; diff --git a/src/replica/duplication/test/mutation_batch_test.cpp b/src/replica/duplication/test/mutation_batch_test.cpp index dbc09a8627..a41aa8ab8d 100644 --- a/src/replica/duplication/test/mutation_batch_test.cpp +++ b/src/replica/duplication/test/mutation_batch_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -55,7 +56,9 @@ class mutation_batch_test : public duplication_test_base } }; -TEST_F(mutation_batch_test, add_mutation_if_valid) +INSTANTIATE_TEST_CASE_P(, mutation_batch_test, ::testing::Values(false, true)); + +TEST_P(mutation_batch_test, add_mutation_if_valid) { auto duplicator = create_test_duplicator(0); mutation_batch batcher(duplicator.get()); @@ -85,7 +88,7 @@ TEST_F(mutation_batch_test, add_mutation_if_valid) ASSERT_EQ(result.size(), 2); } -TEST_F(mutation_batch_test, ignore_non_idempotent_write) +TEST_P(mutation_batch_test, ignore_non_idempotent_write) { auto duplicator = create_test_duplicator(0); mutation_batch batcher(duplicator.get()); @@ -98,7 +101,7 @@ TEST_F(mutation_batch_test, ignore_non_idempotent_write) ASSERT_EQ(result.size(), 0); } -TEST_F(mutation_batch_test, mutation_buffer_commit) +TEST_P(mutation_batch_test, mutation_buffer_commit) { auto duplicator = create_test_duplicator(0); mutation_batch batcher(duplicator.get()); diff --git a/src/replica/duplication/test/replica_duplicator_manager_test.cpp b/src/replica/duplication/test/replica_duplicator_manager_test.cpp index 94d09aeddd..93aed6dd8b 100644 --- a/src/replica/duplication/test/replica_duplicator_manager_test.cpp +++ b/src/replica/duplication/test/replica_duplicator_manager_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -184,24 +185,26 @@ class replica_duplicator_manager_test : public duplication_test_base } }; -TEST_F(replica_duplicator_manager_test, get_duplication_confirms) +INSTANTIATE_TEST_CASE_P(, replica_duplicator_manager_test, ::testing::Values(false, true)); + +TEST_P(replica_duplicator_manager_test, get_duplication_confirms) { test_get_duplication_confirms(); } -TEST_F(replica_duplicator_manager_test, set_confirmed_decree_non_primary) +TEST_P(replica_duplicator_manager_test, set_confirmed_decree_non_primary) { test_set_confirmed_decree_non_primary(); } -TEST_F(replica_duplicator_manager_test, remove_non_existed_duplications) +TEST_P(replica_duplicator_manager_test, remove_non_existed_duplications) { test_remove_non_existed_duplications(); } -TEST_F(replica_duplicator_manager_test, min_confirmed_decree) { test_min_confirmed_decree(); } +TEST_P(replica_duplicator_manager_test, min_confirmed_decree) { test_min_confirmed_decree(); } -TEST_F(replica_duplicator_manager_test, update_checkpoint_prepared) +TEST_P(replica_duplicator_manager_test, update_checkpoint_prepared) { auto r = stub->add_primary_replica(2, 1); duplication_entry ent; diff --git a/src/replica/duplication/test/replica_duplicator_test.cpp b/src/replica/duplication/test/replica_duplicator_test.cpp index 4d60c81b21..87fcd374b9 100644 --- a/src/replica/duplication/test/replica_duplicator_test.cpp +++ b/src/replica/duplication/test/replica_duplicator_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -138,11 +139,13 @@ class replica_duplicator_test : public duplication_test_base } }; -TEST_F(replica_duplicator_test, new_duplicator) { test_new_duplicator(); } +INSTANTIATE_TEST_CASE_P(, replica_duplicator_test, ::testing::Values(false, true)); -TEST_F(replica_duplicator_test, pause_start_duplication) { test_pause_start_duplication(); } +TEST_P(replica_duplicator_test, new_duplicator) { test_new_duplicator(); } -TEST_F(replica_duplicator_test, duplication_progress) +TEST_P(replica_duplicator_test, pause_start_duplication) { test_pause_start_duplication(); } + +TEST_P(replica_duplicator_test, duplication_progress) { auto duplicator = create_test_duplicator(); ASSERT_EQ(duplicator->progress().last_decree, 0); // start duplication from empty plog @@ -171,7 +174,7 @@ TEST_F(replica_duplicator_test, duplication_progress) ASSERT_TRUE(duplicator_for_checkpoint->progress().checkpoint_has_prepared); } -TEST_F(replica_duplicator_test, prapre_dup) +TEST_P(replica_duplicator_test, prapre_dup) { auto duplicator = create_test_duplicator(invalid_decree, 100); replica()->update_expect_last_durable_decree(100); diff --git a/src/replica/duplication/test/replica_follower_test.cpp b/src/replica/duplication/test/replica_follower_test.cpp index f013d61693..a961c36048 100644 --- a/src/replica/duplication/test/replica_follower_test.cpp +++ b/src/replica/duplication/test/replica_follower_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -116,7 +117,9 @@ class replica_follower_test : public duplication_test_base mock_replica_ptr _mock_replica; }; -TEST_F(replica_follower_test, test_init_master_info) +INSTANTIATE_TEST_CASE_P(, replica_follower_test, ::testing::Values(false, true)); + +TEST_P(replica_follower_test, test_init_master_info) { _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterClusterKey, "master"); _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterMetasKey, @@ -140,7 +143,7 @@ TEST_F(replica_follower_test, test_init_master_info) ASSERT_FALSE(_mock_replica->is_duplication_follower()); } -TEST_F(replica_follower_test, test_duplicate_checkpoint) +TEST_P(replica_follower_test, test_duplicate_checkpoint) { _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterClusterKey, "master"); _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterMetasKey, @@ -160,7 +163,7 @@ TEST_F(replica_follower_test, test_duplicate_checkpoint) ASSERT_EQ(follower->duplicate_checkpoint(), ERR_BUSY); } -TEST_F(replica_follower_test, test_async_duplicate_checkpoint_from_master_replica) +TEST_P(replica_follower_test, test_async_duplicate_checkpoint_from_master_replica) { _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterClusterKey, "master"); _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterMetasKey, @@ -182,7 +185,7 @@ TEST_F(replica_follower_test, test_async_duplicate_checkpoint_from_master_replic fail::teardown(); } -TEST_F(replica_follower_test, test_update_master_replica_config) +TEST_P(replica_follower_test, test_update_master_replica_config) { _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterClusterKey, "master"); _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterMetasKey, @@ -229,7 +232,7 @@ TEST_F(replica_follower_test, test_update_master_replica_config) ASSERT_EQ(master_replica_config(follower).pid, p.pid); } -TEST_F(replica_follower_test, test_nfs_copy_checkpoint) +TEST_P(replica_follower_test, test_nfs_copy_checkpoint) { _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterClusterKey, "master"); _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterMetasKey, diff --git a/src/replica/duplication/test/ship_mutation_test.cpp b/src/replica/duplication/test/ship_mutation_test.cpp index d6356418f6..29e17f2a8a 100644 --- a/src/replica/duplication/test/ship_mutation_test.cpp +++ b/src/replica/duplication/test/ship_mutation_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -105,14 +106,16 @@ class ship_mutation_test : public duplication_test_base std::unique_ptr duplicator; }; -TEST_F(ship_mutation_test, ship_mutation_tuple_set) { test_ship_mutation_tuple_set(); } +INSTANTIATE_TEST_CASE_P(, ship_mutation_test, ::testing::Values(false, true)); + +TEST_P(ship_mutation_test, ship_mutation_tuple_set) { test_ship_mutation_tuple_set(); } void retry(pipeline::base *base) { base->schedule([base]() { retry(base); }, 10_s); } -TEST_F(ship_mutation_test, pause) +TEST_P(ship_mutation_test, pause) { auto shipper = mock_ship_mutation(); diff --git a/src/replica/log_file.cpp b/src/replica/log_file.cpp index 973213b3a4..22f2be1e8e 100644 --- a/src/replica/log_file.cpp +++ b/src/replica/log_file.cpp @@ -41,6 +41,7 @@ #include "utils/binary_writer.h" #include "utils/blob.h" #include "utils/crc.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/latency_tracer.h" @@ -178,7 +179,9 @@ log_file::log_file( if (is_read) { int64_t sz; - CHECK(dsn::utils::filesystem::file_size(_path, sz), "fail to get file size of {}.", _path); + CHECK(dsn::utils::filesystem::file_size(_path, dsn::utils::FileDataType::kSensitive, sz), + "fail to get file size of {}.", + _path); _end_offset += sz; } } diff --git a/src/replica/log_file_stream.h b/src/replica/log_file_stream.h index 150b6e824b..b31a5047c6 100644 --- a/src/replica/log_file_stream.h +++ b/src/replica/log_file_stream.h @@ -34,7 +34,6 @@ namespace dsn { namespace replication { -// log_file::file_streamer class log_file::file_streamer { public: diff --git a/src/replica/mutation_cache.cpp b/src/replica/mutation_cache.cpp index 1a7fa4d338..f1e8f25d67 100644 --- a/src/replica/mutation_cache.cpp +++ b/src/replica/mutation_cache.cpp @@ -26,9 +26,6 @@ #include "mutation_cache.h" -// HACK: iwyu suggests each time vector[] is used. -// https://github.com/include-what-you-use/include-what-you-use/issues/166 -// TODO(yingchun): remove this pragma by using mapping.imp // IWYU pragma: no_include #include "consensus_types.h" #include "mutation.h" diff --git a/src/replica/mutation_log_replay.cpp b/src/replica/mutation_log_replay.cpp index 316c12396e..e6f4afadd1 100644 --- a/src/replica/mutation_log_replay.cpp +++ b/src/replica/mutation_log_replay.cpp @@ -140,7 +140,7 @@ namespace replication { if (log == nullptr) { if (err == ERR_HANDLE_EOF || err == ERR_INCOMPLETE_DATA || err == ERR_INVALID_PARAMETERS) { - LOG_DEBUG("skip file {} during log replay", fpath); + LOG_INFO("skip file {} during log replay", fpath); continue; } else { return err; diff --git a/src/replica/replica.h b/src/replica/replica.h index 3933784da0..dece645950 100644 --- a/src/replica/replica.h +++ b/src/replica/replica.h @@ -450,7 +450,7 @@ class replica : public serverlet, public ref_counter, public replica_ba ///////////////////////////////////////////////////////////////// // replica restore from backup - bool read_cold_backup_metadata(const std::string &file, cold_backup_metadata &backup_metadata); + bool read_cold_backup_metadata(const std::string &fname, cold_backup_metadata &backup_metadata); // checkpoint on cold backup media maybe contain useless file, // we should abandon these file base cold_backup_metadata bool remove_useless_file_under_chkpt(const std::string &chkpt_dir, @@ -555,7 +555,7 @@ class replica : public serverlet, public ref_counter, public replica_ba friend class ::pegasus::server::pegasus_server_test_base; friend class ::pegasus::server::rocksdb_wrapper_test; FRIEND_TEST(replica_disk_test, disk_io_error_test); - FRIEND_TEST(replica_error_test, test_auto_trash_of_corruption); + FRIEND_TEST(replica_test, test_auto_trash_of_corruption); // replica configuration, updated by update_local_configuration ONLY replica_configuration _config; diff --git a/src/replica/replica_backup.cpp b/src/replica/replica_backup.cpp index 50a4b45f20..d6c452be9d 100644 --- a/src/replica/replica_backup.cpp +++ b/src/replica/replica_backup.cpp @@ -51,6 +51,7 @@ #include "runtime/api_layer1.h" #include "runtime/task/async_calls.h" #include "utils/autoref_ptr.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -386,10 +387,12 @@ statistic_file_infos_under_dir(const std::string &dir, total_size = 0; file_infos.clear(); + // TODO(yingchun): check if there are any files that are not sensitive (not encrypted). for (std::string &file : sub_files) { std::pair file_info; - if (!utils::filesystem::file_size(file, file_info.second)) { + if (!utils::filesystem::file_size( + file, dsn::utils::FileDataType::kSensitive, file_info.second)) { LOG_ERROR("get file size of {} failed", file); return false; } diff --git a/src/replica/replica_restore.cpp b/src/replica/replica_restore.cpp index f2494482ff..9c255df0b4 100644 --- a/src/replica/replica_restore.cpp +++ b/src/replica/replica_restore.cpp @@ -105,7 +105,8 @@ bool replica::read_cold_backup_metadata(const std::string &fname, } std::string data; - auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &data); + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fname, &data); if (!s.ok()) { LOG_ERROR_PREFIX("read file '{}' failed, err = {}", fname, s.ToString()); return false; diff --git a/src/replica/replica_stub.cpp b/src/replica/replica_stub.cpp index c5bd5000a9..e1815b48bf 100644 --- a/src/replica/replica_stub.cpp +++ b/src/replica/replica_stub.cpp @@ -1754,7 +1754,7 @@ void replica_stub::on_gc_replica(replica_stub_ptr this_, gpid id) CHECK( dsn::utils::filesystem::directory_exists(replica_path), "dir({}) not exist", replica_path); LOG_INFO("start to move replica({}) as garbage, path: {}", id, replica_path); - const auto rename_path = fmt::format("{}.{}.gar", replica_path, dsn_now_us()); + const auto rename_path = fmt::format("{}.{}{}", replica_path, dsn_now_us(), kFolderSuffixGar); if (!dsn::utils::filesystem::rename_path(replica_path, rename_path)) { LOG_WARNING("gc_replica: failed to move directory '{}' to '{}'", replica_path, rename_path); diff --git a/src/replica/replica_stub.h b/src/replica/replica_stub.h index e71e8f864e..e257028db1 100644 --- a/src/replica/replica_stub.h +++ b/src/replica/replica_stub.h @@ -446,7 +446,7 @@ class replica_stub : public serverlet, public ref_counter friend class replica_follower_test; friend class replica_http_service_test; FRIEND_TEST(open_replica_test, open_replica_add_decree_and_ballot_check); - FRIEND_TEST(replica_error_test, test_auto_trash_of_corruption); + FRIEND_TEST(replica_test, test_auto_trash_of_corruption); FRIEND_TEST(replica_test, test_clear_on_failure); FRIEND_TEST(GcSlogFlushFeplicasTest, FlushReplicas); diff --git a/src/replica/replication_app_base.cpp b/src/replica/replication_app_base.cpp index bc84c0c0a6..7bb4a7423b 100644 --- a/src/replica/replication_app_base.cpp +++ b/src/replica/replication_app_base.cpp @@ -52,6 +52,7 @@ #include "utils/binary_writer.h" #include "utils/blob.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/factory_store.h" #include "utils/fail_point.h" #include "utils/filesystem.h" @@ -68,12 +69,14 @@ const std::string replica_init_info::kInitInfo = ".init-info"; namespace { error_code write_blob_to_file(const std::string &fname, const blob &data) { + // TODO(yingchun): consider not encrypt the meta files. std::string tmp_fname = fname + ".tmp"; auto cleanup = defer([tmp_fname]() { utils::filesystem::remove_path(tmp_fname); }); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice(data.data(), data.length()), - tmp_fname, - /* should_sync */ true); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(data.data(), data.length()), + tmp_fname, + /* should_sync */ true); LOG_AND_RETURN_NOT_TRUE( ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "write file {} failed", tmp_fname); LOG_AND_RETURN_NOT_TRUE(ERROR, @@ -119,7 +122,8 @@ error_code replica_init_info::store(const std::string &dir) error_code replica_init_info::load_json(const std::string &fname) { std::string data; - auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &data); + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fname, &data); LOG_AND_RETURN_NOT_TRUE(ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "read file {} failed", fname); LOG_AND_RETURN_NOT_TRUE(ERROR, json::json_forwarder::decode( @@ -148,7 +152,8 @@ std::string replica_init_info::to_string() error_code replica_app_info::load(const std::string &fname) { std::string data; - auto s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fname, &data); + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fname, &data); LOG_AND_RETURN_NOT_TRUE(ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "read file {} failed", fname); binary_reader reader(blob::create_from_bytes(std::move(data))); int magic = 0; diff --git a/src/replica/split/test/replica_split_test.cpp b/src/replica/split/test/replica_split_test.cpp index 4f12f91e00..c0edc3a013 100644 --- a/src/replica/split/test/replica_split_test.cpp +++ b/src/replica/split/test/replica_split_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -558,8 +559,10 @@ class replica_split_test : public replica_test_base learn_state _mock_learn_state; }; +INSTANTIATE_TEST_CASE_P(, replica_split_test, ::testing::Values(false, true)); + // parent_start_split tests -TEST_F(replica_split_test, parent_start_split_tests) +TEST_P(replica_split_test, parent_start_split_tests) { fail::cfg("replica_stub_create_child_replica_if_not_found", "return()"); fail::cfg("replica_child_init_replica", "return()"); @@ -595,7 +598,7 @@ TEST_F(replica_split_test, parent_start_split_tests) } // child_init_replica test -TEST_F(replica_split_test, child_init_replica_test) +TEST_P(replica_split_test, child_init_replica_test) { fail::cfg("replica_stub_split_replica_exec", "return()"); test_child_init_replica(); @@ -605,7 +608,7 @@ TEST_F(replica_split_test, child_init_replica_test) } // parent_check_states tests -TEST_F(replica_split_test, parent_check_states_tests) +TEST_P(replica_split_test, parent_check_states_tests) { fail::cfg("replica_stub_split_replica_exec", "return()"); @@ -625,7 +628,7 @@ TEST_F(replica_split_test, parent_check_states_tests) } // child_copy_prepare_list test -TEST_F(replica_split_test, copy_prepare_list_succeed) +TEST_P(replica_split_test, copy_prepare_list_succeed) { fail::cfg("replica_stub_split_replica_exec", "return()"); fail::cfg("replica_child_learn_states", "return()"); @@ -642,7 +645,7 @@ TEST_F(replica_split_test, copy_prepare_list_succeed) } // child_learn_states tests -TEST_F(replica_split_test, child_learn_states_tests) +TEST_P(replica_split_test, child_learn_states_tests) { generate_child(); @@ -674,7 +677,7 @@ TEST_F(replica_split_test, child_learn_states_tests) } // child_apply_private_logs test -TEST_F(replica_split_test, child_apply_private_logs_succeed) +TEST_P(replica_split_test, child_apply_private_logs_succeed) { fail::cfg("mutation_log_replay_succeed", "return()"); fail::cfg("replication_app_base_apply_mutation", "return()"); @@ -688,7 +691,7 @@ TEST_F(replica_split_test, child_apply_private_logs_succeed) } // child_catch_up_states tests -TEST_F(replica_split_test, child_catch_up_states_tests) +TEST_P(replica_split_test, child_catch_up_states_tests) { fail::cfg("replica_child_notify_catch_up", "return()"); fail::cfg("replication_app_base_apply_mutation", "return()"); @@ -713,7 +716,7 @@ TEST_F(replica_split_test, child_catch_up_states_tests) } // parent_handle_child_catch_up tests -TEST_F(replica_split_test, parent_handle_catch_up_test) +TEST_P(replica_split_test, parent_handle_catch_up_test) { fail::cfg("replica_parent_check_sync_point_commit", "return()"); ballot WRONG_BALLOT = 1; @@ -739,7 +742,7 @@ TEST_F(replica_split_test, parent_handle_catch_up_test) } // update_child_group_partition_count tests -TEST_F(replica_split_test, update_child_group_partition_count_test) +TEST_P(replica_split_test, update_child_group_partition_count_test) { fail::cfg("replica_parent_update_partition_count_request", "return()"); generate_child(); @@ -775,7 +778,7 @@ TEST_F(replica_split_test, update_child_group_partition_count_test) } // on_update_child_group_partition_count tests -TEST_F(replica_split_test, child_update_partition_count_test) +TEST_P(replica_split_test, child_update_partition_count_test) { ballot WRONG_BALLOT = INIT_BALLOT + 1; generate_child(); @@ -802,7 +805,7 @@ TEST_F(replica_split_test, child_update_partition_count_test) } // on_update_child_group_partition_count_reply tests -TEST_F(replica_split_test, parent_on_update_partition_reply_test) +TEST_P(replica_split_test, parent_on_update_partition_reply_test) { fail::cfg("replica_register_child_on_meta", "return()"); generate_child(); @@ -836,7 +839,7 @@ TEST_F(replica_split_test, parent_on_update_partition_reply_test) } // register_child test -TEST_F(replica_split_test, register_child_test) +TEST_P(replica_split_test, register_child_test) { fail::cfg("replica_parent_send_register_request", "return()"); test_register_child_on_meta(); @@ -845,7 +848,7 @@ TEST_F(replica_split_test, register_child_test) } // register_child_reply tests -TEST_F(replica_split_test, register_child_reply_test) +TEST_P(replica_split_test, register_child_reply_test) { fail::cfg("replica_init_group_check", "return()"); fail::cfg("replica_broadcast_group_check", "return()"); @@ -878,7 +881,7 @@ TEST_F(replica_split_test, register_child_reply_test) } // trigger_primary_parent_split unit test -TEST_F(replica_split_test, trigger_primary_parent_split_test) +TEST_P(replica_split_test, trigger_primary_parent_split_test) { fail::cfg("replica_broadcast_group_check", "return()"); generate_child(); @@ -923,7 +926,7 @@ TEST_F(replica_split_test, trigger_primary_parent_split_test) } // trigger_secondary_parent_split unit test -TEST_F(replica_split_test, secondary_handle_split_test) +TEST_P(replica_split_test, secondary_handle_split_test) { generate_child(); @@ -970,7 +973,7 @@ TEST_F(replica_split_test, secondary_handle_split_test) } } -TEST_F(replica_split_test, primary_parent_handle_stop_test) +TEST_P(replica_split_test, primary_parent_handle_stop_test) { fail::cfg("replica_parent_send_notify_stop_request", "return()"); // Test cases: @@ -1002,7 +1005,7 @@ TEST_F(replica_split_test, primary_parent_handle_stop_test) } } -TEST_F(replica_split_test, query_child_state_reply_test) +TEST_P(replica_split_test, query_child_state_reply_test) { fail::cfg("replica_init_group_check", "return()"); fail::cfg("replica_broadcast_group_check", "return()"); @@ -1014,7 +1017,7 @@ TEST_F(replica_split_test, query_child_state_reply_test) ASSERT_TRUE(primary_parent_not_in_split()); } -TEST_F(replica_split_test, check_partition_hash_test) +TEST_P(replica_split_test, check_partition_hash_test) { uint64_t send_to_parent_after_split = 1; uint64_t send_to_child_after_split = 9; diff --git a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp index e78ae81efb..3238366964 100644 --- a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp @@ -58,6 +58,7 @@ #include "utils/autoref_ptr.h" #include "utils/binary_reader.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/utils.h" @@ -186,7 +187,8 @@ void simple_kv_service_impl::recover(const std::string &name, int64_t version) zauto_lock l(_lock); std::unique_ptr rfile; - auto s = rocksdb::Env::Default()->NewSequentialFile(name, &rfile, rocksdb::EnvOptions()); + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(name, &rfile, rocksdb::EnvOptions()); CHECK(s.ok(), "open log file '{}' failed, err = {}", name, s.ToString()); _store.clear(); diff --git a/src/replica/storage/simple_kv/test/run.sh b/src/replica/storage/simple_kv/test/run.sh index e10210fa41..17affedf79 100755 --- a/src/replica/storage/simple_kv/test/run.sh +++ b/src/replica/storage/simple_kv/test/run.sh @@ -118,8 +118,7 @@ if [ ! -z "${cases}" ]; then run_case ${id} echo done - # TODO(yingchun): ENCRYPTION: add enable encryption test. - # TEST_OPTS=${OLD_TEST_OPTS},encrypt_data_at_rest=true + TEST_OPTS=${OLD_TEST_OPTS},encrypt_data_at_rest=true for id in ${cases}; do run_case ${id} echo diff --git a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp index 021496194b..31338567d7 100644 --- a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp @@ -46,6 +46,7 @@ #include "utils/autoref_ptr.h" #include "utils/binary_reader.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/threadpool_code.h" @@ -189,7 +190,8 @@ void simple_kv_service_impl::recover(const std::string &name, int64_t version) dsn::zauto_lock l(_lock); std::unique_ptr rfile; - auto s = rocksdb::Env::Default()->NewSequentialFile(name, &rfile, rocksdb::EnvOptions()); + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(name, &rfile, rocksdb::EnvOptions()); CHECK(s.ok(), "open log file '{}' failed, err = {}", name, s.ToString()); _store.clear(); diff --git a/src/replica/test/log_block_test.cpp b/src/replica/test/log_block_test.cpp index 5e1029ba9b..ada8829e0b 100644 --- a/src/replica/test/log_block_test.cpp +++ b/src/replica/test/log_block_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -39,7 +40,9 @@ class log_block_test : public replica_test_base { }; -TEST_F(log_block_test, constructor) +INSTANTIATE_TEST_CASE_P(, log_block_test, ::testing::Values(false, true)); + +TEST_P(log_block_test, constructor) { log_block block(1); ASSERT_EQ(block.data().size(), 1); @@ -47,7 +50,7 @@ TEST_F(log_block_test, constructor) ASSERT_EQ(block.start_offset(), 1); } -TEST_F(log_block_test, log_block_header) +TEST_P(log_block_test, log_block_header) { log_block block(10); auto hdr = (log_block_header *)block.front().data(); @@ -60,7 +63,9 @@ class log_appender_test : public replica_test_base { }; -TEST_F(log_appender_test, constructor) +INSTANTIATE_TEST_CASE_P(, log_appender_test, ::testing::Values(false, true)); + +TEST_P(log_appender_test, constructor) { log_block block; binary_writer temp_writer; @@ -75,7 +80,7 @@ TEST_F(log_appender_test, constructor) ASSERT_EQ(appender.callbacks().size(), 0); } -TEST_F(log_appender_test, append_mutation) +TEST_P(log_appender_test, append_mutation) { log_appender appender(10); for (int i = 0; i < 5; i++) { @@ -88,7 +93,7 @@ TEST_F(log_appender_test, append_mutation) ASSERT_EQ(appender.blob_count(), 1 + 5 * 2); } -TEST_F(log_appender_test, log_block_not_full) +TEST_P(log_appender_test, log_block_not_full) { log_appender appender(10); for (int i = 0; i < 5; i++) { @@ -106,7 +111,7 @@ TEST_F(log_appender_test, log_block_not_full) ASSERT_EQ(block.data().size(), 1 + 5 * 2); } -TEST_F(log_appender_test, log_block_full) +TEST_P(log_appender_test, log_block_full) { log_appender appender(10); for (int i = 0; i < 1024; i++) { // more than DEFAULT_MAX_BLOCK_BYTES @@ -130,7 +135,7 @@ TEST_F(log_appender_test, log_block_full) ASSERT_EQ(sz, appender.size()); } -TEST_F(log_appender_test, read_log_block) +TEST_P(log_appender_test, read_log_block) { log_appender appender(10); for (int i = 0; i < 1024; i++) { // more than DEFAULT_MAX_BLOCK_BYTES diff --git a/src/replica/test/log_file_test.cpp b/src/replica/test/log_file_test.cpp index 5e0e44cd9b..e0736f15cc 100644 --- a/src/replica/test/log_file_test.cpp +++ b/src/replica/test/log_file_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -56,7 +57,9 @@ class log_file_test : public replica_test_base size_t _start_offset{10}; }; -TEST_F(log_file_test, commit_log_blocks) +INSTANTIATE_TEST_CASE_P(, log_file_test, ::testing::Values(false, true)); + +TEST_P(log_file_test, commit_log_blocks) { // write one block auto appender = std::make_shared(_start_offset); diff --git a/src/replica/test/main.cpp b/src/replica/test/main.cpp index 61d0f998a1..6e414ffddf 100644 --- a/src/replica/test/main.cpp +++ b/src/replica/test/main.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include #include #include #include @@ -24,27 +25,35 @@ #include "replication_service_test_app.h" #include "runtime/app_model.h" #include "runtime/service_app.h" +#include "test_util/test_util.h" #include "utils/error_code.h" int gtest_flags = 0; int gtest_ret = 0; replication_service_test_app *app; -TEST(cold_backup_context, check_backup_on_remote) { app->check_backup_on_remote_test(); } +class cold_backup_context_test : public pegasus::encrypt_data_test_base +{ +}; + +TEST_P(cold_backup_context_test, check_backup_on_remote) { app->check_backup_on_remote_test(); } -TEST(cold_backup_context, read_current_chkpt_file) { app->read_current_chkpt_file_test(); } +TEST_P(cold_backup_context_test, read_current_chkpt_file) { app->read_current_chkpt_file_test(); } -TEST(cold_backup_context, remote_chkpt_dir_exist) { app->remote_chkpt_dir_exist_test(); } +TEST_P(cold_backup_context_test, remote_chkpt_dir_exist) { app->remote_chkpt_dir_exist_test(); } -TEST(cold_backup_context, upload_checkpoint_to_remote) { app->upload_checkpoint_to_remote_test(); } +TEST_P(cold_backup_context_test, upload_checkpoint_to_remote) +{ + app->upload_checkpoint_to_remote_test(); +} -TEST(cold_backup_context, read_backup_metadata) { app->read_backup_metadata_test(); } +TEST_P(cold_backup_context_test, read_backup_metadata) { app->read_backup_metadata_test(); } -TEST(cold_backup_context, on_upload_chkpt_dir) { app->on_upload_chkpt_dir_test(); } +TEST_P(cold_backup_context_test, on_upload_chkpt_dir) { app->on_upload_chkpt_dir_test(); } -TEST(cold_backup_context, write_metadata_file) { app->write_backup_metadata_test(); } +TEST_P(cold_backup_context_test, write_metadata_file) { app->write_backup_metadata_test(); } -TEST(cold_backup_context, write_current_chkpt_file) { app->write_current_chkpt_file_test(); } +TEST_P(cold_backup_context_test, write_current_chkpt_file) { app->write_current_chkpt_file_test(); } error_code replication_service_test_app::start(const std::vector &args) { diff --git a/src/replica/test/mutation_log_learn_test.cpp b/src/replica/test/mutation_log_learn_test.cpp index 6c2fdf71ac..b14701b443 100644 --- a/src/replica/test/mutation_log_learn_test.cpp +++ b/src/replica/test/mutation_log_learn_test.cpp @@ -24,6 +24,7 @@ * THE SOFTWARE. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -56,11 +57,13 @@ class message_ex; namespace replication { -class mutation_log_test : public replica_test_base +class mutation_log_learn_test : public replica_test_base { }; -TEST_F(mutation_log_test, learn) +INSTANTIATE_TEST_CASE_P(, mutation_log_learn_test, ::testing::Values(false, true)); + +TEST_P(mutation_log_learn_test, learn) { std::chrono::steady_clock clock; gpid gpid(1, 1); diff --git a/src/replica/test/mutation_log_test.cpp b/src/replica/test/mutation_log_test.cpp index 4f96d7d57e..1cdaec67f1 100644 --- a/src/replica/test/mutation_log_test.cpp +++ b/src/replica/test/mutation_log_test.cpp @@ -49,6 +49,7 @@ #include "replica/test/mock_utils.h" #include "replica_test_base.h" #include "rrdb/rrdb.code.definition.h" +#include "test_util/test_util.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" #include "utils/blob.h" @@ -86,7 +87,13 @@ static void overwrite_file(const char *file, int offset, const void *buf, int si ASSERT_EQ(ERR_OK, file::close(wfile)); } -TEST(replication_test, log_file) +class replication_test : public pegasus::encrypt_data_test_base +{ +}; + +INSTANTIATE_TEST_CASE_P(, replication_test, ::testing::Values(false, true)); + +TEST_P(replication_test, log_file) { replica_log_info_map mdecrees; gpid gpid(1, 0); @@ -522,20 +529,22 @@ class mutation_log_test : public replica_test_base } }; -TEST_F(mutation_log_test, replay_single_file_1000) { test_replay_single_file(1000); } +INSTANTIATE_TEST_CASE_P(, mutation_log_test, ::testing::Values(false, true)); + +TEST_P(mutation_log_test, replay_single_file_1000) { test_replay_single_file(1000); } -TEST_F(mutation_log_test, replay_single_file_2000) { test_replay_single_file(2000); } +TEST_P(mutation_log_test, replay_single_file_2000) { test_replay_single_file(2000); } -TEST_F(mutation_log_test, replay_single_file_5000) { test_replay_single_file(5000); } +TEST_P(mutation_log_test, replay_single_file_5000) { test_replay_single_file(5000); } -TEST_F(mutation_log_test, replay_single_file_10000) { test_replay_single_file(10000); } +TEST_P(mutation_log_test, replay_single_file_10000) { test_replay_single_file(10000); } -TEST_F(mutation_log_test, replay_single_file_1) { test_replay_single_file(1); } +TEST_P(mutation_log_test, replay_single_file_1) { test_replay_single_file(1); } -TEST_F(mutation_log_test, replay_single_file_10) { test_replay_single_file(10); } +TEST_P(mutation_log_test, replay_single_file_10) { test_replay_single_file(10); } // mutation_log::open -TEST_F(mutation_log_test, open) +TEST_P(mutation_log_test, open) { std::vector mutations; @@ -568,13 +577,13 @@ TEST_F(mutation_log_test, open) } } -TEST_F(mutation_log_test, replay_multiple_files_10000_1mb) { test_replay_multiple_files(10000, 1); } +TEST_P(mutation_log_test, replay_multiple_files_10000_1mb) { test_replay_multiple_files(10000, 1); } -TEST_F(mutation_log_test, replay_multiple_files_20000_1mb) { test_replay_multiple_files(20000, 1); } +TEST_P(mutation_log_test, replay_multiple_files_20000_1mb) { test_replay_multiple_files(20000, 1); } -TEST_F(mutation_log_test, replay_multiple_files_50000_1mb) { test_replay_multiple_files(50000, 1); } +TEST_P(mutation_log_test, replay_multiple_files_50000_1mb) { test_replay_multiple_files(50000, 1); } -TEST_F(mutation_log_test, replay_start_decree) +TEST_P(mutation_log_test, replay_start_decree) { // decree ranges from [1, 30) generate_multiple_log_files(3); @@ -587,7 +596,7 @@ TEST_F(mutation_log_test, replay_start_decree) ASSERT_EQ(mlog->get_log_file_map().size(), 3); } -TEST_F(mutation_log_test, reset_from) +TEST_P(mutation_log_test, reset_from) { std::vector expected; { // writing logs @@ -635,7 +644,7 @@ TEST_F(mutation_log_test, reset_from) // multi-threaded testing. ensure reset_from will wait until // all previous writes complete. -TEST_F(mutation_log_test, reset_from_while_writing) +TEST_P(mutation_log_test, reset_from_while_writing) { std::vector expected; { // writing logs @@ -677,7 +686,7 @@ TEST_F(mutation_log_test, reset_from_while_writing) ASSERT_EQ(actual.size(), expected.size()); } -TEST_F(mutation_log_test, gc_slog) +TEST_P(mutation_log_test, gc_slog) { // Remove the slog dir and create a new one. const std::string slog_dir("./slog_test"); diff --git a/src/replica/test/open_replica_test.cpp b/src/replica/test/open_replica_test.cpp index b4cb088f62..1fa4ebe6fc 100644 --- a/src/replica/test/open_replica_test.cpp +++ b/src/replica/test/open_replica_test.cpp @@ -16,6 +16,7 @@ // under the License. #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -46,7 +47,9 @@ class open_replica_test : public replica_test_base ~open_replica_test() { dsn::utils::filesystem::remove_path("./tmp_dir"); } }; -TEST_F(open_replica_test, open_replica_add_decree_and_ballot_check) +INSTANTIATE_TEST_CASE_P(, open_replica_test, ::testing::Values(false, true)); + +TEST_P(open_replica_test, open_replica_add_decree_and_ballot_check) { app_info ai; ai.app_type = "replica"; diff --git a/src/replica/test/replica_disk_migrate_test.cpp b/src/replica/test/replica_disk_migrate_test.cpp index e90cbf2fa0..376f2d71a2 100644 --- a/src/replica/test/replica_disk_migrate_test.cpp +++ b/src/replica/test/replica_disk_migrate_test.cpp @@ -18,6 +18,7 @@ */ #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -149,7 +150,9 @@ class replica_disk_migrate_test : public replica_disk_test_base } }; -TEST_F(replica_disk_migrate_test, on_migrate_replica) +INSTANTIATE_TEST_CASE_P(, replica_disk_migrate_test, ::testing::Values(false, true)); + +TEST_P(replica_disk_migrate_test, on_migrate_replica) { auto &request = *fake_migrate_rpc.mutable_request(); auto &response = fake_migrate_rpc.response(); @@ -169,7 +172,7 @@ TEST_F(replica_disk_migrate_test, on_migrate_replica) ASSERT_EQ(response.err, ERR_OK); } -TEST_F(replica_disk_migrate_test, migrate_disk_replica_check) +TEST_P(replica_disk_migrate_test, migrate_disk_replica_check) { auto &request = *fake_migrate_rpc.mutable_request(); auto &response = fake_migrate_rpc.response(); @@ -228,7 +231,7 @@ TEST_F(replica_disk_migrate_test, migrate_disk_replica_check) ASSERT_EQ(response.err, ERR_OK); } -TEST_F(replica_disk_migrate_test, disk_migrate_replica_run) +TEST_P(replica_disk_migrate_test, disk_migrate_replica_run) { auto &request = *fake_migrate_rpc.mutable_request(); @@ -293,7 +296,7 @@ TEST_F(replica_disk_migrate_test, disk_migrate_replica_run) ASSERT_EQ(replica_ptr->disk_migrator()->status(), disk_migration_status::IDLE); } -TEST_F(replica_disk_migrate_test, disk_migrate_replica_close) +TEST_P(replica_disk_migrate_test, disk_migrate_replica_close) { auto &request = *fake_migrate_rpc.mutable_request(); request.pid = dsn::gpid(app_info_1.app_id, 2); @@ -308,7 +311,7 @@ TEST_F(replica_disk_migrate_test, disk_migrate_replica_close) ASSERT_TRUE(close_current_replica(fake_migrate_rpc)); } -TEST_F(replica_disk_migrate_test, disk_migrate_replica_update) +TEST_P(replica_disk_migrate_test, disk_migrate_replica_update) { auto &request = *fake_migrate_rpc.mutable_request(); request.pid = dsn::gpid(app_info_1.app_id, 3); @@ -364,7 +367,7 @@ TEST_F(replica_disk_migrate_test, disk_migrate_replica_update) // Test load from new replica dir failed, then fall back to load from origin dir succeed, // and then mark the "new" replica dir as ".gar". -TEST_F(replica_disk_migrate_test, disk_migrate_replica_open) +TEST_P(replica_disk_migrate_test, disk_migrate_replica_open) { gpid test_pid(app_info_1.app_id, 4); diff --git a/src/replica/test/replica_disk_test.cpp b/src/replica/test/replica_disk_test.cpp index 2ecef5adc7..6717ffd402 100644 --- a/src/replica/test/replica_disk_test.cpp +++ b/src/replica/test/replica_disk_test.cpp @@ -17,6 +17,8 @@ * under the License. */ +#include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -41,6 +43,7 @@ #include "replica/test/mock_utils.h" #include "replica_admin_types.h" #include "replica_disk_test_base.h" +#include "runtime/api_layer1.h" #include "runtime/rpc/rpc_holder.h" #include "test_util/test_util.h" #include "utils/autoref_ptr.h" @@ -86,7 +89,9 @@ class replica_disk_test : public replica_disk_test_base } }; -TEST_F(replica_disk_test, on_query_disk_info_all_app) +INSTANTIATE_TEST_CASE_P(, replica_disk_test, ::testing::Values(false, true)); + +TEST_P(replica_disk_test, on_query_disk_info_all_app) { generate_fake_rpc(); stub->on_query_disk_info(fake_query_disk_rpc); @@ -160,7 +165,7 @@ TEST_F(replica_disk_test, on_query_disk_info_all_app) } } -TEST_F(replica_disk_test, on_query_disk_info_app_not_existed) +TEST_P(replica_disk_test, on_query_disk_info_app_not_existed) { generate_fake_rpc(); query_disk_info_request &request = *fake_query_disk_rpc.mutable_request(); @@ -169,7 +174,7 @@ TEST_F(replica_disk_test, on_query_disk_info_app_not_existed) ASSERT_EQ(fake_query_disk_rpc.response().err, ERR_OBJECT_NOT_FOUND); } -TEST_F(replica_disk_test, on_query_disk_info_one_app) +TEST_P(replica_disk_test, on_query_disk_info_one_app) { generate_fake_rpc(); query_disk_info_request &request = *fake_query_disk_rpc.mutable_request(); @@ -195,49 +200,89 @@ TEST_F(replica_disk_test, on_query_disk_info_one_app) } } -TEST_F(replica_disk_test, gc_disk_useless_dir) +TEST_P(replica_disk_test, check_data_dir_removable) { + struct test_case + { + std::string path; + bool expected_removable; + bool expected_invalid; + } tests[] = {{"./replica.0.err", true, true}, + {"./replica.1.gar", true, true}, + {"./replica.2.tmp", true, true}, + {"./replica.3.ori", true, true}, + {"./replica.4.bak", false, true}, + {"./replica.5.abcde", false, false}, + {"./replica.6.x", false, false}, + {"./replica.7.8", false, false}}; + + for (const auto &test : tests) { + EXPECT_EQ(test.expected_removable, is_data_dir_removable(test.path)); + EXPECT_EQ(test.expected_invalid, is_data_dir_invalid(test.path)); + } +} + +TEST_P(replica_disk_test, gc_disk_useless_dir) +{ + PRESERVE_FLAG(gc_disk_error_replica_interval_seconds); + PRESERVE_FLAG(gc_disk_garbage_replica_interval_seconds); + PRESERVE_FLAG(gc_disk_migration_origin_replica_interval_seconds); + PRESERVE_FLAG(gc_disk_migration_tmp_replica_interval_seconds); + FLAGS_gc_disk_error_replica_interval_seconds = 1; FLAGS_gc_disk_garbage_replica_interval_seconds = 1; FLAGS_gc_disk_migration_origin_replica_interval_seconds = 1; FLAGS_gc_disk_migration_tmp_replica_interval_seconds = 1; - std::vector tests{ - "./replica1.err", - "./replica2.err", - "./replica.gar", - "./replica.tmp", - "./replica.ori", - "./replica.bak", - "./replica.1.1", - }; + struct test_case + { + std::string path; + bool expected_exists; + } tests[] = {{"./replica1.err", false}, + {"./replica2.err", false}, + {"./replica.gar", false}, + {"./replica.tmp", false}, + {"./replica.ori", false}, + {"./replica.bak", true}, + {"./replica.1.1", true}, + {"./1.1.pegasus.1234567890.err", false}, + {"./1.2.pegasus.0123456789.gar", false}, + {"./2.1.pegasus.1234567890123456.err", false}, + {"./2.2.pegasus.1234567890abcdef.gar", false}, + {fmt::format("./1.1.pegasus.{}.err", dsn_now_us()), false}, + {fmt::format("./2.1.pegasus.{}.gar", dsn_now_us()), false}, + {fmt::format("./1.2.pegasus.{}.gar", dsn_now_us() + 1000 * 1000 * 1000), true}, + {fmt::format("./2.2.pegasus.{}.err", dsn_now_us() + 1000 * 1000 * 1000), true}}; for (const auto &test : tests) { - utils::filesystem::create_directory(test); - ASSERT_TRUE(utils::filesystem::directory_exists(test)); + // Ensure that every directory does not exist and should be created. + CHECK_TRUE(utils::filesystem::create_directory(test.path)); + ASSERT_TRUE(utils::filesystem::directory_exists(test.path)); } sleep(5); disk_cleaning_report report{}; - dsn::replication::disk_remove_useless_dirs({std::make_shared("test", "./")}, report); + ASSERT_TRUE(dsn::replication::disk_remove_useless_dirs( + {std::make_shared("test", "./")}, report)); for (const auto &test : tests) { - if (!dsn::replication::is_data_dir_removable(test)) { - ASSERT_TRUE(utils::filesystem::directory_exists(test)); - continue; + ASSERT_EQ(test.expected_exists, utils::filesystem::directory_exists(test.path)); + if (test.expected_exists) { + // Delete existing directories, in case that they are mixed with later test cases + // to affect test results. + CHECK_TRUE(dsn::utils::filesystem::remove_path(test.path)); } - ASSERT_FALSE(utils::filesystem::directory_exists(test)); } - ASSERT_EQ(report.remove_dir_count, 5); + ASSERT_EQ(report.remove_dir_count, 11); ASSERT_EQ(report.disk_migrate_origin_count, 1); ASSERT_EQ(report.disk_migrate_tmp_count, 1); - ASSERT_EQ(report.garbage_replica_count, 1); - ASSERT_EQ(report.error_replica_count, 2); + ASSERT_EQ(report.garbage_replica_count, 5); + ASSERT_EQ(report.error_replica_count, 6); } -TEST_F(replica_disk_test, disk_status_test) +TEST_P(replica_disk_test, disk_status_test) { struct disk_status_test { @@ -261,7 +306,7 @@ TEST_F(replica_disk_test, disk_status_test) dn->status = disk_status::NORMAL; } -TEST_F(replica_disk_test, add_new_disk_test) +TEST_P(replica_disk_test, add_new_disk_test) { // Test case: // - invalid params @@ -291,7 +336,7 @@ TEST_F(replica_disk_test, add_new_disk_test) } } -TEST_F(replica_disk_test, disk_io_error_test) +TEST_P(replica_disk_test, disk_io_error_test) { // Disable failure detector to avoid connecting with meta server which is not started. FLAGS_fd_disabled = true; diff --git a/src/replica/test/replica_http_service_test.cpp b/src/replica/test/replica_http_service_test.cpp index 94c682302f..fbb36d285e 100644 --- a/src/replica/test/replica_http_service_test.cpp +++ b/src/replica/test/replica_http_service_test.cpp @@ -16,6 +16,7 @@ // under the License. #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -32,6 +33,7 @@ #include "replica/test/mock_utils.h" #include "replica/test/replica_test_base.h" #include "utils/flags.h" +#include "utils/test_macros.h" using std::map; using std::string; @@ -56,6 +58,15 @@ class replica_http_service_test : public replica_test_base _http_svc = std::make_unique(stub.get()); } + void SetUp() override + { + // Reset config_sync_interval_ms to 30000. + NO_FATALS(test_update_config({{"config_sync_interval_ms", "30000"}}, + R"({"update_status":"ERR_OK"})" + "\n")); + NO_FATALS(test_check_config("config_sync_interval_ms", "30000")); + } + void test_update_config(const map &configs, const string &expect_resp) { http_request req; @@ -86,40 +97,42 @@ class replica_http_service_test : public replica_test_base std::unique_ptr _http_svc; }; -TEST_F(replica_http_service_test, update_config_handler) +INSTANTIATE_TEST_CASE_P(, replica_http_service_test, ::testing::Values(false, true)); + +TEST_P(replica_http_service_test, update_config_handler) { // Test the default value. - test_check_config("config_sync_interval_ms", "30000"); + NO_FATALS(test_check_config("config_sync_interval_ms", "30000")); ASSERT_EQ(30000, FLAGS_config_sync_interval_ms); // Update config failed and value not changed. - test_update_config( + NO_FATALS(test_update_config( {}, R"({"update_status":"ERR_INVALID_PARAMETERS: there should be exactly one config to be updated once"})" - "\n"); - test_check_config("config_sync_interval_ms", "30000"); + "\n")); + NO_FATALS(test_check_config("config_sync_interval_ms", "30000")); ASSERT_EQ(30000, FLAGS_config_sync_interval_ms); // Update config failed and value not changed. - test_update_config( + NO_FATALS(test_update_config( {{"config_sync_interval_ms", "10"}, {"fds_write_limit_rate", "50"}}, R"({"update_status":"ERR_INVALID_PARAMETERS: there should be exactly one config to be updated once"})" - "\n"); - test_check_config("config_sync_interval_ms", "30000"); + "\n")); + NO_FATALS(test_check_config("config_sync_interval_ms", "30000")); ASSERT_EQ(30000, FLAGS_config_sync_interval_ms); // Update config failed and value not changed. - test_update_config({{"config_sync_interval_ms", "-1"}}, - R"({"update_status":"ERR_INVALID_PARAMETERS: -1 is invalid"})" - "\n"); - test_check_config("config_sync_interval_ms", "30000"); + NO_FATALS(test_update_config({{"config_sync_interval_ms", "-1"}}, + R"({"update_status":"ERR_INVALID_PARAMETERS: -1 is invalid"})" + "\n")); + NO_FATALS(test_check_config("config_sync_interval_ms", "30000")); ASSERT_EQ(30000, FLAGS_config_sync_interval_ms); // Update config success and value changed. - test_update_config({{"config_sync_interval_ms", "10"}}, - R"({"update_status":"ERR_OK"})" - "\n"); - test_check_config("config_sync_interval_ms", "10"); + NO_FATALS(test_update_config({{"config_sync_interval_ms", "10"}}, + R"({"update_status":"ERR_OK"})" + "\n")); + NO_FATALS(test_check_config("config_sync_interval_ms", "10")); ASSERT_EQ(10, FLAGS_config_sync_interval_ms); } diff --git a/src/replica/test/replica_learn_test.cpp b/src/replica/test/replica_learn_test.cpp index 8ff02993a1..8cf8e5b662 100644 --- a/src/replica/test/replica_learn_test.cpp +++ b/src/replica/test/replica_learn_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -180,9 +181,11 @@ class replica_learn_test : public duplication_test_base } }; -TEST_F(replica_learn_test, get_learn_start_decree) { test_get_learn_start_decree(); } +INSTANTIATE_TEST_CASE_P(, replica_learn_test, ::testing::Values(false, true)); -TEST_F(replica_learn_test, get_max_gced_decree_for_learn) { test_get_max_gced_decree_for_learn(); } +TEST_P(replica_learn_test, get_learn_start_decree) { test_get_learn_start_decree(); } + +TEST_P(replica_learn_test, get_max_gced_decree_for_learn) { test_get_max_gced_decree_for_learn(); } } // namespace replication } // namespace dsn diff --git a/src/replica/test/replica_test.cpp b/src/replica/test/replica_test.cpp index 40a93cc95f..9b7aea83b4 100644 --- a/src/replica/test/replica_test.cpp +++ b/src/replica/test/replica_test.cpp @@ -16,7 +16,7 @@ // under the License. #include -#include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -61,6 +61,7 @@ #include "runtime/task/task_tracker.h" #include "utils/autoref_ptr.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -185,7 +186,8 @@ class replica_test : public replica_test_base cold_backup::get_current_chkpt_file(backup_root, req.app_name, req.pid, req.backup_id); ASSERT_TRUE(dsn::utils::filesystem::file_exists(current_chkpt_file)); int64_t size = 0; - dsn::utils::filesystem::file_size(current_chkpt_file, size); + dsn::utils::filesystem::file_size( + current_chkpt_file, dsn::utils::FileDataType::kSensitive, size); ASSERT_LT(0, size); } @@ -241,7 +243,7 @@ class replica_test : public replica_test_base // load new max_replica_count from file auto err = replica_info.load(path); - ASSERT_EQ(err, ERR_OK); + ASSERT_EQ(ERR_OK, err); ASSERT_EQ(info, _mock_replica->_app_info); std::cout << "the loaded new app_info is " << info << std::endl; @@ -256,6 +258,8 @@ class replica_test : public replica_test_base std::cout << "the loaded original app_info is " << info << std::endl; } + void test_auto_trash(error_code ec); + public: dsn::app_info _app_info; dsn::gpid _pid; @@ -267,7 +271,9 @@ class replica_test : public replica_test_base const std::string _policy_name; }; -TEST_F(replica_test, write_size_limited) +INSTANTIATE_TEST_CASE_P(, replica_test, ::testing::Values(false, true)); + +TEST_P(replica_test, write_size_limited) { int count = 100; struct dsn::message_header header; @@ -287,7 +293,7 @@ TEST_F(replica_test, write_size_limited) ASSERT_EQ(get_write_size_exceed_threshold_count(), count); } -TEST_F(replica_test, backup_request_qps) +TEST_P(replica_test, backup_request_qps) { // create backup request struct dsn::message_header header; @@ -306,7 +312,7 @@ TEST_F(replica_test, backup_request_qps) ASSERT_GT(get_table_level_backup_request_qps(), 0); } -TEST_F(replica_test, query_data_version_test) +TEST_P(replica_test, query_data_version_test) { replica_http_service http_svc(stub.get()); struct query_data_version_test @@ -333,7 +339,7 @@ TEST_F(replica_test, query_data_version_test) } } -TEST_F(replica_test, query_compaction_test) +TEST_P(replica_test, query_compaction_test) { replica_http_service http_svc(stub.get()); struct query_compaction_test @@ -361,7 +367,7 @@ TEST_F(replica_test, query_compaction_test) } } -TEST_F(replica_test, update_validate_partition_hash_test) +TEST_P(replica_test, update_validate_partition_hash_test) { struct update_validate_partition_hash_test { @@ -384,7 +390,7 @@ TEST_F(replica_test, update_validate_partition_hash_test) } } -TEST_F(replica_test, update_allow_ingest_behind_test) +TEST_P(replica_test, update_allow_ingest_behind_test) { struct update_allow_ingest_behind_test { @@ -407,24 +413,26 @@ TEST_F(replica_test, update_allow_ingest_behind_test) } } -TEST_F(replica_test, test_replica_backup_and_restore) +TEST_P(replica_test, test_replica_backup_and_restore) { // TODO(yingchun): this test last too long time, optimize it! + return; test_on_cold_backup(); auto err = test_find_valid_checkpoint(); ASSERT_EQ(ERR_OK, err); } -TEST_F(replica_test, test_replica_backup_and_restore_with_specific_path) +TEST_P(replica_test, test_replica_backup_and_restore_with_specific_path) { // TODO(yingchun): this test last too long time, optimize it! + return; std::string user_specified_path = "test/backup"; test_on_cold_backup(user_specified_path); auto err = test_find_valid_checkpoint(user_specified_path); ASSERT_EQ(ERR_OK, err); } -TEST_F(replica_test, test_trigger_manual_emergency_checkpoint) +TEST_P(replica_test, test_trigger_manual_emergency_checkpoint) { ASSERT_EQ(_mock_replica->trigger_manual_emergency_checkpoint(100), ERR_OK); ASSERT_TRUE(is_checkpointing()); @@ -451,7 +459,7 @@ TEST_F(replica_test, test_trigger_manual_emergency_checkpoint) _mock_replica->tracker()->wait_outstanding_tasks(); } -TEST_F(replica_test, test_query_last_checkpoint_info) +TEST_P(replica_test, test_query_last_checkpoint_info) { // test no exist gpid auto req = std::make_unique(); @@ -474,7 +482,7 @@ TEST_F(replica_test, test_query_last_checkpoint_info) ASSERT_STR_CONTAINS(resp.base_local_dir, "/data/checkpoint.100"); } -TEST_F(replica_test, test_clear_on_failure) +TEST_P(replica_test, test_clear_on_failure) { // Clear up the remaining state. auto *dn = stub->get_fs_manager()->find_replica_dir(_app_info.app_type, _pid); @@ -497,26 +505,18 @@ TEST_F(replica_test, test_clear_on_failure) ASSERT_FALSE(has_gpid(_pid)); } -class replica_error_test : public replica_test, public testing::WithParamInterface -{ -}; - -INSTANTIATE_TEST_CASE_P(, - replica_error_test, - ::testing::Values(ERR_RDB_CORRUPTION, ERR_DISK_IO_ERROR)); - -TEST_P(replica_error_test, test_auto_trash_of_corruption) +void replica_test::test_auto_trash(error_code ec) { - const auto ec = GetParam(); - // The replica path will only be moved to error path when encounter ERR_RDB_CORRUPTION error. + // The replica path will only be moved to error path when encounter ERR_RDB_CORRUPTION + // error. bool moved_to_err_path = (ec == ERR_RDB_CORRUPTION); // Clear up the remaining state. auto *dn = stub->get_fs_manager()->find_replica_dir(_app_info.app_type, _pid); if (dn != nullptr) { dsn::utils::filesystem::remove_path(dn->replica_dir(_app_info.app_type, _pid)); + dn->holding_replicas.clear(); } - dn->holding_replicas.clear(); // Disable failure detector to avoid connecting with meta server which is not started. FLAGS_fd_disabled = true; @@ -564,7 +564,14 @@ TEST_P(replica_error_test, test_auto_trash_of_corruption) } } -TEST_F(replica_test, update_deny_client_test) +TEST_P(replica_test, test_auto_trash_of_corruption) +{ + NO_FATALS(test_auto_trash(ERR_RDB_CORRUPTION)); +} + +TEST_P(replica_test, test_auto_trash_of_io_error) { NO_FATALS(test_auto_trash(ERR_DISK_IO_ERROR)); } + +TEST_P(replica_test, update_deny_client_test) { struct update_deny_client_test { @@ -583,7 +590,7 @@ TEST_F(replica_test, update_deny_client_test) } } -TEST_F(replica_test, test_update_app_max_replica_count) { test_update_app_max_replica_count(); } +TEST_P(replica_test, test_update_app_max_replica_count) { test_update_app_max_replica_count(); } } // namespace replication } // namespace dsn diff --git a/src/replica/test/replica_test_base.h b/src/replica/test/replica_test_base.h index 3aed53c809..17bb9adf14 100644 --- a/src/replica/test/replica_test_base.h +++ b/src/replica/test/replica_test_base.h @@ -26,20 +26,20 @@ #pragma once -#include "utils/smart_pointers.h" -#include "replica/replication_app_base.h" -#include "utils/filesystem.h" -#include "utils/errors.h" #include -#include "replica/replica_stub.h" - #include "mock_utils.h" +#include "replica/replication_app_base.h" +#include "replica/replica_stub.h" +#include "test_util/test_util.h" +#include "utils/errors.h" +#include "utils/filesystem.h" +#include "utils/smart_pointers.h" namespace dsn { namespace replication { -class replica_stub_test_base : public ::testing::Test +class replica_stub_test_base : public pegasus::encrypt_data_test_base { public: replica_stub_test_base() { stub = std::make_unique(); } diff --git a/src/runtime/test/task_test.cpp b/src/runtime/test/task_test.cpp index 5716c8ce41..1ef5516a14 100644 --- a/src/runtime/test/task_test.cpp +++ b/src/runtime/test/task_test.cpp @@ -25,8 +25,11 @@ #include "aio/file_io.h" #include "runtime/task/task_code.h" #include "runtime/task/task_spec.h" +#include "utils/flags.h" #include "utils/threadpool_code.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + namespace dsn { class disk_file; @@ -65,6 +68,9 @@ class task_test : public ::testing::Test static void test_signal_finished_task() { + // config-test.ini is not encrypted, so set FLAGS_encrypt_data_at_rest = false on force. + FLAGS_encrypt_data_at_rest = false; + disk_file *fp = file::open("config-test.ini", file::FileOpenType::kReadOnly); // this aio task is enqueued into read-queue of disk_engine diff --git a/src/server/config.ini b/src/server/config.ini index 543fa316f6..99ca1ea1c7 100644 --- a/src/server/config.ini +++ b/src/server/config.ini @@ -400,6 +400,7 @@ stateful = true # The HTTP port exposed to Prometheus for pulling metrics from pegasus server. prometheus_port = 9091 + encrypt_data_at_rest = false [pegasus.collector] available_detect_app = stat diff --git a/src/server/config.min.ini b/src/server/config.min.ini index 947cc5c91e..02a1f68f08 100644 --- a/src/server/config.min.ini +++ b/src/server/config.min.ini @@ -158,6 +158,7 @@ perf_counter_sink = # The HTTP port exposed to Prometheus for pulling metrics from pegasus server. prometheus_port = @PROMETHEUS_PORT@ + encrypt_data_at_rest = false [pegasus.collector] available_detect_app = stat diff --git a/src/server/hotkey_collector.cpp b/src/server/hotkey_collector.cpp index 0574652f28..618c2c18ff 100644 --- a/src/server/hotkey_collector.cpp +++ b/src/server/hotkey_collector.cpp @@ -196,7 +196,7 @@ inline void hotkey_collector::change_state_by_result() if (!_result.hot_hash_key.empty()) { change_state_to_finished(); LOG_ERROR_PREFIX("Find the hotkey: {}", - pegasus::utils::c_escape_string(_result.hot_hash_key)); + pegasus::utils::c_escape_sensitive_string(_result.hot_hash_key)); } break; default: @@ -277,7 +277,7 @@ void hotkey_collector::on_start_detect(dsn::replication::detect_hotkey_response hint = fmt::format("{} hotkey result has been found: {}, you can send a stop rpc to " "restart hotkey detection", dsn::enum_to_string(_hotkey_type), - pegasus::utils::c_escape_string(_result.hot_hash_key)); + pegasus::utils::c_escape_sensitive_string(_result.hot_hash_key)); break; case hotkey_collector_state::STOPPED: change_state_to_coarse_detecting(); @@ -314,6 +314,9 @@ void hotkey_collector::query_result(dsn::replication::detect_hotkey_response &re LOG_INFO_PREFIX(hint); } else { resp.err = dsn::ERR_OK; + // Hot key should not be encrypted, thus use `c_escape_string` instead of + // `c_escape_sensitive_string` (otherwise it would be overwritten with + // ""). resp.__set_hotkey_result(pegasus::utils::c_escape_string(_result.hot_hash_key)); } } diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 9bc325ba9d..32d6b0e3ce 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -77,6 +76,7 @@ #include "utils/blob.h" #include "utils/chrono_literals.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" @@ -369,8 +369,8 @@ void pegasus_server_impl::on_get(get_rpc rpc) LOG_ERROR_PREFIX("rocksdb get failed for get from {}: hash_key = \"{}\", sort_key = " "\"{}\", error = {}", rpc.remote_address(), - ::pegasus::utils::c_escape_string(hash_key), - ::pegasus::utils::c_escape_string(sort_key), + ::pegasus::utils::c_escape_sensitive_string(hash_key), + ::pegasus::utils::c_escape_sensitive_string(sort_key), status.ToString()); } else if (!status.IsNotFound()) { LOG_ERROR_PREFIX("rocksdb get failed for get from {}: error = {}", @@ -393,8 +393,8 @@ void pegasus_server_impl::on_get(get_rpc rpc) "hash_key = {}, sort_key = {}, return = {}, " "value_size = {}, time_used = {} ns", rpc.remote_address(), - ::pegasus::utils::c_escape_string(hash_key), - ::pegasus::utils::c_escape_string(sort_key), + ::pegasus::utils::c_escape_sensitive_string(hash_key), + ::pegasus::utils::c_escape_sensitive_string(sort_key), status.ToString(), value.size(), time_used); @@ -508,17 +508,17 @@ void pegasus_server_impl::on_multi_get(multi_get_rpc rpc) "sort_key_filter_pattern = \"{}\", final_start = \"{}\" ({}), final_stop = " "\"{}\" ({})", rpc.remote_address(), - ::pegasus::utils::c_escape_string(request.hash_key), - ::pegasus::utils::c_escape_string(request.start_sortkey), + ::pegasus::utils::c_escape_sensitive_string(request.hash_key), + ::pegasus::utils::c_escape_sensitive_string(request.start_sortkey), request.start_inclusive ? "inclusive" : "exclusive", - ::pegasus::utils::c_escape_string(request.stop_sortkey), + ::pegasus::utils::c_escape_sensitive_string(request.stop_sortkey), request.stop_inclusive ? "inclusive" : "exclusive", ::dsn::apps::_filter_type_VALUES_TO_NAMES.find(request.sort_key_filter_type) ->second, - ::pegasus::utils::c_escape_string(request.sort_key_filter_pattern), - ::pegasus::utils::c_escape_string(start), + ::pegasus::utils::c_escape_sensitive_string(request.sort_key_filter_pattern), + ::pegasus::utils::c_escape_sensitive_string(start), start_inclusive ? "inclusive" : "exclusive", - ::pegasus::utils::c_escape_string(stop), + ::pegasus::utils::c_escape_sensitive_string(stop), stop_inclusive ? "inclusive" : "exclusive"); } resp.error = rocksdb::Status::kOk; @@ -685,7 +685,7 @@ void pegasus_server_impl::on_multi_get(multi_get_rpc rpc) LOG_ERROR_PREFIX("rocksdb scan failed for multi_get from {}: hash_key = \"{}\", " "reverse = {}, error = {}", rpc.remote_address(), - ::pegasus::utils::c_escape_string(request.hash_key), + ::pegasus::utils::c_escape_sensitive_string(request.hash_key), request.reverse ? "true" : "false", it->status().ToString()); } else { @@ -730,12 +730,13 @@ void pegasus_server_impl::on_multi_get(multi_get_rpc rpc) // print log if (!status.ok()) { if (FLAGS_rocksdb_verbose_log) { - LOG_ERROR_PREFIX("rocksdb get failed for multi_get from {}: hash_key = \"{}\", " - "sort_key = \"{}\", error = {}", - rpc.remote_address(), - ::pegasus::utils::c_escape_string(request.hash_key), - ::pegasus::utils::c_escape_string(request.sort_keys[i]), - status.ToString()); + LOG_ERROR_PREFIX( + "rocksdb get failed for multi_get from {}: hash_key = \"{}\", " + "sort_key = \"{}\", error = {}", + rpc.remote_address(), + ::pegasus::utils::c_escape_sensitive_string(request.hash_key), + ::pegasus::utils::c_escape_sensitive_string(request.sort_keys[i]), + status.ToString()); } else if (!status.IsNotFound()) { LOG_ERROR_PREFIX("rocksdb get failed for multi_get from {}: error = {}", rpc.remote_address(), @@ -803,13 +804,13 @@ void pegasus_server_impl::on_multi_get(multi_get_rpc rpc) "result_count = {}, result_size = {}, iteration_count = {}, " "expire_count = {}, filter_count = {}, time_used = {} ns", rpc.remote_address(), - ::pegasus::utils::c_escape_string(request.hash_key), - ::pegasus::utils::c_escape_string(request.start_sortkey), + ::pegasus::utils::c_escape_sensitive_string(request.hash_key), + ::pegasus::utils::c_escape_sensitive_string(request.start_sortkey), request.start_inclusive ? "inclusive" : "exclusive", - ::pegasus::utils::c_escape_string(request.stop_sortkey), + ::pegasus::utils::c_escape_sensitive_string(request.stop_sortkey), request.stop_inclusive ? "inclusive" : "exclusive", ::dsn::apps::_filter_type_VALUES_TO_NAMES.find(request.sort_key_filter_type)->second, - ::pegasus::utils::c_escape_string(request.sort_key_filter_pattern), + ::pegasus::utils::c_escape_sensitive_string(request.sort_key_filter_pattern), request.max_kv_count, request.max_kv_size, request.reverse ? "true" : "false", @@ -894,8 +895,8 @@ void pegasus_server_impl::on_batch_get(batch_get_rpc rpc) LOG_ERROR_PREFIX( "rocksdb data expired for batch_get from {}, hash_key = {}, sort_key = {}", rpc.remote_address().to_string(), - pegasus::utils::c_escape_string(hash_key), - pegasus::utils::c_escape_string(sort_key)); + pegasus::utils::c_escape_sensitive_string(hash_key), + pegasus::utils::c_escape_sensitive_string(sort_key)); } continue; } @@ -1011,7 +1012,7 @@ void pegasus_server_impl::on_sortkey_count(sortkey_count_rpc rpc) LOG_ERROR_PREFIX( "rocksdb scan failed for sortkey_count from {}: hash_key = \"{}\", error = {}", rpc.remote_address(), - ::pegasus::utils::c_escape_string(hash_key), + ::pegasus::utils::c_escape_sensitive_string(hash_key), it->status().ToString()); } else { LOG_ERROR_PREFIX("rocksdb scan failed for sortkey_count from {}: error = {}", @@ -1071,8 +1072,8 @@ void pegasus_server_impl::on_ttl(ttl_rpc rpc) LOG_ERROR_PREFIX("rocksdb get failed for ttl from {}: hash_key = \"{}\", sort_key = " "\"{}\", error = {}", rpc.remote_address(), - ::pegasus::utils::c_escape_string(hash_key), - ::pegasus::utils::c_escape_string(sort_key), + ::pegasus::utils::c_escape_sensitive_string(hash_key), + ::pegasus::utils::c_escape_sensitive_string(sort_key), status.ToString()); } else if (!status.IsNotFound()) { LOG_ERROR_PREFIX("rocksdb get failed for ttl from {}: error = {}", @@ -1181,9 +1182,9 @@ void pegasus_server_impl::on_get_scanner(get_scanner_rpc rpc) LOG_WARNING_PREFIX("empty key range for get_scanner from {}: start_key = \"{}\" ({}), " "stop_key = \"{}\" ({})", rpc.remote_address(), - ::pegasus::utils::c_escape_string(request.start_key), + ::pegasus::utils::c_escape_sensitive_string(request.start_key), request.start_inclusive ? "inclusive" : "exclusive", - ::pegasus::utils::c_escape_string(request.stop_key), + ::pegasus::utils::c_escape_sensitive_string(request.stop_key), request.stop_inclusive ? "inclusive" : "exclusive"); } resp.error = rocksdb::Status::kOk; @@ -1288,9 +1289,9 @@ void pegasus_server_impl::on_get_scanner(get_scanner_rpc rpc) "({}), stop_key = \"{}\" ({}), batch_size = {}, read_count = {}, " "error = {}", rpc.remote_address(), - ::pegasus::utils::c_escape_string(start), + ::pegasus::utils::c_escape_sensitive_string(start), request.start_inclusive ? "inclusive" : "exclusive", - ::pegasus::utils::c_escape_string(stop), + ::pegasus::utils::c_escape_sensitive_string(stop), request.stop_inclusive ? "inclusive" : "exclusive", batch_count, count, @@ -1459,7 +1460,7 @@ void pegasus_server_impl::on_scan(scan_rpc rpc) "\"{}\" ({}), batch_size = {}, read_count = {}, error = {}", rpc.remote_address(), request.context_id, - ::pegasus::utils::c_escape_string(stop), + ::pegasus::utils::c_escape_sensitive_string(stop), stop_inclusive ? "inclusive" : "exclusive", batch_count, count, @@ -1638,7 +1639,7 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) rocksdb::ConfigOptions config_options; // Set `ignore_unknown_options` true for forward compatibility. config_options.ignore_unknown_options = true; - config_options.env = rocksdb::Env::Default(); + config_options.env = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); auto status = rocksdb::LoadLatestOptions(config_options, rdb_path, &loaded_db_opt, &loaded_cf_descs); if (!status.ok()) { @@ -1683,7 +1684,7 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) config_options.ignore_unsupported_options = true; config_options.sanity_level = rocksdb::ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible; - config_options.env = rocksdb::Env::Default(); + config_options.env = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); auto s = rocksdb::CheckOptionsCompatibility(config_options, rdb_path, _db_opts, column_families); if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) { @@ -3449,31 +3450,32 @@ std::string pegasus_server_impl::dump_write_request(dsn::message_ex *request) ::dsn::blob hash_key, sort_key; pegasus_restore_key(put.key, hash_key, sort_key); return fmt::format("put: hash_key={}, sort_key={}", - pegasus::utils::c_escape_string(hash_key), - pegasus::utils::c_escape_string(sort_key)); + pegasus::utils::c_escape_sensitive_string(hash_key), + pegasus::utils::c_escape_sensitive_string(sort_key)); } if (rpc_code == dsn::apps::RPC_RRDB_RRDB_MULTI_PUT) { auto multi_put = multi_put_rpc(request).request(); return fmt::format("multi_put: hash_key={}, multi_put_count={}", - pegasus::utils::c_escape_string(multi_put.hash_key), + pegasus::utils::c_escape_sensitive_string(multi_put.hash_key), multi_put.kvs.size()); } if (rpc_code == dsn::apps::RPC_RRDB_RRDB_CHECK_AND_SET) { auto check_and_set = check_and_set_rpc(request).request(); return fmt::format("check_and_set: hash_key={}, check_sort_key={}, set_sort_key={}", - pegasus::utils::c_escape_string(check_and_set.hash_key), - pegasus::utils::c_escape_string(check_and_set.check_sort_key), - pegasus::utils::c_escape_string(check_and_set.set_sort_key)); + pegasus::utils::c_escape_sensitive_string(check_and_set.hash_key), + pegasus::utils::c_escape_sensitive_string(check_and_set.check_sort_key), + pegasus::utils::c_escape_sensitive_string(check_and_set.set_sort_key)); } if (rpc_code == dsn::apps::RPC_RRDB_RRDB_CHECK_AND_MUTATE) { auto check_and_mutate = check_and_mutate_rpc(request).request(); - return fmt::format("check_and_mutate: hash_key={}, check_sort_key={}, set_value_count={}", - pegasus::utils::c_escape_string(check_and_mutate.hash_key), - pegasus::utils::c_escape_string(check_and_mutate.check_sort_key), - check_and_mutate.mutate_list.size()); + return fmt::format( + "check_and_mutate: hash_key={}, check_sort_key={}, set_value_count={}", + pegasus::utils::c_escape_sensitive_string(check_and_mutate.hash_key), + pegasus::utils::c_escape_sensitive_string(check_and_mutate.check_sort_key), + check_and_mutate.mutate_list.size()); } return "default"; diff --git a/src/server/pegasus_server_impl_init.cpp b/src/server/pegasus_server_impl_init.cpp index 528b2bd8e9..855d6c6710 100644 --- a/src/server/pegasus_server_impl_init.cpp +++ b/src/server/pegasus_server_impl_init.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include @@ -52,6 +51,7 @@ #include "server/pegasus_read_service.h" #include "server/pegasus_server_write.h" // IWYU pragma: keep #include "server/range_read_limiter.h" +#include "utils/env.h" #include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/strings.h" @@ -423,7 +423,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) _rng_rd_opts.rocksdb_iteration_threshold_time_ms = FLAGS_rocksdb_iteration_threshold_time_ms; // init rocksdb::DBOptions - _db_opts.env = rocksdb::Env::Default(); + _db_opts.env = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); _db_opts.create_if_missing = true; // atomic flush data CF and meta CF, aim to keep consistency of 'last flushed decree' in meta CF // and data in data CF. diff --git a/src/server/pegasus_server_write.cpp b/src/server/pegasus_server_write.cpp index 1c6399cc27..b00f016ed2 100644 --- a/src/server/pegasus_server_write.cpp +++ b/src/server/pegasus_server_write.cpp @@ -172,8 +172,8 @@ void pegasus_server_write::request_key_check(int64_t decree, "decree: {}, code: {}, hash_key: {}, sort_key: {}", decree, msg->local_rpc_code.to_string(), - utils::c_escape_string(hash_key), - utils::c_escape_string(sort_key)); + utils::c_escape_sensitive_string(hash_key), + utils::c_escape_sensitive_string(sort_key)); } } diff --git a/src/server/pegasus_write_service_impl.h b/src/server/pegasus_write_service_impl.h index 69b71c6ba4..fa4c44e4ab 100644 --- a/src/server/pegasus_write_service_impl.h +++ b/src/server/pegasus_write_service_impl.h @@ -217,7 +217,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base LOG_ERROR_PREFIX("incr failed: decree = {}, error = " "old value \"{}\" is not an integer or out of range", decree, - utils::c_escape_string(old_value)); + utils::c_escape_sensitive_string(old_value)); resp.error = rocksdb::Status::kInvalidArgument; // we should write empty record to update rocksdb's last flushed decree return empty_put(decree); @@ -290,8 +290,8 @@ class pegasus_write_service::impl : public dsn::replication::replica_base LOG_ERROR_ROCKSDB("Error to GetCheckValue for CheckAndSet decree: {}, hash_key: {}, " "check_sort_key: {}", decree, - utils::c_escape_string(update.hash_key), - utils::c_escape_string(update.check_sort_key)); + utils::c_escape_sensitive_string(update.hash_key), + utils::c_escape_sensitive_string(update.check_sort_key)); resp.error = err; return resp.error; } @@ -410,8 +410,8 @@ class pegasus_write_service::impl : public dsn::replication::replica_base LOG_ERROR_ROCKSDB("Error to GetCheckValue for CheckAndMutate decree: {}, hash_key: {}, " "check_sort_key: {}", decree, - utils::c_escape_string(update.hash_key), - utils::c_escape_string(update.check_sort_key)); + utils::c_escape_sensitive_string(update.hash_key), + utils::c_escape_sensitive_string(update.check_sort_key)); resp.error = err; return resp.error; } @@ -650,7 +650,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base LOG_ERROR_PREFIX("check failed: decree = {}, error = " "check value \"{}\" is not an integer or out of range", decree, - utils::c_escape_string(value)); + utils::c_escape_sensitive_string(value)); invalid_argument = true; return false; } @@ -660,7 +660,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base LOG_ERROR_PREFIX("check failed: decree = {}, error = " "check operand \"{}\" is not an integer or out of range", decree, - utils::c_escape_string(check_operand)); + utils::c_escape_sensitive_string(check_operand)); invalid_argument = true; return false; } diff --git a/src/server/rocksdb_wrapper.cpp b/src/server/rocksdb_wrapper.cpp index 59cd199d27..3fe882a420 100644 --- a/src/server/rocksdb_wrapper.cpp +++ b/src/server/rocksdb_wrapper.cpp @@ -96,8 +96,8 @@ int rocksdb_wrapper::get(dsn::string_view raw_key, /*out*/ db_get_context *ctx) LOG_ERROR_ROCKSDB("Get", s.ToString(), "hash_key: {}, sort_key: {}", - utils::c_escape_string(hash_key), - utils::c_escape_string(sort_key)); + utils::c_escape_sensitive_string(hash_key), + utils::c_escape_sensitive_string(sort_key)); return s.code(); } @@ -156,8 +156,8 @@ int rocksdb_wrapper::write_batch_put_ctx(const db_write_context &ctx, s.ToString(), "decree: {}, hash_key: {}, sort_key: {}, expire_ts: {}", ctx.decree, - utils::c_escape_string(hash_key), - utils::c_escape_string(sort_key), + utils::c_escape_sensitive_string(hash_key), + utils::c_escape_sensitive_string(sort_key), expire_sec); } return s.code(); @@ -203,8 +203,8 @@ int rocksdb_wrapper::write_batch_delete(int64_t decree, dsn::string_view raw_key s.ToString(), "decree: {}, hash_key: {}, sort_key: {}", decree, - utils::c_escape_string(hash_key), - utils::c_escape_string(sort_key)); + utils::c_escape_sensitive_string(hash_key), + utils::c_escape_sensitive_string(sort_key)); } return s.code(); } diff --git a/src/server/test/capacity_unit_calculator_test.cpp b/src/server/test/capacity_unit_calculator_test.cpp index 93db546e14..98c6825977 100644 --- a/src/server/test/capacity_unit_calculator_test.cpp +++ b/src/server/test/capacity_unit_calculator_test.cpp @@ -18,6 +18,7 @@ */ #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -157,9 +158,11 @@ class capacity_unit_calculator_test : public pegasus_server_test_base } }; -TEST_F(capacity_unit_calculator_test, init) { test_init(); } +INSTANTIATE_TEST_CASE_P(, capacity_unit_calculator_test, ::testing::Values(false, true)); -TEST_F(capacity_unit_calculator_test, get) +TEST_P(capacity_unit_calculator_test, init) { test_init(); } + +TEST_P(capacity_unit_calculator_test, get) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); msg->header->context.u.is_backup_request = false; @@ -199,7 +202,7 @@ TEST_F(capacity_unit_calculator_test, get) _cal->reset(); } -TEST_F(capacity_unit_calculator_test, multi_get) +TEST_P(capacity_unit_calculator_test, multi_get) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); msg->header->context.u.is_backup_request = false; @@ -231,7 +234,7 @@ TEST_F(capacity_unit_calculator_test, multi_get) _cal->reset(); } -TEST_F(capacity_unit_calculator_test, scan) +TEST_P(capacity_unit_calculator_test, scan) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); msg->header->context.u.is_backup_request = false; @@ -266,7 +269,7 @@ TEST_F(capacity_unit_calculator_test, scan) _cal->reset(); } -TEST_F(capacity_unit_calculator_test, sortkey_count) +TEST_P(capacity_unit_calculator_test, sortkey_count) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); msg->header->context.u.is_backup_request = false; @@ -282,7 +285,7 @@ TEST_F(capacity_unit_calculator_test, sortkey_count) } } -TEST_F(capacity_unit_calculator_test, ttl) +TEST_P(capacity_unit_calculator_test, ttl) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); msg->header->context.u.is_backup_request = false; @@ -298,7 +301,7 @@ TEST_F(capacity_unit_calculator_test, ttl) } } -TEST_F(capacity_unit_calculator_test, put) +TEST_P(capacity_unit_calculator_test, put) { for (int i = 0; i < MAX_ROCKSDB_STATUS_CODE; i++) { _cal->add_put_cu(i, key, dsn::blob::create_from_bytes(std::string(4097, ' '))); @@ -312,7 +315,7 @@ TEST_F(capacity_unit_calculator_test, put) } } -TEST_F(capacity_unit_calculator_test, remove) +TEST_P(capacity_unit_calculator_test, remove) { for (int i = 0; i < MAX_ROCKSDB_STATUS_CODE; i++) { _cal->add_remove_cu(i, key); @@ -326,7 +329,7 @@ TEST_F(capacity_unit_calculator_test, remove) } } -TEST_F(capacity_unit_calculator_test, multi_put) +TEST_P(capacity_unit_calculator_test, multi_put) { std::vector<::dsn::apps::key_value> kvs; @@ -348,7 +351,7 @@ TEST_F(capacity_unit_calculator_test, multi_put) } } -TEST_F(capacity_unit_calculator_test, multi_remove) +TEST_P(capacity_unit_calculator_test, multi_remove) { std::vector<::dsn::blob> keys; @@ -370,7 +373,7 @@ TEST_F(capacity_unit_calculator_test, multi_remove) } } -TEST_F(capacity_unit_calculator_test, incr) +TEST_P(capacity_unit_calculator_test, incr) { for (int i = 0; i < MAX_ROCKSDB_STATUS_CODE; i++) { _cal->add_incr_cu(i, key); @@ -388,7 +391,7 @@ TEST_F(capacity_unit_calculator_test, incr) } } -TEST_F(capacity_unit_calculator_test, check_and_set) +TEST_P(capacity_unit_calculator_test, check_and_set) { dsn::blob cas_hash_key = dsn::blob::create_from_bytes("hash_key"); dsn::blob check_sort_key = dsn::blob::create_from_bytes("check_sort_key"); @@ -420,7 +423,7 @@ TEST_F(capacity_unit_calculator_test, check_and_set) _cal->reset(); } -TEST_F(capacity_unit_calculator_test, check_and_mutate) +TEST_P(capacity_unit_calculator_test, check_and_mutate) { dsn::blob cam_hash_key = dsn::blob::create_from_bytes("hash_key"); dsn::blob check_sort_key = dsn::blob::create_from_bytes("check_sort_key"); @@ -457,7 +460,7 @@ TEST_F(capacity_unit_calculator_test, check_and_mutate) _cal->reset(); } -TEST_F(capacity_unit_calculator_test, backup_request_bytes) +TEST_P(capacity_unit_calculator_test, backup_request_bytes) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); diff --git a/src/server/test/hotkey_collector_test.cpp b/src/server/test/hotkey_collector_test.cpp index be98def07d..9553123b49 100644 --- a/src/server/test/hotkey_collector_test.cpp +++ b/src/server/test/hotkey_collector_test.cpp @@ -18,6 +18,7 @@ #include "server/hotkey_collector.h" #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -123,7 +124,9 @@ class coarse_collector_test : public pegasus_server_test_base dsn::task_tracker _tracker; }; -TEST_F(coarse_collector_test, coarse_collector) +INSTANTIATE_TEST_CASE_P(, coarse_collector_test, ::testing::Values(false, true)); + +TEST_P(coarse_collector_test, coarse_collector) { detect_hotkey_result result; @@ -178,7 +181,9 @@ class fine_collector_test : public pegasus_server_test_base dsn::task_tracker _tracker; }; -TEST_F(fine_collector_test, fine_collector) +INSTANTIATE_TEST_CASE_P(, fine_collector_test, ::testing::Values(false, true)); + +TEST_P(fine_collector_test, fine_collector) { detect_hotkey_result result; @@ -286,13 +291,15 @@ class hotkey_collector_test : public pegasus_server_test_base dsn::task_tracker _tracker; }; -TEST_F(hotkey_collector_test, hotkey_type) +INSTANTIATE_TEST_CASE_P(, hotkey_collector_test, ::testing::Values(false, true)); + +TEST_P(hotkey_collector_test, hotkey_type) { ASSERT_EQ(get_collector_type(get_read_collector()), dsn::replication::hotkey_type::READ); ASSERT_EQ(get_collector_type(get_write_collector()), dsn::replication::hotkey_type::WRITE); } -TEST_F(hotkey_collector_test, state_transform) +TEST_P(hotkey_collector_test, state_transform) { auto collector = get_read_collector(); ASSERT_EQ(get_collector_stat(collector), hotkey_collector_state::STOPPED); @@ -360,7 +367,7 @@ TEST_F(hotkey_collector_test, state_transform) _tracker.wait_outstanding_tasks(); } -TEST_F(hotkey_collector_test, data_completeness) +TEST_P(hotkey_collector_test, data_completeness) { dsn::replication::detect_hotkey_response resp; on_detect_hotkey(generate_control_rpc(dsn::replication::hotkey_type::READ, diff --git a/src/server/test/hotspot_partition_test.cpp b/src/server/test/hotspot_partition_test.cpp index d978fc831b..97f4252bc7 100644 --- a/src/server/test/hotspot_partition_test.cpp +++ b/src/server/test/hotspot_partition_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include @@ -122,7 +123,9 @@ class hotspot_partition_test : public pegasus_server_test_base void clear_calculator_histories() { calculator._partitions_stat_histories.clear(); } }; -TEST_F(hotspot_partition_test, hotspot_partition_policy) +INSTANTIATE_TEST_CASE_P(, hotspot_partition_test, ::testing::Values(false, true)); + +TEST_P(hotspot_partition_test, hotspot_partition_policy) { // Insert normal scenario data to test std::vector test_rows = generate_row_data(); @@ -175,7 +178,7 @@ TEST_F(hotspot_partition_test, hotspot_partition_policy) clear_calculator_histories(); } -TEST_F(hotspot_partition_test, send_detect_hotkey_request) +TEST_P(hotspot_partition_test, send_detect_hotkey_request) { const int READ_HOT_PARTITION = 7; const int WRITE_HOT_PARTITION = 0; diff --git a/src/server/test/manual_compact_service_test.cpp b/src/server/test/manual_compact_service_test.cpp index d5d3f8b7c7..ff980dfd62 100644 --- a/src/server/test/manual_compact_service_test.cpp +++ b/src/server/test/manual_compact_service_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -110,7 +111,9 @@ class manual_compact_service_test : public pegasus_server_test_base } }; -TEST_F(manual_compact_service_test, check_compact_disabled) +INSTANTIATE_TEST_CASE_P(, manual_compact_service_test, ::testing::Values(false, true)); + +TEST_P(manual_compact_service_test, check_compact_disabled) { std::map envs; check_compact_disabled(envs, false); @@ -134,7 +137,7 @@ TEST_F(manual_compact_service_test, check_compact_disabled) check_compact_disabled(envs, false); } -TEST_F(manual_compact_service_test, check_once_compact) +TEST_P(manual_compact_service_test, check_once_compact) { // suppose compacted at 1500000000 set_compact_time(compacted_ts); @@ -167,7 +170,7 @@ TEST_F(manual_compact_service_test, check_once_compact) check_once_compact(envs, true); } -TEST_F(manual_compact_service_test, check_periodic_compact) +TEST_P(manual_compact_service_test, check_periodic_compact) { std::map envs; @@ -245,7 +248,7 @@ TEST_F(manual_compact_service_test, check_periodic_compact) check_periodic_compact(envs, false); } -TEST_F(manual_compact_service_test, extract_manual_compact_opts) +TEST_P(manual_compact_service_test, extract_manual_compact_opts) { // init _db max level set_num_level(7); @@ -283,7 +286,7 @@ TEST_F(manual_compact_service_test, extract_manual_compact_opts) ASSERT_EQ(out.target_level, -1); } -TEST_F(manual_compact_service_test, check_manual_compact_state_0_interval) +TEST_P(manual_compact_service_test, check_manual_compact_state_0_interval) { FLAGS_manual_compact_min_interval_seconds = 0; @@ -299,7 +302,7 @@ TEST_F(manual_compact_service_test, check_manual_compact_state_0_interval) check_manual_compact_state(false, "2nd start not ok"); } -TEST_F(manual_compact_service_test, check_manual_compact_state_1h_interval) +TEST_P(manual_compact_service_test, check_manual_compact_state_1h_interval) { FLAGS_manual_compact_min_interval_seconds = 3600; diff --git a/src/server/test/pegasus_compression_options_test.cpp b/src/server/test/pegasus_compression_options_test.cpp index b33eeeac14..16f9689bdf 100644 --- a/src/server/test/pegasus_compression_options_test.cpp +++ b/src/server/test/pegasus_compression_options_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -101,7 +102,9 @@ class pegasus_compression_options_test : public pegasus_server_test_base } }; -TEST_F(pegasus_compression_options_test, compression_type_convert_ok) +INSTANTIATE_TEST_CASE_P(, pegasus_compression_options_test, ::testing::Values(false, true)); + +TEST_P(pegasus_compression_options_test, compression_type_convert_ok) { compression_type_convert("none", none); compression_type_convert("snappy", snappy); @@ -109,7 +112,7 @@ TEST_F(pegasus_compression_options_test, compression_type_convert_ok) compression_type_convert("zstd", zstd); } -TEST_F(pegasus_compression_options_test, compression_type_convert_not_support) +TEST_P(pegasus_compression_options_test, compression_type_convert_not_support) { rocksdb::CompressionType tmp_type; ASSERT_FALSE(compression_str_to_type("not_support_zip", tmp_type)); @@ -122,7 +125,7 @@ TEST_F(pegasus_compression_options_test, compression_type_convert_not_support) ASSERT_EQ("", compression_type_to_str(rocksdb::kDisableCompressionOption)); } -TEST_F(pegasus_compression_options_test, compression_types_convert_ok) +TEST_P(pegasus_compression_options_test, compression_types_convert_ok) { // Old style. compression_types_convert_ok("none", {none, none, none, none, none, none, none}); @@ -143,7 +146,7 @@ TEST_F(pegasus_compression_options_test, compression_types_convert_ok) {none, lz4, snappy, zstd, lz4, snappy, zstd}); } -TEST_F(pegasus_compression_options_test, compression_types_convert_fail) +TEST_P(pegasus_compression_options_test, compression_types_convert_fail) { // Old style. compression_types_convert_fail("none1"); @@ -157,7 +160,7 @@ TEST_F(pegasus_compression_options_test, compression_types_convert_fail) compression_types_convert_fail("per_levelsnappy"); } -TEST_F(pegasus_compression_options_test, check_rocksdb_compression_types_default) +TEST_P(pegasus_compression_options_test, check_rocksdb_compression_types_default) { start(); check_db_compression_types({none, none, lz4, lz4, lz4, lz4}, "start with default"); diff --git a/src/server/test/pegasus_mutation_duplicator_test.cpp b/src/server/test/pegasus_mutation_duplicator_test.cpp index 168793236f..3d143e1893 100644 --- a/src/server/test/pegasus_mutation_duplicator_test.cpp +++ b/src/server/test/pegasus_mutation_duplicator_test.cpp @@ -20,6 +20,7 @@ #include "server/pegasus_mutation_duplicator.h" #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -288,7 +289,9 @@ class pegasus_mutation_duplicator_test : public pegasus_server_test_base } }; -TEST_F(pegasus_mutation_duplicator_test, get_hash_from_request) +INSTANTIATE_TEST_CASE_P(, pegasus_mutation_duplicator_test, ::testing::Values(false, true)); + +TEST_P(pegasus_mutation_duplicator_test, get_hash_from_request) { std::string hash_key("hash"); std::string sort_key("sort"); @@ -336,7 +339,7 @@ TEST_F(pegasus_mutation_duplicator_test, get_hash_from_request) // Verifies that calls on `get_hash_key_from_request` won't make // message unable to read. (if `get_hash_key_from_request` doesn't // copy the message internally, it will.) -TEST_F(pegasus_mutation_duplicator_test, read_after_get_hash_key) +TEST_P(pegasus_mutation_duplicator_test, read_after_get_hash_key) { std::string hash_key("hash"); std::string sort_key("sort"); @@ -358,18 +361,18 @@ TEST_F(pegasus_mutation_duplicator_test, read_after_get_hash_key) ASSERT_EQ(rpc.request().key.to_string(), raw_key.to_string()); } -TEST_F(pegasus_mutation_duplicator_test, duplicate) { test_duplicate(); } +TEST_P(pegasus_mutation_duplicator_test, duplicate) { test_duplicate(); } -TEST_F(pegasus_mutation_duplicator_test, duplicate_failed) { test_duplicate_failed(); } +TEST_P(pegasus_mutation_duplicator_test, duplicate_failed) { test_duplicate_failed(); } -TEST_F(pegasus_mutation_duplicator_test, duplicate_isolated_hashkeys) +TEST_P(pegasus_mutation_duplicator_test, duplicate_isolated_hashkeys) { test_duplicate_isolated_hashkeys(); } -TEST_F(pegasus_mutation_duplicator_test, create_duplicator) { test_create_duplicator(); } +TEST_P(pegasus_mutation_duplicator_test, create_duplicator) { test_create_duplicator(); } -TEST_F(pegasus_mutation_duplicator_test, duplicate_duplicate) +TEST_P(pegasus_mutation_duplicator_test, duplicate_duplicate) { replica_base replica(dsn::gpid(1, 1), "fake_replica", "temp"); auto duplicator = new_mutation_duplicator(&replica, "onebox2", "temp"); diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 718446821d..90c2276b60 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -144,19 +144,21 @@ class pegasus_server_impl_test : public pegasus_server_test_base } }; -TEST_F(pegasus_server_impl_test, test_table_level_slow_query) +INSTANTIATE_TEST_CASE_P(, pegasus_server_impl_test, ::testing::Values(false, true)); + +TEST_P(pegasus_server_impl_test, test_table_level_slow_query) { ASSERT_EQ(dsn::ERR_OK, start()); test_table_level_slow_query(); } -TEST_F(pegasus_server_impl_test, default_data_version) +TEST_P(pegasus_server_impl_test, default_data_version) { ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_EQ(_server->_pegasus_data_version, 1); } -TEST_F(pegasus_server_impl_test, test_open_db_with_latest_options) +TEST_P(pegasus_server_impl_test, test_open_db_with_latest_options) { // open a new db with no app env. ASSERT_EQ(dsn::ERR_OK, start()); @@ -176,7 +178,7 @@ TEST_F(pegasus_server_impl_test, test_open_db_with_latest_options) ASSERT_EQ(opts.disable_auto_compactions, _server->_db->GetOptions().disable_auto_compactions); } -TEST_F(pegasus_server_impl_test, test_open_db_with_app_envs) +TEST_P(pegasus_server_impl_test, test_open_db_with_app_envs) { std::map envs; envs[ROCKSDB_ENV_USAGE_SCENARIO_KEY] = ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD; @@ -184,19 +186,19 @@ TEST_F(pegasus_server_impl_test, test_open_db_with_app_envs) ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario); } -TEST_F(pegasus_server_impl_test, test_open_db_with_rocksdb_envs) +TEST_P(pegasus_server_impl_test, test_open_db_with_rocksdb_envs) { // Hint: Verify the set_rocksdb_options_before_creating function by boolean is_restart=false. test_open_db_with_rocksdb_envs(false); } -TEST_F(pegasus_server_impl_test, test_restart_db_with_rocksdb_envs) +TEST_P(pegasus_server_impl_test, test_restart_db_with_rocksdb_envs) { // Hint: Verify the reset_rocksdb_options function by boolean is_restart=true. test_open_db_with_rocksdb_envs(true); } -TEST_F(pegasus_server_impl_test, test_stop_db_twice) +TEST_P(pegasus_server_impl_test, test_stop_db_twice) { ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_TRUE(_server->_is_open); @@ -212,7 +214,7 @@ TEST_F(pegasus_server_impl_test, test_stop_db_twice) ASSERT_TRUE(_server->_db == nullptr); } -TEST_F(pegasus_server_impl_test, test_update_user_specified_compaction) +TEST_P(pegasus_server_impl_test, test_update_user_specified_compaction) { _server->_user_specified_compaction = ""; std::map envs; @@ -226,7 +228,7 @@ TEST_F(pegasus_server_impl_test, test_update_user_specified_compaction) ASSERT_EQ(user_specified_compaction, _server->_user_specified_compaction); } -TEST_F(pegasus_server_impl_test, test_load_from_duplication_data) +TEST_P(pegasus_server_impl_test, test_load_from_duplication_data) { auto origin_file = fmt::format("{}/{}", _server->duplication_dir(), "checkpoint"); dsn::utils::filesystem::create_directory(_server->duplication_dir()); diff --git a/src/server/test/pegasus_server_test_base.h b/src/server/test/pegasus_server_test_base.h index 7b1fe225d3..5cea01f3de 100644 --- a/src/server/test/pegasus_server_test_base.h +++ b/src/server/test/pegasus_server_test_base.h @@ -24,9 +24,13 @@ #include #include #include "common/fs_manager.h" +#include "utils/flags.h" #include "replica/replica_stub.h" +#include "test_util/test_util.h" #include "utils/filesystem.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + namespace pegasus { namespace server { @@ -39,7 +43,7 @@ class mock_pegasus_server_impl : public pegasus_server_impl MOCK_CONST_METHOD0(is_duplication_follower, bool()); }; -class pegasus_server_test_base : public ::testing::Test +class pegasus_server_test_base : public pegasus::encrypt_data_test_base { public: pegasus_server_test_base() @@ -49,7 +53,14 @@ class pegasus_server_test_base : public ::testing::Test _replica_stub = new dsn::replication::replica_stub(); _replica_stub->get_fs_manager()->initialize({"test_dir"}, {"test_tag"}); - _gpid = dsn::gpid(100, 1); + // Use different gpid for encryption and non-encryption test to avoid reopening a rocksdb + // instance with different encryption option. + if (FLAGS_encrypt_data_at_rest) { + _gpid = dsn::gpid(100, 0); + } else { + _gpid = dsn::gpid(100, 1); + } + dsn::app_info app_info; app_info.app_type = "pegasus"; diff --git a/src/server/test/pegasus_server_write_test.cpp b/src/server/test/pegasus_server_write_test.cpp index 3a9cfc2b1d..d32ef3ff09 100644 --- a/src/server/test/pegasus_server_write_test.cpp +++ b/src/server/test/pegasus_server_write_test.cpp @@ -18,6 +18,7 @@ */ #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -140,7 +141,9 @@ class pegasus_server_write_test : public pegasus_server_test_base } }; -TEST_F(pegasus_server_write_test, batch_writes) { test_batch_writes(); } +INSTANTIATE_TEST_CASE_P(, pegasus_server_write_test, ::testing::Values(false, true)); + +TEST_P(pegasus_server_write_test, batch_writes) { test_batch_writes(); } } // namespace server } // namespace pegasus diff --git a/src/server/test/pegasus_write_service_impl_test.cpp b/src/server/test/pegasus_write_service_impl_test.cpp index 0d32540fda..d61ffe64ac 100644 --- a/src/server/test/pegasus_write_service_impl_test.cpp +++ b/src/server/test/pegasus_write_service_impl_test.cpp @@ -18,6 +18,7 @@ */ #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -87,7 +88,9 @@ class incr_test : public pegasus_write_service_impl_test dsn::apps::incr_response resp; }; -TEST_F(incr_test, incr_on_absent_record) +INSTANTIATE_TEST_CASE_P(, incr_test, ::testing::Values(false, true)); + +TEST_P(incr_test, incr_on_absent_record) { // ensure key is absent db_get_context get_ctx; @@ -102,7 +105,7 @@ TEST_F(incr_test, incr_on_absent_record) ASSERT_TRUE(get_ctx.found); } -TEST_F(incr_test, negative_incr_and_zero_incr) +TEST_P(incr_test, negative_incr_and_zero_incr) { req.increment = -100; ASSERT_EQ(0, _write_impl->incr(0, req, resp)); @@ -117,7 +120,7 @@ TEST_F(incr_test, negative_incr_and_zero_incr) ASSERT_EQ(resp.new_value, -101); } -TEST_F(incr_test, invalid_incr) +TEST_P(incr_test, invalid_incr) { single_set(req.key, dsn::blob::create_from_bytes("abc")); @@ -134,7 +137,7 @@ TEST_F(incr_test, invalid_incr) ASSERT_EQ(resp.new_value, 100); } -TEST_F(incr_test, fail_on_get) +TEST_P(incr_test, fail_on_get) { dsn::fail::setup(); dsn::fail::cfg("db_get", "100%1*return()"); @@ -147,7 +150,7 @@ TEST_F(incr_test, fail_on_get) dsn::fail::teardown(); } -TEST_F(incr_test, fail_on_put) +TEST_P(incr_test, fail_on_put) { dsn::fail::setup(); dsn::fail::cfg("db_write_batch_put", "100%1*return()"); @@ -160,7 +163,7 @@ TEST_F(incr_test, fail_on_put) dsn::fail::teardown(); } -TEST_F(incr_test, incr_on_expire_record) +TEST_P(incr_test, incr_on_expire_record) { // make the key expired req.expire_ts_seconds = 1; diff --git a/src/server/test/pegasus_write_service_test.cpp b/src/server/test/pegasus_write_service_test.cpp index f277551bc6..30f2e7f8ea 100644 --- a/src/server/test/pegasus_write_service_test.cpp +++ b/src/server/test/pegasus_write_service_test.cpp @@ -18,6 +18,7 @@ */ #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -224,13 +225,15 @@ class pegasus_write_service_test : public pegasus_server_test_base } }; -TEST_F(pegasus_write_service_test, multi_put) { test_multi_put(); } +INSTANTIATE_TEST_CASE_P(, pegasus_write_service_test, ::testing::Values(false, true)); -TEST_F(pegasus_write_service_test, multi_remove) { test_multi_remove(); } +TEST_P(pegasus_write_service_test, multi_put) { test_multi_put(); } -TEST_F(pegasus_write_service_test, batched_writes) { test_batched_writes(); } +TEST_P(pegasus_write_service_test, multi_remove) { test_multi_remove(); } -TEST_F(pegasus_write_service_test, duplicate_not_batched) +TEST_P(pegasus_write_service_test, batched_writes) { test_batched_writes(); } + +TEST_P(pegasus_write_service_test, duplicate_not_batched) { std::string hash_key = "hash_key"; constexpr int kv_num = 100; @@ -280,7 +283,7 @@ TEST_F(pegasus_write_service_test, duplicate_not_batched) } } -TEST_F(pegasus_write_service_test, duplicate_batched) +TEST_P(pegasus_write_service_test, duplicate_batched) { std::string hash_key = "hash_key"; constexpr int kv_num = 100; @@ -314,7 +317,7 @@ TEST_F(pegasus_write_service_test, duplicate_batched) } } -TEST_F(pegasus_write_service_test, illegal_duplicate_request) +TEST_P(pegasus_write_service_test, illegal_duplicate_request) { std::string hash_key = "hash_key"; std::string sort_key = "sort_key"; diff --git a/src/server/test/rocksdb_wrapper_test.cpp b/src/server/test/rocksdb_wrapper_test.cpp index 86542c64fe..372517c8fa 100644 --- a/src/server/test/rocksdb_wrapper_test.cpp +++ b/src/server/test/rocksdb_wrapper_test.cpp @@ -18,6 +18,7 @@ */ #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -103,7 +104,9 @@ class rocksdb_wrapper_test : public pegasus_server_test_base } }; -TEST_F(rocksdb_wrapper_test, get) +INSTANTIATE_TEST_CASE_P(, rocksdb_wrapper_test, ::testing::Values(false, true)); + +TEST_P(rocksdb_wrapper_test, get) { // not found db_get_context get_ctx1; @@ -135,7 +138,7 @@ TEST_F(rocksdb_wrapper_test, get) ASSERT_EQ(user_value, value); } -TEST_F(rocksdb_wrapper_test, put_verify_timetag) +TEST_P(rocksdb_wrapper_test, put_verify_timetag) { set_app_duplicating(); @@ -212,7 +215,7 @@ TEST_F(rocksdb_wrapper_test, put_verify_timetag) } // verify timetag on data version v0 -TEST_F(rocksdb_wrapper_test, verify_timetag_compatible_with_version_0) +TEST_P(rocksdb_wrapper_test, verify_timetag_compatible_with_version_0) { const_cast(_rocksdb_wrapper->_pegasus_data_version) = 0; // old version diff --git a/src/test/bench_test/config.cpp b/src/test/bench_test/config.cpp index e307c79bb7..8276142162 100644 --- a/src/test/bench_test/config.cpp +++ b/src/test/bench_test/config.cpp @@ -19,11 +19,11 @@ #include "config.h" -#include +#include "utils/env.h" namespace pegasus { namespace test { -config::config() { env = rocksdb::Env::Default(); } +config::config() { env = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); } } // namespace test } // namespace pegasus diff --git a/src/test/function_test/base_api/test_range_read.cpp b/src/test/function_test/base_api/test_range_read.cpp index 1a0742fcb4..5827d8da83 100644 --- a/src/test/function_test/base_api/test_range_read.cpp +++ b/src/test/function_test/base_api/test_range_read.cpp @@ -109,7 +109,6 @@ class range_read_test : public test_util std::map expect_kvs_; }; -// TODO(yingchun): use TEST_P to refact TEST_F(range_read_test, multiget_test) { pegasus::pegasus_client::multi_get_options options; diff --git a/src/test/function_test/bulk_load/test_bulk_load.cpp b/src/test/function_test/bulk_load/test_bulk_load.cpp index 28e52858f3..b4076f60b1 100644 --- a/src/test/function_test/bulk_load/test_bulk_load.cpp +++ b/src/test/function_test/bulk_load/test_bulk_load.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include "base/pegasus_const.h" #include "block_service/local/local_service.h" @@ -46,11 +47,15 @@ #include "test/function_test/utils/test_util.h" #include "utils/blob.h" #include "utils/enum_helper.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/errors.h" #include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/test_macros.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + using namespace ::dsn; using namespace ::dsn::replication; using namespace pegasus; @@ -98,10 +103,11 @@ class bulk_load_test : public test_util void generate_bulk_load_info(const bulk_load_info &bli, const std::string &bulk_load_info_path) { auto value = dsn::json::json_forwarder::encode(bli); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice(value.data(), value.length()), - bulk_load_info_path, - /* should_sync */ true); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(value.data(), value.length()), + bulk_load_info_path, + /* should_sync */ true); ASSERT_TRUE(s.ok()) << s.ToString(); } @@ -110,15 +116,17 @@ class bulk_load_test : public test_util void generate_bulk_load_info_meta(const std::string &bulk_load_info_path) { dist::block_service::file_metadata fm; - ASSERT_TRUE(utils::filesystem::file_size(bulk_load_info_path, fm.size)); + ASSERT_TRUE(utils::filesystem::file_size( + bulk_load_info_path, dsn::utils::FileDataType::kSensitive, fm.size)); ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(bulk_load_info_path, fm.md5)); std::string value = nlohmann::json(fm).dump(); auto bulk_load_info_meta_path = fmt::format("{}/{}/{}/.bulk_load_info.meta", kLocalBulkLoadRoot, kCluster, app_name_); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice(value), - bulk_load_info_meta_path, - /* should_sync */ true); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(value), + bulk_load_info_meta_path, + /* should_sync */ true); ASSERT_TRUE(s.ok()) << s.ToString(); } @@ -132,6 +140,15 @@ class bulk_load_test : public test_util NO_FATALS(run_cmd_from_project_root( fmt::format("cp -r {}/{} {}", kSourceFilesRoot, kBulkLoad, kLocalServiceRoot))); + if (FLAGS_encrypt_data_at_rest) { + std::vector src_files; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(kLocalServiceRoot, src_files, true)); + for (const auto &src_file : src_files) { + auto s = dsn::utils::encrypt_file(src_file); + ASSERT_TRUE(s.ok()) << s.ToString(); + } + } + // Generate 'bulk_load_info'. auto bulk_load_info_path = fmt::format("{}/{}/{}/bulk_load_info", kLocalBulkLoadRoot, kCluster, app_name_); diff --git a/src/test/function_test/config.ini b/src/test/function_test/config.ini index 5b76a05a26..01c7bae703 100644 --- a/src/test/function_test/config.ini +++ b/src/test/function_test/config.ini @@ -80,6 +80,9 @@ mycluster = 127.0.0.1:34601,127.0.0.1:34602,127.0.0.1:34603 onebox = 127.0.0.1:34601,127.0.0.1:34602,127.0.0.1:34603 single_master_cluster = 127.0.0.1:34601 +[pegasus.server] +encrypt_data_at_rest = false + [function_test.throttle_test] - throttle_test_medium_value_kb = 20 - throttle_test_large_value_kb = 50 +throttle_test_medium_value_kb = 20 +throttle_test_large_value_kb = 50 diff --git a/src/test_util/test_util.cpp b/src/test_util/test_util.cpp index 935d50631a..a4dbf41deb 100644 --- a/src/test_util/test_util.cpp +++ b/src/test_util/test_util.cpp @@ -41,10 +41,11 @@ namespace pegasus { void create_local_test_file(const std::string &full_name, dsn::replication::file_meta *fm) { ASSERT_NE(fm, nullptr); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice("write some data."), - full_name, - /* should_sync */ true); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice("write some data."), + full_name, + /* should_sync */ true); ASSERT_TRUE(s.ok()) << s.ToString(); fm->name = full_name; ASSERT_EQ(dsn::ERR_OK, dsn::utils::filesystem::md5sum(full_name, fm->md5)); diff --git a/src/test_util/test_util.h b/src/test_util/test_util.h index 23b1624be7..90930c226c 100644 --- a/src/test_util/test_util.h +++ b/src/test_util/test_util.h @@ -19,15 +19,20 @@ #pragma once +#include +#include #include #include #include #include -#include #include -#include "fmt/core.h" #include "runtime/api_layer1.h" +#include "utils/env.h" +// IWYU refused to include "utils/defer.h" everywhere, both in .h and .cpp files. +// However, once "utils/defer.h" is not included, it is inevitable that compilation +// will fail since dsn::defer is referenced. Thus force IWYU to keep it. +#include "utils/defer.h" // IWYU pragma: keep #include "utils/flags.h" #include "utils/test_macros.h" @@ -39,13 +44,32 @@ class file_meta; DSN_DECLARE_bool(encrypt_data_at_rest); +// Save the current value of a flag and restore it at the end of the function. +#define PRESERVE_FLAG(name) \ + auto PRESERVED_FLAGS_##name = FLAGS_##name; \ + auto PRESERVED_FLAGS_##name##_cleanup = \ + dsn::defer([PRESERVED_FLAGS_##name]() { FLAGS_##name = PRESERVED_FLAGS_##name; }) + namespace pegasus { // A base parameterized test class for testing enable/disable encryption at rest. class encrypt_data_test_base : public testing::TestWithParam { public: - encrypt_data_test_base() { FLAGS_encrypt_data_at_rest = GetParam(); } + encrypt_data_test_base() + { + FLAGS_encrypt_data_at_rest = GetParam(); + // The size of an actual encrypted file should plus kEncryptionHeaderkSize bytes if consider + // it as kNonSensitive. + if (FLAGS_encrypt_data_at_rest) { + _extra_encrypted_file_size = dsn::utils::kEncryptionHeaderkSize; + } + } + + uint64_t extra_encrypted_file_size() const { return _extra_encrypted_file_size; } + +private: + uint64_t _extra_encrypted_file_size = 0; }; class stop_watch diff --git a/src/utils/alloc.h b/src/utils/alloc.h index 542f4efb51..16045b3b5b 100644 --- a/src/utils/alloc.h +++ b/src/utils/alloc.h @@ -17,7 +17,11 @@ #pragma once -#include +#include // IWYU pragma: keep +#include +#include +#include +#include #include "utils/ports.h" @@ -25,12 +29,6 @@ // where CACHELINE_SIZE is defined. #ifdef CACHELINE_SIZE -#include -#include // IWYU pragma: keep -#include -#include -#include - #ifndef NDEBUG #include "utils/fmt_logging.h" #endif diff --git a/src/utils/command_manager.cpp b/src/utils/command_manager.cpp index 38d84054b5..ab2e22f790 100644 --- a/src/utils/command_manager.cpp +++ b/src/utils/command_manager.cpp @@ -179,7 +179,7 @@ command_manager::~command_manager() { _cmds.clear(); CHECK(_handlers.empty(), - "All commands must be deregistered before command_manager is destroyed, however {} is " + "All commands must be deregistered before command_manager is destroyed, however '{}' is " "still registered", _handlers.begin()->first); } diff --git a/src/utils/filesystem.cpp b/src/utils/filesystem.cpp index 211aefbecb..48528e61fd 100644 --- a/src/utils/filesystem.cpp +++ b/src/utils/filesystem.cpp @@ -491,7 +491,8 @@ bool create_file(const std::string &path) } std::unique_ptr wfile; - auto s = rocksdb::Env::Default()->ReopenWritableFile(path, &wfile, rocksdb::EnvOptions()); + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->ReopenWritableFile(path, &wfile, rocksdb::EnvOptions()); if (dsn_unlikely(!s.ok())) { LOG_WARNING("fail to create file {}, err={}", path, s.ToString()); return false; @@ -678,7 +679,8 @@ error_code md5sum(const std::string &file_path, /*out*/ std::string &result) } std::unique_ptr sfile; - auto s = rocksdb::Env::Default()->NewSequentialFile(file_path, &sfile, rocksdb::EnvOptions()); + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(file_path, &sfile, rocksdb::EnvOptions()); if (!sfile) { LOG_ERROR("md5sum error: open file {} failed, err={}", file_path, s.ToString()); return ERR_FILE_OPERATION_FAILED; @@ -870,17 +872,20 @@ bool check_dir_rw(const std::string &path, std::string &err_msg) static const std::string kFname = "read_write_test_file"; std::string fpath = path_combine(path, kFname); auto cleanup = defer([&fpath]() { remove_path(fpath); }); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice(kTestValue), - fpath, - /* should_sync */ true); + // Use kSensitive to test encryption functionality as well. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(kTestValue), + fpath, + /* should_sync */ true); if (dsn_unlikely(!s.ok())) { err_msg = fmt::format("fail to write file {}, err={}", fpath, s.ToString()); return false; } std::string read_data; - s = rocksdb::ReadFileToString(rocksdb::Env::Default(), fpath, &read_data); + s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fpath, &read_data); if (dsn_unlikely(!s.ok())) { err_msg = fmt::format("fail to read file {}, err={}", fpath, s.ToString()); return false; diff --git a/src/utils/flags.h b/src/utils/flags.h index 355fa14ded..a0b9deb46a 100644 --- a/src/utils/flags.h +++ b/src/utils/flags.h @@ -51,7 +51,7 @@ struct hash // Example: // DSN_DEFINE_string(core, filename, "my_file.txt", "The file to read"); // DSN_DEFINE_validator(filename, [](const char *fname){ return is_file(fname); }); -// auto fptr = file::open(FLAGS_filename, O_RDONLY | O_BINARY, 0); +// auto fptr = file::open(FLAGS_filename, file::FileOpenType::kReadOnly); #define DSN_DECLARE_VARIABLE(type, name) extern type FLAGS_##name diff --git a/src/utils/macros.h b/src/utils/macros.h new file mode 100644 index 0000000000..c44c45e252 --- /dev/null +++ b/src/utils/macros.h @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +// Suppose there is a macro defined as: +// #define FOO 123 +// +// Once we need the value represented by FOO to be a string, i.e. "123", just do: +// STRINGIFY(FOO) +// +// See https://gcc.gnu.org/onlinedocs/gcc-4.8.5/cpp/Stringification.html for details. +#define STRINGIFY_HELPER(x) #x +#define STRINGIFY(x) STRINGIFY_HELPER(x) diff --git a/src/utils/metrics.h b/src/utils/metrics.h index 27c6355f32..75afc31d89 100644 --- a/src/utils/metrics.h +++ b/src/utils/metrics.h @@ -46,6 +46,7 @@ #include "utils/enum_helper.h" #include "utils/fmt_logging.h" #include "utils/long_adder.h" +#include "utils/macros.h" #include "utils/nth_element.h" #include "utils/ports.h" #include "utils/singleton.h" @@ -973,8 +974,6 @@ struct kth_percentile_property double decimal; }; -#define STRINGIFY_HELPER(x) #x -#define STRINGIFY(x) STRINGIFY_HELPER(x) #define STRINGIFY_KTH_PERCENTILE_NAME(kth) STRINGIFY(KTH_PERCENTILE_NAME(kth)) #define KTH_TO_DECIMAL(kth) 0.##kth #define KTH_PERCENTILE_PROPERTY_LIST(kth) \ diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index 16c4e95e14..f4c90789a8 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -182,6 +182,10 @@ simple_logger::simple_logger(const char *log_dir) create_log_file(); + // TODO(yingchun): simple_logger is destroyed after command_manager, so will cause crash like + // "assertion expression: [_handlers.empty()] All commands must be deregistered before + // command_manager is destroyed, however 'flush-log' is still registered". + // We need to fix it. _cmds.emplace_back(::dsn::command_manager::instance().register_command( {"flush-log"}, "flush-log - flush log to stderr or log file", diff --git a/src/utils/strings.h b/src/utils/strings.h index 7584df99bc..1759856f01 100644 --- a/src/utils/strings.h +++ b/src/utils/strings.h @@ -114,6 +114,7 @@ std::string get_last_component(const std::string &input, const char splitters[]) char *trim_string(char *s); +// TODO(yingchun): unify the following functions with the ones in utils::filesystem // calculate the md5 checksum of buffer std::string string_md5(const char *buffer, unsigned int length); diff --git a/src/utils/test/TokenBucketTest.cpp b/src/utils/test/TokenBucketTest.cpp index 74db80227f..8ad5b32a7e 100644 --- a/src/utils/test/TokenBucketTest.cpp +++ b/src/utils/test/TokenBucketTest.cpp @@ -18,7 +18,7 @@ #include // IWYU pragma: no_include -#include +// IWYU pragma: no_include // IWYU pragma: no_include #include #include diff --git a/src/utils/test/env.cpp b/src/utils/test/env.cpp index 49e389729f..6b82bed768 100644 --- a/src/utils/test/env.cpp +++ b/src/utils/test/env.cpp @@ -41,8 +41,8 @@ #include #include #include -#include #include +#include #include #include @@ -87,16 +87,6 @@ TEST(env_test, get_env) class env_file_test : public pegasus::encrypt_data_test_base { -public: - env_file_test() : pegasus::encrypt_data_test_base() - { - // The size of an actual encrypted file should plus kEncryptionHeaderkSize bytes if consider - // it as kNonSensitive. - if (FLAGS_encrypt_data_at_rest) { - _extra_size = dsn::utils::kEncryptionHeaderkSize; - } - } - uint64_t _extra_size = 0; }; INSTANTIATE_TEST_CASE_P(, env_file_test, ::testing::Values(false, true)); @@ -134,7 +124,7 @@ TEST_P(env_file_test, encrypt_file_2_files) ASSERT_EQ(kFileContentSize, wfile_size); ASSERT_TRUE(dsn::utils::filesystem::file_size( kEncryptedFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); - ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + ASSERT_EQ(kFileContentSize + extra_encrypted_file_size(), wfile_size); // Check file content. std::string data; s = rocksdb::ReadFileToString(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), @@ -174,7 +164,7 @@ TEST_P(env_file_test, encrypt_file_1_file) ASSERT_EQ(kFileContentSize, wfile_size); ASSERT_TRUE(dsn::utils::filesystem::file_size( kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); - ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + ASSERT_EQ(kFileContentSize + extra_encrypted_file_size(), wfile_size); // Check file content. std::string data; s = rocksdb::ReadFileToString( @@ -204,7 +194,7 @@ TEST_P(env_file_test, copy_file) ASSERT_EQ(kFileContentSize, wfile_size); ASSERT_TRUE(dsn::utils::filesystem::file_size( kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); - ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + ASSERT_EQ(kFileContentSize + extra_encrypted_file_size(), wfile_size); // Check copy_file(src_fname, dst_fname, total_size). // Loop twice to check overwrite. @@ -218,7 +208,7 @@ TEST_P(env_file_test, copy_file) ASSERT_EQ(kFileContentSize, wfile_size); ASSERT_TRUE(dsn::utils::filesystem::file_size( kCopyFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); - ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + ASSERT_EQ(kFileContentSize + extra_encrypted_file_size(), wfile_size); // Check file content. std::string data; s = rocksdb::ReadFileToString( @@ -249,7 +239,7 @@ TEST_P(env_file_test, copy_file_by_size) ASSERT_EQ(kFileContentSize, wfile_size); ASSERT_TRUE(dsn::utils::filesystem::file_size( kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); - ASSERT_EQ(kFileContentSize + _extra_size, wfile_size); + ASSERT_EQ(kFileContentSize + extra_encrypted_file_size(), wfile_size); // Check copy_file_by_size(src_fname, dst_fname, limit_size). struct test_case @@ -271,7 +261,7 @@ TEST_P(env_file_test, copy_file_by_size) ASSERT_EQ(test.expect_size, actual_size); ASSERT_TRUE(dsn::utils::filesystem::file_size( kCopyFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); - ASSERT_EQ(test.expect_size + _extra_size, wfile_size); + ASSERT_EQ(test.expect_size + extra_encrypted_file_size(), wfile_size); // Check file content. std::string data; s = rocksdb::ReadFileToString( diff --git a/src/utils/test/file_utils.cpp b/src/utils/test/file_utils.cpp index c4cadeb959..42066d2bd6 100644 --- a/src/utils/test/file_utils.cpp +++ b/src/utils/test/file_utils.cpp @@ -343,7 +343,7 @@ class file_utils : public pegasus::encrypt_data_test_base void file_utils_test_cleanup() {} }; -INSTANTIATE_TEST_CASE_P(, file_utils, ::testing::Values(false)); +INSTANTIATE_TEST_CASE_P(, file_utils, ::testing::Values(false, true)); TEST_P(file_utils, basic) { diff --git a/src/utils/test/utils.cpp b/src/utils/test/utils.cpp index c372b79ff7..756f7f5de0 100644 --- a/src/utils/test/utils.cpp +++ b/src/utils/test/utils.cpp @@ -34,7 +34,7 @@ */ // IWYU pragma: no_include -#include +// IWYU pragma: no_include // IWYU pragma: no_include #include #include