Merge branch 'master' into dev

Signed-off-by: saica.go <zephyird@gmail.com>

.bazelrc

@@ -7,4 +7,20 @@ build:gcc7-later --cxxopt -faligned-new
 build --incompatible_blacklisted_protos_requires_proto_info=false
 build --copt=-fdiagnostics-color=always
 
+build:sanitize-common --strip=never
+build:sanitize-common --copt -O1
+build:sanitize-common --copt -g
+build:sanitize-common --copt -fno-omit-frame-pointer
+
+build:asan --config=sanitize-common
+build:asan --copt -fsanitize=address
+build:asan --copt -DADDRESS_SANITIZER
+build:asan --linkopt -fsanitize=address
+
+build:asan --config=sanitize-common
+build:msan --copt -fsanitize=memory
+build:msan --copt -fsanitize=undefined
+build:msan --linkopt -fsanitize=address
+build:msan --linkopt -fsanitize=undefined
+
 run --copt=-fdiagnostics-color=always

.clang-format

@@ -4,6 +4,7 @@ BasedOnStyle:  Google
 AccessModifierOffset: -3
 DerivePointerAlignment: false
 IndentWidth:     4
+AlignAfterOpenBracket: DontAlign
 ---
 Language: Proto
 BasedOnStyle:  Google

.github/workflows/clang-format.yaml → .github/workflows/clang-format.yml

@@ -9,12 +9,12 @@ jobs:
     name: Clang Formatter
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           # fetch everything to be able to compare with any ref
           fetch-depth: 0
 
-      - name: Check 
+      - name: Check
         env:
           LANG: "C.UTF-8"
           LC_ALL: "C.UTF-8"

.github/workflows/cppcheck.yml

@@ -0,0 +1,24 @@
+name: Static Checker
+
+on: pull_request
+
+jobs:
+  cppcheck:
+    name: CppCheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: chmorgan/cppcheck-action@v1.4
+        with:
+          enable: all
+          std: c++11
+          inconclusive: disable
+          output_file: ./cppcheck_report.txt
+          other_options: "-j4 --suppressions-list=util/cppcheck/cppcheck.suppressions --error-exitcode=1 -itest -icurvefs/test -inebd/test -inbd/test -icurvefs_python -icurvesnapshot_python -ithirdparties"
+
+      - name: Show cppcheck report
+        if: failure()
+        run: |
+          cat ./cppcheck_report.txt
+          exit 1

.gitignore

@@ -82,6 +82,9 @@ runlog/
 !curve-snapshotcloneserver-nginx/app/lib
 !nebd/nebd-package/usr/bin
 
+# build output
+build
+
 # mac
 .DS_Store
 
@@ -166,3 +169,11 @@ test/integration/snapshotcloneserver/config/*.conf
 
 *.deb
 *.whl
+
+*.class
+curvefs/sdk/java/target/
+curvefs/sdk/java/native/build
+curvefs/sdk/libcurvefs/examples/bin
+curvefs/sdk/output/
+hadoop-test/
+nnbench-test/

.obm.cfg

@@ -1,2 +1,2 @@
-container_name: curve-build-playground-master
-container_image: opencurvedocker/curve-base:build-debian9
+container_name: curve-build-playground.master
+container_image: opencurvedocker/curve-base:build-debian11

Makefile

@@ -1,6 +1,6 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build dep install image playground check test docker
+.PHONY: list build dep ci-list ci-build ci-dep install image playground check test docker format
 
 stor?=""
 prefix?= "$(PWD)/projects"
@@ -13,6 +13,14 @@ os?= "debian11"
 ci?=0
 commit_id="HEAD^"
 define help_msg
+## build curvebs
+	make build stor=bs dep=1
+	make dep stor=bs && make build stor=bs
+
+## build curvefs
+	make build stor=fs dep=1
+    make dep stor=fs && make build stor=fs
+
 ## list
 Usage:
     make list stor=bs/fs
@@ -21,20 +29,25 @@ Examples:
 
 ## build
 Usage:
-    make build stor=bs/fs only=TARGET dep=0/1 release=0/1 os=OS
+    make build stor=bs/fs only=TARGET1,...,TARGETx dep=0/1 release=0/1 os=OS
 Examples:
     make build stor=bs only=//src/chunkserver:chunkserver
-    make build stor=bs only=src/* dep=0
-    make build stor=fs only=test/* os=debian9
-    make build stor=fs release=1
+    make build stor=bs only=src/*,test/* dep=0
+    make build stor=fs only=test/* os=debian11
 
+    make build stor=fs release=1
+Note:
+    Extra build options can be specified using BUILD_OPTS environment variable, which will be passed to bazel build command.
 
 ## dep
+## configure dependency(before build)
 Usage:
     make dep stor=bs/fs
 Examples:
     make dep stor=bs
 
+## ci-list/build/dep
+## use the same way above, but in the container
 
 ## install
 Usage:
@@ -49,7 +62,21 @@ Examples:
 Usage:
     make image stor=bs/fs tag=TAG os=OS
 Examples:
-    make image stor=bs tag=opencurvedocker/curvebs:v1.2 os=debian9
+    make image stor=bs tag=opencurvedocker/curvebs:v1.2 os=debian11
+
+
+## package
+Usage:
+    make <tar|deb> release=0/1 dep=0/1 os=OS
+Examples:
+    make deb
+    make tar release=1 dep=1 os=debian11
+
+
+## playground
+## create/run a container, changes outside will be mapped into the container
+Usage/Example:
+	make playground
 endef
 export help_msg
 
@@ -65,8 +92,11 @@ build:
 dep:
 	@bash util/build.sh --stor=$(stor) --only="" --dep=1
 
+ci-list:
+	@bash util/build_in_image.sh --stor=$(stor) --list
+
 ci-build:
-	@bash util/build_in_image.sh --stor=$(stor) --only=$(only) --dep=$(dep) --release=$(release) --ci=$(ci) --os=$(os)
+	@bash util/build_in_image.sh --stor=$(stor) --only=$(only) --dep=$(dep) --release=$(release) --ci=$(ci) --os=$(os) --sanitizer=$(sanitizer)
 
 ci-dep:
 	@bash util/build_in_image.sh --stor=$(stor) --only="" --dep=1
@@ -77,8 +107,11 @@ install:
 image:
 	@bash util/image.sh $(stor) $(tag) $(os)
 
+tar deb:
+	@RELEASE=$(release) DEP=$(dep) OS=$(os) bash util/package.sh $@
+
 playground:
-	@bash util/playground.sh
+	@bash util/playground.sh --version=$(version)
 
 check:
 	@bash util/check.sh $(stor)
@@ -88,5 +121,12 @@ test:
 
 docker:
 	@bash util/docker.sh --os=$(os) --ci=$(ci)
+
 format:
 	@bash util/format.sh $(commit_id)
+
+init-hadoop:
+	@bash util/init-hadoop.sh
+
+sdk:
+	@bash util/sdk.sh

WORKSPACE

@@ -142,7 +142,7 @@ git_repository(
     patches = [
         "//:thirdparties/brpc/brpc.patch",
         "//:thirdparties/brpc/fix-gcc11.patch",
-        "//:thirdparties/brpc/0001-bvar-warning-on-conflict-bvar-name.patch", 
+        "//:thirdparties/brpc/0001-bvar-warning-on-conflict-bvar-name.patch",
     ],
     patch_args = ["-p1"],
 )
@@ -192,7 +192,6 @@ new_local_repository(
     path = "thirdparties/memcache/libmemcached-1.1.2",
 )
 
-
 http_archive(
     name = "aws",
     urls = ["https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz"],
@@ -259,6 +258,21 @@ http_archive(
   build_file = "//:thirdparties/spdlog.BUILD",
 )
 
+# incbin
+new_git_repository(
+    name = "incbin",
+    remote = "https://github.com/graphitemaster/incbin.git",
+    commit = "6e576cae5ab5810f25e2631f2e0b80cbe7dc8cbf",
+    build_file = "//:thirdparties/incbin.BUILD",
+)
+
+# config
+new_local_repository(
+    name = "config",
+    build_file = "//:thirdparties/config.BUILD",
+    path = "thirdparties/config",
+)
+
 # Bazel platform rules.
 http_archive(
     name = "platforms",
@@ -274,6 +288,13 @@ new_local_repository(
     path = "thirdparties/rocksdb",
 )
 
+# jni
+new_local_repository(
+    name = "jni",
+    build_file = "//:thirdparties/jni.BUILD",
+    path = "thirdparties",
+)
+
 # Hedron's Compile Commands Extractor for Bazel
 # https://github.com/hedronvision/bazel-compile-commands-extractor
 http_archive(

conf/chunkserver.conf

@@ -194,7 +194,7 @@ rconcurrentapply.queuedepth=1
 # 是否开启从chunkfilepool获取chunk，一般是true
 chunkfilepool.enable_get_chunk_from_pool=true
 # chunkfilepool目录
-chunkfilepool.chunk_file_pool_dir=./0/  # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__
+chunkfilepool.chunk_file_pool_dir=./0/chunks  # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__
 # chunkfilepool meta文件路径
 chunkfilepool.meta_path=./chunkfilepool.meta  # __CURVEADM_TEMPLATE__ ${prefix}/data/chunkfilepool.meta __CURVEADM_TEMPLATE__
 # chunkfilepool meta文件大小
@@ -207,6 +207,14 @@ chunkfilepool.clean.enable=true
 chunkfilepool.clean.bytes_per_write=4096
 # The throttle iops for cleaning chunk (4KB/IO)
 chunkfilepool.clean.throttle_iops=500
+# Whether allocate filePool by percent of disk size.
+chunkfilepool.allocated_by_percent=true
+# Preallocate storage percent of total disk
+chunkfilepool.allocate_percent=80
+# Preallocate storage size of chunkfilepool (None/KB/MB/GB/TB)
+chunkfilepool.chunk_file_pool_size=1GB
+# The thread num for format chunks
+chunkfilepool.thread_num=1
 
 #
 # WAL file pool
@@ -229,6 +237,14 @@ walfilepool.metapage_size=4096
 walfilepool.meta_file_size=4096
 # WAL filepool get chunk最大重试次数
 walfilepool.retry_times=5
+# Whether allocate filePool by percent of disk size.
+walfilepool.allocated_by_percent=true
+# Preallocate storage percent of total disk
+walfilepool.allocate_percent=90
+# Preallocate storage size size of walfilepool (None/KB/MB/GB/TB)
+walfilepool.wal_file_pool_size=0
+# The thread num for format chunks
+walfilepool.thread_num=1
 
 #
 # trash settings

conf/chunkserver.conf.example

@@ -186,7 +186,7 @@ rconcurrentapply.queuedepth=1
 # 是否开启从chunkfilepool获取chunk，一般是true
 chunkfilepool.enable_get_chunk_from_pool=true
 # chunkfilepool目录
-chunkfilepool.chunk_file_pool_dir=./0/
+chunkfilepool.chunk_file_pool_dir=./0/chunks
 # chunkfilepool meta文件路径
 #chunkfilepool.meta_path=./chunkfilepool.meta
 # chunkfilepool meta文件大小
@@ -199,6 +199,14 @@ chunkfilepool.clean.enable=true
 chunkfilepool.clean.bytes_per_write=4096
 # The throttle iops for cleaning chunk (4KB/IO)
 chunkfilepool.clean.throttle_iops=500
+# Whether allocate filePool by percent of disk size.
+chunkfilepool.allocated_by_percent=true
+# Preallocate storage percent of total disk
+chunkfilepool.allocate_percent=80
+# Preallocate storage size of chunkfilepool (None/KB/MB/GB/TB)
+chunkfilepool.chunk_file_pool_size=1GB
+# The thread num for format chunks
+chunkfilepool.thread_num=1
 
 #
 # WAL file pool
@@ -221,6 +229,14 @@ walfilepool.metapage_size=4096
 walfilepool.meta_file_size=4096
 # WAL filepool get chunk最大重试次数
 walfilepool.retry_times=5
+# Whether allocate filePool by percent of disk size.
+walfilepool.allocated_by_percent=true
+# Preallocate storage percent of total disk
+walfilepool.allocate_percent=10
+# Preallocate storage size size of walfilepool (None/KB/MB/GB/TB)
+walfilepool.wal_file_pool_size=0
+# The thread num for format chunks
+walfilepool.thread_num=1
 
 #
 # trash settings

curve-ansible/README.md

@@ -30,8 +30,8 @@ chunkserver_num=13
 ### 3、包管理
 curve-ansible同时提供了debian包和tar包的方式安装curve软件包。默认使用tar包方式安装，如果想用debian包的方式，需要在inventory或者命令行中指定install_with_deb=true。（debian的方式目前不支持格式化指定盘，只支持格式化全部ATA盘）
 
-- debian包的方式下，需要使用mk-deb.sh打debian包，并将debian包上传到debian源中，使其能够用apt-get命令安装。
-- tar包的方式下，tar包可以是来自curve github release，也可以是自己执行mk-tar.sh脚本获取到的。
+- debian包的方式下，需要使用 make deb 打debian包，并将debian包上传到debian源中，使其能够用apt-get命令安装。
+- tar包的方式下，tar包可以是来自curve github release，也可以是自己执行 make tar 获取到的。
 
 ## 三、使用命令
 ### 1、集群部署