Add Bench CMD for engines and replicas

app/cmd/benchmark.go

@@ -0,0 +1,81 @@
+package cmd
+
+import (
+	"fmt"
+	replicaClient "github.com/longhorn/longhorn-engine/pkg/replica/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func BenchmarkCmd() cli.Command {
+	return cli.Command{
+		Name: "bench",
+		Flags: []cli.Flag{
+			cli.StringFlag{
+				Name:  "instance-type",
+				Usage: "Longhorn instance type, engine or replica",
+			},
+			cli.StringFlag{
+				Name:  "bench-type,b",
+				Usage: "The type can be <seq>/<rand>-<iops>/<bandwidth>/<latency>-<read>/<write>. For example, seq-bandwidth-write.",
+			},
+			cli.IntFlag{
+				Name:  "thread,t",
+				Value: 1,
+				Usage: "The concurrent thread count. For latency related benchmarks, this value will be forcibly set to 1.",
+			},
+			cli.Int64Flag{
+				Name:  "size",
+				Value: 1 << 30,
+				Usage: "The test size. " +
+					"If there are multi-thread enabled, this cmd will evenly split the test size for each thread. And each thread should have at least one block handled." +
+					"For bandwidth benchmarks, the block size is 1MB. Other benchmarks use 4K-block instead.",
+			},
+		},
+		Usage: "Benchmark engine or replica IO performance. When benchmarking engine with this cmd, the frontend iSCSI will be bypassed. When benchmarking replica, both the frontend iSCSI and the engine-replica connection will be bypassed.",
+		Action: func(c *cli.Context) {
+			if err := bench(c); err != nil {
+				logrus.WithError(err).Fatalf("Error running iops command")
+			}
+		},
+	}
+}
+
+func bench(c *cli.Context) error {
+	instanceType := c.String("instance-type")
+	benchType := c.String("bench-type")
+	threadCnt := c.Int("thread")
+	size := c.Int64("size")
+
+	var output string
+
+	switch instanceType {
+	case "engine":
+		controllerClient, err := getControllerClient(c)
+		if err != nil {
+			return err
+		}
+		defer controllerClient.Close()
+		output, err = controllerClient.VolumeBench(benchType, threadCnt, size)
+		if err != nil {
+			return err
+		}
+	case "replica":
+		url := c.GlobalString("url")
+		volumeName := c.GlobalString("volume-name")
+		repClient, err := replicaClient.NewReplicaClient(url, volumeName, "")
+		if err != nil {
+			return err
+		}
+		defer repClient.Close()
+		output, err = repClient.Bench(benchType, threadCnt, size)
+		if err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("invalid bench type %s", benchType)
+	}
+
+	fmt.Println(output)
+	return nil
+}

go.mod

@@ -2,6 +2,8 @@ module github.com/longhorn/longhorn-engine
 
 go 1.22.2
 
+replace github.com/longhorn/types => github.com/shuo-wu/types v0.0.0-20240429183325-93bff58b40b8
+
 require (
 	github.com/docker/go-units v0.5.0
 	github.com/gofrs/flock v0.8.1

go.sum

@@ -14,8 +14,6 @@ github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 h1:SjZ2GvvOononHOpK
 github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8/go.mod h1:uEyr4WpAH4hio6LFriaPkL938XnrvLpNPmQHBdrmbIE=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -33,8 +31,6 @@ github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0
 github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
 github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q=
 github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc=
-github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
-github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
@@ -70,42 +66,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/longhorn/backupstore v0.0.0-20240417071544-3bd377eeefeb h1:GkAG0P7QI32PYgjLBV6VvaEAM/Z9dKyPohccU1js+Uk=
-github.com/longhorn/backupstore v0.0.0-20240417071544-3bd377eeefeb/go.mod h1:4cbJWtlrD2cGTQxQLtdlPTYopiJiusXH7CpOBrn/s3k=
-github.com/longhorn/backupstore v0.0.0-20240426093637-ac3867f121c0 h1:sb9HK8uWApqrBbpjnxlNSmIB5f4oi3UqfeRHVniPcqo=
-github.com/longhorn/backupstore v0.0.0-20240426093637-ac3867f121c0/go.mod h1:NlOHCyQiGjn9TS1HUfQlBESttDPzi9x1Vs38geRV9SU=
 github.com/longhorn/backupstore v0.0.0-20240427164602-c8721e8ea31a h1:cwHDzWi0zkgg/SHtvEhWIYLvZHJvCp5twS8qBOUt/Nc=
 github.com/longhorn/backupstore v0.0.0-20240427164602-c8721e8ea31a/go.mod h1:NlOHCyQiGjn9TS1HUfQlBESttDPzi9x1Vs38geRV9SU=
-github.com/longhorn/go-common-libs v0.0.0-20240411093823-b8862efb8e03 h1:RN7mq4FrbHcAeemI5tDha9u4X+RSRrPugD1cY1FHdvo=
-github.com/longhorn/go-common-libs v0.0.0-20240411093823-b8862efb8e03/go.mod h1:7onp+E4hSg2DnB40dJU0Y7adrvykGg6jHxOb48imPGg=
-github.com/longhorn/go-common-libs v0.0.0-20240420123020-ed4ab0cfdbea h1:v0bayAUkxSDAwknvPFwnpOSrfmRZHdVoHrfDF617hZA=
-github.com/longhorn/go-common-libs v0.0.0-20240420123020-ed4ab0cfdbea/go.mod h1:3V1ZXTenmy7nCaQ555gtvp785fur1jK+seQ5gTUCEcI=
-github.com/longhorn/go-common-libs v0.0.0-20240426033718-77710d2242df h1:q2a/IL7zoPkYxIMsz46E6uWDamu9z4k+h/4u4pHsdqE=
-github.com/longhorn/go-common-libs v0.0.0-20240426033718-77710d2242df/go.mod h1:qppGSy9WsxH1C9T6yhSMvG53ynpasyKOH/n9e1jr8mw=
 github.com/longhorn/go-common-libs v0.0.0-20240427164621-70d1933bfa90 h1:uBVee6AulPsgSvsvaxdj4xIdgkIvjIqA0j7ZD+Hg/Mc=
 github.com/longhorn/go-common-libs v0.0.0-20240427164621-70d1933bfa90/go.mod h1:qppGSy9WsxH1C9T6yhSMvG53ynpasyKOH/n9e1jr8mw=
-github.com/longhorn/go-iscsi-helper v0.0.0-20240329081212-26b6406b3b2b h1:Cp8eBy4tKAaqOfycyLcCKIzhJd+K04/WfE9/OIALOaA=
-github.com/longhorn/go-iscsi-helper v0.0.0-20240329081212-26b6406b3b2b/go.mod h1:tynRAH865oFPQQw4rY4wgL7a/JEYQsAhy+DXglx51eI=
-github.com/longhorn/go-iscsi-helper v0.0.0-20240417063713-e20db15bb989 h1:4ddaq015o2NoMQeNqlKOZArN/aSr1IEbK9hExDCVFVM=
-github.com/longhorn/go-iscsi-helper v0.0.0-20240417063713-e20db15bb989/go.mod h1:4fs0E8kJuGqlYjXbpsQF0yEA6oOsgfDCxZAnLJuPaOU=
-github.com/longhorn/go-iscsi-helper v0.0.0-20240426065530-1ff1e567f43d h1:NeeDP9AgHbYkBfYRdWboguMMPk0I+8SZWxx1cAaGf0I=
-github.com/longhorn/go-iscsi-helper v0.0.0-20240426065530-1ff1e567f43d/go.mod h1:d9t3gtE+UPjescbCFluXd4xBc8OQT/JrC2cdkk2IXWQ=
 github.com/longhorn/go-iscsi-helper v0.0.0-20240427164656-e9439c0018ce h1:PxKniE9F6IZ2DMKfmxDsbqeAxQI1TZhnw7/HOBMs1Is=
 github.com/longhorn/go-iscsi-helper v0.0.0-20240427164656-e9439c0018ce/go.mod h1:d9t3gtE+UPjescbCFluXd4xBc8OQT/JrC2cdkk2IXWQ=
-github.com/longhorn/sparse-tools v0.0.0-20240228120902-ce8c4c2e71ca h1:dECamLpXIlL7GRmiruGseb5xO6hGAWyu2xYm9D46mb8=
-github.com/longhorn/sparse-tools v0.0.0-20240228120902-ce8c4c2e71ca/go.mod h1:pvlUkVwRGojXhcTkkzksOe4i7GVk59P2PbJjHIB2Yj0=
-github.com/longhorn/sparse-tools v0.0.0-20240424162924-2651ad40ad19 h1:/bSLCJxmmtq+alVHtvPl5eDXgMHqBOzOwcCCJxtRts0=
-github.com/longhorn/sparse-tools v0.0.0-20240424162924-2651ad40ad19/go.mod h1:pvlUkVwRGojXhcTkkzksOe4i7GVk59P2PbJjHIB2Yj0=
 github.com/longhorn/sparse-tools v0.0.0-20240427164751-a7b9f1b2c8a8 h1:lwtmZEomiv8uchwo9JIyoo+lK8J3cLCm7/qzpn6wmzo=
 github.com/longhorn/sparse-tools v0.0.0-20240427164751-a7b9f1b2c8a8/go.mod h1:pvlUkVwRGojXhcTkkzksOe4i7GVk59P2PbJjHIB2Yj0=
-github.com/longhorn/types v0.0.0-20240417064442-e7df610ea802 h1:evjIqn8Ta4toYsdcfvUBXOyV64ZEo72CYe+Vfd+3STQ=
-github.com/longhorn/types v0.0.0-20240417064442-e7df610ea802/go.mod h1:pqT+7B8T+nkyUZYe8tL3CwPDCHjkbe3mHUDY5ntJFyQ=
-github.com/longhorn/types v0.0.0-20240417112740-a0d8514936b8 h1:M9TQLN379VNN3JsuLuKHDRIy/11jYeIuxYLFuekjcPw=
-github.com/longhorn/types v0.0.0-20240417112740-a0d8514936b8/go.mod h1:pqT+7B8T+nkyUZYe8tL3CwPDCHjkbe3mHUDY5ntJFyQ=
-github.com/longhorn/types v0.0.0-20240424162824-4995e1e42438 h1:nSuJa62e6F9YsP06Js4jnKr0AsvEMGwBRLm9fXTlmns=
-github.com/longhorn/types v0.0.0-20240424162824-4995e1e42438/go.mod h1:pqT+7B8T+nkyUZYe8tL3CwPDCHjkbe3mHUDY5ntJFyQ=
-github.com/longhorn/types v0.0.0-20240427164854-38dbed8528d3 h1:7YDGJTwro/kCcMmnCWM1UIL4JgervR6MT++71PvcbGY=
-github.com/longhorn/types v0.0.0-20240427164854-38dbed8528d3/go.mod h1:pqT+7B8T+nkyUZYe8tL3CwPDCHjkbe3mHUDY5ntJFyQ=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -155,6 +123,8 @@ github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhV
 github.com/shirou/gopsutil/v3 v3.24.3/go.mod h1:JpND7O217xa72ewWz9zN2eIIkPWsDN/3pl0H8Qt0uwg=
 github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
+github.com/shuo-wu/types v0.0.0-20240429183325-93bff58b40b8 h1:74+dyJJaM7OLyv4I0E5MP3n1OIllJsW/m7z2XJaLd5I=
+github.com/shuo-wu/types v0.0.0-20240429183325-93bff58b40b8/go.mod h1:pqT+7B8T+nkyUZYe8tL3CwPDCHjkbe3mHUDY5ntJFyQ=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/slok/goresilience v0.2.0 h1:dagdIiWlhTm7BK/r/LRKz+zvw0SCNk+nHf7obdsbzxQ=
@@ -176,8 +146,6 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
-github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk=
-github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA=
 github.com/urfave/cli v1.22.15 h1:nuqt+pdC/KqswQKhETJjo7pvn/k4xMUxgW6liI7XpnM=
 github.com/urfave/cli v1.22.15/go.mod h1:wSan1hmo5zeyLGBjRJbzRTNk8gwoYa2B9n4q9dmRIc0=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -199,10 +167,6 @@ golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
 golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -273,12 +237,8 @@ gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
 gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
 k8s.io/apimachinery v0.27.1 h1:EGuZiLI95UQQcClhanryclaQE6xjg1Bts6/L3cD7zyc=
 k8s.io/apimachinery v0.27.1/go.mod h1:5ikh59fK3AJ287GUvpUsryoMFtH9zj/ARfWCo3AyXTM=
-k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
-k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
 k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
 k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
-k8s.io/mount-utils v0.29.3 h1:iEcqPP7Vv8UClH8nnMfovtmy/04fIloRW9JuSXykoZ0=
-k8s.io/mount-utils v0.29.3/go.mod h1:9IWJTMe8tG0MYMLEp60xK9GYVeCdA3g4LowmnVi+t9Y=
 k8s.io/mount-utils v0.30.0 h1:EceYTNYVabfpdtIAHC4KgMzoZkm1B8ovZ1J666mYZQI=
 k8s.io/mount-utils v0.30.0/go.mod h1:9sCVmwGLcV1MPvbZ+rToMDnl1QcGozy+jBPd0MsQLIo=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=

main.go

@@ -146,6 +146,7 @@ func longhornCli() {
 		cmd.FrontendCmd(),
 		cmd.SystemBackupCmd(),
 		cmd.ProfilerCmd(),
+		cmd.BenchmarkCmd(),
 		VersionCmd(),
 	}
 	a.CommandNotFound = cmdNotFound

pkg/controller/client/controller_client.go

@@ -261,6 +261,23 @@ func (c *ControllerClient) VolumeSnapshotMaxSizeSet(size int64) error {
 	return nil
 }
 
+func (c *ControllerClient) VolumeBench(benchType string, thread int, size int64) (string, error) {
+	controllerServiceClient := c.getControllerServiceClient()
+	ctx, cancel := context.WithTimeout(context.Background(), GRPCServiceTimeout)
+	defer cancel()
+
+	resp, err := controllerServiceClient.VolumeBench(ctx, &enginerpc.VolumeBenchRequest{
+		BenchType: benchType,
+		Thread:    int32(thread),
+		Size:      size,
+	})
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to bench %s engine for volume %v", benchType, c.VolumeName)
+	}
+
+	return resp.Output, nil
+}
+
 func (c *ControllerClient) ReplicaList() ([]*types.ControllerReplicaInfo, error) {
 	controllerServiceClient := c.getControllerServiceClient()
 	ctx, cancel := context.WithTimeout(context.Background(), GRPCServiceTimeout)

pkg/controller/control.go

@@ -1285,3 +1285,14 @@ func (c *Controller) GetLatestMetics() *enginerpc.Metrics {
 func getAverageLatency(totalLatency, iops uint64) uint64 {
 	return totalLatency / iops
 }
+
+func (c *Controller) Bench(benchType string, thread int, size int64) (output string, err error) {
+	if size%4096 != 0 {
+		return "", fmt.Errorf("failed to bench volume engine with size %v, because it is not multiple of volume block size 4096", size)
+	}
+	if size > c.size {
+		return "", fmt.Errorf("failed to bench volume engine with size %v, because it is greater than the engine size %v", size, c.size)
+	}
+
+	return util.Bench(benchType, thread, size, c.WriteAt, c.ReadAt)
+}

pkg/controller/rpc/server.go

@@ -202,6 +202,15 @@ func (cs *ControllerServer) VolumeSnapshotMaxSizeSet(ctx context.Context, req *e
 	return cs.getVolume(), nil
 }
 
+func (cs *ControllerServer) VolumeBench(ctx context.Context, req *enginerpc.VolumeBenchRequest) (*enginerpc.VolumeBenchResponse, error) {
+	output, err := cs.c.Bench(req.BenchType, int(req.Thread), req.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	return &enginerpc.VolumeBenchResponse{Output: output}, nil
+}
+
 func (cs *ControllerServer) ReplicaList(ctx context.Context, req *emptypb.Empty) (*enginerpc.ReplicaListReply, error) {
 	return &enginerpc.ReplicaListReply{
 		Replicas: cs.listControllerReplica(),

pkg/replica/client/client.go

@@ -790,3 +790,23 @@ func (c *ReplicaClient) SnapshotHashLockState() (bool, error) {
 
 	return resp.IsLocked, nil
 }
+
+func (c *ReplicaClient) Bench(benchType string, thread int, size int64) (string, error) {
+	replicaServiceClient, err := c.getReplicaServiceClient()
+	if err != nil {
+		return "", err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), GRPCServiceCommonTimeout)
+	defer cancel()
+
+	resp, err := replicaServiceClient.ReplicaBench(ctx, &enginerpc.ReplicaBenchRequest{
+		BenchType: benchType,
+		Thread:    int32(thread),
+		Size:      size,
+	})
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to bench replica %v", c.replicaServiceURL)
+	}
+
+	return resp.Output, nil
+}

pkg/replica/replica.go

@@ -1259,6 +1259,17 @@ func (r *Replica) Expand(size int64) (err error) {
 	return nil
 }
 
+func (r *Replica) Bench(benchType string, thread int, size int64) (output string, err error) {
+	if size%diskutil.VolumeSectorSize != 0 {
+		return "", fmt.Errorf("failed to bench volume replica with size %v, because it is not multiple of volume sector size %v", size, diskutil.VolumeSectorSize)
+	}
+	if size > r.info.Size {
+		return "", fmt.Errorf("failed to bench volume replica with size %v, because it is greater than the replica size %v", size, r.info.Size)
+	}
+
+	return util.Bench(benchType, thread, size, r.WriteAt, r.ReadAt)
+}
+
 func (r *Replica) WriteAt(buf []byte, offset int64) (int, error) {
 	if r.readOnly {
 		return 0, fmt.Errorf("cannot write on read-only replica")

pkg/replica/rpc/server.go

@@ -194,6 +194,15 @@ func (rs *ReplicaServer) ReplicaExpand(ctx context.Context, req *enginerpc.Repli
 	return &enginerpc.ReplicaExpandResponse{Replica: rs.getReplica()}, nil
 }
 
+func (rs *ReplicaServer) ReplicaBench(ctx context.Context, req *enginerpc.ReplicaBenchRequest) (*enginerpc.ReplicaBenchResponse, error) {
+	output, err := rs.s.Bench(req.BenchType, int(req.Thread), req.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	return &enginerpc.ReplicaBenchResponse{Output: output}, nil
+}
+
 func (rs *ReplicaServer) DiskRemove(ctx context.Context, req *enginerpc.DiskRemoveRequest) (*enginerpc.DiskRemoveResponse, error) {
 	if err := rs.s.RemoveDiffDisk(req.Name, req.Force); err != nil {
 		return nil, err

pkg/replica/server.go

@@ -232,6 +232,19 @@ func (s *Server) Expand(size int64) error {
 	return s.r.Expand(size)
 }
 
+func (s *Server) Bench(benchType string, thread int, size int64) (string, error) {
+	s.Lock()
+	if s.r == nil {
+		s.Unlock()
+		return "", nil
+	}
+	s.Unlock()
+
+	logrus.Infof("Replica server starts to bench %s with %v thread, test size %v", benchType, thread, size)
+
+	return s.r.Bench(benchType, thread, size)
+}
+
 func (s *Server) RemoveDiffDisk(name string, force bool) error {
 	s.Lock()
 	defer s.Unlock()