From beff382355cf19b76e625c6a0a53fe18b26920b8 Mon Sep 17 00:00:00 2001 From: Wine93 Date: Mon, 17 Jan 2022 18:21:30 +0800 Subject: [PATCH] improve(bs): create nbd device by automatic. --- cli/command/map.go | 2 +- configs/bs/cluster/topology.yaml | 15 ++++----------- internal/task/step/shell.go | 12 ++++++------ internal/task/task/bs/format.go | 18 +++++++++--------- internal/task/task/bs/map.go | 22 ++++++++++++++++++++++ pkg/module/module.go | 1 - pkg/module/shell.go | 8 ++++++++ 7 files changed, 50 insertions(+), 28 deletions(-) diff --git a/cli/command/map.go b/cli/command/map.go index af0eeda86..296439041 100644 --- a/cli/command/map.go +++ b/cli/command/map.go @@ -66,7 +66,7 @@ func NewMapCommand(curveadm *cli.CurveAdm) *cobra.Command { flags := cmd.Flags() flags.StringVarP(&options.filename, "conf", "c", "client.yaml", "Specify client configuration file") - flags.BoolVarP(&options.create, "create", "", false, "") + flags.BoolVarP(&options.create, "create", "", false, "Create volume iff not exist") flags.StringVarP(&options.size, "size", "", "10GB", "Specify volume size") return cmd diff --git a/configs/bs/cluster/topology.yaml b/configs/bs/cluster/topology.yaml index 59c1e2935..02ada65b9 100644 --- a/configs/bs/cluster/topology.yaml +++ b/configs/bs/cluster/topology.yaml @@ -6,6 +6,10 @@ global: container_image: opencurvedocker/curvebs:v1.2 log_dir: /home/${user}/logs/${service_role}${service_replica_sequence} data_dir: /home/${user}/data/${service_role}${service_replica_sequence} + s3.nos_address: <> + s3.snapshot_bucket_name: <> + s3.ak: <> + s3.sk: <> variable: machine1: 10.0.1.1 machine2: 10.0.1.2 @@ -35,22 +39,15 @@ chunkserver_services: config: listen.ip: ${service_host} listen.port: 82${format_replica_sequence} # 8200,8201,8202 - global.enable_external_server: true data_dir: /data/chunkserver${service_replica_sequence} # /data/chunkserver0, /data/chunksever1 copysets: 100 deploy: - host: ${machine1} replica: 3 - config: - listen.external_ip: 10.0.2.0 - host: ${machine2} replica: 3 - config: - listen.external_ip: 10.0.2.1 - host: ${machine3} replica: 3 - config: - listen.external_ip: 10.0.2.2 snapshotclone_services: config: @@ -58,10 +55,6 @@ snapshotclone_services: listen.port: 5555 listen.dummy_port: 8081 listen.proxy_port: 8080 - s3.nos_address: <> - s3.snapshot_bucket_name: <> - s3.ak: <> - s3.sk: <> deploy: - host: ${machine1} - host: ${machine2} diff --git a/internal/task/step/shell.go b/internal/task/step/shell.go index ba0632bc5..a6b49ad6e 100644 --- a/internal/task/step/shell.go +++ b/internal/task/step/shell.go @@ -103,7 +103,7 @@ func (s *CreateDirectory) Execute(ctx *context.Context) error { _, err := cmd.Execute(module.ExecOption{ ExecWithSudo: s.ExecWithSudo, ExecInLocal: s.ExecInLocal, - ExexSudoAlias: s.ExecSudoAlias, + ExecSudoAlias: s.ExecSudoAlias, }) if err != nil { return err @@ -124,7 +124,7 @@ func (s *RemoveFile) Execute(ctx *context.Context) error { out, err := cmd.Execute(module.ExecOption{ ExecWithSudo: s.ExecWithSudo, ExecInLocal: s.ExecInLocal, - ExexSudoAlias: s.ExecSudoAlias, + ExecSudoAlias: s.ExecSudoAlias, }) // device busy: maybe directory is mount point out = strings.TrimSuffix(out, "\n") @@ -143,7 +143,7 @@ func (s *CreateFilesystem) Execute(ctx *context.Context) error { _, err := cmd.Execute(module.ExecOption{ ExecWithSudo: s.ExecWithSudo, ExecInLocal: s.ExecInLocal, - ExexSudoAlias: s.ExecSudoAlias, + ExecSudoAlias: s.ExecSudoAlias, }) return err } @@ -153,7 +153,7 @@ func (s *MountFilesystem) Execute(ctx *context.Context) error { _, err := cmd.Execute(module.ExecOption{ ExecWithSudo: s.ExecWithSudo, ExecInLocal: s.ExecInLocal, - ExexSudoAlias: s.ExecSudoAlias, + ExecSudoAlias: s.ExecSudoAlias, }) return err } @@ -168,7 +168,7 @@ func (s *UmountFilesystem) Execute(ctx *context.Context) error { out, err := cmd.Execute(module.ExecOption{ ExecWithSudo: s.ExecWithSudo, ExecInLocal: s.ExecInLocal, - ExexSudoAlias: s.ExecSudoAlias, + ExecSudoAlias: s.ExecSudoAlias, }) out = strings.TrimSuffix(out, "\n") @@ -191,7 +191,7 @@ func (s *ShowDiskFree) Execute(ctx *context.Context) error { out, err := cmd.Execute(module.ExecOption{ ExecWithSudo: s.ExecWithSudo, ExecInLocal: s.ExecInLocal, - ExexSudoAlias: s.ExecSudoAlias, + ExecSudoAlias: s.ExecSudoAlias, }) if err != nil { return err diff --git a/internal/task/task/bs/format.go b/internal/task/task/bs/format.go index 608892941..dfb8e846e 100644 --- a/internal/task/task/bs/format.go +++ b/internal/task/task/bs/format.go @@ -82,8 +82,8 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *format.FormatConfig) Quiet: true, Filter: fmt.Sprintf("name=%s", containerName), Out: &oldContainerId, - ExecInLocal: false, ExecWithSudo: true, + ExecInLocal: false, ExecSudoAlias: curveadm.SudoAlias(), }) t.AddStep(&step2SkipFormat{ @@ -95,34 +95,34 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *format.FormatConfig) Directorys: []string{device}, IgnoreUmounted: true, IgnoreNotFound: true, - ExecInLocal: false, ExecWithSudo: true, + ExecInLocal: false, ExecSudoAlias: curveadm.SudoAlias(), }) t.AddStep(&step.CreateDirectory{ Paths: []string{mountPoint}, - ExecInLocal: false, ExecWithSudo: true, + ExecInLocal: false, ExecSudoAlias: curveadm.SudoAlias(), }) t.AddStep(&step.CreateFilesystem{ // mkfs.ext4 MOUNT_POINT Device: device, - ExecInLocal: false, ExecWithSudo: true, + ExecInLocal: false, ExecSudoAlias: curveadm.SudoAlias(), }) t.AddStep(&step.MountFilesystem{ Source: device, Directory: mountPoint, - ExecInLocal: false, ExecWithSudo: true, + ExecInLocal: false, ExecSudoAlias: curveadm.SudoAlias(), }) // 3: run container to format chunkfile pool t.AddStep(&step.PullImage{ Image: fc.GetContainerIamge(), - ExecInLocal: false, ExecWithSudo: true, + ExecInLocal: false, ExecSudoAlias: curveadm.SudoAlias(), }) t.AddStep(&step.CreateContainer{ @@ -133,22 +133,22 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *format.FormatConfig) Remove: true, Volumes: []step.Volume{{HostPath: mountPoint, ContainerPath: chunkfilePoolRootDir}}, Out: &containerId, - ExecInLocal: false, ExecWithSudo: true, + ExecInLocal: false, ExecSudoAlias: curveadm.SudoAlias(), }) t.AddStep(&step.InstallFile{ ContainerId: &containerId, ContainerDestPath: formatScriptPath, Content: &formatScript, - ExecInLocal: false, ExecWithSudo: true, + ExecInLocal: false, ExecSudoAlias: curveadm.SudoAlias(), }) t.AddStep(&step.StartContainer{ ContainerId: &containerId, - ExecInLocal: false, ExecWithSudo: true, + ExecInLocal: false, ExecSudoAlias: curveadm.SudoAlias(), }) return t, nil diff --git a/internal/task/task/bs/map.go b/internal/task/task/bs/map.go index 2fd61764c..d5ad51680 100644 --- a/internal/task/task/bs/map.go +++ b/internal/task/task/bs/map.go @@ -32,6 +32,7 @@ import ( "github.com/opencurve/curveadm/internal/task/step" "github.com/opencurve/curveadm/internal/task/task" "github.com/opencurve/curveadm/internal/utils" + "github.com/opencurve/curveadm/pkg/module" ) const ( @@ -52,6 +53,12 @@ type ( user string volume string } + + step2CreateNBDDevice struct { + execWithSudo bool + execInLocal bool + execSudoAlias string + } ) func (s *step2CheckNEBDClient) Execute(ctx *context.Context) error { @@ -62,6 +69,16 @@ func (s *step2CheckNEBDClient) Execute(ctx *context.Context) error { return nil } +func (s *step2CreateNBDDevice) Execute(ctx *context.Context) error { + cmd := ctx.Module().Shell().ModProbe("nbd", "nbds_max=64") + _, err := cmd.Execute(module.ExecOption{ + ExecWithSudo: s.execWithSudo, + ExecInLocal: s.execInLocal, + ExecSudoAlias: s.execSudoAlias, + }) + return err +} + func formatImage(user, volume string) string { return fmt.Sprintf(FORMAT_IMAGE, volume, user) } @@ -97,6 +114,11 @@ func NewMapTask(curveadm *cli.CurveAdm, cc *client.ClientConfig) (*task.Task, er user: user, volume: volume, }) + t.AddStep(&step2CreateNBDDevice{ + execWithSudo: true, + execInLocal: false, + execSudoAlias: curveadm.SudoAlias(), + }) t.AddStep(&step.PullImage{ Image: cc.GetContainerImage(), ExecWithSudo: true, diff --git a/pkg/module/module.go b/pkg/module/module.go index 8cebf6840..874ad5db0 100644 --- a/pkg/module/module.go +++ b/pkg/module/module.go @@ -37,7 +37,6 @@ type ExecOption struct { ExecWithSudo bool ExecInLocal bool ExecSudoAlias string - ExexSudoAlias string } type Module struct { diff --git a/pkg/module/shell.go b/pkg/module/shell.go index 81cfc72c1..456c03cae 100644 --- a/pkg/module/shell.go +++ b/pkg/module/shell.go @@ -45,6 +45,7 @@ const ( TEMPLATE_MOUNT = "mount {{.options}} {{.source}} {{.directory}}" TEMPLATE_UMOUNT = "umount {{.options}} {{.directory}}" TEMPLATE_DISKFREE = "df {{.options}} {{.files}}" + TEMPLATE_MODPROBE = "modprobe {{.options}} {{.modulename}} {{.arguments}}" TEMPLATE_COMMAND = "bash -c '{{.command}}'" TEMPLATE_EXEC_SCEIPT = "{{.scriptPath}} {{.arguments}}" ) @@ -127,6 +128,13 @@ func (s *Shell) DiskFree(file ...string) *Shell { return s } +func (s *Shell) ModProbe(modulename string, args ...string) *Shell { + s.tmpl = template.Must(template.New("ModProbe").Parse(TEMPLATE_MODPROBE)) + s.data["modulename"] = modulename + s.data["arguments"] = strings.Join(args, " ") + return s +} + func (s *Shell) Command(command string) *Shell { s.tmpl = template.Must(template.New("Command").Parse(TEMPLATE_COMMAND)) s.data["command"] = command