Skip to content

Commit

Permalink
improve(bs): create nbd device by automatic.
Browse files Browse the repository at this point in the history
  • Loading branch information
@Wine93
Wine93 committed Jan 17, 2022
1 parent f3fea80 commit beff382
Show file tree
Hide file tree
Showing 7 changed files with 50 additions and 28 deletions.
2 changes: 1 addition & 1 deletion cli/command/map.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func NewMapCommand(curveadm *cli.CurveAdm) *cobra.Command {

flags := cmd.Flags()
flags.StringVarP(&options.filename, "conf", "c", "client.yaml", "Specify client configuration file")
flags.BoolVarP(&options.create, "create", "", false, "")
flags.BoolVarP(&options.create, "create", "", false, "Create volume iff not exist")
flags.StringVarP(&options.size, "size", "", "10GB", "Specify volume size")

return cmd
Expand Down
15 changes: 4 additions & 11 deletions configs/bs/cluster/topology.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ global:
container_image: opencurvedocker/curvebs:v1.2
log_dir: /home/${user}/logs/${service_role}${service_replica_sequence}
data_dir: /home/${user}/data/${service_role}${service_replica_sequence}
s3.nos_address: <>
s3.snapshot_bucket_name: <>
s3.ak: <>
s3.sk: <>
variable:
machine1: 10.0.1.1
machine2: 10.0.1.2
Expand Down Expand Up @@ -35,33 +39,22 @@ chunkserver_services:
config:
listen.ip: ${service_host}
listen.port: 82${format_replica_sequence} # 8200,8201,8202
global.enable_external_server: true
data_dir: /data/chunkserver${service_replica_sequence} # /data/chunkserver0, /data/chunksever1
copysets: 100
deploy:
- host: ${machine1}
replica: 3
config:
listen.external_ip: 10.0.2.0
- host: ${machine2}
replica: 3
config:
listen.external_ip: 10.0.2.1
- host: ${machine3}
replica: 3
config:
listen.external_ip: 10.0.2.2

snapshotclone_services:
config:
listen.ip: ${service_host}
listen.port: 5555
listen.dummy_port: 8081
listen.proxy_port: 8080
s3.nos_address: <>
s3.snapshot_bucket_name: <>
s3.ak: <>
s3.sk: <>
deploy:
- host: ${machine1}
- host: ${machine2}
Expand Down
12 changes: 6 additions & 6 deletions internal/task/step/shell.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func (s *CreateDirectory) Execute(ctx *context.Context) error {
_, err := cmd.Execute(module.ExecOption{
ExecWithSudo: s.ExecWithSudo,
ExecInLocal: s.ExecInLocal,
ExexSudoAlias: s.ExecSudoAlias,
ExecSudoAlias: s.ExecSudoAlias,
})
if err != nil {
return err
Expand All @@ -124,7 +124,7 @@ func (s *RemoveFile) Execute(ctx *context.Context) error {
out, err := cmd.Execute(module.ExecOption{
ExecWithSudo: s.ExecWithSudo,
ExecInLocal: s.ExecInLocal,
ExexSudoAlias: s.ExecSudoAlias,
ExecSudoAlias: s.ExecSudoAlias,
})
// device busy: maybe directory is mount point
out = strings.TrimSuffix(out, "\n")
Expand All @@ -143,7 +143,7 @@ func (s *CreateFilesystem) Execute(ctx *context.Context) error {
_, err := cmd.Execute(module.ExecOption{
ExecWithSudo: s.ExecWithSudo,
ExecInLocal: s.ExecInLocal,
ExexSudoAlias: s.ExecSudoAlias,
ExecSudoAlias: s.ExecSudoAlias,
})
return err
}
Expand All @@ -153,7 +153,7 @@ func (s *MountFilesystem) Execute(ctx *context.Context) error {
_, err := cmd.Execute(module.ExecOption{
ExecWithSudo: s.ExecWithSudo,
ExecInLocal: s.ExecInLocal,
ExexSudoAlias: s.ExecSudoAlias,
ExecSudoAlias: s.ExecSudoAlias,
})
return err
}
Expand All @@ -168,7 +168,7 @@ func (s *UmountFilesystem) Execute(ctx *context.Context) error {
out, err := cmd.Execute(module.ExecOption{
ExecWithSudo: s.ExecWithSudo,
ExecInLocal: s.ExecInLocal,
ExexSudoAlias: s.ExecSudoAlias,
ExecSudoAlias: s.ExecSudoAlias,
})

out = strings.TrimSuffix(out, "\n")
Expand All @@ -191,7 +191,7 @@ func (s *ShowDiskFree) Execute(ctx *context.Context) error {
out, err := cmd.Execute(module.ExecOption{
ExecWithSudo: s.ExecWithSudo,
ExecInLocal: s.ExecInLocal,
ExexSudoAlias: s.ExecSudoAlias,
ExecSudoAlias: s.ExecSudoAlias,
})
if err != nil {
return err
Expand Down
18 changes: 9 additions & 9 deletions internal/task/task/bs/format.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,8 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *format.FormatConfig)
Quiet: true,
Filter: fmt.Sprintf("name=%s", containerName),
Out: &oldContainerId,
ExecInLocal: false,
ExecWithSudo: true,
ExecInLocal: false,
ExecSudoAlias: curveadm.SudoAlias(),
})
t.AddStep(&step2SkipFormat{
Expand All @@ -95,34 +95,34 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *format.FormatConfig)
Directorys: []string{device},
IgnoreUmounted: true,
IgnoreNotFound: true,
ExecInLocal: false,
ExecWithSudo: true,
ExecInLocal: false,
ExecSudoAlias: curveadm.SudoAlias(),
})
t.AddStep(&step.CreateDirectory{
Paths: []string{mountPoint},
ExecInLocal: false,
ExecWithSudo: true,
ExecInLocal: false,
ExecSudoAlias: curveadm.SudoAlias(),
})
t.AddStep(&step.CreateFilesystem{ // mkfs.ext4 MOUNT_POINT
Device: device,
ExecInLocal: false,
ExecWithSudo: true,
ExecInLocal: false,
ExecSudoAlias: curveadm.SudoAlias(),
})
t.AddStep(&step.MountFilesystem{
Source: device,
Directory: mountPoint,
ExecInLocal: false,
ExecWithSudo: true,
ExecInLocal: false,
ExecSudoAlias: curveadm.SudoAlias(),
})
// 3: run container to format chunkfile pool
t.AddStep(&step.PullImage{
Image: fc.GetContainerIamge(),
ExecInLocal: false,
ExecWithSudo: true,
ExecInLocal: false,
ExecSudoAlias: curveadm.SudoAlias(),
})
t.AddStep(&step.CreateContainer{
Expand All @@ -133,22 +133,22 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *format.FormatConfig)
Remove: true,
Volumes: []step.Volume{{HostPath: mountPoint, ContainerPath: chunkfilePoolRootDir}},
Out: &containerId,
ExecInLocal: false,
ExecWithSudo: true,
ExecInLocal: false,
ExecSudoAlias: curveadm.SudoAlias(),
})
t.AddStep(&step.InstallFile{
ContainerId: &containerId,
ContainerDestPath: formatScriptPath,
Content: &formatScript,
ExecInLocal: false,
ExecWithSudo: true,
ExecInLocal: false,
ExecSudoAlias: curveadm.SudoAlias(),
})
t.AddStep(&step.StartContainer{
ContainerId: &containerId,
ExecInLocal: false,
ExecWithSudo: true,
ExecInLocal: false,
ExecSudoAlias: curveadm.SudoAlias(),
})
return t, nil
Expand Down
22 changes: 22 additions & 0 deletions internal/task/task/bs/map.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/opencurve/curveadm/internal/task/step"
"github.com/opencurve/curveadm/internal/task/task"
"github.com/opencurve/curveadm/internal/utils"
"github.com/opencurve/curveadm/pkg/module"
)

const (
Expand All @@ -52,6 +53,12 @@ type (
user string
volume string
}

step2CreateNBDDevice struct {
execWithSudo bool
execInLocal bool
execSudoAlias string
}
)

func (s *step2CheckNEBDClient) Execute(ctx *context.Context) error {
Expand All @@ -62,6 +69,16 @@ func (s *step2CheckNEBDClient) Execute(ctx *context.Context) error {
return nil
}

func (s *step2CreateNBDDevice) Execute(ctx *context.Context) error {
cmd := ctx.Module().Shell().ModProbe("nbd", "nbds_max=64")
_, err := cmd.Execute(module.ExecOption{
ExecWithSudo: s.execWithSudo,
ExecInLocal: s.execInLocal,
ExecSudoAlias: s.execSudoAlias,
})
return err
}

func formatImage(user, volume string) string {
return fmt.Sprintf(FORMAT_IMAGE, volume, user)
}
Expand Down Expand Up @@ -97,6 +114,11 @@ func NewMapTask(curveadm *cli.CurveAdm, cc *client.ClientConfig) (*task.Task, er
user: user,
volume: volume,
})
t.AddStep(&step2CreateNBDDevice{
execWithSudo: true,
execInLocal: false,
execSudoAlias: curveadm.SudoAlias(),
})
t.AddStep(&step.PullImage{
Image: cc.GetContainerImage(),
ExecWithSudo: true,
Expand Down
1 change: 0 additions & 1 deletion pkg/module/module.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ type ExecOption struct {
ExecWithSudo bool
ExecInLocal bool
ExecSudoAlias string
ExexSudoAlias string
}

type Module struct {
Expand Down
8 changes: 8 additions & 0 deletions pkg/module/shell.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ const (
TEMPLATE_MOUNT = "mount {{.options}} {{.source}} {{.directory}}"
TEMPLATE_UMOUNT = "umount {{.options}} {{.directory}}"
TEMPLATE_DISKFREE = "df {{.options}} {{.files}}"
TEMPLATE_MODPROBE = "modprobe {{.options}} {{.modulename}} {{.arguments}}"
TEMPLATE_COMMAND = "bash -c '{{.command}}'"
TEMPLATE_EXEC_SCEIPT = "{{.scriptPath}} {{.arguments}}"
)
Expand Down Expand Up @@ -127,6 +128,13 @@ func (s *Shell) DiskFree(file ...string) *Shell {
return s
}

func (s *Shell) ModProbe(modulename string, args ...string) *Shell {
s.tmpl = template.Must(template.New("ModProbe").Parse(TEMPLATE_MODPROBE))
s.data["modulename"] = modulename
s.data["arguments"] = strings.Join(args, " ")
return s
}

func (s *Shell) Command(command string) *Shell {
s.tmpl = template.Must(template.New("Command").Parse(TEMPLATE_COMMAND))
s.data["command"] = command
Expand Down

0 comments on commit beff382

Please sign in to comment.