diff --git a/cluster-provision/gocli/cmd/provision.go b/cluster-provision/gocli/cmd/provision.go index e86858eca8..7f5adfcbf9 100644 --- a/cluster-provision/gocli/cmd/provision.go +++ b/cluster-provision/gocli/cmd/provision.go @@ -2,33 +2,32 @@ package cmd import ( "fmt" - "io/ioutil" - "os" - "os/signal" "path/filepath" - "strconv" "strings" - "github.com/alessio/shellescape" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" - "github.com/docker/docker/pkg/archive" "github.com/docker/go-connections/nat" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/net/context" - containers2 "kubevirt.io/kubevirtci/cluster-provision/gocli/containers" "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/utils" - "kubevirt.io/kubevirtci/cluster-provision/gocli/docker" + "kubevirt.io/kubevirtci/cluster-provision/gocli/providers" ) +const ( + baseLinuxPhase = "quay.io/kubevirtci/centos9" + baseK8sPhase = "quay.io/kubevirtci/centos9:2408130400-bb670376" +) + +var versionMap = map[string]string{ + "1.30": "1.30.2", + "1.29": "1.29.6", + "1.28": "1.28.11", + "1.31": "1.31.0", +} + // NewProvisionCommand provision given cluster func NewProvisionCommand() *cobra.Command { - provision := &cobra.Command{ Use: "provision", Short: "provision starts a given cluster", @@ -38,10 +37,10 @@ func NewProvisionCommand() *cobra.Command { provision.Flags().StringP("memory", "m", "3096M", "amount of ram per node") provision.Flags().UintP("cpu", "c", 2, "number of cpu cores per node") provision.Flags().String("qemu-args", "", "additional qemu args to pass through to the nodes") - provision.Flags().Bool("random-ports", false, "expose all ports on random localhost ports") + provision.Flags().Bool("random-ports", true, "expose all ports on random localhost ports") provision.Flags().Bool("slim", false, "create slim provider (uncached images)") - provision.Flags().Uint("vnc-port", 0, "port on localhost for vnc") - provision.Flags().Uint("ssh-port", 0, "port on localhost for ssh server") + provision.Flags().Uint16("vnc-port", 0, "port on localhost for vnc") + provision.Flags().Uint16("ssh-port", 0, "port on localhost for ssh server") provision.Flags().String("container-suffix", "", "use additional suffix for the provisioned container image") provision.Flags().String("phases", "linux,k8s", "phases to run, possible values: linux,k8s linux k8s") provision.Flags().StringArray("additional-persistent-kernel-arguments", []string{}, "additional persistent kernel arguments applied after provision") @@ -51,330 +50,88 @@ func NewProvisionCommand() *cobra.Command { func provisionCluster(cmd *cobra.Command, args []string) (retErr error) { var base string - packagePath := args[0] - versionBytes, err := os.ReadFile(filepath.Join(packagePath, "version")) - if err != nil { - return err - } - version := strings.TrimSpace(string(versionBytes)) - baseBytes, err := os.ReadFile(filepath.Join(packagePath, "base")) - if err != nil { - return err + versionNoMinor := args[0] + + v, ok := versionMap[versionNoMinor] + if !ok { + return fmt.Errorf("Invalid version passed, exiting!") + } + + opts := []providers.KubevirtProviderOption{} + flags := cmd.Flags() + for flagName, flagConfig := range providers.ProvisionFlagMap { + switch flagConfig.FlagType { + case "string": + flagVal, err := flags.GetString(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + case "bool": + flagVal, err := flags.GetBool(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + + case "uint": + flagVal, err := flags.GetUint(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + case "uint16": + flagVal, err := flags.GetUint16(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + case "[]string": + flagVal, err := flags.GetStringArray(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + } } + phases, err := cmd.Flags().GetString("phases") if err != nil { return err } if strings.Contains(phases, "linux") { - base = fmt.Sprintf("quay.io/kubevirtci/%s", strings.TrimSpace(string(baseBytes))) + base = baseLinuxPhase } else { - k8sPath := fmt.Sprintf("%s/../", packagePath) - baseImageBytes, err := os.ReadFile(filepath.Join(k8sPath, "base-image")) - if err != nil { - return err - } - base = strings.TrimSpace(string(baseImageBytes)) + base = baseK8sPhase } containerSuffix, err := cmd.Flags().GetString("container-suffix") if err != nil { return err } - name := filepath.Base(packagePath) + name := filepath.Base(versionNoMinor) if len(containerSuffix) > 0 { name = fmt.Sprintf("%s-%s", name, containerSuffix) } - prefix := fmt.Sprintf("k8s-%s-provision", name) - target := fmt.Sprintf("quay.io/kubevirtci/k8s-%s", name) - scripts := filepath.Join(packagePath) - - if phases == "linux" { - target = base + "-base" - } - - memory, err := cmd.Flags().GetString("memory") - if err != nil { - return err - } - - randomPorts, err := cmd.Flags().GetBool("random-ports") - if err != nil { - return err - } - - slim, err := cmd.Flags().GetBool("slim") - if err != nil { - return err - } portMap := nat.PortMap{} utils.AppendTCPIfExplicit(portMap, utils.PortSSH, cmd.Flags(), "ssh-port") utils.AppendTCPIfExplicit(portMap, utils.PortVNC, cmd.Flags(), "vnc-port") - qemuArgs, err := cmd.Flags().GetString("qemu-args") - if err != nil { - return err - } - - cpu, err := cmd.Flags().GetUint("cpu") - if err != nil { - return err - } - cli, err := client.NewClientWithOpts(client.FromEnv) if err != nil { return err } - ctx := context.Background() - - stop := make(chan error, 10) - containers, volumes, done := docker.NewCleanupHandler(cli, stop, cmd.OutOrStderr(), true) - - defer func() { - stop <- retErr - <-done - }() - go func() { - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt) - <-interrupt - stop <- fmt.Errorf("Interrupt received, clean up") - }() + ctx, cancel := context.WithCancel(context.Background()) - // Pull the base image - err = docker.ImagePull(cli, ctx, base, types.ImagePullOptions{}) + kp := providers.NewKubevirtProvider(versionNoMinor, base, cli, opts) + err = kp.Provision(ctx, cancel, portMap, v) if err != nil { - panic(err) - } - - // Start dnsmasq - dnsmasq, err := containers2.DNSMasq(cli, ctx, &containers2.DNSMasqOptions{ - ClusterImage: base, - SecondaryNicsCount: 0, - RandomPorts: randomPorts, - PortMap: portMap, - Prefix: prefix, - NodeCount: 1, - }) - if err != nil { - return err - } - containers <- dnsmasq.ID - if err := cli.ContainerStart(ctx, dnsmasq.ID, container.StartOptions{}); err != nil { return err } - nodeName := nodeNameFromIndex(1) - nodeNum := fmt.Sprintf("%02d", 1) - - vol, err := cli.VolumeCreate(ctx, volume.CreateOptions{ - Name: fmt.Sprintf("%s-%s", prefix, nodeName), - }) - if err != nil { - return err - } - volumes <- vol.Name - registryVol, err := cli.VolumeCreate(ctx, volume.CreateOptions{ - Name: fmt.Sprintf("%s-%s", prefix, "registry"), - }) - if err != nil { - return err - } - - if len(qemuArgs) > 0 { - qemuArgs = "--qemu-args " + qemuArgs - } - node, err := cli.ContainerCreate(ctx, &container.Config{ - Image: base, - Env: []string{ - fmt.Sprintf("NODE_NUM=%s", nodeNum), - }, - Volumes: map[string]struct{}{ - "/var/run/disk": {}, - "/var/lib/registry": {}, - }, - Cmd: []string{"/bin/bash", "-c", fmt.Sprintf("/vm.sh --memory %s --cpu %s %s", memory, strconv.Itoa(int(cpu)), qemuArgs)}, - }, &container.HostConfig{ - Mounts: []mount.Mount{ - { - Type: "volume", - Source: vol.Name, - Target: "/var/run/disk", - }, - { - Type: "volume", - Source: registryVol.Name, - Target: "/var/lib/registry", - }, - }, - Privileged: true, - NetworkMode: container.NetworkMode("container:" + dnsmasq.ID), - }, nil, nil, nodeContainer(prefix, nodeName)) - if err != nil { - return err - } - containers <- node.ID - if err := cli.ContainerStart(ctx, node.ID, container.StartOptions{}); err != nil { - return err - } - - // copy provider scripts - err = copyDirectory(ctx, cli, node.ID, scripts, "/scripts") - if err != nil { - return err - } - - // Wait for ssh.sh script to exist - err = _cmd(cli, nodeContainer(prefix, nodeName), "while [ ! -f /ssh_ready ] ; do sleep 1; done", "checking for ssh.sh script") - if err != nil { - return err - } - - // Wait for the VM to be up - err = _cmd(cli, nodeContainer(prefix, nodeName), "ssh.sh echo VM is up", "waiting for node to come up") - if err != nil { - return err - } - - envVars := fmt.Sprintf("version=%s slim=%t", version, slim) - if strings.Contains(phases, "linux") { - err = performPhase(cli, nodeContainer(prefix, nodeName), "/scripts/provision.sh", envVars) - if err != nil { - return err - } - } - if strings.Contains(phases, "k8s") { - // copy provider scripts - err = copyDirectory(ctx, cli, node.ID, scripts, "/scripts") - if err != nil { - return err - } - err = _cmd(cli, nodeContainer(prefix, nodeName), "if [ -f /scripts/extra-pre-pull-images ]; then scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key -P 22 /scripts/extra-pre-pull-images vagrant@192.168.66.101:/tmp/extra-pre-pull-images; fi", "copying /scripts/extra-pre-pull-images if existing") - if err != nil { - return err - } - err = _cmd(cli, nodeContainer(prefix, nodeName), "if [ -f /scripts/fetch-images.sh ]; then scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key -P 22 /scripts/fetch-images.sh vagrant@192.168.66.101:/tmp/fetch-images.sh; fi", "copying /scripts/fetch-images.sh if existing") - if err != nil { - return err - } - - err = _cmd(cli, nodeContainer(prefix, nodeName), "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key vagrant@192.168.66.101 'mkdir -p /tmp/ceph /tmp/cnao /tmp/nfs-csi /tmp/nodeports /tmp/prometheus /tmp/whereabouts /tmp/kwok'", "Create required manifest directories before copy") - if err != nil { - return err - } - // Copy manifests to the VM - err = _cmd(cli, nodeContainer(prefix, nodeName), "scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key -P 22 /scripts/manifests/* vagrant@192.168.66.101:/tmp", "copying manifests to the VM") - if err != nil { - return err - } - - err = performPhase(cli, nodeContainer(prefix, nodeName), "/scripts/k8s_provision.sh", envVars) - if err != nil { - return err - } - } - - _cmd(cli, nodeContainer(prefix, nodeName), "ssh.sh sudo shutdown now -h", "shutting down the node") - err = _cmd(cli, nodeContainer(prefix, nodeName), "rm /usr/local/bin/ssh.sh", "removing the ssh.sh script") - if err != nil { - return err - } - err = _cmd(cli, nodeContainer(prefix, nodeName), "rm /ssh_ready", "removing the ssh_ready mark") - if err != nil { - return err - } - logrus.Info("waiting for the node to stop") - okChan, errChan := cli.ContainerWait(ctx, nodeContainer(prefix, nodeName), container.WaitConditionNotRunning) - select { - case <-okChan: - case err := <-errChan: - if err != nil { - return fmt.Errorf("waiting for the node to stop failed: %v", err) - } - } - - logrus.Info("preparing additional persistent kernel arguments after initial provision") - additionalKernelArguments, err := cmd.Flags().GetStringArray("additional-persistent-kernel-arguments") - if err != nil { - return err - } - - dir, err := ioutil.TempDir("", "gocli") - if err != nil { - return fmt.Errorf("failed creating a temporary directory: %v", err) - } - defer os.RemoveAll(dir) - if err := ioutil.WriteFile(filepath.Join(dir, "additional.kernel.args"), []byte(shellescape.QuoteCommand(additionalKernelArguments)), 0666); err != nil { - return fmt.Errorf("failed creating additional.kernel.args file: %v", err) - } - if err := copyDirectory(ctx, cli, node.ID, dir, "/"); err != nil { - return fmt.Errorf("failed copying additional kernel arguments into the container: %v", err) - } - - logrus.Infof("Commiting the node as %s", target) - _, err = cli.ContainerCommit(ctx, node.ID, container.CommitOptions{ - Reference: target, - Comment: "PROVISION SUCCEEDED", - Author: "gocli", - Changes: nil, - Pause: false, - Config: nil, - }) - if err != nil { - return fmt.Errorf("commiting the node failed: %v", err) - } - return nil } - -func copyDirectory(ctx context.Context, cli *client.Client, containerID string, sourceDirectory string, targetDirectory string) error { - srcInfo, err := archive.CopyInfoSourcePath(sourceDirectory, false) - if err != nil { - return err - } - - srcArchive, err := archive.TarResource(srcInfo) - if err != nil { - return err - } - defer srcArchive.Close() - - dstInfo := archive.CopyInfo{Path: targetDirectory} - - dstDir, preparedArchive, err := archive.PrepareArchiveCopy(srcArchive, srcInfo, dstInfo) - if err != nil { - return err - } - defer preparedArchive.Close() - - err = cli.CopyToContainer(ctx, containerID, dstDir, preparedArchive, types.CopyToContainerOptions{AllowOverwriteDirWithFile: false}) - if err != nil { - return err - } - return nil -} - -func _cmd(cli *client.Client, container string, cmd string, description string) error { - logrus.Info(description) - success, err := docker.Exec(cli, container, []string{"/bin/bash", "-c", cmd}, os.Stdout) - if err != nil { - return fmt.Errorf("%s failed: %v", description, err) - } else if !success { - return fmt.Errorf("%s failed", cmd) - } - return nil -} - -func performPhase(cli *client.Client, container string, script string, envVars string) error { - err := _cmd(cli, container, fmt.Sprintf("test -f %s", script), "checking provision scripts") - if err != nil { - return err - } - - return _cmd(cli, container, - fmt.Sprintf("ssh.sh sudo %s /bin/bash < %s", envVars, script), - fmt.Sprintf("provisioning the node (%s)", script)) -} diff --git a/cluster-provision/gocli/providers/base_provider.go b/cluster-provision/gocli/providers/base_provider.go index b627f57838..6a62f80c52 100644 --- a/cluster-provision/gocli/providers/base_provider.go +++ b/cluster-provision/gocli/providers/base_provider.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "os/signal" "path/filepath" @@ -18,7 +19,9 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" + "github.com/docker/docker/pkg/archive" "github.com/docker/go-connections/nat" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/resource" @@ -32,6 +35,7 @@ import ( dockerproxy "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/docker-proxy" etcd "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/etcd" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/istio" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/k8sprovision" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/ksm" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/labelnodes" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/multus" @@ -39,6 +43,7 @@ import ( "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/node01" nodesprovision "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/nodes" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/prometheus" + provisionopt "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/provision" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/psa" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/realtime" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/rookceph" @@ -200,7 +205,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc } containers <- node.ID - if err := kp.Docker.ContainerStart(ctx, node.ID, types.ContainerStartOptions{}); err != nil { + if err := kp.Docker.ContainerStart(ctx, node.ID, container.StartOptions{}); err != nil { return err } @@ -239,12 +244,17 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc return err } - err = sshClient.CopyRemoteFile("/etc/kubernetes/admin.conf", ".kubeconfig") + kubeConf, err := os.Create(".kubeconfig") if err != nil { return err } - config, err := k8s.InitConfig(".kubeconfig", kp.APIServerPort) + err = sshClient.CopyRemoteFile("/etc/kubernetes/admin.conf", kubeConf) + if err != nil { + return err + } + + config, err := k8s.NewConfig(".kubeconfig", kp.APIServerPort) if err != nil { return err } @@ -267,6 +277,196 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc return nil } +func (kp *KubevirtProvider) Provision(ctx context.Context, cancel context.CancelFunc, portMap nat.PortMap, k8sVersion string) (retErr error) { + prefix := fmt.Sprintf("k8s-%s-provision", kp.Version) + target := fmt.Sprintf("quay.io/kubevirtci/k8s-%s", kp.Version) + if kp.Phases == "linux" { + target = kp.Image + "-base" + } + kp.Version = prefix + + stop := make(chan error, 10) + containers, volumes, done := docker.NewCleanupHandler(kp.Docker, stop, os.Stdout, true) + + defer func() { + stop <- retErr + <-done + }() + + go kp.handleInterrupt(cancel, stop) + + err := docker.ImagePull(kp.Docker, ctx, kp.Image, types.ImagePullOptions{}) + if err != nil { + return err + } + + dnsmasq, err := kp.runDNSMasq(ctx, portMap) + if err != nil { + return err + } + + kp.DNSMasq = dnsmasq + containers <- dnsmasq + + dm, err := kp.Docker.ContainerInspect(context.Background(), dnsmasq) + if err != nil { + return err + } + + sshPort, err := utils.GetPublicPort(utils.PortSSH, dm.NetworkSettings.Ports) + if err != nil { + return err + } + + nodeName := kp.nodeNameFromIndex(1) + nodeNum := fmt.Sprintf("%02d", 1) + + vol, err := kp.Docker.VolumeCreate(ctx, volume.CreateOptions{ + Name: fmt.Sprintf("%s-%s", prefix, nodeName), + }) + if err != nil { + return err + } + volumes <- vol.Name + registryVol, err := kp.Docker.VolumeCreate(ctx, volume.CreateOptions{ + Name: fmt.Sprintf("%s-%s", prefix, "registry"), + }) + if err != nil { + return err + } + + node, err := kp.Docker.ContainerCreate(ctx, &container.Config{ + Image: kp.Image, + Env: []string{ + fmt.Sprintf("NODE_NUM=%s", nodeNum), + }, + Volumes: map[string]struct{}{ + "/var/run/disk": {}, + "/var/lib/registry": {}, + }, + Cmd: []string{"/bin/bash", "-c", fmt.Sprintf("/vm.sh --memory %s --cpu %s %s", kp.Memory, strconv.Itoa(int(kp.CPU)), kp.QemuArgs)}, + }, &container.HostConfig{ + Mounts: []mount.Mount{ + { + Type: "volume", + Source: vol.Name, + Target: "/var/run/disk", + }, + { + Type: "volume", + Source: registryVol.Name, + Target: "/var/lib/registry", + }, + }, + Privileged: true, + NetworkMode: container.NetworkMode("container:" + kp.DNSMasq), + }, nil, nil, kp.nodeContainer(kp.Version, nodeName)) + if err != nil { + return err + } + containers <- node.ID + if err := kp.Docker.ContainerStart(ctx, node.ID, container.StartOptions{}); err != nil { + return err + } + + // Wait for ssh.sh script to exist + _, err = docker.Exec(kp.Docker, kp.nodeContainer(kp.Version, nodeName), []string{"bin/bash", "-c", "while [ ! -f /ssh_ready ] ; do sleep 1; done", "checking for ssh.sh script"}, os.Stdout) + if err != nil { + return err + } + + sshClient, err := libssh.NewSSHClient(sshPort, 1, false) + if err != nil { + return err + } + + rootkey := rootkey.NewRootKey(sshClient) + if err = rootkey.Exec(); err != nil { + fmt.Println(err) + } + + sshClient, err = libssh.NewSSHClient(sshPort, 1, true) + if err != nil { + return err + } + + if strings.Contains(kp.Phases, "linux") { + provisionOpt := provisionopt.NewLinuxProvisioner(sshClient) + if err = provisionOpt.Exec(); err != nil { + return err + } + } + + if strings.Contains(kp.Phases, "k8s") { + // copy provider scripts + if err = sshClient.Command("mkdir -p /tmp/ceph /tmp/cnao /tmp/nfs-csi /tmp/nodeports /tmp/prometheus /tmp/whereabouts /tmp/kwok"); err != nil { + return err + } + // Copy manifests to the VM + success, err := docker.Exec(kp.Docker, kp.nodeContainer(kp.Version, nodeName), []string{"/bin/bash", "-c", "scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key -P 22 /scripts/manifests/* root@192.168.66.101:/tmp"}, os.Stdout) + if err != nil { + return err + } + + if !success { + return fmt.Errorf("error copying manifests to node") + } + + provisionK8sOpt := k8sprovision.NewK8sProvisioner(sshClient, k8sVersion, kp.Slim) + if err = provisionK8sOpt.Exec(); err != nil { + return err + } + } + + if err = sshClient.Command("sudo shutdown now -h"); err != nil { + return err + } + + _, err = docker.Exec(kp.Docker, kp.nodeContainer(kp.Version, nodeName), []string{"rm", "/ssh_ready"}, io.Discard) + if err != nil { + return err + } + + logrus.Info("waiting for the node to stop") + okChan, errChan := kp.Docker.ContainerWait(ctx, kp.nodeContainer(kp.Version, nodeName), container.WaitConditionNotRunning) + select { + case <-okChan: + case err := <-errChan: + if err != nil { + return fmt.Errorf("waiting for the node to stop failed: %v", err) + } + } + + if len(kp.AdditionalKernelArgs) > 0 { + dir, err := os.MkdirTemp("", "gocli") + if err != nil { + return fmt.Errorf("failed creating a temporary directory: %v", err) + } + defer os.RemoveAll(dir) + if err := os.WriteFile(filepath.Join(dir, "additional.kernel.args"), []byte(shellescape.QuoteCommand(kp.AdditionalKernelArgs)), 0666); err != nil { + return fmt.Errorf("failed creating additional.kernel.args file: %v", err) + } + if err := kp.copyDirectory(ctx, kp.Docker, node.ID, dir, "/"); err != nil { + return fmt.Errorf("failed copying additional kernel arguments into the container: %v", err) + } + } + + logrus.Infof("Commiting the node as %s", target) + _, err = kp.Docker.ContainerCommit(ctx, node.ID, container.CommitOptions{ + Reference: target, + Comment: "PROVISION SUCCEEDED", + Author: "gocli", + Changes: nil, + Pause: false, + Config: nil, + }) + if err != nil { + return fmt.Errorf("commiting the node failed: %v", err) + } + + return nil +} + func (kp *KubevirtProvider) runDNSMasq(ctx context.Context, portMap nat.PortMap) (string, error) { dnsmasqMounts := []mount.Mount{} _, err := os.Stat("/lib/modules") @@ -312,7 +512,7 @@ func (kp *KubevirtProvider) runDNSMasq(ctx context.Context, portMap nat.PortMap) Mounts: dnsmasqMounts, }, nil, nil, kp.Version+"-dnsmasq") - if err := kp.Docker.ContainerStart(ctx, dnsmasq.ID, types.ContainerStartOptions{}); err != nil { + if err := kp.Docker.ContainerStart(ctx, dnsmasq.ID, container.StartOptions{}); err != nil { return "", err } return dnsmasq.ID, nil @@ -333,7 +533,7 @@ func (kp *KubevirtProvider) runRegistry(ctx context.Context) (string, error) { return "", err } - if err := kp.Docker.ContainerStart(ctx, registry.ID, types.ContainerStartOptions{}); err != nil { + if err := kp.Docker.ContainerStart(ctx, registry.ID, container.StartOptions{}); err != nil { return "", err } @@ -368,7 +568,7 @@ func (kp *KubevirtProvider) runNFSGanesha(ctx context.Context) (string, error) { return "", err } - if err := kp.Docker.ContainerStart(ctx, nfsGanesha.ID, types.ContainerStartOptions{}); err != nil { + if err := kp.Docker.ContainerStart(ctx, nfsGanesha.ID, container.StartOptions{}); err != nil { return "", err } return nfsGanesha.ID, nil @@ -380,7 +580,7 @@ func (kp *KubevirtProvider) provisionNode(sshClient libssh.Client, nodeIdx int) if kp.EnableFIPS { for _, cmd := range []string{"sudo fips-mode-setup --enable", "sudo reboot"} { - if _, err := sshClient.Command(cmd, true); err != nil { + if err := sshClient.Command(cmd); err != nil { return fmt.Errorf("Starting fips mode failed: %s", err) } } @@ -413,14 +613,8 @@ func (kp *KubevirtProvider) provisionNode(sshClient libssh.Client, nodeIdx int) opts = append(opts, bvfio) } - if kp.SingleStack { - if _, err := sshClient.Command("touch /home/vagrant/single_stack", false); err != nil { - return fmt.Errorf("provisioning node %d failed (setting singleStack phase): %s", nodeIdx, err) - } - } - if kp.EnableAudit { - if _, err := sshClient.Command("touch /home/vagrant/enable_audit", false); err != nil { + if err := sshClient.Command("touch /home/vagrant/enable_audit"); err != nil { return fmt.Errorf("provisioning node %d failed (setting enableAudit phase): %s", nodeIdx, err) } } @@ -431,7 +625,7 @@ func (kp *KubevirtProvider) provisionNode(sshClient libssh.Client, nodeIdx int) } if nodeIdx == 1 { - n := node01.NewNode01Provisioner(sshClient) + n := node01.NewNode01Provisioner(sshClient, kp.SingleStack) opts = append(opts, n) } else { @@ -444,7 +638,7 @@ func (kp *KubevirtProvider) provisionNode(sshClient libssh.Client, nodeIdx int) bindVfioOpt := bindvfio.NewBindVfioOpt(sshClient, gpuDeviceID) opts = append(opts, bindVfioOpt) } - n := nodesprovision.NewNodesProvisioner(sshClient) + n := nodesprovision.NewNodesProvisioner(sshClient, kp.SingleStack) opts = append(opts, n) } @@ -476,7 +670,7 @@ func (kp *KubevirtProvider) provisionK8sOpts(sshClient libssh.Client) error { opts = append(opts, labelnodes.NewNodeLabler(sshClient, labelSelector)) if kp.CDI { - opts = append(opts, cdi.NewCdiOpt(kp.Client, kp.CDIVersion)) + opts = append(opts, cdi.NewCdiOpt(kp.Client, sshClient, kp.CDIVersion)) } if kp.AAQ { @@ -673,6 +867,33 @@ func (kp *KubevirtProvider) getPCIDeviceIOMMUGroup(address string) (string, erro return iommuGroup, nil } +func (kp *KubevirtProvider) copyDirectory(ctx context.Context, cli *client.Client, containerID string, sourceDirectory string, targetDirectory string) error { + srcInfo, err := archive.CopyInfoSourcePath(sourceDirectory, false) + if err != nil { + return err + } + + srcArchive, err := archive.TarResource(srcInfo) + if err != nil { + return err + } + defer srcArchive.Close() + + dstInfo := archive.CopyInfo{Path: targetDirectory} + + dstDir, preparedArchive, err := archive.PrepareArchiveCopy(srcArchive, srcInfo, dstInfo) + if err != nil { + return err + } + defer preparedArchive.Close() + + err = cli.CopyToContainer(ctx, containerID, dstDir, preparedArchive, types.CopyToContainerOptions{AllowOverwriteDirWithFile: false}) + if err != nil { + return err + } + return nil +} + func (kp *KubevirtProvider) handleInterrupt(cancel context.CancelFunc, stop chan error) { interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, os.Interrupt)