diff --git a/.github/ISSUE_TEMPLATE/__en-US.yaml b/.github/ISSUE_TEMPLATE/__en-US.yaml index fbf268625988..b749463a514f 100644 --- a/.github/ISSUE_TEMPLATE/__en-US.yaml +++ b/.github/ISSUE_TEMPLATE/__en-US.yaml @@ -47,8 +47,6 @@ body: - VirtualBox - None (Baremetal) - SSH - - VMware - - Parallels - QEMU validations: required: false diff --git a/.github/ISSUE_TEMPLATE/ru.yaml b/.github/ISSUE_TEMPLATE/ru.yaml index 4e7cae7befde..607c7a827b95 100644 --- a/.github/ISSUE_TEMPLATE/ru.yaml +++ b/.github/ISSUE_TEMPLATE/ru.yaml @@ -45,8 +45,6 @@ body: - VirtualBox - None (Baremetal) - SSH - - VMware - - Parallels - QEMU validations: required: false diff --git a/cmd/drivers/hyperkit/main.go b/cmd/drivers/hyperkit/main.go index 4bff1b3d622e..dfe888680e4f 100644 --- a/cmd/drivers/hyperkit/main.go +++ b/cmd/drivers/hyperkit/main.go @@ -22,7 +22,7 @@ import ( "fmt" "os" - "github.com/docker/machine/libmachine/drivers/plugin" + "k8s.io/minikube/pkg/libmachine/drivers/plugin" "k8s.io/minikube/pkg/drivers/hyperkit" ) diff --git a/cmd/drivers/kvm/main.go b/cmd/drivers/kvm/main.go index 7251f68365a8..228d7a33376a 100644 --- a/cmd/drivers/kvm/main.go +++ b/cmd/drivers/kvm/main.go @@ -22,7 +22,7 @@ import ( "fmt" "os" - "github.com/docker/machine/libmachine/drivers/plugin" + "k8s.io/minikube/pkg/libmachine/drivers/plugin" "k8s.io/minikube/pkg/drivers/kvm" ) diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go index 26efa837cea3..39158b31211b 100644 --- a/cmd/minikube/cmd/config/profile_list.go +++ b/cmd/minikube/cmd/config/profile_list.go @@ -33,7 +33,7 @@ import ( "k8s.io/minikube/pkg/minikube/reason" "k8s.io/minikube/pkg/minikube/style" - "github.com/docker/machine/libmachine" + "k8s.io/minikube/pkg/libmachine" "github.com/olekukonko/tablewriter" "github.com/olekukonko/tablewriter/tw" "github.com/spf13/cobra" diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 67be68c5868f..0eb073d0b64d 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -26,11 +26,11 @@ import ( "strings" "time" - "github.com/docker/machine/libmachine/mcnerror" + "k8s.io/minikube/pkg/libmachine/mcnerror" "github.com/mitchellh/go-ps" "github.com/pkg/errors" - "github.com/docker/machine/libmachine" + "k8s.io/minikube/pkg/libmachine" "github.com/spf13/cobra" "github.com/spf13/viper" "k8s.io/klog/v2" diff --git a/cmd/minikube/cmd/delete_test.go b/cmd/minikube/cmd/delete_test.go index 945f844a7239..71824b673cdc 100644 --- a/cmd/minikube/cmd/delete_test.go +++ b/cmd/minikube/cmd/delete_test.go @@ -25,7 +25,7 @@ import ( "strings" "testing" - "github.com/docker/machine/libmachine" + "k8s.io/minikube/pkg/libmachine" "github.com/google/go-cmp/cmp" "github.com/otiai10/copy" "github.com/spf13/viper" diff --git a/cmd/minikube/cmd/logs.go b/cmd/minikube/cmd/logs.go index 86a70e3c6346..416c32a34835 100644 --- a/cmd/minikube/cmd/logs.go +++ b/cmd/minikube/cmd/logs.go @@ -19,7 +19,7 @@ package cmd import ( "os" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/state" "github.com/spf13/cobra" "github.com/spf13/viper" "k8s.io/klog/v2" diff --git a/cmd/minikube/cmd/podman-env.go b/cmd/minikube/cmd/podman-env.go index e282000542c7..9ba29598ef56 100644 --- a/cmd/minikube/cmd/podman-env.go +++ b/cmd/minikube/cmd/podman-env.go @@ -23,8 +23,8 @@ import ( "os/exec" "strings" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/ssh" "github.com/spf13/cobra" "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/command" diff --git a/cmd/minikube/cmd/podman-env_test.go b/cmd/minikube/cmd/podman-env_test.go index 7215fe9a1fd0..b03503af8185 100644 --- a/cmd/minikube/cmd/podman-env_test.go +++ b/cmd/minikube/cmd/podman-env_test.go @@ -20,7 +20,7 @@ import ( "bytes" "testing" - "github.com/docker/machine/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/ssh" "github.com/google/go-cmp/cmp" ) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 020cd198a9c6..fc616106d654 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -36,7 +36,7 @@ import ( "github.com/Delta456/box-cli-maker/v2" "github.com/blang/semver/v4" "github.com/docker/go-connections/nat" - "github.com/docker/machine/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/ssh" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go index 3a391d41f1c2..dcde72d31fb3 100644 --- a/cmd/minikube/cmd/start_test.go +++ b/cmd/minikube/cmd/start_test.go @@ -320,9 +320,7 @@ func TestBaseImageFlagDriverCombo(t *testing.T) { {driver.KVM2, false}, {driver.VirtualBox, false}, {driver.HyperKit, false}, - {driver.VMware, false}, {driver.HyperV, false}, - {driver.Parallels, false}, {"something_invalid", false}, {"", false}, } diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index c0784692240d..3e36a544d7df 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -25,8 +25,8 @@ import ( "text/template" "time" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "github.com/spf13/cobra" "k8s.io/klog/v2" diff --git a/cmd/minikube/cmd/stop.go b/cmd/minikube/cmd/stop.go index 3b3e1beed779..ab731ce3a4e4 100644 --- a/cmd/minikube/cmd/stop.go +++ b/cmd/minikube/cmd/stop.go @@ -21,8 +21,8 @@ import ( "runtime" "time" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/mcnerror" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/mcnerror" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/viper" diff --git a/cmd/minikube/main.go b/cmd/minikube/main.go index eb3fa841e2e7..5e38fc612798 100644 --- a/cmd/minikube/main.go +++ b/cmd/minikube/main.go @@ -42,7 +42,7 @@ import ( // Force exp dependency _ "golang.org/x/exp/ebnf" - mlog "github.com/docker/machine/libmachine/log" + mlog "k8s.io/minikube/pkg/libmachine/log" "github.com/google/slowjam/pkg/stacklog" "github.com/pkg/profile" diff --git a/go.mod b/go.mod index ca4f45e0ac33..4c95cf989cc9 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,9 @@ require ( cloud.google.com/go/storage v1.56.1 github.com/Delta456/box-cli-maker/v2 v2.3.0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.29.0 - github.com/Parallels/docker-machine-parallels/v2 v2.0.1 github.com/VividCortex/godaemon v1.0.0 github.com/Xuanwo/go-locale v1.1.3 + github.com/aregm/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 github.com/blang/semver/v4 v4.0.0 github.com/briandowns/spinner v1.23.2 github.com/cenkalti/backoff/v4 v4.3.0 @@ -22,7 +22,6 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.6.0 github.com/docker/go-units v0.5.0 - github.com/docker/machine v0.16.2 github.com/elazarl/goproxy v1.7.2 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 github.com/google/go-cmp v0.7.0 @@ -32,6 +31,7 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/go-getter v1.8.0 github.com/hashicorp/go-retryablehttp v0.7.8 + github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 github.com/hooklift/iso9660 v1.0.0 github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 github.com/johanneswuerbach/nfsexports v0.0.0-20200318065542-c48c3734757f @@ -40,12 +40,12 @@ require ( github.com/juju/mutex/v2 v2.0.0 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/klauspost/cpuid v1.2.0 - github.com/machine-drivers/docker-machine-driver-vmware v0.1.5 github.com/mattbaird/jsonpatch v0.0.0-20200820163806-098863c1fc24 github.com/mattn/go-isatty v0.0.20 github.com/mitchellh/go-ps v1.0.0 github.com/moby/hyperkit v0.0.0-20210108224842-2f061e447e14 github.com/moby/patternmatcher v0.6.0 + github.com/moby/term v0.5.0 github.com/olekukonko/tablewriter v1.0.9 github.com/opencontainers/cgroups v0.0.4 github.com/opencontainers/go-digest v1.0.0 @@ -56,6 +56,7 @@ require ( github.com/pkg/profile v1.7.0 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 + github.com/sayboras/dockerclient v1.0.0 github.com/shirou/gopsutil/v3 v3.24.5 github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.9 @@ -103,7 +104,6 @@ require ( github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/VividCortex/ewma v1.2.0 // indirect - github.com/aregm/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 // indirect github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.29.15 // indirect @@ -163,18 +163,19 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/gookit/color v1.5.2 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.65 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect + github.com/juju/loggo v1.0.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect @@ -192,7 +193,6 @@ require ( github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect - github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/muesli/reflow v0.3.0 // indirect @@ -212,7 +212,6 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.7.0 // indirect - github.com/sayboras/dockerclient v1.0.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -254,9 +253,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) - -replace ( - github.com/Parallels/docker-machine-parallels/v2 => github.com/minikube-machine/machine-driver-parallels/v2 v2.0.2-0.20240730142131-ada9375ea417 - github.com/docker/machine => github.com/minikube-machine/machine v0.0.0-20240815173309-ffb6b643c381 - github.com/machine-drivers/docker-machine-driver-vmware => github.com/minikube-machine/machine-driver-vmware v0.1.6-0.20230701123042-a391c48b14d5 -) diff --git a/go.sum b/go.sum index 2d0abe27143d..9cbd00a9b580 100644 --- a/go.sum +++ b/go.sum @@ -35,7 +35,6 @@ cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8W dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= @@ -68,7 +67,6 @@ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= @@ -128,15 +126,11 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/briandowns/spinner v1.23.2 h1:Zc6ecUnI+YzLmJniCfDNaMbW0Wid1d5+qcTq4L2FW8w= github.com/briandowns/spinner v1.23.2/go.mod h1:LaZeM4wm2Ywy6vO571mvhQNRcWfRUnXOs0RcKV0wYKM= -github.com/bugsnag/bugsnag-go v0.0.0-20151120182711-02e952891c52/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= -github.com/bugsnag/panicwrap v0.0.0-20160118154447-aceac81c6e2f/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/c4milo/gotoolkit v0.0.0-20190525173301-67483a18c17a h1:+uvtaGSLJh0YpLLHCQ9F+UVGy4UOS542hsjj8wBjvH0= github.com/c4milo/gotoolkit v0.0.0-20190525173301-67483a18c17a/go.mod h1:txokOny9wavBtq2PWuHmj1P+eFwpCsj+gQeNNANChfU= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -166,7 +160,6 @@ github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -184,16 +177,12 @@ github.com/docker/cli v28.4.0+incompatible h1:RBcf3Kjw2pMtwui5V0DIMdyeab8glEw5QY github.com/docker/cli v28.4.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v0.0.0-20180621001606-093424bec097/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v17.12.0-ce-rc1.0.20181225093023-5ddb1d410a8b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v17.12.0-ce-rc1.0.20190115220918-5ec31380a5d3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= -github.com/docker/go-units v0.0.0-20151230175859-0bbddae09c5a/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -308,7 +297,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -370,7 +358,6 @@ github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVU github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= -github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -437,7 +424,6 @@ github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYW github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -475,12 +461,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/minikube-machine/machine v0.0.0-20240815173309-ffb6b643c381 h1:9walvLKjkCrvs9GlFwFO/qvwtZV9O5ujkgqOMAp1nOA= -github.com/minikube-machine/machine v0.0.0-20240815173309-ffb6b643c381/go.mod h1:rC8K+60rTqfkrL3a6X/UZeQgP5Gz1Y5czIkWDZcvBuQ= -github.com/minikube-machine/machine-driver-parallels/v2 v2.0.2-0.20240730142131-ada9375ea417 h1:f+neTRGCtvmW3Tm1V72vWpoTPuNOnXSQsHZdYOryfGM= -github.com/minikube-machine/machine-driver-parallels/v2 v2.0.2-0.20240730142131-ada9375ea417/go.mod h1:NKwI5KryEmEHMZVj80t9JQcfXWZp4/ZYNBuw4C5sQ9E= -github.com/minikube-machine/machine-driver-vmware v0.1.6-0.20230701123042-a391c48b14d5 h1:1z7xOzfMO4aBR9+2nYjlhRXX1773fX60HTS0QGpGRPU= -github.com/minikube-machine/machine-driver-vmware v0.1.6-0.20230701123042-a391c48b14d5/go.mod h1:HifYFOWR0bAMN4hWtaSADClogvtPy/jV0aRC5alhrKo= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= @@ -507,7 +487,6 @@ github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/moby/term v0.0.0-20200416134343-063f2cd0b49d/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -603,7 +582,6 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= @@ -618,11 +596,8 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -635,7 +610,6 @@ github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cA github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= @@ -669,7 +643,6 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= @@ -729,7 +702,6 @@ go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1 golang.org/x/build v0.0.0-20190927031335-2835ba2e683f h1:hXVePvSFG7tPGX4Pwk1d10ePFfoTCc0QmISfpKOHsS8= golang.org/x/build v0.0.0-20190927031335-2835ba2e683f/go.mod h1:fYw7AShPAhGMdXqA9gRadk/CcMsvLlClpE5oBwnS3dM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190424203555-c05e17bb3b2d/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -737,7 +709,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= @@ -838,7 +809,6 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -894,7 +864,6 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -985,9 +954,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 99d6110dd297..2b71134fc231 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -29,7 +29,7 @@ import ( "time" "github.com/blang/semver/v4" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "github.com/spf13/viper" diff --git a/pkg/drivers/common/common.go b/pkg/drivers/common/common.go index 847eb6679e32..1191d370a295 100644 --- a/pkg/drivers/common/common.go +++ b/pkg/drivers/common/common.go @@ -27,11 +27,11 @@ import ( "strings" "syscall" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/log" - "github.com/docker/machine/libmachine/mcnflag" - "github.com/docker/machine/libmachine/mcnutils" - "github.com/docker/machine/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnflag" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/ssh" "github.com/pkg/errors" "k8s.io/klog/v2" diff --git a/pkg/drivers/common/virtiofs/virtiofs.go b/pkg/drivers/common/virtiofs/virtiofs.go index 05b9139b0fa3..5fc11f4d76e4 100644 --- a/pkg/drivers/common/virtiofs/virtiofs.go +++ b/pkg/drivers/common/virtiofs/virtiofs.go @@ -22,7 +22,7 @@ import ( "path/filepath" "strings" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "github.com/google/uuid" ) diff --git a/pkg/drivers/common/vmnet/vmnet.go b/pkg/drivers/common/vmnet/vmnet.go index 2c120713806b..ac200e82e9d0 100644 --- a/pkg/drivers/common/vmnet/vmnet.go +++ b/pkg/drivers/common/vmnet/vmnet.go @@ -30,8 +30,8 @@ import ( "strings" "syscall" - "github.com/docker/machine/libmachine/log" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/state" "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/process" diff --git a/pkg/drivers/hyperkit/driver.go b/pkg/drivers/hyperkit/driver.go index e03ff56f5a30..4bc5b7f9c142 100644 --- a/pkg/drivers/hyperkit/driver.go +++ b/pkg/drivers/hyperkit/driver.go @@ -30,9 +30,9 @@ import ( "syscall" "time" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/log" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/state" "github.com/johanneswuerbach/nfsexports" ps "github.com/mitchellh/go-ps" hyperkit "github.com/moby/hyperkit/go" diff --git a/pkg/drivers/hyperv/hyperv.go b/pkg/drivers/hyperv/hyperv.go new file mode 100644 index 000000000000..91e27ba81942 --- /dev/null +++ b/pkg/drivers/hyperv/hyperv.go @@ -0,0 +1,509 @@ +package hyperv + +import ( + "encoding/json" + "fmt" + "net" + "os" + "strings" + "time" + + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnflag" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/state" +) + +type Driver struct { + *drivers.BaseDriver + Boot2DockerURL string + VSwitch string + DiskSize int + MemSize int + CPU int + MacAddr string + VLanID int + DisableDynamicMemory bool +} + +const ( + defaultDiskSize = 20000 + defaultMemory = 1024 + defaultCPU = 1 + defaultVLanID = 0 + defaultDisableDynamicMemory = false + defaultSwitchID = "c08cb7b8-9b3c-408e-8e30-5e16a3aeb444" +) + +// NewDriver creates a new Hyper-v driver with default settings. +func NewDriver(hostName, storePath string) *Driver { + return &Driver{ + DiskSize: defaultDiskSize, + MemSize: defaultMemory, + CPU: defaultCPU, + DisableDynamicMemory: defaultDisableDynamicMemory, + BaseDriver: &drivers.BaseDriver{ + MachineName: hostName, + StorePath: storePath, + }, + } +} + +// GetCreateFlags registers the flags this driver adds to +// "docker hosts create" +func (d *Driver) GetCreateFlags() []mcnflag.Flag { + return []mcnflag.Flag{ + mcnflag.StringFlag{ + Name: "hyperv-boot2docker-url", + Usage: "URL of the boot2docker ISO. Defaults to the latest available version.", + EnvVar: "HYPERV_BOOT2DOCKER_URL", + }, + mcnflag.StringFlag{ + Name: "hyperv-virtual-switch", + Usage: "Virtual switch name. Defaults to first found.", + EnvVar: "HYPERV_VIRTUAL_SWITCH", + }, + mcnflag.IntFlag{ + Name: "hyperv-disk-size", + Usage: "Maximum size of dynamically expanding disk in MB.", + Value: defaultDiskSize, + EnvVar: "HYPERV_DISK_SIZE", + }, + mcnflag.IntFlag{ + Name: "hyperv-memory", + Usage: "Memory size for host in MB.", + Value: defaultMemory, + EnvVar: "HYPERV_MEMORY", + }, + mcnflag.IntFlag{ + Name: "hyperv-cpu-count", + Usage: "number of CPUs for the machine", + Value: defaultCPU, + EnvVar: "HYPERV_CPU_COUNT", + }, + mcnflag.StringFlag{ + Name: "hyperv-static-macaddress", + Usage: "Hyper-V network adapter's static MAC address.", + EnvVar: "HYPERV_STATIC_MACADDRESS", + }, + mcnflag.IntFlag{ + Name: "hyperv-vlan-id", + Usage: "Hyper-V network adapter's VLAN ID if any", + Value: defaultVLanID, + EnvVar: "HYPERV_VLAN_ID", + }, + mcnflag.BoolFlag{ + Name: "hyperv-disable-dynamic-memory", + Usage: "Disable dynamic memory management setting", + EnvVar: "HYPERV_DISABLE_DYNAMIC_MEMORY", + }, + } +} + +func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error { + d.Boot2DockerURL = flags.String("hyperv-boot2docker-url") + d.VSwitch = flags.String("hyperv-virtual-switch") + d.DiskSize = flags.Int("hyperv-disk-size") + d.MemSize = flags.Int("hyperv-memory") + d.CPU = flags.Int("hyperv-cpu-count") + d.MacAddr = flags.String("hyperv-static-macaddress") + d.VLanID = flags.Int("hyperv-vlan-id") + d.SSHUser = "docker" + d.DisableDynamicMemory = flags.Bool("hyperv-disable-dynamic-memory") + d.SetSwarmConfigFromFlags(flags) + + return nil +} + +func (d *Driver) GetSSHHostname() (string, error) { + return d.GetIP() +} + +// DriverName returns the name of the driver +func (d *Driver) DriverName() string { + return "hyperv" +} + +func (d *Driver) GetURL() (string, error) { + ip, err := d.GetIP() + if err != nil { + return "", err + } + + if ip == "" { + return "", nil + } + + return fmt.Sprintf("tcp://%s", net.JoinHostPort(ip, "2376")), nil +} + +func (d *Driver) GetState() (state.State, error) { + stdout, err := cmdOut("(", "Hyper-V\\Get-VM", d.MachineName, ").state") + if err != nil { + return state.None, fmt.Errorf("Failed to find the VM status") + } + + resp := parseLines(stdout) + if len(resp) < 1 { + return state.None, nil + } + + switch resp[0] { + case "Running": + return state.Running, nil + case "Off": + return state.Stopped, nil + default: + return state.None, nil + } +} + +// PreCreateCheck checks that the machine creation process can be started safely. +func (d *Driver) PreCreateCheck() error { + // Check that powershell was found + if powershell == "" { + return ErrPowerShellNotFound + } + + // Check that hyperv is installed + if err := hypervAvailable(); err != nil { + return err + } + + // Check that the user is an Administrator + isAdmin, err := isAdministrator() + if err != nil { + return err + } + if !isAdmin { + return ErrNotAdministrator + } + + // Check that there is a virtual switch already configured + if _, err := d.chooseVirtualSwitch(); err != nil { + return err + } + + // Downloading boot2docker to cache should be done here to make sure + // that a download failure will not leave a machine half created. + b2dutils := mcnutils.NewB2dUtils(d.StorePath) + err = b2dutils.UpdateISOCache(d.Boot2DockerURL) + return err +} + +func (d *Driver) Create() error { + b2dutils := mcnutils.NewB2dUtils(d.StorePath) + if err := b2dutils.CopyIsoToMachineDir(d.Boot2DockerURL, d.MachineName); err != nil { + return err + } + + log.Infof("Creating SSH key...") + if err := ssh.GenerateSSHKey(d.GetSSHKeyPath()); err != nil { + return err + } + + log.Infof("Creating VM...") + if d.VSwitch == "" { + defaultVSwitch, err := d.chooseVirtualSwitch() + if err != nil { + return err + } + d.VSwitch = defaultVSwitch + } + log.Infof("Using switch %q", d.VSwitch) + + diskImage, err := d.generateDiskImage() + if err != nil { + return err + } + + if err := cmd("Hyper-V\\New-VM", + d.MachineName, + "-Path", fmt.Sprintf("'%s'", d.ResolveStorePath(".")), + "-SwitchName", quote(d.VSwitch), + "-MemoryStartupBytes", toMb(d.MemSize)); err != nil { + return err + } + if d.DisableDynamicMemory { + if err := cmd("Hyper-V\\Set-VMMemory", + "-VMName", d.MachineName, + "-DynamicMemoryEnabled", "$false"); err != nil { + return err + } + } + + if d.CPU > 1 { + if err := cmd("Hyper-V\\Set-VMProcessor", + d.MachineName, + "-Count", fmt.Sprintf("%d", d.CPU)); err != nil { + return err + } + } + + if d.MacAddr != "" { + if err := cmd("Hyper-V\\Set-VMNetworkAdapter", + "-VMName", d.MachineName, + "-StaticMacAddress", fmt.Sprintf("\"%s\"", d.MacAddr)); err != nil { + return err + } + } + + if d.VLanID > 0 { + if err := cmd("Hyper-V\\Set-VMNetworkAdapterVlan", + "-VMName", d.MachineName, + "-Access", + "-VlanId", fmt.Sprintf("%d", d.VLanID)); err != nil { + return err + } + } + + if err := cmd("Hyper-V\\Set-VMDvdDrive", + "-VMName", d.MachineName, + "-Path", quote(d.ResolveStorePath("boot2docker.iso"))); err != nil { + return err + } + + if err := cmd("Hyper-V\\Add-VMHardDiskDrive", + "-VMName", d.MachineName, + "-Path", quote(diskImage)); err != nil { + return err + } + + log.Infof("Starting VM...") + return d.Start() +} + +func (d *Driver) chooseVirtualSwitch() (string, error) { + type Switch struct { + ID string + Name string + SwitchType int + } + + getHyperVSwitches := func(filters []string) ([]Switch, error) { + cmd := []string{"Hyper-V\\Get-VMSwitch", "Select Id, Name, SwitchType"} + cmd = append(cmd, filters...) + stdout, err := cmdOut(fmt.Sprintf("[Console]::OutputEncoding = [Text.Encoding]::UTF8; ConvertTo-Json @(%s)", strings.Join(cmd, "|"))) + if err != nil { + return nil, err + } + + var switches []Switch + err = json.Unmarshal([]byte(strings.NewReplacer("\r", "").Replace(stdout)), &switches) + if err != nil { + return nil, err + } + + return switches, nil + } + + if d.VSwitch == "" { + // prefer Default Switch over external switches + switches, err := getHyperVSwitches([]string{fmt.Sprintf("Where-Object {($_.SwitchType -eq 'External') -or ($_.Id -eq '%s')}", defaultSwitchID), "Sort-Object -Property SwitchType"}) + if err != nil { + return "", fmt.Errorf("unable to get available hyperv switches") + } + + if len(switches) < 1 { + return "", fmt.Errorf("no External vswitch nor Default Switch found. A valid vswitch must be available for this command to run. Check https://docs.docker.com/machine/drivers/hyper-v/") + } + + return switches[0].Name, nil + } + + // prefer external switches (using descending order) + switches, err := getHyperVSwitches([]string{fmt.Sprintf("Where-Object {$_.Name -eq '%s'}", d.VSwitch), "Sort-Object -Property SwitchType -Descending"}) + if err != nil { + return "", fmt.Errorf("unable to get available hyperv switches") + } + + if len(switches) < 1 { + return "", fmt.Errorf("vswitch %q not found", d.VSwitch) + } + + return switches[0].Name, nil +} + +// waitForIP waits until the host has a valid IP +func (d *Driver) waitForIP() (string, error) { + log.Infof("Waiting for host to start...") + + for { + ip, _ := d.GetIP() + if ip != "" { + return ip, nil + } + + time.Sleep(1 * time.Second) + } +} + +// waitStopped waits until the host is stopped +func (d *Driver) waitStopped() error { + log.Infof("Waiting for host to stop...") + + for { + s, err := d.GetState() + if err != nil { + return err + } + + if s != state.Running { + return nil + } + + time.Sleep(1 * time.Second) + } +} + +// Start starts an host +func (d *Driver) Start() error { + if err := cmd("Hyper-V\\Start-VM", d.MachineName); err != nil { + return err + } + + ip, err := d.waitForIP() + if err != nil { + return err + } + + d.IPAddress = ip + + return nil +} + +// Stop stops an host +func (d *Driver) Stop() error { + if err := cmd("Hyper-V\\Stop-VM", d.MachineName); err != nil { + return err + } + + if err := d.waitStopped(); err != nil { + return err + } + + d.IPAddress = "" + + return nil +} + +// Remove removes an host +func (d *Driver) Remove() error { + s, err := d.GetState() + if err != nil { + return err + } + + if s == state.Running { + if err := d.Kill(); err != nil { + return err + } + } + + return cmd("Hyper-V\\Remove-VM", d.MachineName, "-Force") +} + +// Restart stops and starts an host +func (d *Driver) Restart() error { + err := d.Stop() + if err != nil { + return err + } + + return d.Start() +} + +// Kill force stops an host +func (d *Driver) Kill() error { + if err := cmd("Hyper-V\\Stop-VM", d.MachineName, "-TurnOff"); err != nil { + return err + } + + if err := d.waitStopped(); err != nil { + return err + } + + d.IPAddress = "" + + return nil +} + +func (d *Driver) GetIP() (string, error) { + s, err := d.GetState() + if err != nil { + return "", err + } + if s != state.Running { + return "", drivers.ErrHostIsNotRunning + } + + stdout, err := cmdOut("((", "Hyper-V\\Get-VM", d.MachineName, ").networkadapters[0]).ipaddresses[0]") + if err != nil { + return "", err + } + + resp := parseLines(stdout) + if len(resp) < 1 { + return "", fmt.Errorf("IP not found") + } + + return resp[0], nil +} + +func (d *Driver) publicSSHKeyPath() string { + return d.GetSSHKeyPath() + ".pub" +} + +// generateDiskImage creates a small fixed vhd, put the tar in, convert to dynamic, then resize +func (d *Driver) generateDiskImage() (string, error) { + diskImage := d.ResolveStorePath("disk.vhd") + fixed := d.ResolveStorePath("fixed.vhd") + + // Resizing vhds requires administrator privileges + // incase the user is only a hyper-v admin then create the disk at the target size to avoid resizing. + isWindowsAdmin, err := isWindowsAdministrator() + if err != nil { + return "", err + } + fixedDiskSize := "10MB" + if !isWindowsAdmin { + fixedDiskSize = toMb(d.DiskSize) + } + + log.Infof("Creating VHD") + if err := cmd("Hyper-V\\New-VHD", "-Path", quote(fixed), "-SizeBytes", fixedDiskSize, "-Fixed"); err != nil { + return "", err + } + + tarBuf, err := mcnutils.MakeDiskImage(d.publicSSHKeyPath()) + if err != nil { + return "", err + } + + file, err := os.OpenFile(fixed, os.O_WRONLY, 0644) + if err != nil { + return "", err + } + defer file.Close() + + file.Seek(0, os.SEEK_SET) + _, err = file.Write(tarBuf.Bytes()) + if err != nil { + return "", err + } + file.Close() + + if err := cmd("Hyper-V\\Convert-VHD", "-Path", quote(fixed), "-DestinationPath", quote(diskImage), "-VHDType", "Dynamic", "-DeleteSource"); err != nil { + return "", err + } + + if isWindowsAdmin { + if err := cmd("Hyper-V\\Resize-VHD", "-Path", quote(diskImage), "-SizeBytes", toMb(d.DiskSize)); err != nil { + return "", err + } + } + + return diskImage, nil +} diff --git a/pkg/drivers/hyperv/powershell.go b/pkg/drivers/hyperv/powershell.go new file mode 100644 index 000000000000..e916aaa0579d --- /dev/null +++ b/pkg/drivers/hyperv/powershell.go @@ -0,0 +1,114 @@ +package hyperv + +import ( + "bufio" + "bytes" + "errors" + "os/exec" + "strings" + + "fmt" + + "k8s.io/minikube/pkg/libmachine/log" +) + +var powershell string + +var ( + ErrPowerShellNotFound = errors.New("Powershell was not found in the path") + ErrNotAdministrator = errors.New("Hyper-v commands have to be run as an Administrator") + ErrNotInstalled = errors.New("Hyper-V PowerShell Module is not available") +) + +func init() { + powershell, _ = exec.LookPath("powershell.exe") +} + +func cmdOut(args ...string) (string, error) { + args = append([]string{"-NoProfile", "-NonInteractive"}, args...) + cmd := exec.Command(powershell, args...) + log.Debugf("[executing ==>] : %v %v", powershell, strings.Join(args, " ")) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + log.Debugf("[stdout =====>] : %s", stdout.String()) + log.Debugf("[stderr =====>] : %s", stderr.String()) + return stdout.String(), err +} + +func cmd(args ...string) error { + _, err := cmdOut(args...) + return err +} + +func parseLines(stdout string) []string { + resp := []string{} + + s := bufio.NewScanner(strings.NewReader(stdout)) + for s.Scan() { + resp = append(resp, s.Text()) + } + + return resp +} + +func hypervAvailable() error { + stdout, err := cmdOut("@(Get-Module -ListAvailable hyper-v).Name | Get-Unique") + if err != nil { + return err + } + + resp := parseLines(stdout) + if resp == nil || len(resp) == 0 || resp[0] != "Hyper-V" { + return ErrNotInstalled + } + + return nil +} + +func isAdministrator() (bool, error) { + hypervAdmin := isHypervAdministrator() + + if hypervAdmin { + return true, nil + } + + windowsAdmin, err := isWindowsAdministrator() + + if err != nil { + return false, err + } + + return windowsAdmin, nil +} + +func isHypervAdministrator() bool { + stdout, err := cmdOut(`@([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole(([System.Security.Principal.SecurityIdentifier]::new("S-1-5-32-578")))`) + if err != nil { + log.Debug(err) + return false + } + + resp := parseLines(stdout) + return resp[0] == "True" +} + +func isWindowsAdministrator() (bool, error) { + stdout, err := cmdOut(`@([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")`) + if err != nil { + return false, err + } + + resp := parseLines(stdout) + return resp[0] == "True", nil +} + +func quote(text string) string { + return fmt.Sprintf("'%s'", text) +} + +func toMb(value int) string { + return fmt.Sprintf("%dMB", value) +} diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index f8e35de5f51d..3d6ad0c5590f 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -27,9 +27,9 @@ import ( "sync" "time" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/ssh" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 6370cb30ad48..2de2f7d6af66 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -30,7 +30,7 @@ import ( "time" "github.com/blang/semver/v4" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" diff --git a/pkg/drivers/krunkit/krunkit.go b/pkg/drivers/krunkit/krunkit.go index cbbeadb573cb..2ce5a5e50217 100644 --- a/pkg/drivers/krunkit/krunkit.go +++ b/pkg/drivers/krunkit/krunkit.go @@ -33,11 +33,11 @@ import ( "syscall" "time" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/log" - "github.com/docker/machine/libmachine/mcnutils" - "github.com/docker/machine/libmachine/ssh" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" diff --git a/pkg/drivers/kvm/domain.go b/pkg/drivers/kvm/domain.go index eb9602e7cdb4..6439c00509db 100644 --- a/pkg/drivers/kvm/domain.go +++ b/pkg/drivers/kvm/domain.go @@ -23,7 +23,7 @@ import ( "fmt" "text/template" - "github.com/docker/machine/libmachine/log" + "k8s.io/minikube/pkg/libmachine/log" "github.com/pkg/errors" "libvirt.org/go/libvirt" ) diff --git a/pkg/drivers/kvm/gpu.go b/pkg/drivers/kvm/gpu.go index 5e981efc83f8..e44459891d42 100644 --- a/pkg/drivers/kvm/gpu.go +++ b/pkg/drivers/kvm/gpu.go @@ -26,7 +26,7 @@ import ( "strings" "text/template" - "github.com/docker/machine/libmachine/log" + "k8s.io/minikube/pkg/libmachine/log" ) var sysFsPCIDevicesPath = "/sys/bus/pci/devices/" diff --git a/pkg/drivers/kvm/kvm.go b/pkg/drivers/kvm/kvm.go index 11a107e2662b..b1ea2993fb8d 100644 --- a/pkg/drivers/kvm/kvm.go +++ b/pkg/drivers/kvm/kvm.go @@ -25,9 +25,9 @@ import ( "syscall" "time" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/log" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/minikube/pkg/drivers/common" "k8s.io/minikube/pkg/util/retry" diff --git a/pkg/drivers/kvm/network.go b/pkg/drivers/kvm/network.go index 6e407c2dee88..7d217936206c 100644 --- a/pkg/drivers/kvm/network.go +++ b/pkg/drivers/kvm/network.go @@ -26,7 +26,7 @@ import ( "text/template" "time" - "github.com/docker/machine/libmachine/log" + "k8s.io/minikube/pkg/libmachine/log" "github.com/pkg/errors" "k8s.io/minikube/pkg/network" "k8s.io/minikube/pkg/util/retry" diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go index d3fa25e3dede..18bfb0146fce 100644 --- a/pkg/drivers/none/none.go +++ b/pkg/drivers/none/none.go @@ -20,8 +20,8 @@ import ( "fmt" "os/exec" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" knet "k8s.io/apimachinery/pkg/util/net" "k8s.io/klog/v2" diff --git a/pkg/drivers/qemu/qemu.go b/pkg/drivers/qemu/qemu.go index 763a23158442..3f22632c639a 100644 --- a/pkg/drivers/qemu/qemu.go +++ b/pkg/drivers/qemu/qemu.go @@ -33,11 +33,11 @@ import ( "syscall" "time" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/log" - "github.com/docker/machine/libmachine/mcnutils" - "github.com/docker/machine/libmachine/ssh" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" diff --git a/pkg/drivers/ssh/ssh.go b/pkg/drivers/ssh/ssh.go index 9c4a7cb9ef7d..3849770e151d 100644 --- a/pkg/drivers/ssh/ssh.go +++ b/pkg/drivers/ssh/ssh.go @@ -28,11 +28,11 @@ import ( "golang.org/x/crypto/ssh" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/engine" - "github.com/docker/machine/libmachine/log" - "github.com/docker/machine/libmachine/mcnutils" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/common" diff --git a/pkg/drivers/vfkit/vfkit.go b/pkg/drivers/vfkit/vfkit.go index 88fbd96ad006..796bff13b25f 100644 --- a/pkg/drivers/vfkit/vfkit.go +++ b/pkg/drivers/vfkit/vfkit.go @@ -35,11 +35,11 @@ import ( "syscall" "time" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/log" - "github.com/docker/machine/libmachine/mcnutils" - "github.com/docker/machine/libmachine/ssh" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" diff --git a/pkg/drivers/virtualbox/disk.go b/pkg/drivers/virtualbox/disk.go new file mode 100644 index 000000000000..01083ce1b6a8 --- /dev/null +++ b/pkg/drivers/virtualbox/disk.go @@ -0,0 +1,141 @@ +package virtualbox + +import ( + "fmt" + "io" + "os" + "os/exec" + + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnutils" +) + +type VirtualDisk struct { + UUID string + Path string +} + +type DiskCreator interface { + Create(size int, publicSSHKeyPath, diskPath string) error +} + +func NewDiskCreator() DiskCreator { + return &defaultDiskCreator{} +} + +type defaultDiskCreator struct{} + +// Make a boot2docker VM disk image. +func (c *defaultDiskCreator) Create(size int, publicSSHKeyPath, diskPath string) error { + log.Debugf("Creating %d MB hard disk image...", size) + + tarBuf, err := mcnutils.MakeDiskImage(publicSSHKeyPath) + if err != nil { + return err + } + + log.Debug("Calling inner createDiskImage") + + return createDiskImage(diskPath, size, tarBuf) +} + +// createDiskImage makes a disk image at dest with the given size in MB. If r is +// not nil, it will be read as a raw disk image to convert from. +func createDiskImage(dest string, size int, r io.Reader) error { + // Convert a raw image from stdin to the dest VMDK image. + sizeBytes := int64(size) << 20 // usually won't fit in 32-bit int (max 2GB) + // FIXME: why isn't this just using the vbm*() functions? + cmd := exec.Command(vboxManageCmd, "convertfromraw", "stdin", dest, + fmt.Sprintf("%d", sizeBytes), "--format", "VMDK") + + log.Debug(cmd) + + if os.Getenv("MACHINE_DEBUG") != "" { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } + + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + + log.Debug("Starting command") + + if err := cmd.Start(); err != nil { + return err + } + + log.Debug("Copying to stdin") + + n, err := io.Copy(stdin, r) + if err != nil { + return err + } + + log.Debug("Filling zeroes") + + // The total number of bytes written to stdin must match sizeBytes, or + // VBoxManage.exe on Windows will fail. Fill remaining with zeros. + if left := sizeBytes - n; left > 0 { + if err := zeroFill(stdin, left); err != nil { + return err + } + } + + log.Debug("Closing STDIN") + + // cmd won't exit until the stdin is closed. + if err := stdin.Close(); err != nil { + return err + } + + log.Debug("Waiting on cmd") + + return cmd.Wait() +} + +// zeroFill writes n zero bytes into w. +func zeroFill(w io.Writer, n int64) error { + const blocksize = 32 << 10 + zeros := make([]byte, blocksize) + var k int + var err error + for n > 0 { + if n > blocksize { + k, err = w.Write(zeros) + } else { + k, err = w.Write(zeros[:n]) + } + if err != nil { + return err + } + n -= int64(k) + } + return nil +} + +func getVMDiskInfo(name string, vbox VBoxManager) (*VirtualDisk, error) { + out, err := vbox.vbmOut("showvminfo", name, "--machinereadable") + if err != nil { + return nil, err + } + + disk := &VirtualDisk{} + + err = parseKeyValues(out, reEqualQuoteLine, func(key, val string) error { + switch key { + case "SATA-1-0": + disk.Path = val + case "SATA-ImageUUID-1-0": + disk.UUID = val + } + + return nil + }) + if err != nil { + return nil, err + } + + return disk, nil +} diff --git a/pkg/drivers/virtualbox/ip.go b/pkg/drivers/virtualbox/ip.go new file mode 100644 index 000000000000..3ee0a1892304 --- /dev/null +++ b/pkg/drivers/virtualbox/ip.go @@ -0,0 +1,36 @@ +package virtualbox + +import ( + "time" + + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/mcnutils" +) + +// IPWaiter waits for an IP to be configured. +type IPWaiter interface { + Wait(d *Driver) error +} + +func NewIPWaiter() IPWaiter { + return &sshIPWaiter{} +} + +type sshIPWaiter struct{} + +func (w *sshIPWaiter) Wait(d *Driver) error { + // Wait for SSH over NAT to be available before returning to user + if err := drivers.WaitForSSH(d); err != nil { + return err + } + + // Bail if we don't get an IP from DHCP after a given number of seconds. + if err := mcnutils.WaitForSpecific(d.hostOnlyIPAvailable, 5, 4*time.Second); err != nil { + return err + } + + var err error + d.IPAddress, err = d.GetIP() + + return err +} diff --git a/pkg/drivers/virtualbox/misc.go b/pkg/drivers/virtualbox/misc.go new file mode 100644 index 000000000000..8e6770c6fccb --- /dev/null +++ b/pkg/drivers/virtualbox/misc.go @@ -0,0 +1,112 @@ +package virtualbox + +import ( + "bufio" + "math/rand" + "os" + + "time" + + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/ssh" +) + +// B2DUpdater describes the interactions with b2d. +type B2DUpdater interface { + UpdateISOCache(storePath, isoURL string) error + CopyIsoToMachineDir(storePath, machineName, isoURL string) error +} + +func NewB2DUpdater() B2DUpdater { + return &b2dUtilsUpdater{} +} + +type b2dUtilsUpdater struct{} + +func (u *b2dUtilsUpdater) CopyIsoToMachineDir(storePath, machineName, isoURL string) error { + return mcnutils.NewB2dUtils(storePath).CopyIsoToMachineDir(isoURL, machineName) +} + +func (u *b2dUtilsUpdater) UpdateISOCache(storePath, isoURL string) error { + return mcnutils.NewB2dUtils(storePath).UpdateISOCache(isoURL) +} + +// SSHKeyGenerator describes the generation of ssh keys. +type SSHKeyGenerator interface { + Generate(path string) error +} + +func NewSSHKeyGenerator() SSHKeyGenerator { + return &defaultSSHKeyGenerator{} +} + +type defaultSSHKeyGenerator struct{} + +func (g *defaultSSHKeyGenerator) Generate(path string) error { + return ssh.GenerateSSHKey(path) +} + +// LogsReader describes the reading of VBox.log +type LogsReader interface { + Read(path string) ([]string, error) +} + +func NewLogsReader() LogsReader { + return &vBoxLogsReader{} +} + +type vBoxLogsReader struct{} + +func (c *vBoxLogsReader) Read(path string) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return []string{}, err + } + + defer file.Close() + + lines := []string{} + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + + return lines, nil +} + +// RandomInter returns random int values. +type RandomInter interface { + RandomInt(n int) int +} + +func NewRandomInter() RandomInter { + src := rand.NewSource(time.Now().UnixNano()) + + return &defaultRandomInter{ + rand: rand.New(src), + } +} + +type defaultRandomInter struct { + rand *rand.Rand +} + +func (d *defaultRandomInter) RandomInt(n int) int { + return d.rand.Intn(n) +} + +// Sleeper sleeps for given duration. +type Sleeper interface { + Sleep(d time.Duration) +} + +func NewSleeper() Sleeper { + return &defaultSleeper{} +} + +type defaultSleeper struct{} + +func (s *defaultSleeper) Sleep(d time.Duration) { + time.Sleep(d) +} diff --git a/pkg/drivers/virtualbox/network.go b/pkg/drivers/virtualbox/network.go new file mode 100644 index 000000000000..ac172066ad18 --- /dev/null +++ b/pkg/drivers/virtualbox/network.go @@ -0,0 +1,415 @@ +package virtualbox + +import ( + "errors" + "fmt" + "net" + "regexp" + "strings" + "time" + + "runtime" + + "k8s.io/minikube/pkg/libmachine/log" +) + +const ( + buggyNetmask = "0f000000" + dhcpPrefix = "HostInterfaceNetworking-" +) + +var ( + reHostOnlyAdapterCreated = regexp.MustCompile(`Interface '(.+)' was successfully created`) + errNewHostOnlyAdapterNotVisible = errors.New("The host-only adapter we just created is not visible. This is a well known VirtualBox bug. You might want to uninstall it and reinstall at least version 5.0.12 that is is supposed to fix this issue") +) + +// Host-only network. +type hostOnlyNetwork struct { + Name string + GUID string + DHCP bool + IPv4 net.IPNet + HwAddr net.HardwareAddr + Medium string + Status string + NetworkName string // referenced in DHCP.NetworkName +} + +// HostInterfaces returns host network interface info. By default delegates to net.Interfaces() +type HostInterfaces interface { + Interfaces() ([]net.Interface, error) + Addrs(iface *net.Interface) ([]net.Addr, error) +} + +func NewHostInterfaces() HostInterfaces { + return &defaultHostInterfaces{} +} + +type defaultHostInterfaces struct { +} + +func (ni *defaultHostInterfaces) Interfaces() ([]net.Interface, error) { + return net.Interfaces() +} + +func (ni *defaultHostInterfaces) Addrs(iface *net.Interface) ([]net.Addr, error) { + return iface.Addrs() +} + +// Save changes the configuration of the host-only network. +func (n *hostOnlyNetwork) Save(vbox VBoxManager) error { + if err := n.SaveIPv4(vbox); err != nil { + return err + } + + if n.DHCP { + vbox.vbm("hostonlyif", "ipconfig", n.Name, "--dhcp") // not implemented as of VirtualBox 4.3 + } + + return nil +} + +// SaveIPv4 changes the ipv4 configuration of the host-only network. +func (n *hostOnlyNetwork) SaveIPv4(vbox VBoxManager) error { + if n.IPv4.IP != nil && n.IPv4.Mask != nil { + if runtime.GOOS == "windows" { + log.Warn("Windows might ask for the permission to configure a network adapter. Sometimes, such confirmation window is minimized in the taskbar.") + } + + if err := vbox.vbm("hostonlyif", "ipconfig", n.Name, "--ip", n.IPv4.IP.String(), "--netmask", net.IP(n.IPv4.Mask).String()); err != nil { + return err + } + } + + return nil +} + +// createHostonlyAdapter creates a new host-only network. +func createHostonlyAdapter(vbox VBoxManager) (*hostOnlyNetwork, error) { + if runtime.GOOS == "windows" { + log.Warn("Windows might ask for the permission to create a network adapter. Sometimes, such confirmation window is minimized in the taskbar.") + } + + out, err := vbox.vbmOut("hostonlyif", "create") + if err != nil { + return nil, err + } + + res := reHostOnlyAdapterCreated.FindStringSubmatch(string(out)) + if res == nil { + return nil, errors.New("Failed to create host-only adapter") + } + + return &hostOnlyNetwork{Name: res[1]}, nil +} + +// listHostOnlyAdapters gets all host-only adapters in a map keyed by NetworkName. +func listHostOnlyAdapters(vbox VBoxManager) (map[string]*hostOnlyNetwork, error) { + out, err := vbox.vbmOut("list", "hostonlyifs") + if err != nil { + return nil, err + } + + byName := map[string]*hostOnlyNetwork{} + byIP := map[string]*hostOnlyNetwork{} + n := &hostOnlyNetwork{} + + err = parseKeyValues(out, reColonLine, func(key, val string) error { + switch key { + case "Name": + n.Name = val + case "GUID": + n.GUID = val + case "DHCP": + n.DHCP = (val != "Disabled") + case "IPAddress": + n.IPv4.IP = net.ParseIP(val) + case "NetworkMask": + n.IPv4.Mask = parseIPv4Mask(val) + case "HardwareAddress": + mac, err := net.ParseMAC(val) + if err != nil { + return err + } + n.HwAddr = mac + case "MediumType": + n.Medium = val + case "Status": + n.Status = val + case "VBoxNetworkName": + n.NetworkName = val + + if _, present := byName[n.NetworkName]; present { + return fmt.Errorf("VirtualBox is configured with multiple host-only adapters with the same name %q. Please remove one", n.NetworkName) + } + byName[n.NetworkName] = n + + if len(n.IPv4.IP) != 0 { + if _, present := byIP[n.IPv4.IP.String()]; present { + return fmt.Errorf("VirtualBox is configured with multiple host-only adapters with the same IP %q. Please remove one", n.IPv4.IP) + } + byIP[n.IPv4.IP.String()] = n + } + + n = &hostOnlyNetwork{} + } + + return nil + }) + if err != nil { + return nil, err + } + + return byName, nil +} + +func getHostOnlyAdapter(nets map[string]*hostOnlyNetwork, hostIP net.IP, netmask net.IPMask) *hostOnlyNetwork { + for _, n := range nets { + // Second part of this conditional handles a race where + // VirtualBox returns us the incorrect netmask value for the + // newly created adapter. + if hostIP.Equal(n.IPv4.IP) && + (netmask.String() == n.IPv4.Mask.String() || n.IPv4.Mask.String() == buggyNetmask) { + log.Debugf("Found: %s", n.Name) + return n + } + } + + log.Debug("Not found") + return nil +} + +func getOrCreateHostOnlyNetwork(hostIP net.IP, netmask net.IPMask, nets map[string]*hostOnlyNetwork, vbox VBoxManager) (*hostOnlyNetwork, error) { + // Search for an existing host-only adapter. + hostOnlyAdapter := getHostOnlyAdapter(nets, hostIP, netmask) + if hostOnlyAdapter != nil { + return hostOnlyAdapter, nil + } + + // No existing host-only adapter found. Create a new one. + _, err := createHostonlyAdapter(vbox) + if err != nil { + // Sometimes the host-only adapter fails to create. See https://www.virtualbox.org/ticket/14040 + // BUT, it is created in fact! So let's wait until it appears last in the list + log.Warnf("Creating a new host-only adapter produced an error: %s", err) + log.Warn("This is a known VirtualBox bug. Let's try to recover anyway...") + } + + // It can take some time for an adapter to appear. Let's poll. + hostOnlyAdapter, err = waitForNewHostOnlyNetwork(nets, vbox) + if err != nil { + // Sometimes, Vbox says it created it but then it cannot be found... + return nil, errNewHostOnlyAdapterNotVisible + } + + log.Warnf("Found a new host-only adapter: %q", hostOnlyAdapter.Name) + + hostOnlyAdapter.IPv4.IP = hostIP + hostOnlyAdapter.IPv4.Mask = netmask + if err := hostOnlyAdapter.Save(vbox); err != nil { + return nil, err + } + + return hostOnlyAdapter, nil +} + +func waitForNewHostOnlyNetwork(oldNets map[string]*hostOnlyNetwork, vbox VBoxManager) (*hostOnlyNetwork, error) { + for i := 0; i < 10; i++ { + time.Sleep(1 * time.Second) + + newNets, err := listHostOnlyAdapters(vbox) + if err != nil { + return nil, err + } + + for name, latestNet := range newNets { + if _, present := oldNets[name]; !present { + return latestNet, nil + } + } + } + + return nil, errors.New("Failed to find a new host-only adapter") +} + +// DHCP server info. +type dhcpServer struct { + NetworkName string + IPv4 net.IPNet + LowerIP net.IP + UpperIP net.IP + Enabled bool +} + +// removeOrphanDHCPServers removed the DHCP servers linked to no host-only adapter +func removeOrphanDHCPServers(vbox VBoxManager) error { + dhcps, err := listDHCPServers(vbox) + if err != nil { + return err + } + + if len(dhcps) == 0 { + return nil + } + + log.Debug("Removing orphan DHCP servers...") + + nets, err := listHostOnlyAdapters(vbox) + if err != nil { + return err + } + + for name := range dhcps { + if strings.HasPrefix(name, dhcpPrefix) { + if _, present := nets[name]; !present { + if err := vbox.vbm("dhcpserver", "remove", "--netname", name); err != nil { + log.Warnf("Unable to remove orphan dhcp server %q: %s", name, err) + } + } + } + } + + return nil +} + +// addHostOnlyDHCPServer adds a DHCP server to a host-only network. +func addHostOnlyDHCPServer(ifname string, d dhcpServer, vbox VBoxManager) error { + name := dhcpPrefix + ifname + + dhcps, err := listDHCPServers(vbox) + if err != nil { + return err + } + + // On some platforms (OSX), creating a host-only adapter adds a default dhcpserver, + // while on others (Windows?) it does not. + command := "add" + if dhcp, ok := dhcps[name]; ok { + command = "modify" + if (dhcp.IPv4.IP.Equal(d.IPv4.IP)) && (dhcp.IPv4.Mask.String() == d.IPv4.Mask.String()) && (dhcp.LowerIP.Equal(d.LowerIP)) && (dhcp.UpperIP.Equal(d.UpperIP)) && (dhcp.Enabled == d.Enabled) { + // dhcp is up to date + return nil + } + } + + args := []string{"dhcpserver", command, + "--netname", name, + "--ip", d.IPv4.IP.String(), + "--netmask", net.IP(d.IPv4.Mask).String(), + "--lowerip", d.LowerIP.String(), + "--upperip", d.UpperIP.String(), + } + if d.Enabled { + args = append(args, "--enable") + } else { + args = append(args, "--disable") + } + + if runtime.GOOS == "windows" { + log.Warn("Windows might ask for the permission to configure a dhcp server. Sometimes, such confirmation window is minimized in the taskbar.") + } + + return vbox.vbm(args...) +} + +// listDHCPServers lists all DHCP server settings in a map keyed by DHCP.NetworkName. +func listDHCPServers(vbox VBoxManager) (map[string]*dhcpServer, error) { + out, err := vbox.vbmOut("list", "dhcpservers") + if err != nil { + return nil, err + } + + m := map[string]*dhcpServer{} + dhcp := &dhcpServer{} + + err = parseKeyValues(out, reColonLine, func(key, val string) error { + switch key { + case "NetworkName": + dhcp = &dhcpServer{} + m[val] = dhcp + dhcp.NetworkName = val + case "IP": + dhcp.IPv4.IP = net.ParseIP(val) + case "upperIPAddress": + dhcp.UpperIP = net.ParseIP(val) + case "lowerIPAddress": + dhcp.LowerIP = net.ParseIP(val) + case "NetworkMask": + dhcp.IPv4.Mask = parseIPv4Mask(val) + case "Enabled": + dhcp.Enabled = (val == "Yes") + } + + return nil + }) + if err != nil { + return nil, err + } + + return m, nil +} + +// listHostInterfaces returns a map of net.IPNet addresses of host interfaces that are "UP" and not loopback adapters +// and not virtualbox host-only networks (given by excludeNets), keyed by CIDR string. +func listHostInterfaces(hif HostInterfaces, excludeNets map[string]*hostOnlyNetwork) (map[string]*net.IPNet, error) { + ifaces, err := hif.Interfaces() + if err != nil { + return nil, err + } + m := map[string]*net.IPNet{} + + for _, iface := range ifaces { + addrs, err := hif.Addrs(&iface) + if err != nil { + return nil, err + } + + // Check if an address of the interface is in the list of excluded addresses + ifaceExcluded := false + for _, a := range addrs { + switch ipnet := a.(type) { + case *net.IPNet: + _, excluded := excludeNets[ipnet.String()] + if excluded { + ifaceExcluded = true + break + } + } + } + + // If excluded, or not up, or a loopback interface, skip the interface + if ifaceExcluded || iface.Flags&net.FlagUp == 0 || iface.Flags&net.FlagLoopback != 0 { + continue + } + + // This is a host interface, so add all its addresses to the map + for _, a := range addrs { + switch ipnet := a.(type) { + case *net.IPNet: + m[ipnet.String()] = ipnet + } + } + } + return m, nil +} + +// checkIPNetCollision returns true if any host interfaces conflict with the host-only network mask passed as a parameter. +// This works with IPv4 or IPv6 ip addresses. +func checkIPNetCollision(hostonly *net.IPNet, hostIfaces map[string]*net.IPNet) (bool, error) { + for _, ifaceNet := range hostIfaces { + if hostonly.IP.Equal(ifaceNet.IP.Mask(ifaceNet.Mask)) { + return true, nil + } + } + return false, nil +} + +// parseIPv4Mask parses IPv4 netmask written in IP form (e.g. 255.255.255.0). +// This function should really belong to the net package. +func parseIPv4Mask(s string) net.IPMask { + mask := net.ParseIP(s) + if mask == nil { + return nil + } + return net.IPv4Mask(mask[12], mask[13], mask[14], mask[15]) +} diff --git a/pkg/drivers/virtualbox/vbm.go b/pkg/drivers/virtualbox/vbm.go new file mode 100644 index 000000000000..e1ba9c4bd219 --- /dev/null +++ b/pkg/drivers/virtualbox/vbm.go @@ -0,0 +1,165 @@ +package virtualbox + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "os/exec" + "regexp" + "strings" + + "strconv" + + "time" + + "k8s.io/minikube/pkg/libmachine/log" +) + +const ( + retryCountOnObjectNotReadyError = 5 + objectNotReady = "error: The object is not ready" + retryDelay = 100 * time.Millisecond +) + +var ( + reColonLine = regexp.MustCompile(`(.+):\s+(.*)`) + reEqualLine = regexp.MustCompile(`(.+)=(.*)`) + reEqualQuoteLine = regexp.MustCompile(`"(.+)"="(.*)"`) + reMachineNotFound = regexp.MustCompile(`Could not find a registered machine named '(.+)'`) + + ErrMachineNotExist = errors.New("machine does not exist") + ErrVBMNotFound = errors.New("VBoxManage not found. Make sure VirtualBox is installed and VBoxManage is in the path") + + vboxManageCmd = detectVBoxManageCmd() +) + +// VBoxManager defines the interface to communicate to VirtualBox. +type VBoxManager interface { + vbm(args ...string) error + + vbmOut(args ...string) (string, error) + + vbmOutErr(args ...string) (string, string, error) +} + +// VBoxCmdManager communicates with VirtualBox through the commandline using `VBoxManage`. +type VBoxCmdManager struct { + runCmd func(cmd *exec.Cmd) error +} + +// NewVBoxManager creates a VBoxManager instance. +func NewVBoxManager() *VBoxCmdManager { + return &VBoxCmdManager{ + runCmd: func(cmd *exec.Cmd) error { return cmd.Run() }, + } +} + +func (v *VBoxCmdManager) vbm(args ...string) error { + _, _, err := v.vbmOutErr(args...) + return err +} + +func (v *VBoxCmdManager) vbmOut(args ...string) (string, error) { + stdout, _, err := v.vbmOutErr(args...) + return stdout, err +} + +func (v *VBoxCmdManager) vbmOutErr(args ...string) (string, string, error) { + return v.vbmOutErrRetry(retryCountOnObjectNotReadyError, args...) +} + +func (v *VBoxCmdManager) vbmOutErrRetry(retry int, args ...string) (string, string, error) { + cmd := exec.Command(vboxManageCmd, args...) + log.Debugf("COMMAND: %v %v", vboxManageCmd, strings.Join(args, " ")) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := v.runCmd(cmd) + stderrStr := stderr.String() + if len(args) > 0 { + log.Debugf("STDOUT:\n{\n%v}", stdout.String()) + log.Debugf("STDERR:\n{\n%v}", stderrStr) + } + + if err != nil { + if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound { + err = ErrVBMNotFound + } + } + + // Sometimes, we just need to retry... + if retry > 1 { + if strings.Contains(stderrStr, objectNotReady) { + time.Sleep(retryDelay) + return v.vbmOutErrRetry(retry-1, args...) + } + } + + if err == nil || strings.HasPrefix(err.Error(), "exit status ") { + // VBoxManage will sometimes not set the return code, but has a fatal error + // such as VBoxManage.exe: error: VT-x is not available. (VERR_VMX_NO_VMX) + if strings.Contains(stderrStr, "error:") { + err = fmt.Errorf("%v %v failed:\n%v", vboxManageCmd, strings.Join(args, " "), stderrStr) + } + } + + return stdout.String(), stderrStr, err +} + +func checkVBoxManageVersion(version string) error { + major, minor, err := parseVersion(version) + if (err != nil) || (major < 4) || (major == 4 && minor <= 2) { + return fmt.Errorf("We support Virtualbox starting with version 5. Your VirtualBox install is %q. Please upgrade at https://www.virtualbox.org", version) + } + + if major < 5 { + log.Warnf("You are using version %s of VirtualBox. If you encounter issues, you might want to upgrade to version 5 at https://www.virtualbox.org", version) + } + + return nil +} + +func parseVersion(version string) (int, int, error) { + parts := strings.Split(version, ".") + if len(parts) < 2 { + return 0, 0, fmt.Errorf("Invalid version: %q", version) + } + + major, err := strconv.Atoi(parts[0]) + if err != nil { + return 0, 0, fmt.Errorf("Invalid version: %q", version) + } + + minor, err := strconv.Atoi(parts[1]) + if err != nil { + return 0, 0, fmt.Errorf("Invalid version: %q", version) + } + + return major, minor, err +} + +func parseKeyValues(stdOut string, regexp *regexp.Regexp, callback func(key, val string) error) error { + r := strings.NewReader(stdOut) + s := bufio.NewScanner(r) + + for s.Scan() { + line := s.Text() + if line == "" { + continue + } + + res := regexp.FindStringSubmatch(line) + if res == nil { + continue + } + + key, val := res[1], res[2] + if err := callback(key, val); err != nil { + return err + } + } + + return s.Err() +} diff --git a/pkg/drivers/virtualbox/virtualbox.go b/pkg/drivers/virtualbox/virtualbox.go new file mode 100644 index 000000000000..d4304886319a --- /dev/null +++ b/pkg/drivers/virtualbox/virtualbox.go @@ -0,0 +1,1042 @@ +package virtualbox + +import ( + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "time" + + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnflag" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/state" +) + +const ( + defaultCPU = 1 + defaultMemory = 1024 + defaultBoot2DockerURL = "" + defaultBoot2DockerImportVM = "" + defaultHostOnlyCIDR = "192.168.99.1/24" + defaultHostOnlyNictype = "82540EM" + defaultHostOnlyPromiscMode = "deny" + defaultUIType = "headless" + defaultHostOnlyNoDHCP = false + defaultDiskSize = 20000 + defaultDNSProxy = true + defaultDNSResolver = false + defaultHostLoopbackReachable = true +) + +var ( + ErrUnableToGenerateRandomIP = errors.New("unable to generate random IP") + ErrMustEnableVTX = errors.New("This computer doesn't have VT-X/AMD-v enabled. Enabling it in the BIOS is mandatory") + ErrNotCompatibleWithHyperV = errors.New("This computer is running Hyper-V. VirtualBox won't boot a 64bits VM when Hyper-V is activated. Either use Hyper-V as a driver, or disable the Hyper-V hypervisor. (To skip this check, use --virtualbox-no-vtx-check)") + ErrNetworkAddrCidr = errors.New("host-only cidr must be specified with a host address, not a network address") + ErrNetworkAddrCollision = errors.New("host-only cidr conflicts with the network address of a host interface") +) + +type Driver struct { + *drivers.BaseDriver + VBoxManager + HostInterfaces + b2dUpdater B2DUpdater + sshKeyGenerator SSHKeyGenerator + diskCreator DiskCreator + logsReader LogsReader + ipWaiter IPWaiter + randomInter RandomInter + sleeper Sleeper + version int + CPU int + Memory int + DiskSize int + NatNicType string + Boot2DockerURL string + Boot2DockerImportVM string + HostDNSResolver bool + HostLoopbackReachable bool + HostOnlyCIDR string + HostOnlyNicType string + HostOnlyPromiscMode string + UIType string + HostOnlyNoDHCP bool + NoShare bool + DNSProxy bool + NoVTXCheck bool + ShareFolder string +} + +// NewDriver creates a new VirtualBox driver with default settings. +func NewDriver(hostName, storePath string) *Driver { + return &Driver{ + VBoxManager: NewVBoxManager(), + b2dUpdater: NewB2DUpdater(), + sshKeyGenerator: NewSSHKeyGenerator(), + diskCreator: NewDiskCreator(), + logsReader: NewLogsReader(), + ipWaiter: NewIPWaiter(), + randomInter: NewRandomInter(), + sleeper: NewSleeper(), + HostInterfaces: NewHostInterfaces(), + Memory: defaultMemory, + CPU: defaultCPU, + DiskSize: defaultDiskSize, + NatNicType: defaultHostOnlyNictype, + HostOnlyCIDR: defaultHostOnlyCIDR, + HostOnlyNicType: defaultHostOnlyNictype, + HostOnlyPromiscMode: defaultHostOnlyPromiscMode, + UIType: defaultUIType, + HostOnlyNoDHCP: defaultHostOnlyNoDHCP, + DNSProxy: defaultDNSProxy, + HostDNSResolver: defaultDNSResolver, + HostLoopbackReachable: defaultHostLoopbackReachable, + BaseDriver: &drivers.BaseDriver{ + MachineName: hostName, + StorePath: storePath, + }, + } +} + +// GetCreateFlags registers the flags this driver adds to +// "docker hosts create" +func (d *Driver) GetCreateFlags() []mcnflag.Flag { + return []mcnflag.Flag{ + mcnflag.IntFlag{ + Name: "virtualbox-memory", + Usage: "Size of memory for host in MB", + Value: defaultMemory, + EnvVar: "VIRTUALBOX_MEMORY_SIZE", + }, + mcnflag.IntFlag{ + Name: "virtualbox-cpu-count", + Usage: "number of CPUs for the machine (-1 to use the number of CPUs available)", + Value: defaultCPU, + EnvVar: "VIRTUALBOX_CPU_COUNT", + }, + mcnflag.IntFlag{ + Name: "virtualbox-disk-size", + Usage: "Size of disk for host in MB", + Value: defaultDiskSize, + EnvVar: "VIRTUALBOX_DISK_SIZE", + }, + mcnflag.StringFlag{ + Name: "virtualbox-boot2docker-url", + Usage: "The URL of the boot2docker image. Defaults to the latest available version", + Value: defaultBoot2DockerURL, + EnvVar: "VIRTUALBOX_BOOT2DOCKER_URL", + }, + mcnflag.StringFlag{ + Name: "virtualbox-import-boot2docker-vm", + Usage: "The name of a Boot2Docker VM to import", + Value: defaultBoot2DockerImportVM, + EnvVar: "VIRTUALBOX_BOOT2DOCKER_IMPORT_VM", + }, + mcnflag.BoolFlag{ + Name: "virtualbox-host-dns-resolver", + Usage: "Use the host DNS resolver", + EnvVar: "VIRTUALBOX_HOST_DNS_RESOLVER", + }, + mcnflag.BoolFlag{ + Name: "virtualbox-host-loopback-reachable", + Usage: "Enable host loopback interface accessibility", + EnvVar: "VIRTUALBOX_HOST_LOOPBACK_REACHABLE", + }, + mcnflag.StringFlag{ + Name: "virtualbox-nat-nictype", + Usage: "Specify the Network Adapter Type", + Value: defaultHostOnlyNictype, + EnvVar: "VIRTUALBOX_NAT_NICTYPE", + }, + mcnflag.StringFlag{ + Name: "virtualbox-hostonly-cidr", + Usage: "Specify the Host Only CIDR", + Value: defaultHostOnlyCIDR, + EnvVar: "VIRTUALBOX_HOSTONLY_CIDR", + }, + mcnflag.StringFlag{ + Name: "virtualbox-hostonly-nictype", + Usage: "Specify the Host Only Network Adapter Type", + Value: defaultHostOnlyNictype, + EnvVar: "VIRTUALBOX_HOSTONLY_NIC_TYPE", + }, + mcnflag.StringFlag{ + Name: "virtualbox-hostonly-nicpromisc", + Usage: "Specify the Host Only Network Adapter Promiscuous Mode", + Value: defaultHostOnlyPromiscMode, + EnvVar: "VIRTUALBOX_HOSTONLY_NIC_PROMISC", + }, + mcnflag.StringFlag{ + Name: "virtualbox-ui-type", + Usage: "Specify the UI Type: (gui|sdl|headless|separate)", + Value: defaultUIType, + EnvVar: "VIRTUALBOX_UI_TYPE", + }, + mcnflag.BoolFlag{ + Name: "virtualbox-hostonly-no-dhcp", + Usage: "Disable the Host Only DHCP Server", + EnvVar: "VIRTUALBOX_HOSTONLY_NO_DHCP", + }, + mcnflag.BoolFlag{ + Name: "virtualbox-no-share", + Usage: "Disable the mount of your home directory", + EnvVar: "VIRTUALBOX_NO_SHARE", + }, + mcnflag.BoolFlag{ + Name: "virtualbox-no-dns-proxy", + Usage: "Disable proxying all DNS requests to the host", + EnvVar: "VIRTUALBOX_NO_DNS_PROXY", + }, + mcnflag.BoolFlag{ + Name: "virtualbox-no-vtx-check", + Usage: "Disable checking for the availability of hardware virtualization before the vm is started", + EnvVar: "VIRTUALBOX_NO_VTX_CHECK", + }, + mcnflag.StringFlag{ + EnvVar: "VIRTUALBOX_SHARE_FOLDER", + Name: "virtualbox-share-folder", + Usage: "Mount the specified directory instead of the default home location. Format: dir:name", + }, + } +} + +func (d *Driver) GetSSHHostname() (string, error) { + return "127.0.0.1", nil +} + +func (d *Driver) GetSSHUsername() string { + if d.SSHUser == "" { + d.SSHUser = "docker" + } + + return d.SSHUser +} + +// DriverName returns the name of the driver +func (d *Driver) DriverName() string { + return "virtualbox" +} + +func (d *Driver) GetURL() (string, error) { + ip, err := d.GetIP() + if err != nil { + return "", err + } + if ip == "" { + return "", nil + } + return fmt.Sprintf("tcp://%s:2376", ip), nil +} + +func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error { + d.CPU = flags.Int("virtualbox-cpu-count") + d.Memory = flags.Int("virtualbox-memory") + d.DiskSize = flags.Int("virtualbox-disk-size") + d.Boot2DockerURL = flags.String("virtualbox-boot2docker-url") + d.SetSwarmConfigFromFlags(flags) + d.SSHUser = "docker" + d.Boot2DockerImportVM = flags.String("virtualbox-import-boot2docker-vm") + d.HostDNSResolver = flags.Bool("virtualbox-host-dns-resolver") + d.HostLoopbackReachable = flags.Bool("virtualbox-host-loopback-reachable") + d.NatNicType = flags.String("virtualbox-nat-nictype") + d.HostOnlyCIDR = flags.String("virtualbox-hostonly-cidr") + d.HostOnlyNicType = flags.String("virtualbox-hostonly-nictype") + d.HostOnlyPromiscMode = flags.String("virtualbox-hostonly-nicpromisc") + d.UIType = flags.String("virtualbox-ui-type") + d.HostOnlyNoDHCP = flags.Bool("virtualbox-hostonly-no-dhcp") + d.NoShare = flags.Bool("virtualbox-no-share") + d.DNSProxy = !flags.Bool("virtualbox-no-dns-proxy") + d.NoVTXCheck = flags.Bool("virtualbox-no-vtx-check") + d.ShareFolder = flags.String("virtualbox-share-folder") + + return nil +} + +// PreCreateCheck checks that VBoxManage exists and works +func (d *Driver) PreCreateCheck() error { + // Check that VBoxManage exists and works + version, err := d.vbmOut("--version") + if err != nil { + return err + } + + // Check that VBoxManage is of a supported version + if err = checkVBoxManageVersion(strings.TrimSpace(version)); err != nil { + return err + } + + d.version, _, _ = parseVersion(strings.TrimSpace(version)) + + if !d.NoVTXCheck { + if isHyperVInstalled() { + return ErrNotCompatibleWithHyperV + } + + if d.IsVTXDisabled() { + return ErrMustEnableVTX + } + } + + // Downloading boot2docker to cache should be done here to make sure + // that a download failure will not leave a machine half created. + if err := d.b2dUpdater.UpdateISOCache(d.StorePath, d.Boot2DockerURL); err != nil { + return err + } + + // Check that Host-only interfaces are ok + if _, err = listHostOnlyAdapters(d.VBoxManager); err != nil { + return err + } + + return nil +} + +func (d *Driver) Create() error { + if err := d.CreateVM(); err != nil { + return err + } + + log.Info("Starting the VM...") + return d.Start() +} + +func (d *Driver) CreateVM() error { + if err := d.b2dUpdater.CopyIsoToMachineDir(d.StorePath, d.MachineName, d.Boot2DockerURL); err != nil { + return err + } + + log.Info("Creating VirtualBox VM...") + + // import b2d VM if requested + if d.Boot2DockerImportVM != "" { + name := d.Boot2DockerImportVM + + // make sure vm is stopped + _ = d.vbm("controlvm", name, "poweroff") + + diskInfo, err := getVMDiskInfo(name, d.VBoxManager) + if err != nil { + return err + } + + if _, err := os.Stat(diskInfo.Path); err != nil { + return err + } + + if err := d.vbm("clonehd", diskInfo.Path, d.diskPath()); err != nil { + return err + } + + log.Debugf("Importing VM settings...") + vmInfo, err := getVMInfo(name, d.VBoxManager) + if err != nil { + return err + } + + d.CPU = vmInfo.CPUs + d.Memory = vmInfo.Memory + + log.Debugf("Importing SSH key...") + keyPath := filepath.Join(mcnutils.GetHomeDir(), ".ssh", "id_boot2docker") + if err := mcnutils.CopyFile(keyPath, d.GetSSHKeyPath()); err != nil { + return err + } + } else { + log.Infof("Creating SSH key...") + if err := d.sshKeyGenerator.Generate(d.GetSSHKeyPath()); err != nil { + return err + } + + log.Debugf("Creating disk image...") + if err := d.diskCreator.Create(d.DiskSize, d.publicSSHKeyPath(), d.diskPath()); err != nil { + return err + } + } + + if err := d.vbm("createvm", + "--basefolder", d.ResolveStorePath("."), + "--name", d.MachineName, + "--register"); err != nil { + return err + } + + log.Debugf("VM CPUS: %d", d.CPU) + log.Debugf("VM Memory: %d", d.Memory) + + cpus := d.CPU + if cpus < 1 { + cpus = int(runtime.NumCPU()) + } + if cpus > 32 { + cpus = 32 + } + + hostDNSResolver := "off" + if d.HostDNSResolver { + hostDNSResolver = "on" + } + + hostLoopbackReachable := "off" + if d.HostLoopbackReachable { + hostLoopbackReachable = "on" + } + + dnsProxy := "off" + if d.DNSProxy { + dnsProxy = "on" + } + + var modifyFlags = []string{ + "modifyvm", d.MachineName, + "--firmware", "bios", + "--bioslogofadein", "off", + "--bioslogofadeout", "off", + "--bioslogodisplaytime", "0", + "--biosbootmenu", "disabled", + "--ostype", "Linux26_64", + "--cpus", fmt.Sprintf("%d", cpus), + "--memory", fmt.Sprintf("%d", d.Memory), + "--acpi", "on", + "--ioapic", "on", + "--rtcuseutc", "on", + "--natdnshostresolver1", hostDNSResolver, + "--natdnsproxy1", dnsProxy, + "--cpuhotplug", "off", + "--pae", "on", + "--hpet", "on", + "--hwvirtex", "on", + "--nestedpaging", "on", + "--largepages", "on", + "--vtxvpid", "on", + "--accelerate3d", "off", + "--boot1", "dvd"} + + if d.version > 6 { + modifyFlags = append(modifyFlags, "--natlocalhostreachable1", hostLoopbackReachable) + } + + if runtime.GOOS == "windows" && runtime.GOARCH == "386" { + modifyFlags = append(modifyFlags, "--longmode", "on") + } + + if err := d.vbm(modifyFlags...); err != nil { + return err + } + + if err := d.vbm("modifyvm", d.MachineName, + "--nic1", "nat", + "--nictype1", d.NatNicType, + "--cableconnected1", "on"); err != nil { + return err + } + + if err := d.vbm("storagectl", d.MachineName, + "--name", "SATA", + "--add", "sata", + "--hostiocache", "on"); err != nil { + return err + } + + if err := d.vbm("storageattach", d.MachineName, + "--storagectl", "SATA", + "--port", "0", + "--device", "0", + "--type", "dvddrive", + "--medium", d.ResolveStorePath("boot2docker.iso")); err != nil { + return err + } + + if err := d.vbm("storageattach", d.MachineName, + "--storagectl", "SATA", + "--port", "1", + "--device", "0", + "--type", "hdd", + "--medium", d.diskPath()); err != nil { + return err + } + + // let VBoxService do nice magic automounting (when it's used) + if err := d.vbm("guestproperty", "set", d.MachineName, "/VirtualBox/GuestAdd/SharedFolders/MountPrefix", "/"); err != nil { + return err + } + if err := d.vbm("guestproperty", "set", d.MachineName, "/VirtualBox/GuestAdd/SharedFolders/MountDir", "/"); err != nil { + return err + } + + shareName, shareDir := getShareDriveAndName() + + if d.ShareFolder != "" { + shareDir, shareName = parseShareFolder(d.ShareFolder) + } + + if shareDir != "" && !d.NoShare { + log.Debugf("setting up shareDir '%s' -> '%s'", shareDir, shareName) + if _, err := os.Stat(shareDir); err != nil && !os.IsNotExist(err) { + return err + } else if !os.IsNotExist(err) { + if shareName == "" { + // parts of the VBox internal code are buggy with share names that start with "/" + shareName = strings.TrimLeft(shareDir, "/") + // TODO do some basic Windows -> MSYS path conversion + // ie, s!^([a-z]+):[/\\]+!\1/!; s!\\!/!g + } + + // woo, shareDir exists! let's carry on! + if err := d.vbm("sharedfolder", "add", d.MachineName, "--name", shareName, "--hostpath", shareDir, "--automount"); err != nil { + return err + } + + // enable symlinks + if err := d.vbm("setextradata", d.MachineName, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/"+shareName, "1"); err != nil { + return err + } + } + } + + return nil +} + +func parseShareFolder(shareFolder string) (string, string) { + split := strings.Split(shareFolder, ":") + shareDir := strings.Join(split[:len(split)-1], ":") + shareName := split[len(split)-1] + return shareDir, shareName +} + +func (d *Driver) hostOnlyIPAvailable() bool { + ip, err := d.GetIP() + if err != nil { + log.Debugf("ERROR getting IP: %s", err) + return false + } + if ip == "" { + log.Debug("Strangely, there was no error attempting to get the IP, but it was still empty.") + return false + } + + log.Debugf("IP is %s", ip) + return true +} + +func (d *Driver) Start() error { + s, err := d.GetState() + if err != nil { + return err + } + + var hostOnlyAdapter *hostOnlyNetwork + if s == state.Stopped { + log.Infof("Check network to re-create if needed...") + + if hostOnlyAdapter, err = d.setupHostOnlyNetwork(d.MachineName); err != nil { + return fmt.Errorf("Error setting up host only network on machine start: %s", err) + } + } + + switch s { + case state.Stopped, state.Saved: + d.SSHPort, err = setPortForwarding(d, 1, "ssh", "tcp", 22, d.SSHPort) + if err != nil { + return err + } + + if err := d.vbm("startvm", d.MachineName, "--type", d.UIType); err != nil { + if lines, readErr := d.readVBoxLog(); readErr == nil && len(lines) > 0 { + return fmt.Errorf("Unable to start the VM: %s\nDetails: %s", err, lines[len(lines)-1]) + } + return fmt.Errorf("Unable to start the VM: %s", err) + } + case state.Paused: + if err := d.vbm("controlvm", d.MachineName, "resume", "--type", d.UIType); err != nil { + return err + } + log.Infof("Resuming VM ...") + default: + log.Infof("VM not in restartable state") + } + + if !d.NoVTXCheck { + // Verify that VT-X is not disabled in the started VM + vtxIsDisabled, err := d.IsVTXDisabledInTheVM() + if err != nil { + return fmt.Errorf("Checking if hardware virtualization is enabled failed: %s", err) + } + + if vtxIsDisabled { + return ErrMustEnableVTX + } + } + + log.Infof("Waiting for an IP...") + if err := d.ipWaiter.Wait(d); err != nil { + return err + } + + if hostOnlyAdapter == nil { + return nil + } + + // Check that the host-only adapter we just created can still be found + // Sometimes it is corrupted after the VM is started. + nets, err := listHostOnlyAdapters(d.VBoxManager) + if err != nil { + return err + } + + ip, network, err := parseAndValidateCIDR(d.HostOnlyCIDR) + if err != nil { + return err + } + + err = validateNoIPCollisions(d.HostInterfaces, network, nets) + if err != nil { + return err + } + + hostOnlyNet := getHostOnlyAdapter(nets, ip, network.Mask) + if hostOnlyNet != nil { + // OK, we found a valid host-only adapter + return nil + } + + // This happens a lot on windows. The adapter has an invalid IP and the VM has the same IP + log.Warn("The host-only adapter is corrupted. Let's stop the VM, fix the host-only adapter and restart the VM") + if err := d.Stop(); err != nil { + return err + } + + // We have to be sure the host-only adapter is not used by the VM + d.sleeper.Sleep(5 * time.Second) + + log.Debugf("Fixing %+v...", hostOnlyAdapter) + if err := hostOnlyAdapter.SaveIPv4(d.VBoxManager); err != nil { + return err + } + + // We have to be sure the adapter is updated before starting the VM + d.sleeper.Sleep(5 * time.Second) + + if err := d.vbm("startvm", d.MachineName, "--type", d.UIType); err != nil { + return fmt.Errorf("Unable to start the VM: %s", err) + } + + log.Infof("Waiting for an IP...") + return d.ipWaiter.Wait(d) +} + +func (d *Driver) Stop() error { + currentState, err := d.GetState() + if err != nil { + return err + } + + if currentState == state.Paused { + if err := d.vbm("controlvm", d.MachineName, "resume"); err != nil { // , "--type", d.UIType + return err + } + log.Infof("Resuming VM ...") + } + + if err := d.vbm("controlvm", d.MachineName, "acpipowerbutton"); err != nil { + return err + } + for { + s, err := d.GetState() + if err != nil { + return err + } + if s == state.Running { + d.sleeper.Sleep(1 * time.Second) + } else { + break + } + } + + d.IPAddress = "" + + return nil +} + +// Restart restarts a machine which is known to be running. +func (d *Driver) Restart() error { + if err := d.Stop(); err != nil { + return fmt.Errorf("Problem stopping the VM: %s", err) + } + + if err := d.Start(); err != nil { + return fmt.Errorf("Problem starting the VM: %s", err) + } + + d.IPAddress = "" + + return d.ipWaiter.Wait(d) +} + +func (d *Driver) Kill() error { + return d.vbm("controlvm", d.MachineName, "poweroff") +} + +func (d *Driver) Remove() error { + s, err := d.GetState() + if err == ErrMachineNotExist { + return nil + } + if err != nil { + return err + } + + if s != state.Stopped && s != state.Saved { + if err := d.Kill(); err != nil { + return err + } + } + + return d.vbm("unregistervm", "--delete", d.MachineName) +} + +func (d *Driver) GetState() (state.State, error) { + stdout, stderr, err := d.vbmOutErr("showvminfo", d.MachineName, "--machinereadable") + if err != nil { + if reMachineNotFound.FindString(stderr) != "" { + return state.Error, ErrMachineNotExist + } + return state.Error, err + } + re := regexp.MustCompile(`(?m)^VMState="(\w+)"`) + groups := re.FindStringSubmatch(stdout) + if len(groups) < 1 { + return state.None, nil + } + switch groups[1] { + case "running": + return state.Running, nil + case "paused": + return state.Paused, nil + case "saved": + return state.Saved, nil + case "poweroff", "aborted": + return state.Stopped, nil + } + return state.None, nil +} + +func (d *Driver) getHostOnlyMACAddress() (string, error) { + // Return the MAC address of the host-only adapter + // assigned to this machine. The returned address + // is lower-cased and does not contain colons. + + stdout, stderr, err := d.vbmOutErr("showvminfo", d.MachineName, "--machinereadable") + if err != nil { + if reMachineNotFound.FindString(stderr) != "" { + return "", ErrMachineNotExist + } + return "", err + } + + // First, we get the number of the host-only interface + re := regexp.MustCompile(`(?m)^hostonlyadapter([\d]+)`) + groups := re.FindStringSubmatch(stdout) + if len(groups) < 2 { + return "", errors.New("Machine does not have a host-only adapter") + } + + // Then we grab the MAC address based on that number + adapterNumber := groups[1] + re = regexp.MustCompile(fmt.Sprintf("(?m)^macaddress%s=\"(.*)\"", adapterNumber)) + groups = re.FindStringSubmatch(stdout) + if len(groups) < 2 { + return "", fmt.Errorf("Could not find MAC address for adapter %v", adapterNumber) + } + + return strings.ToLower(groups[1]), nil +} + +func (d *Driver) parseIPForMACFromIPAddr(ipAddrOutput string, macAddress string) (string, error) { + // Given the output of "ip addr show" on the VM, return the IPv4 address + // of the interface with the given MAC address. + + lines := strings.Split(ipAddrOutput, "\n") + returnNextIP := false + + for _, line := range lines { + line = strings.TrimSpace(line) + + if strings.HasPrefix(line, "link") { // line contains MAC address + vals := strings.Split(line, " ") + if len(vals) >= 2 { + macBlock := vals[1] + macWithoutColons := strings.Replace(macBlock, ":", "", -1) + if macWithoutColons == macAddress { // we are in the correct device block + returnNextIP = true + } + } + } else if strings.HasPrefix(line, "inet") && !strings.HasPrefix(line, "inet6") && returnNextIP { + vals := strings.Split(line, " ") + if len(vals) >= 2 { + return vals[1][:strings.Index(vals[1], "/")], nil + } + } + } + + return "", fmt.Errorf("Could not find matching IP for MAC address %v", macAddress) +} + +func (d *Driver) GetIP() (string, error) { + // DHCP is used to get the IP, so virtualbox hosts don't have IPs unless + // they are running + s, err := d.GetState() + if err != nil { + return "", err + } + if s != state.Running { + return "", drivers.ErrHostIsNotRunning + } + + macAddress, err := d.getHostOnlyMACAddress() + if err != nil { + return "", err + } + + log.Debugf("Host-only MAC: %s\n", macAddress) + + output, err := drivers.RunSSHCommandFromDriver(d, "ip addr show") + if err != nil { + return "", err + } + + log.Debugf("SSH returned: %s\nEND SSH\n", output) + + ipAddress, err := d.parseIPForMACFromIPAddr(output, macAddress) + if err != nil { + return "", err + } + + return ipAddress, nil +} + +func (d *Driver) publicSSHKeyPath() string { + return d.GetSSHKeyPath() + ".pub" +} + +func (d *Driver) diskPath() string { + return d.ResolveStorePath("disk.vmdk") +} + +func (d *Driver) setupHostOnlyNetwork(machineName string) (*hostOnlyNetwork, error) { + hostOnlyCIDR := d.HostOnlyCIDR + + // This is to assist in migrating from version 0.2 to 0.3 format + // it should be removed in a later release + if hostOnlyCIDR == "" { + hostOnlyCIDR = defaultHostOnlyCIDR + } + + ip, network, err := parseAndValidateCIDR(hostOnlyCIDR) + if err != nil { + return nil, err + } + + nets, err := listHostOnlyAdapters(d.VBoxManager) + if err != nil { + return nil, err + } + + err = validateNoIPCollisions(d.HostInterfaces, network, nets) + if err != nil { + return nil, err + } + + log.Debugf("Searching for hostonly interface for IPv4: %s and Mask: %s", ip, network.Mask) + hostOnlyAdapter, err := getOrCreateHostOnlyNetwork(ip, network.Mask, nets, d.VBoxManager) + if err != nil { + return nil, err + } + + if err := removeOrphanDHCPServers(d.VBoxManager); err != nil { + return nil, err + } + + dhcpAddr, err := getRandomIPinSubnet(d, ip) + if err != nil { + return nil, err + } + + lowerIP, upperIP := getDHCPAddressRange(dhcpAddr, network) + + log.Debugf("Adding/Modifying DHCP server %q with address range %q - %q...", dhcpAddr, lowerIP, upperIP) + + dhcp := dhcpServer{} + dhcp.IPv4.IP = dhcpAddr + dhcp.IPv4.Mask = network.Mask + dhcp.LowerIP = lowerIP + dhcp.UpperIP = upperIP + dhcp.Enabled = !d.HostOnlyNoDHCP + if err := addHostOnlyDHCPServer(hostOnlyAdapter.Name, dhcp, d.VBoxManager); err != nil { + return nil, err + } + + if err := d.vbm("modifyvm", machineName, + "--nic2", "hostonly", + "--nictype2", d.HostOnlyNicType, + "--nicpromisc2", d.HostOnlyPromiscMode, + "--hostonlyadapter2", hostOnlyAdapter.Name, + "--cableconnected2", "on"); err != nil { + return nil, err + } + + return hostOnlyAdapter, nil +} + +func getDHCPAddressRange(dhcpAddr net.IP, network *net.IPNet) (lowerIP net.IP, upperIP net.IP) { + nAddr := network.IP.To4() + ones, bits := network.Mask.Size() + + if ones <= 24 { + // For a /24 subnet, use the original behavior of allowing the address range + // between x.x.x.100 and x.x.x.254. + lowerIP = net.IPv4(nAddr[0], nAddr[1], nAddr[2], byte(100)) + upperIP = net.IPv4(nAddr[0], nAddr[1], nAddr[2], byte(254)) + return + } + + // Start the lowerIP range one address above the selected DHCP address. + lowerIP = net.IPv4(nAddr[0], nAddr[1], nAddr[2], dhcpAddr.To4()[3]+1) + + // The highest D-part of the address A.B.C.D in this subnet is at 2^n - 1, + // where n is the number of available bits in the subnet. Since the highest + // address is reserved for subnet broadcast, the highest *assignable* address + // is at (2^n - 1) - 1 == 2^n - 2. + maxAssignableSubnetAddress := (byte)((1 << (uint)(bits-ones)) - 2) + upperIP = net.IPv4(nAddr[0], nAddr[1], nAddr[2], maxAssignableSubnetAddress) + return +} + +func parseAndValidateCIDR(hostOnlyCIDR string) (net.IP, *net.IPNet, error) { + ip, network, err := net.ParseCIDR(hostOnlyCIDR) + if err != nil { + return nil, nil, err + } + + networkAddress := network.IP.To4() + if ip.Equal(networkAddress) { + return nil, nil, ErrNetworkAddrCidr + } + + return ip, network, nil +} + +// validateNoIPCollisions ensures no conflicts between the host's network interfaces and the vbox host-only network that +// will be used for machine vm instances. +func validateNoIPCollisions(hif HostInterfaces, hostOnlyNet *net.IPNet, currHostOnlyNets map[string]*hostOnlyNetwork) error { + hostOnlyByCIDR := map[string]*hostOnlyNetwork{} + //listHostOnlyAdapters returns a map w/ virtualbox net names as key. Rekey to CIDRs + for _, n := range currHostOnlyNets { + ipnet := net.IPNet{IP: n.IPv4.IP, Mask: n.IPv4.Mask} + hostOnlyByCIDR[ipnet.String()] = n + } + + m, err := listHostInterfaces(hif, hostOnlyByCIDR) + if err != nil { + return err + } + + collision, err := checkIPNetCollision(hostOnlyNet, m) + if err != nil { + return err + } + + if collision { + return ErrNetworkAddrCollision + } + return nil +} + +// Select an available port, trying the specified +// port first, falling back on an OS selected port. +func getAvailableTCPPort(port int) (int, error) { + for i := 0; i <= 10; i++ { + ln, err := net.Listen("tcp4", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + return 0, err + } + defer ln.Close() + addr := ln.Addr().String() + addrParts := strings.SplitN(addr, ":", 2) + p, err := strconv.Atoi(addrParts[1]) + if err != nil { + return 0, err + } + if p != 0 { + port = p + return port, nil + } + port = 0 // Throw away the port hint before trying again + time.Sleep(1) + } + return 0, fmt.Errorf("unable to allocate tcp port") +} + +// Setup a NAT port forwarding entry. +func setPortForwarding(d *Driver, interfaceNum int, mapName, protocol string, guestPort, desiredHostPort int) (int, error) { + actualHostPort, err := getAvailableTCPPort(desiredHostPort) + if err != nil { + return -1, err + } + if desiredHostPort != actualHostPort && desiredHostPort != 0 { + log.Debugf("NAT forwarding host port for guest port %d (%s) changed from %d to %d", + guestPort, mapName, desiredHostPort, actualHostPort) + } + cmd := fmt.Sprintf("--natpf%d", interfaceNum) + d.vbm("modifyvm", d.MachineName, cmd, "delete", mapName) + if err := d.vbm("modifyvm", d.MachineName, + cmd, fmt.Sprintf("%s,%s,127.0.0.1,%d,,%d", mapName, protocol, actualHostPort, guestPort)); err != nil { + return -1, err + } + return actualHostPort, nil +} + +// getRandomIPinSubnet returns a pseudo-random net.IP in the same +// subnet as the IP passed +func getRandomIPinSubnet(d *Driver, baseIP net.IP) (net.IP, error) { + var dhcpAddr net.IP + + nAddr := baseIP.To4() + // select pseudo-random DHCP addr; make sure not to clash with the host + // only try 5 times and bail if no random received + for i := 0; i < 5; i++ { + n := d.randomInter.RandomInt(24) + 1 + if byte(n) != nAddr[3] { + dhcpAddr = net.IPv4(nAddr[0], nAddr[1], nAddr[2], byte(n)) + break + } + } + + if dhcpAddr == nil { + return nil, ErrUnableToGenerateRandomIP + } + + return dhcpAddr, nil +} + +func detectVBoxManageCmdInPath() string { + cmd := "VBoxManage" + if path, err := exec.LookPath(cmd); err == nil { + return path + } + return cmd +} + +func (d *Driver) readVBoxLog() ([]string, error) { + logPath := filepath.Join(d.ResolveStorePath(d.MachineName), "Logs", "VBox.log") + log.Debugf("Checking vm logs: %s", logPath) + + return d.logsReader.Read(logPath) +} diff --git a/pkg/drivers/virtualbox/virtualbox_darwin.go b/pkg/drivers/virtualbox/virtualbox_darwin.go new file mode 100644 index 000000000000..0c225bde6d8e --- /dev/null +++ b/pkg/drivers/virtualbox/virtualbox_darwin.go @@ -0,0 +1,13 @@ +package virtualbox + +func detectVBoxManageCmd() string { + return detectVBoxManageCmdInPath() +} + +func getShareDriveAndName() (string, string) { + return "Users", "/Users" +} + +func isHyperVInstalled() bool { + return false +} diff --git a/pkg/drivers/virtualbox/virtualbox_freebsd.go b/pkg/drivers/virtualbox/virtualbox_freebsd.go new file mode 100644 index 000000000000..be1a658d6ed6 --- /dev/null +++ b/pkg/drivers/virtualbox/virtualbox_freebsd.go @@ -0,0 +1,20 @@ +package virtualbox + +import "path/filepath" + +func detectVBoxManageCmd() string { + return detectVBoxManageCmdInPath() +} + +func getShareDriveAndName() (string, string) { + path, err := filepath.EvalSymlinks("/home") + if err != nil { + path = "/home" + } + + return "hosthome", path +} + +func isHyperVInstalled() bool { + return false +} diff --git a/pkg/drivers/virtualbox/virtualbox_linux.go b/pkg/drivers/virtualbox/virtualbox_linux.go new file mode 100644 index 000000000000..f41429e2ec61 --- /dev/null +++ b/pkg/drivers/virtualbox/virtualbox_linux.go @@ -0,0 +1,13 @@ +package virtualbox + +func detectVBoxManageCmd() string { + return detectVBoxManageCmdInPath() +} + +func getShareDriveAndName() (string, string) { + return "hosthome", "/home" +} + +func isHyperVInstalled() bool { + return false +} diff --git a/pkg/drivers/virtualbox/virtualbox_openbsd.go b/pkg/drivers/virtualbox/virtualbox_openbsd.go new file mode 100644 index 000000000000..f41429e2ec61 --- /dev/null +++ b/pkg/drivers/virtualbox/virtualbox_openbsd.go @@ -0,0 +1,13 @@ +package virtualbox + +func detectVBoxManageCmd() string { + return detectVBoxManageCmdInPath() +} + +func getShareDriveAndName() (string, string) { + return "hosthome", "/home" +} + +func isHyperVInstalled() bool { + return false +} diff --git a/pkg/drivers/virtualbox/virtualbox_windows.go b/pkg/drivers/virtualbox/virtualbox_windows.go new file mode 100644 index 000000000000..d77f85976dc5 --- /dev/null +++ b/pkg/drivers/virtualbox/virtualbox_windows.go @@ -0,0 +1,105 @@ +package virtualbox + +import ( + "strings" + + "fmt" + "os" + "os/exec" + "path/filepath" + + "k8s.io/minikube/pkg/libmachine/log" + "golang.org/x/sys/windows/registry" +) + +// cmdOutput runs a shell command and returns its output. +func cmdOutput(name string, args ...string) (string, error) { + cmd := exec.Command(name, args...) + log.Debugf("COMMAND: %v %v", name, strings.Join(args, " ")) + + stdout, err := cmd.Output() + if err != nil { + return "", err + } + + log.Debugf("STDOUT:\n{\n%v}", string(stdout)) + + return string(stdout), nil +} + +func detectVBoxManageCmd() string { + cmd := "VBoxManage" + if p := os.Getenv("VBOX_INSTALL_PATH"); p != "" { + if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil { + return path + } + } + + if p := os.Getenv("VBOX_MSI_INSTALL_PATH"); p != "" { + if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil { + return path + } + } + + // Look in default installation path for VirtualBox version > 5 + if path, err := exec.LookPath(filepath.Join("C:\\Program Files\\Oracle\\VirtualBox", cmd)); err == nil { + return path + } + + // Look in windows registry + if p, err := findVBoxInstallDirInRegistry(); err == nil { + if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil { + return path + } + } + + return detectVBoxManageCmdInPath() //fallback to path +} + +func findVBoxInstallDirInRegistry() (string, error) { + registryKey, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Oracle\VirtualBox`, registry.QUERY_VALUE) + if err != nil { + errorMessage := fmt.Sprintf("Can't find VirtualBox registry entries, is VirtualBox really installed properly? %s", err) + log.Debugf(errorMessage) + return "", fmt.Errorf(errorMessage) + } + + defer registryKey.Close() + + installDir, _, err := registryKey.GetStringValue("InstallDir") + if err != nil { + errorMessage := fmt.Sprintf("Can't find InstallDir registry key within VirtualBox registries entries, is VirtualBox really installed properly? %s", err) + log.Debugf(errorMessage) + return "", fmt.Errorf(errorMessage) + } + + return installDir, nil +} + +func getShareDriveAndName() (string, string) { + return "c/Users", "\\\\?\\c:\\Users" +} + +func isHyperVInstalled() bool { + // check if hyper-v is installed + _, err := exec.LookPath("vmms.exe") + if err != nil { + errmsg := "Hyper-V is not installed." + log.Debugf(errmsg, err) + return false + } + + // check to see if a hypervisor is present. if hyper-v is installed and enabled, + // display an error explaining the incompatibility between virtualbox and hyper-v. + output, err := cmdOutput("wmic", "computersystem", "get", "hypervisorpresent") + + if err != nil { + errmsg := "Could not check to see if Hyper-V is running." + log.Debugf(errmsg, err) + return false + } + + enabled := strings.Contains(output, "TRUE") + return enabled + +} diff --git a/pkg/drivers/virtualbox/vm.go b/pkg/drivers/virtualbox/vm.go new file mode 100644 index 000000000000..cd95ada76727 --- /dev/null +++ b/pkg/drivers/virtualbox/vm.go @@ -0,0 +1,41 @@ +package virtualbox + +import "strconv" + +type VM struct { + CPUs int + Memory int +} + +func getVMInfo(name string, vbox VBoxManager) (*VM, error) { + out, err := vbox.vbmOut("showvminfo", name, "--machinereadable") + if err != nil { + return nil, err + } + + vm := &VM{} + + err = parseKeyValues(out, reEqualLine, func(key, val string) error { + switch key { + case "cpus": + v, err := strconv.Atoi(val) + if err != nil { + return err + } + vm.CPUs = v + case "memory": + v, err := strconv.Atoi(val) + if err != nil { + return err + } + vm.Memory = v + } + + return nil + }) + if err != nil { + return nil, err + } + + return vm, nil +} diff --git a/pkg/drivers/virtualbox/vtx.go b/pkg/drivers/virtualbox/vtx.go new file mode 100644 index 000000000000..7adb631fe965 --- /dev/null +++ b/pkg/drivers/virtualbox/vtx.go @@ -0,0 +1,28 @@ +package virtualbox + +import "strings" + +// IsVTXDisabledInTheVM checks if VT-X is disabled in the started vm. +func (d *Driver) IsVTXDisabledInTheVM() (bool, error) { + lines, err := d.readVBoxLog() + if err != nil { + return true, err + } + + for _, line := range lines { + if strings.Contains(line, "VT-x is disabled") && !strings.Contains(line, "Falling back to raw-mode: VT-x is disabled in the BIOS for all CPU modes") { + return true, nil + } + if strings.Contains(line, "the host CPU does NOT support HW virtualization") { + return true, nil + } + if strings.Contains(line, "VERR_VMX_UNABLE_TO_START_VM") { + return true, nil + } + if strings.Contains(line, "Power up failed") && strings.Contains(line, "VERR_VMX_NO_VMX") { + return true, nil + } + } + + return false, nil +} diff --git a/pkg/drivers/virtualbox/vtx_intel.go b/pkg/drivers/virtualbox/vtx_intel.go new file mode 100644 index 000000000000..8aca2c9b7db1 --- /dev/null +++ b/pkg/drivers/virtualbox/vtx_intel.go @@ -0,0 +1,14 @@ +// +build 386 amd64 + +package virtualbox + +import "github.com/aregm/cpuid" + +// IsVTXDisabled checks if VT-x is disabled in the CPU. +func (d *Driver) IsVTXDisabled() bool { + if cpuid.HasFeature(cpuid.VMX) || cpuid.HasExtraFeature(cpuid.SVM) { + return false + } + + return true +} diff --git a/pkg/drivers/virtualbox/vtx_other.go b/pkg/drivers/virtualbox/vtx_other.go new file mode 100644 index 000000000000..5dbfd17553b1 --- /dev/null +++ b/pkg/drivers/virtualbox/vtx_other.go @@ -0,0 +1,8 @@ +// +build !386,!amd64 + +package virtualbox + +// IsVTXDisabled checks if VT-x is disabled in the CPU. +func (d *Driver) IsVTXDisabled() bool { + return true +} diff --git a/pkg/gvisor/disable.go b/pkg/gvisor/disable.go index 6e0fe95a1e6b..f711f97b09da 100644 --- a/pkg/gvisor/disable.go +++ b/pkg/gvisor/disable.go @@ -21,7 +21,7 @@ import ( "os" "path/filepath" - "github.com/docker/machine/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/mcnutils" "github.com/pkg/errors" ) diff --git a/pkg/gvisor/enable.go b/pkg/gvisor/enable.go index 6599812c2962..910285cd2d72 100644 --- a/pkg/gvisor/enable.go +++ b/pkg/gvisor/enable.go @@ -28,7 +28,7 @@ import ( "runtime" "syscall" - "github.com/docker/machine/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/mcnutils" "github.com/pkg/errors" ) diff --git a/pkg/libmachine/auth/auth.go b/pkg/libmachine/auth/auth.go new file mode 100644 index 000000000000..86ae79d2326e --- /dev/null +++ b/pkg/libmachine/auth/auth.go @@ -0,0 +1,18 @@ +package auth + +type Options struct { + CertDir string + CaCertPath string + CaPrivateKeyPath string + CaCertRemotePath string + ServerCertPath string + ServerKeyPath string + ClientKeyPath string + ServerCertRemotePath string + ServerKeyRemotePath string + ClientCertPath string + ServerCertSANs []string + // StorePath is left in for historical reasons, but not really meant to + // be used directly. + StorePath string +} diff --git a/pkg/libmachine/cert/bootstrap.go b/pkg/libmachine/cert/bootstrap.go new file mode 100644 index 000000000000..09ed206c83e0 --- /dev/null +++ b/pkg/libmachine/cert/bootstrap.go @@ -0,0 +1,136 @@ +package cert + +import ( + "errors" + "fmt" + "os" + + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnutils" +) + +func createCACert(authOptions *auth.Options, caOrg string, bits int) error { + caCertPath := authOptions.CaCertPath + caPrivateKeyPath := authOptions.CaPrivateKeyPath + + log.Infof("Creating CA: %s", caCertPath) + + // check if the key path exists; if so, error + if _, err := os.Stat(caPrivateKeyPath); err == nil { + return errors.New("certificate authority key already exists") + } + + if err := GenerateCACertificate(caCertPath, caPrivateKeyPath, caOrg, bits); err != nil { + return fmt.Errorf("generating CA certificate failed: %s", err) + } + + return nil +} + +func createCert(authOptions *auth.Options, org string, bits int) error { + certDir := authOptions.CertDir + caCertPath := authOptions.CaCertPath + caPrivateKeyPath := authOptions.CaPrivateKeyPath + clientCertPath := authOptions.ClientCertPath + clientKeyPath := authOptions.ClientKeyPath + + log.Infof("Creating client certificate: %s", clientCertPath) + + if _, err := os.Stat(certDir); err != nil { + if os.IsNotExist(err) { + if err := os.Mkdir(certDir, 0700); err != nil { + return fmt.Errorf("failure creating machine client cert dir: %s", err) + } + } else { + return err + } + } + + // check if the key path exists; if so, error + if _, err := os.Stat(clientKeyPath); err == nil { + return errors.New("client key already exists") + } + + // Used to generate the client certificate. + certOptions := &Options{ + Hosts: []string{""}, + CertFile: clientCertPath, + KeyFile: clientKeyPath, + CAFile: caCertPath, + CAKeyFile: caPrivateKeyPath, + Org: org, + Bits: bits, + SwarmMaster: false, + } + + if err := GenerateCert(certOptions); err != nil { + return fmt.Errorf("failure generating client certificate: %s", err) + } + + return nil +} + +func BootstrapCertificates(authOptions *auth.Options) error { + certDir := authOptions.CertDir + caCertPath := authOptions.CaCertPath + clientCertPath := authOptions.ClientCertPath + clientKeyPath := authOptions.ClientKeyPath + caPrivateKeyPath := authOptions.CaPrivateKeyPath + + // TODO: I'm not super happy about this use of "org", the user should + // have to specify it explicitly instead of implicitly basing it on + // $USER. + caOrg := mcnutils.GetUsername() + org := caOrg + "." + + bits := 2048 + + if _, err := os.Stat(certDir); err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(certDir, 0700); err != nil { + return fmt.Errorf("creating machine certificate dir failed: %s", err) + } + } else { + return err + } + } + + if _, err := os.Stat(caCertPath); os.IsNotExist(err) { + if err := createCACert(authOptions, caOrg, bits); err != nil { + return err + } + } else { + current, err := CheckCertificateDate(caCertPath) + if err != nil { + return err + } + if !current { + log.Info("CA certificate is outdated and needs to be regenerated") + os.Remove(caPrivateKeyPath) + if err := createCACert(authOptions, caOrg, bits); err != nil { + return err + } + } + } + + if _, err := os.Stat(clientCertPath); os.IsNotExist(err) { + if err := createCert(authOptions, org, bits); err != nil { + return err + } + } else { + current, err := CheckCertificateDate(clientCertPath) + if err != nil { + return err + } + if !current { + log.Info("Client certificate is outdated and needs to be regenerated") + os.Remove(clientKeyPath) + if err := createCert(authOptions, org, bits); err != nil { + return err + } + } + } + + return nil +} diff --git a/pkg/libmachine/cert/cert.go b/pkg/libmachine/cert/cert.go new file mode 100644 index 000000000000..0d2260337983 --- /dev/null +++ b/pkg/libmachine/cert/cert.go @@ -0,0 +1,294 @@ +package cert + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "io/ioutil" + "math/big" + "net" + "os" + "time" + + "errors" + + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/log" +) + +var defaultGenerator = NewX509CertGenerator() + +type Options struct { + Hosts []string + CertFile, KeyFile, CAFile, CAKeyFile, Org string + Bits int + SwarmMaster bool +} + +type Generator interface { + GenerateCACertificate(certFile, keyFile, org string, bits int) error + GenerateCert(opts *Options) error + ReadTLSConfig(addr string, authOptions *auth.Options) (*tls.Config, error) + ValidateCertificate(addr string, authOptions *auth.Options) (bool, error) +} + +type X509CertGenerator struct{} + +func NewX509CertGenerator() Generator { + return &X509CertGenerator{} +} + +func GenerateCACertificate(certFile, keyFile, org string, bits int) error { + return defaultGenerator.GenerateCACertificate(certFile, keyFile, org, bits) +} + +func GenerateCert(opts *Options) error { + return defaultGenerator.GenerateCert(opts) +} + +func ValidateCertificate(addr string, authOptions *auth.Options) (bool, error) { + return defaultGenerator.ValidateCertificate(addr, authOptions) +} + +func ReadTLSConfig(addr string, authOptions *auth.Options) (*tls.Config, error) { + return defaultGenerator.ReadTLSConfig(addr, authOptions) +} + +func SetCertGenerator(cg Generator) { + defaultGenerator = cg +} + +func (xcg *X509CertGenerator) getTLSConfig(caCert, cert, key []byte, allowInsecure bool) (*tls.Config, error) { + // TLS config + var tlsConfig tls.Config + tlsConfig.InsecureSkipVerify = allowInsecure + certPool := x509.NewCertPool() + + ok := certPool.AppendCertsFromPEM(caCert) + if !ok { + return &tlsConfig, errors.New("There was an error reading certificate") + } + + tlsConfig.RootCAs = certPool + keypair, err := tls.X509KeyPair(cert, key) + if err != nil { + return &tlsConfig, err + } + tlsConfig.Certificates = []tls.Certificate{keypair} + + return &tlsConfig, nil +} + +func (xcg *X509CertGenerator) newCertificate(org string) (*x509.Certificate, error) { + now := time.Now() + // need to set notBefore slightly in the past to account for time + // skew in the VMs otherwise the certs sometimes are not yet valid + notBefore := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute()-5, 0, 0, time.Local) + notAfter := notBefore.Add(time.Hour * 24 * 1080) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, err + } + + return &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{org}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageKeyAgreement, + BasicConstraintsValid: true, + }, nil + +} + +// GenerateCACertificate generates a new certificate authority from the specified org +// and bit size and stores the resulting certificate and key file +// in the arguments. +func (xcg *X509CertGenerator) GenerateCACertificate(certFile, keyFile, org string, bits int) error { + template, err := xcg.newCertificate(org) + if err != nil { + return err + } + + template.IsCA = true + template.KeyUsage |= x509.KeyUsageCertSign + template.KeyUsage |= x509.KeyUsageKeyEncipherment + template.KeyUsage |= x509.KeyUsageKeyAgreement + + priv, err := rsa.GenerateKey(rand.Reader, bits) + if err != nil { + return err + } + + derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv) + if err != nil { + return err + } + + certOut, err := os.Create(certFile) + if err != nil { + return err + } + + pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + certOut.Close() + + keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + + } + + pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + keyOut.Close() + + return nil +} + +// GenerateCert generates a new certificate signed using the provided +// certificate authority files and stores the result in the certificate +// file and key provided. The provided host names are set to the +// appropriate certificate fields. +func (xcg *X509CertGenerator) GenerateCert(opts *Options) error { + template, err := xcg.newCertificate(opts.Org) + if err != nil { + return err + } + // client + if len(opts.Hosts) == 1 && opts.Hosts[0] == "" { + template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + template.KeyUsage = x509.KeyUsageDigitalSignature + } else { // server + template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + if opts.SwarmMaster { + // Extend the Swarm master's server certificate + // permissions to also be able to connect to downstream + // nodes as a client. + template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageClientAuth) + } + for _, h := range opts.Hosts { + if ip := net.ParseIP(h); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, h) + } + } + } + + tlsCert, err := tls.LoadX509KeyPair(opts.CAFile, opts.CAKeyFile) + if err != nil { + return err + } + + priv, err := rsa.GenerateKey(rand.Reader, opts.Bits) + if err != nil { + return err + } + + x509Cert, err := x509.ParseCertificate(tlsCert.Certificate[0]) + if err != nil { + return err + } + + derBytes, err := x509.CreateCertificate(rand.Reader, template, x509Cert, &priv.PublicKey, tlsCert.PrivateKey) + if err != nil { + return err + } + + certOut, err := os.Create(opts.CertFile) + if err != nil { + return err + } + + pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + certOut.Close() + + keyOut, err := os.OpenFile(opts.KeyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + + pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + keyOut.Close() + + return nil +} + +// ReadTLSConfig reads the tls config for a machine. +func (xcg *X509CertGenerator) ReadTLSConfig(addr string, authOptions *auth.Options) (*tls.Config, error) { + caCertPath := authOptions.CaCertPath + clientCertPath := authOptions.ClientCertPath + clientKeyPath := authOptions.ClientKeyPath + + log.Debugf("Reading CA certificate from %s", caCertPath) + caCert, err := ioutil.ReadFile(caCertPath) + if err != nil { + return nil, err + } + + log.Debugf("Reading client certificate from %s", clientCertPath) + clientCert, err := ioutil.ReadFile(clientCertPath) + if err != nil { + return nil, err + } + + log.Debugf("Reading client key from %s", clientKeyPath) + clientKey, err := ioutil.ReadFile(clientKeyPath) + if err != nil { + return nil, err + } + + return xcg.getTLSConfig(caCert, clientCert, clientKey, false) +} + +// ValidateCertificate validate the certificate installed on the vm. +func (xcg *X509CertGenerator) ValidateCertificate(addr string, authOptions *auth.Options) (bool, error) { + tlsConfig, err := xcg.ReadTLSConfig(addr, authOptions) + if err != nil { + return false, err + } + + dialer := &net.Dialer{ + Timeout: time.Second * 20, + } + + _, err = tls.DialWithDialer(dialer, "tcp", addr, tlsConfig) + if err != nil { + return false, err + } + + return true, nil +} + +func CheckCertificateDate(certPath string) (bool, error) { + log.Debugf("Reading certificate data from %s", certPath) + certBytes, err := ioutil.ReadFile(certPath) + if err != nil { + return false, err + } + + log.Debug("Decoding PEM data...") + pemBlock, _ := pem.Decode(certBytes) + if pemBlock == nil { + return false, errors.New("Failed to decode PEM data") + } + + log.Debug("Parsing certificate...") + cert, err := x509.ParseCertificate(pemBlock.Bytes) + if err != nil { + return false, err + } + if time.Now().After(cert.NotAfter) { + return false, nil + } + + return true, nil +} diff --git a/pkg/libmachine/check/check.go b/pkg/libmachine/check/check.go new file mode 100644 index 000000000000..a0203032e290 --- /dev/null +++ b/pkg/libmachine/check/check.go @@ -0,0 +1,118 @@ +package check + +import ( + "errors" + "fmt" + "net/url" + "strings" + + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/cert" + "k8s.io/minikube/pkg/libmachine/host" +) + +var ( + DefaultConnChecker ConnChecker + ErrSwarmNotStarted = errors.New("Connection to Swarm cannot be checked but the certs are valid. Maybe swarm is not started") +) + +func init() { + DefaultConnChecker = &MachineConnChecker{} +} + +// ErrCertInvalid for when the cert is computed to be invalid. +type ErrCertInvalid struct { + wrappedErr error + hostURL string +} + +func (e ErrCertInvalid) Error() string { + return fmt.Sprintf(`There was an error validating certificates for host %q: %s +You can attempt to regenerate them using 'docker-machine regenerate-certs [name]'. +Be advised that this will trigger a Docker daemon restart which might stop running containers. +`, e.hostURL, e.wrappedErr) +} + +type ConnChecker interface { + Check(*host.Host, bool) (dockerHost string, authOptions *auth.Options, err error) +} + +type MachineConnChecker struct{} + +func (mcc *MachineConnChecker) Check(h *host.Host, swarm bool) (string, *auth.Options, error) { + dockerHost, err := h.Driver.GetURL() + if err != nil { + return "", &auth.Options{}, err + } + + dockerURL := dockerHost + if swarm { + dockerURL, err = parseSwarm(dockerHost, h) + if err != nil { + return "", &auth.Options{}, err + } + } + + u, err := url.Parse(dockerURL) + if err != nil { + return "", &auth.Options{}, fmt.Errorf("Error parsing URL: %s", err) + } + + authOptions := h.AuthOptions() + + if err := checkCert(u.Host, authOptions); err != nil { + if swarm { + // Connection to the swarm port cannot be checked. Maybe it's just the swarm containers that are down + // TODO: check the containers and restart them + // Let's check the non-swarm connection to give a better error message to the user. + if _, _, err := mcc.Check(h, false); err == nil { + return "", &auth.Options{}, ErrSwarmNotStarted + } + } + + return "", &auth.Options{}, fmt.Errorf("Error checking and/or regenerating the certs: %s", err) + } + + return dockerURL, authOptions, nil +} + +func checkCert(hostURL string, authOptions *auth.Options) error { + valid, err := cert.ValidateCertificate(hostURL, authOptions) + if !valid || err != nil { + return ErrCertInvalid{ + wrappedErr: err, + hostURL: hostURL, + } + } + + return nil +} + +// TODO: This could use a unit test. +func parseSwarm(hostURL string, h *host.Host) (string, error) { + swarmOptions := h.HostOptions.SwarmOptions + + if !swarmOptions.Master { + return "", fmt.Errorf("%q is not a swarm master. The --swarm flag is intended for use with swarm masters", h.Name) + } + + u, err := url.Parse(swarmOptions.Host) + if err != nil { + return "", fmt.Errorf("There was an error parsing the url: %s", err) + } + parts := strings.Split(u.Host, ":") + swarmPort := parts[1] + + // get IP of machine to replace in case swarm host is 0.0.0.0 + mURL, err := url.Parse(hostURL) + if err != nil { + return "", fmt.Errorf("There was an error parsing the url: %s", err) + } + + mParts := strings.Split(mURL.Host, ":") + machineIP := mParts[0] + + hostURL = fmt.Sprintf("tcp://%s:%s", machineIP, swarmPort) + + return hostURL, nil +} diff --git a/pkg/libmachine/drivers/base.go b/pkg/libmachine/drivers/base.go new file mode 100644 index 000000000000..4ed066432c2a --- /dev/null +++ b/pkg/libmachine/drivers/base.go @@ -0,0 +1,94 @@ +package drivers + +import ( + "errors" + "path/filepath" +) + +const ( + DefaultSSHUser = "root" + DefaultSSHPort = 22 + DefaultEngineInstallURL = "https://get.docker.com" +) + +// BaseDriver - Embed this struct into drivers to provide the common set +// of fields and functions. +type BaseDriver struct { + IPAddress string + MachineName string + SSHUser string + SSHPort int + SSHKeyPath string + StorePath string + SwarmMaster bool + SwarmHost string + SwarmDiscovery string +} + +// DriverName returns the name of the driver +func (d *BaseDriver) DriverName() string { + return "unknown" +} + +// GetMachineName returns the machine name +func (d *BaseDriver) GetMachineName() string { + return d.MachineName +} + +// GetIP returns the ip +func (d *BaseDriver) GetIP() (string, error) { + if d.IPAddress == "" { + return "", errors.New("IP address is not set") + } + return d.IPAddress, nil +} + +// GetSSHKeyPath returns the ssh key path +func (d *BaseDriver) GetSSHKeyPath() string { + if d.SSHKeyPath == "" { + d.SSHKeyPath = d.ResolveStorePath("id_rsa") + } + return d.SSHKeyPath +} + +// GetSSHPort returns the ssh port, 22 if not specified +func (d *BaseDriver) GetSSHPort() (int, error) { + if d.SSHPort == 0 { + d.SSHPort = DefaultSSHPort + } + + return d.SSHPort, nil +} + +// GetSSHUsername returns the ssh user name, root if not specified +func (d *BaseDriver) GetSSHUsername() string { + if d.SSHUser == "" { + d.SSHUser = DefaultSSHUser + } + return d.SSHUser +} + +// PreCreateCheck is called to enforce pre-creation steps +func (d *BaseDriver) PreCreateCheck() error { + return nil +} + +// ResolveStorePath returns the store path where the machine is +func (d *BaseDriver) ResolveStorePath(file string) string { + return filepath.Join(d.StorePath, "machines", d.MachineName, file) +} + +// SetSwarmConfigFromFlags configures the driver for swarm +func (d *BaseDriver) SetSwarmConfigFromFlags(flags DriverOptions) { + d.SwarmMaster = flags.Bool("swarm-master") + d.SwarmHost = flags.String("swarm-host") + d.SwarmDiscovery = flags.String("swarm-discovery") +} + +func EngineInstallURLFlagSet(flags DriverOptions) bool { + return EngineInstallURLSet(flags.String("engine-install-url")) +} + +func EngineInstallURLSet(url string) bool { + return url != DefaultEngineInstallURL && url != "" +} diff --git a/pkg/libmachine/drivers/check.go b/pkg/libmachine/drivers/check.go new file mode 100644 index 000000000000..78076c17bffd --- /dev/null +++ b/pkg/libmachine/drivers/check.go @@ -0,0 +1,84 @@ +package drivers + +import "k8s.io/minikube/pkg/libmachine/mcnflag" + +// CheckDriverOptions implements DriverOptions and is used to validate flag parsing +type CheckDriverOptions struct { + FlagsValues map[string]interface{} + CreateFlags []mcnflag.Flag + InvalidFlags []string +} + +func (o *CheckDriverOptions) String(key string) string { + for _, flag := range o.CreateFlags { + if flag.String() == key { + f, ok := flag.(mcnflag.StringFlag) + if !ok { + o.InvalidFlags = append(o.InvalidFlags, flag.String()) + } + + value, present := o.FlagsValues[key].(string) + if present { + return value + } + return f.Value + } + } + + return "" +} + +func (o *CheckDriverOptions) StringSlice(key string) []string { + for _, flag := range o.CreateFlags { + if flag.String() == key { + f, ok := flag.(mcnflag.StringSliceFlag) + if !ok { + o.InvalidFlags = append(o.InvalidFlags, flag.String()) + } + + value, present := o.FlagsValues[key].([]string) + if present { + return value + } + return f.Value + } + } + + return nil +} + +func (o *CheckDriverOptions) Int(key string) int { + for _, flag := range o.CreateFlags { + if flag.String() == key { + f, ok := flag.(mcnflag.IntFlag) + if !ok { + o.InvalidFlags = append(o.InvalidFlags, flag.String()) + } + + value, present := o.FlagsValues[key].(int) + if present { + return value + } + return f.Value + } + } + + return 0 +} + +func (o *CheckDriverOptions) Bool(key string) bool { + for _, flag := range o.CreateFlags { + if flag.String() == key { + _, ok := flag.(mcnflag.BoolFlag) + if !ok { + o.InvalidFlags = append(o.InvalidFlags, flag.String()) + } + } + } + + value, present := o.FlagsValues[key].(bool) + if present { + return value + } + return false +} diff --git a/pkg/libmachine/drivers/drivers.go b/pkg/libmachine/drivers/drivers.go new file mode 100644 index 000000000000..8268d4c0c1f0 --- /dev/null +++ b/pkg/libmachine/drivers/drivers.go @@ -0,0 +1,109 @@ +package drivers + +import ( + "errors" + + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnflag" + "k8s.io/minikube/pkg/libmachine/state" +) + +// Driver defines how a host is created and controlled. Different types of +// driver represent different ways hosts can be created (e.g. different +// hypervisors, different cloud providers) +type Driver interface { + // Create a host using the driver's config + Create() error + + // DriverName returns the name of the driver + DriverName() string + + // GetCreateFlags returns the mcnflag.Flag slice representing the flags + // that can be set, their descriptions and defaults. + GetCreateFlags() []mcnflag.Flag + + // GetIP returns an IP or hostname that this host is available at + // e.g. 1.2.3.4 or docker-host-d60b70a14d3a.cloudapp.net + GetIP() (string, error) + + // GetMachineName returns the name of the machine + GetMachineName() string + + // GetSSHHostname returns hostname for use with ssh + GetSSHHostname() (string, error) + + // GetSSHKeyPath returns key path for use with ssh + GetSSHKeyPath() string + + // GetSSHPort returns port for use with ssh + GetSSHPort() (int, error) + + // GetSSHUsername returns username for use with ssh + GetSSHUsername() string + + // GetURL returns a Docker compatible host URL for connecting to this host + // e.g. tcp://1.2.3.4:2376 + GetURL() (string, error) + + // GetState returns the state that the host is in (running, stopped, etc) + GetState() (state.State, error) + + // Kill stops a host forcefully + Kill() error + + // PreCreateCheck allows for pre-create operations to make sure a driver is ready for creation + PreCreateCheck() error + + // Remove a host + Remove() error + + // Restart a host. This may just call Stop(); Start() if the provider does not + // have any special restart behaviour. + Restart() error + + // SetConfigFromFlags configures the driver with the object that was returned + // by RegisterCreateFlags + SetConfigFromFlags(opts DriverOptions) error + + // Start a host + Start() error + + // Stop a host gracefully + Stop() error +} + +var ErrHostIsNotRunning = errors.New("Host is not running") + +type DriverOptions interface { + String(key string) string + StringSlice(key string) []string + Int(key string) int + Bool(key string) bool +} + +func MachineInState(d Driver, desiredState state.State) func() bool { + return func() bool { + currentState, err := d.GetState() + if err != nil { + log.Debugf("Error getting machine state: %s", err) + } + if currentState == desiredState { + return true + } + return false + } +} + +// MustBeRunning will return an error if the machine is not in a running state. +func MustBeRunning(d Driver) error { + s, err := d.GetState() + if err != nil { + return err + } + + if s != state.Running { + return ErrHostIsNotRunning + } + + return nil +} diff --git a/pkg/libmachine/drivers/notsupported.go b/pkg/libmachine/drivers/notsupported.go new file mode 100644 index 000000000000..67bc3c63c827 --- /dev/null +++ b/pkg/libmachine/drivers/notsupported.go @@ -0,0 +1,89 @@ +package drivers + +import ( + "fmt" + + "k8s.io/minikube/pkg/libmachine/mcnflag" + "k8s.io/minikube/pkg/libmachine/state" +) + +type DriverNotSupported struct { + *BaseDriver + Name string +} + +type NotSupported struct { + DriverName string +} + +func (e NotSupported) Error() string { + return fmt.Sprintf("Driver %q not supported on this platform.", e.DriverName) +} + +// NewDriverNotSupported creates a placeholder Driver that replaces +// a driver that is not supported on a given platform. eg fusion on linux. +func NewDriverNotSupported(driverName, hostName, storePath string) Driver { + return &DriverNotSupported{ + BaseDriver: &BaseDriver{ + MachineName: hostName, + StorePath: storePath, + }, + Name: driverName, + } +} + +func (d *DriverNotSupported) DriverName() string { + return d.Name +} + +func (d *DriverNotSupported) PreCreateCheck() error { + return NotSupported{d.DriverName()} +} + +func (d *DriverNotSupported) GetCreateFlags() []mcnflag.Flag { + return nil +} + +func (d *DriverNotSupported) SetConfigFromFlags(flags DriverOptions) error { + return NotSupported{d.DriverName()} +} + +func (d *DriverNotSupported) GetURL() (string, error) { + return "", NotSupported{d.DriverName()} +} + +func (d *DriverNotSupported) GetSSHHostname() (string, error) { + return "", NotSupported{d.DriverName()} +} + +func (d *DriverNotSupported) GetState() (state.State, error) { + return state.Error, NotSupported{d.DriverName()} +} + +func (d *DriverNotSupported) Create() error { + return NotSupported{d.DriverName()} +} + +func (d *DriverNotSupported) Remove() error { + return NotSupported{d.DriverName()} +} + +func (d *DriverNotSupported) Start() error { + return NotSupported{d.DriverName()} +} + +func (d *DriverNotSupported) Stop() error { + return NotSupported{d.DriverName()} +} + +func (d *DriverNotSupported) Restart() error { + return NotSupported{d.DriverName()} +} + +func (d *DriverNotSupported) Kill() error { + return NotSupported{d.DriverName()} +} + +func (d *DriverNotSupported) Upgrade() error { + return NotSupported{d.DriverName()} +} diff --git a/pkg/libmachine/drivers/plugin/localbinary/plugin.go b/pkg/libmachine/drivers/plugin/localbinary/plugin.go new file mode 100644 index 000000000000..9bd856da267e --- /dev/null +++ b/pkg/libmachine/drivers/plugin/localbinary/plugin.go @@ -0,0 +1,250 @@ +package localbinary + +import ( + "bufio" + "fmt" + "io" + "os" + "os/exec" + "strings" + "time" + + "k8s.io/minikube/pkg/libmachine/log" +) + +var ( + // Timeout where we will bail if we're not able to properly contact the + // plugin server. + defaultTimeout = 10 * time.Second + CurrentBinaryIsDockerMachine = false + CoreDrivers = []string{"amazonec2", "azure", "digitalocean", + "exoscale", "generic", "google", "hyperv", "none", "openstack", + "rackspace", "softlayer", "virtualbox", "vmwarefusion", + "vmwarevcloudair", "vmwarevsphere"} +) + +const ( + pluginOut = "(%s) %s" + pluginErr = "(%s) DBG | %s" + PluginEnvKey = "MACHINE_PLUGIN_TOKEN" + PluginEnvVal = "42" + PluginEnvDriverName = "MACHINE_PLUGIN_DRIVER_NAME" +) + +type PluginStreamer interface { + // Return a channel for receiving the output of the stream line by + // line. + // + // It happens to be the case that we do this all inside of the main + // plugin struct today, but that may not be the case forever. + AttachStream(*bufio.Scanner) <-chan string +} + +type PluginServer interface { + // Get the address where the plugin server is listening. + Address() (string, error) + + // Serve kicks off the plugin server. + Serve() error + + // Close shuts down the initialized server. + Close() error +} + +type McnBinaryExecutor interface { + // Execute the driver plugin. Returns scanners for plugin binary + // stdout and stderr. + Start() (*bufio.Scanner, *bufio.Scanner, error) + + // Stop reading from the plugins in question. + Close() error +} + +// DriverPlugin interface wraps the underlying mechanics of starting a driver +// plugin server and then figuring out where it can be dialed. +type DriverPlugin interface { + PluginServer + PluginStreamer +} + +type Plugin struct { + Executor McnBinaryExecutor + Addr string + MachineName string + addrCh chan string + stopCh chan struct{} + timeout time.Duration +} + +type Executor struct { + pluginStdout, pluginStderr io.ReadCloser + DriverName string + cmd *exec.Cmd + binaryPath string +} + +type ErrPluginBinaryNotFound struct { + driverName string + driverPath string +} + +func (e ErrPluginBinaryNotFound) Error() string { + return fmt.Sprintf("Driver %q not found. Do you have the plugin binary %q accessible in your PATH?", e.driverName, e.driverPath) +} + +// driverPath finds the path of a driver binary by its name. +// + If the driver is a core driver, there is no separate driver binary. We reuse current binary if it's `docker-machine` +// or we assume `docker-machine` is in the PATH. +// + If the driver is NOT a core driver, then the separate binary must be in the PATH and it's name must be +// `docker-machine-driver-driverName` +func driverPath(driverName string) string { + for _, coreDriver := range CoreDrivers { + if coreDriver == driverName { + if CurrentBinaryIsDockerMachine { + return os.Args[0] + } + + return "docker-machine" + } + } + + return fmt.Sprintf("docker-machine-driver-%s", driverName) +} + +func NewPlugin(driverName string) (*Plugin, error) { + driverPath := driverPath(driverName) + binaryPath, err := exec.LookPath(driverPath) + if err != nil { + return nil, ErrPluginBinaryNotFound{driverName, driverPath} + } + + log.Debugf("Found binary path at %s", binaryPath) + + return &Plugin{ + stopCh: make(chan struct{}), + addrCh: make(chan string, 1), + Executor: &Executor{ + DriverName: driverName, + binaryPath: binaryPath, + }, + }, nil +} + +func (lbe *Executor) Start() (*bufio.Scanner, *bufio.Scanner, error) { + var err error + + log.Debugf("Launching plugin server for driver %s", lbe.DriverName) + + lbe.cmd = exec.Command(lbe.binaryPath) + + lbe.pluginStdout, err = lbe.cmd.StdoutPipe() + if err != nil { + return nil, nil, fmt.Errorf("Error getting cmd stdout pipe: %s", err) + } + + lbe.pluginStderr, err = lbe.cmd.StderrPipe() + if err != nil { + return nil, nil, fmt.Errorf("Error getting cmd stderr pipe: %s", err) + } + + outScanner := bufio.NewScanner(lbe.pluginStdout) + errScanner := bufio.NewScanner(lbe.pluginStderr) + + os.Setenv(PluginEnvKey, PluginEnvVal) + os.Setenv(PluginEnvDriverName, lbe.DriverName) + + if err := lbe.cmd.Start(); err != nil { + return nil, nil, fmt.Errorf("Error starting plugin binary: %s", err) + } + + return outScanner, errScanner, nil +} + +func (lbe *Executor) Close() error { + if err := lbe.cmd.Wait(); err != nil { + return fmt.Errorf("Error waiting for binary close: %s", err) + } + + return nil +} + +func stream(scanner *bufio.Scanner, streamOutCh chan<- string, stopCh <-chan struct{}) { + for scanner.Scan() { + line := scanner.Text() + if err := scanner.Err(); err != nil { + log.Warnf("Scanning stream: %s", err) + } + select { + case streamOutCh <- strings.Trim(line, "\n"): + case <-stopCh: + return + } + } +} + +func (lbp *Plugin) AttachStream(scanner *bufio.Scanner) <-chan string { + streamOutCh := make(chan string) + go stream(scanner, streamOutCh, lbp.stopCh) + return streamOutCh +} + +func (lbp *Plugin) execServer() error { + outScanner, errScanner, err := lbp.Executor.Start() + if err != nil { + return err + } + + // Scan just one line to get the address, then send it to the relevant + // channel. + outScanner.Scan() + addr := outScanner.Text() + if err := outScanner.Err(); err != nil { + return fmt.Errorf("Reading plugin address failed: %s", err) + } + + lbp.addrCh <- strings.TrimSpace(addr) + + stdOutCh := lbp.AttachStream(outScanner) + stdErrCh := lbp.AttachStream(errScanner) + + for { + select { + case out := <-stdOutCh: + log.Infof(pluginOut, lbp.MachineName, out) + case err := <-stdErrCh: + log.Debugf(pluginErr, lbp.MachineName, err) + case <-lbp.stopCh: + if err := lbp.Executor.Close(); err != nil { + return fmt.Errorf("Error closing local plugin binary: %s", err) + } + return nil + } + } +} + +func (lbp *Plugin) Serve() error { + return lbp.execServer() +} + +func (lbp *Plugin) Address() (string, error) { + if lbp.Addr == "" { + if lbp.timeout == 0 { + lbp.timeout = defaultTimeout + } + + select { + case lbp.Addr = <-lbp.addrCh: + log.Debugf("Plugin server listening at address %s", lbp.Addr) + close(lbp.addrCh) + return lbp.Addr, nil + case <-time.After(lbp.timeout): + return "", fmt.Errorf("Failed to dial the plugin server in %s", lbp.timeout) + } + } + return lbp.Addr, nil +} + +func (lbp *Plugin) Close() error { + close(lbp.stopCh) + return nil +} diff --git a/pkg/libmachine/drivers/plugin/register_driver.go b/pkg/libmachine/drivers/plugin/register_driver.go new file mode 100644 index 000000000000..dacd9a4409a8 --- /dev/null +++ b/pkg/libmachine/drivers/plugin/register_driver.go @@ -0,0 +1,63 @@ +package plugin + +import ( + "fmt" + "net" + "net/http" + "net/rpc" + "os" + "time" + + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers/plugin/localbinary" + "k8s.io/minikube/pkg/libmachine/drivers/rpc" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/version" +) + +var ( + heartbeatTimeout = 10 * time.Second +) + +func RegisterDriver(d drivers.Driver) { + if os.Getenv(localbinary.PluginEnvKey) != localbinary.PluginEnvVal { + fmt.Fprintf(os.Stderr, `This is a Docker Machine plugin binary. +Plugin binaries are not intended to be invoked directly. +Please use this plugin through the main 'docker-machine' binary. +(API version: %d) +`, version.APIVersion) + os.Exit(1) + } + + log.SetDebug(true) + os.Setenv("MACHINE_DEBUG", "1") + + rpcd := rpcdriver.NewRPCServerDriver(d) + rpc.RegisterName(rpcdriver.RPCServiceNameV0, rpcd) + rpc.RegisterName(rpcdriver.RPCServiceNameV1, rpcd) + rpc.HandleHTTP() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + fmt.Fprintf(os.Stderr, "Error loading RPC server: %s\n", err) + os.Exit(1) + } + defer listener.Close() + + fmt.Println(listener.Addr()) + + go http.Serve(listener, nil) + + for { + select { + case <-rpcd.CloseCh: + log.Debug("Closing plugin on server side") + os.Exit(0) + case <-rpcd.HeartbeatCh: + continue + case <-time.After(heartbeatTimeout): + // TODO: Add heartbeat retry logic + os.Exit(1) + } + } +} diff --git a/pkg/libmachine/drivers/rpc/client_driver.go b/pkg/libmachine/drivers/rpc/client_driver.go new file mode 100644 index 000000000000..6c3132d230ca --- /dev/null +++ b/pkg/libmachine/drivers/rpc/client_driver.go @@ -0,0 +1,367 @@ +package rpcdriver + +import ( + "fmt" + "net/rpc" + "sync" + "time" + + "io" + + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers/plugin/localbinary" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnflag" + "k8s.io/minikube/pkg/libmachine/state" + "k8s.io/minikube/pkg/libmachine/version" +) + +var ( + heartbeatInterval = 5 * time.Second +) + +type RPCClientDriverFactory interface { + NewRPCClientDriver(driverName string, rawDriver []byte) (*RPCClientDriver, error) + io.Closer +} + +type DefaultRPCClientDriverFactory struct { + openedDrivers []*RPCClientDriver + openedDriversLock sync.Locker +} + +func NewRPCClientDriverFactory() RPCClientDriverFactory { + return &DefaultRPCClientDriverFactory{ + openedDrivers: []*RPCClientDriver{}, + openedDriversLock: &sync.Mutex{}, + } +} + +type RPCClientDriver struct { + plugin localbinary.DriverPlugin + heartbeatDoneCh chan bool + Client *InternalClient +} + +type RPCCall struct { + ServiceMethod string + Args interface{} + Reply interface{} +} + +type InternalClient struct { + MachineName string + RPCClient *rpc.Client + rpcServiceName string +} + +const ( + RPCServiceNameV0 = `RpcServerDriver` + RPCServiceNameV1 = `RPCServerDriver` + + HeartbeatMethod = `.Heartbeat` + GetVersionMethod = `.GetVersion` + CloseMethod = `.Close` + GetCreateFlagsMethod = `.GetCreateFlags` + SetConfigRawMethod = `.SetConfigRaw` + GetConfigRawMethod = `.GetConfigRaw` + DriverNameMethod = `.DriverName` + SetConfigFromFlagsMethod = `.SetConfigFromFlags` + GetURLMethod = `.GetURL` + GetMachineNameMethod = `.GetMachineName` + GetIPMethod = `.GetIP` + GetSSHHostnameMethod = `.GetSSHHostname` + GetSSHKeyPathMethod = `.GetSSHKeyPath` + GetSSHPortMethod = `.GetSSHPort` + GetSSHUsernameMethod = `.GetSSHUsername` + GetStateMethod = `.GetState` + PreCreateCheckMethod = `.PreCreateCheck` + CreateMethod = `.Create` + RemoveMethod = `.Remove` + StartMethod = `.Start` + StopMethod = `.Stop` + RestartMethod = `.Restart` + KillMethod = `.Kill` + UpgradeMethod = `.Upgrade` +) + +func (ic *InternalClient) Call(serviceMethod string, args interface{}, reply interface{}) error { + if serviceMethod != HeartbeatMethod { + log.Debugf("(%s) Calling %+v", ic.MachineName, serviceMethod) + } + return ic.RPCClient.Call(ic.rpcServiceName+serviceMethod, args, reply) +} + +func (ic *InternalClient) switchToV0() { + ic.rpcServiceName = RPCServiceNameV0 +} + +func NewInternalClient(rpcclient *rpc.Client) *InternalClient { + return &InternalClient{ + RPCClient: rpcclient, + rpcServiceName: RPCServiceNameV1, + } +} + +func (f *DefaultRPCClientDriverFactory) Close() error { + f.openedDriversLock.Lock() + defer f.openedDriversLock.Unlock() + + for _, openedDriver := range f.openedDrivers { + if err := openedDriver.close(); err != nil { + // No need to display an error. + // There's nothing we can do and it doesn't add value to the user. + } + } + f.openedDrivers = []*RPCClientDriver{} + + return nil +} + +func (f *DefaultRPCClientDriverFactory) NewRPCClientDriver(driverName string, rawDriver []byte) (*RPCClientDriver, error) { + mcnName := "" + + p, err := localbinary.NewPlugin(driverName) + if err != nil { + return nil, err + } + + go func() { + if err := p.Serve(); err != nil { + // TODO: Is this best approach? + log.Warn(err) + return + } + }() + + addr, err := p.Address() + if err != nil { + return nil, fmt.Errorf("Error attempting to get plugin server address for RPC: %s", err) + } + + rpcclient, err := rpc.DialHTTP("tcp", addr) + if err != nil { + return nil, err + } + + c := &RPCClientDriver{ + Client: NewInternalClient(rpcclient), + heartbeatDoneCh: make(chan bool), + } + + f.openedDriversLock.Lock() + f.openedDrivers = append(f.openedDrivers, c) + f.openedDriversLock.Unlock() + + var serverVersion int + if err := c.Client.Call(GetVersionMethod, struct{}{}, &serverVersion); err != nil { + // this is the first call we make to the server. We try to play nice with old pre 0.5.1 client, + // by gracefully trying old RPCServiceName, we do this only once, and keep the result for future calls. + log.Debugf(err.Error()) + log.Debugf("Client (%s) with %s does not work, re-attempting with %s", c.Client.MachineName, RPCServiceNameV1, RPCServiceNameV0) + c.Client.switchToV0() + if err := c.Client.Call(GetVersionMethod, struct{}{}, &serverVersion); err != nil { + return nil, err + } + } + + if serverVersion != version.APIVersion { + return nil, fmt.Errorf("Driver binary uses an incompatible API version (%d)", serverVersion) + } + log.Debug("Using API Version ", serverVersion) + + go func(c *RPCClientDriver) { + for { + select { + case <-c.heartbeatDoneCh: + return + case <-time.After(heartbeatInterval): + if err := c.Client.Call(HeartbeatMethod, struct{}{}, nil); err != nil { + log.Warnf("Wrapper Docker Machine process exiting due to closed plugin server (%s)", err) + if err := c.close(); err != nil { + log.Warn(err) + } + } + } + } + }(c) + + if err := c.SetConfigRaw(rawDriver); err != nil { + return nil, err + } + + mcnName = c.GetMachineName() + p.MachineName = mcnName + c.Client.MachineName = mcnName + c.plugin = p + + return c, nil +} + +func (c *RPCClientDriver) MarshalJSON() ([]byte, error) { + return c.GetConfigRaw() +} + +func (c *RPCClientDriver) UnmarshalJSON(data []byte) error { + return c.SetConfigRaw(data) +} + +func (c *RPCClientDriver) close() error { + c.heartbeatDoneCh <- true + close(c.heartbeatDoneCh) + + log.Debug("Making call to close driver server") + + if err := c.Client.Call(CloseMethod, struct{}{}, nil); err != nil { + log.Debugf("Failed to make call to close driver server: %s", err) + } else { + log.Debug("Successfully made call to close driver server") + } + + log.Debug("Making call to close connection to plugin binary") + + return c.plugin.Close() +} + +// Helper method to make requests which take no arguments and return simply a +// string, e.g. "GetIP". +func (c *RPCClientDriver) rpcStringCall(method string) (string, error) { + var info string + + if err := c.Client.Call(method, struct{}{}, &info); err != nil { + return "", err + } + + return info, nil +} + +func (c *RPCClientDriver) GetCreateFlags() []mcnflag.Flag { + var flags []mcnflag.Flag + + if err := c.Client.Call(GetCreateFlagsMethod, struct{}{}, &flags); err != nil { + log.Warnf("Error attempting call to get create flags: %s", err) + } + + return flags +} + +func (c *RPCClientDriver) SetConfigRaw(data []byte) error { + return c.Client.Call(SetConfigRawMethod, data, nil) +} + +func (c *RPCClientDriver) GetConfigRaw() ([]byte, error) { + var data []byte + + if err := c.Client.Call(GetConfigRawMethod, struct{}{}, &data); err != nil { + return nil, err + } + + return data, nil +} + +// DriverName returns the name of the driver +func (c *RPCClientDriver) DriverName() string { + driverName, err := c.rpcStringCall(DriverNameMethod) + if err != nil { + log.Warnf("Error attempting call to get driver name: %s", err) + } + + return driverName +} + +func (c *RPCClientDriver) SetConfigFromFlags(flags drivers.DriverOptions) error { + return c.Client.Call(SetConfigFromFlagsMethod, &flags, nil) +} + +func (c *RPCClientDriver) GetURL() (string, error) { + return c.rpcStringCall(GetURLMethod) +} + +func (c *RPCClientDriver) GetMachineName() string { + name, err := c.rpcStringCall(GetMachineNameMethod) + if err != nil { + log.Warnf("Error attempting call to get machine name: %s", err) + } + + return name +} + +func (c *RPCClientDriver) GetIP() (string, error) { + return c.rpcStringCall(GetIPMethod) +} + +func (c *RPCClientDriver) GetSSHHostname() (string, error) { + return c.rpcStringCall(GetSSHHostnameMethod) +} + +// GetSSHKeyPath returns the key path +// TODO: This method doesn't even make sense to have with RPC. +func (c *RPCClientDriver) GetSSHKeyPath() string { + path, err := c.rpcStringCall(GetSSHKeyPathMethod) + if err != nil { + log.Warnf("Error attempting call to get SSH key path: %s", err) + } + + return path +} + +func (c *RPCClientDriver) GetSSHPort() (int, error) { + var port int + + if err := c.Client.Call(GetSSHPortMethod, struct{}{}, &port); err != nil { + return 0, err + } + + return port, nil +} + +func (c *RPCClientDriver) GetSSHUsername() string { + username, err := c.rpcStringCall(GetSSHUsernameMethod) + if err != nil { + log.Warnf("Error attempting call to get SSH username: %s", err) + } + + return username +} + +func (c *RPCClientDriver) GetState() (state.State, error) { + var s state.State + + if err := c.Client.Call(GetStateMethod, struct{}{}, &s); err != nil { + return state.Error, err + } + + return s, nil +} + +func (c *RPCClientDriver) PreCreateCheck() error { + return c.Client.Call(PreCreateCheckMethod, struct{}{}, nil) +} + +func (c *RPCClientDriver) Create() error { + return c.Client.Call(CreateMethod, struct{}{}, nil) +} + +func (c *RPCClientDriver) Remove() error { + return c.Client.Call(RemoveMethod, struct{}{}, nil) +} + +func (c *RPCClientDriver) Start() error { + return c.Client.Call(StartMethod, struct{}{}, nil) +} + +func (c *RPCClientDriver) Stop() error { + return c.Client.Call(StopMethod, struct{}{}, nil) +} + +func (c *RPCClientDriver) Restart() error { + return c.Client.Call(RestartMethod, struct{}{}, nil) +} + +func (c *RPCClientDriver) Kill() error { + return c.Client.Call(KillMethod, struct{}{}, nil) +} + +func (c *RPCClientDriver) Upgrade() error { + return c.Client.Call(UpgradeMethod, struct{}{}, nil) +} diff --git a/pkg/libmachine/drivers/rpc/server_driver.go b/pkg/libmachine/drivers/rpc/server_driver.go new file mode 100644 index 000000000000..66af07e19f0d --- /dev/null +++ b/pkg/libmachine/drivers/rpc/server_driver.go @@ -0,0 +1,227 @@ +package rpcdriver + +import ( + "encoding/gob" + "encoding/json" + "fmt" + "runtime/debug" + + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnflag" + "k8s.io/minikube/pkg/libmachine/state" + "k8s.io/minikube/pkg/libmachine/version" +) + +type Stacker interface { + Stack() []byte +} + +type StandardStack struct{} + +func (ss *StandardStack) Stack() []byte { + return debug.Stack() +} + +var ( + stdStacker Stacker = &StandardStack{} +) + +func init() { + gob.Register(new(RPCFlags)) + gob.Register(new(mcnflag.IntFlag)) + gob.Register(new(mcnflag.StringFlag)) + gob.Register(new(mcnflag.StringSliceFlag)) + gob.Register(new(mcnflag.BoolFlag)) +} + +type RPCFlags struct { + Values map[string]interface{} +} + +func (r RPCFlags) Get(key string) interface{} { + val, ok := r.Values[key] + if !ok { + log.Warnf("Trying to access option %s which does not exist", key) + log.Warn("THIS ***WILL*** CAUSE UNEXPECTED BEHAVIOR") + } + return val +} + +func (r RPCFlags) String(key string) string { + val, ok := r.Get(key).(string) + if !ok { + log.Warnf("Type assertion did not go smoothly to string for key %s", key) + } + return val +} + +func (r RPCFlags) StringSlice(key string) []string { + val, ok := r.Get(key).([]string) + if !ok { + log.Warnf("Type assertion did not go smoothly to string slice for key %s", key) + } + return val +} + +func (r RPCFlags) Int(key string) int { + val, ok := r.Get(key).(int) + if !ok { + log.Warnf("Type assertion did not go smoothly to int for key %s", key) + } + return val +} + +func (r RPCFlags) Bool(key string) bool { + val, ok := r.Get(key).(bool) + if !ok { + log.Warnf("Type assertion did not go smoothly to bool for key %s", key) + } + return val +} + +type RPCServerDriver struct { + ActualDriver drivers.Driver + CloseCh chan bool + HeartbeatCh chan bool +} + +func NewRPCServerDriver(d drivers.Driver) *RPCServerDriver { + return &RPCServerDriver{ + ActualDriver: d, + CloseCh: make(chan bool), + HeartbeatCh: make(chan bool), + } +} + +func (r *RPCServerDriver) Close(_, _ *struct{}) error { + r.CloseCh <- true + return nil +} + +func (r *RPCServerDriver) GetVersion(_ *struct{}, reply *int) error { + *reply = version.APIVersion + return nil +} + +func (r *RPCServerDriver) GetConfigRaw(_ *struct{}, reply *[]byte) error { + driverData, err := json.Marshal(r.ActualDriver) + if err != nil { + return err + } + + *reply = driverData + + return nil +} + +func (r *RPCServerDriver) GetCreateFlags(_ *struct{}, reply *[]mcnflag.Flag) error { + *reply = r.ActualDriver.GetCreateFlags() + return nil +} + +func (r *RPCServerDriver) SetConfigRaw(data []byte, _ *struct{}) error { + return json.Unmarshal(data, &r.ActualDriver) +} + +func trapPanic(err *error) { + if r := recover(); r != nil { + *err = fmt.Errorf("Panic in the driver: %s\n%s", r.(error), stdStacker.Stack()) + } +} + +func (r *RPCServerDriver) Create(_, _ *struct{}) (err error) { + // In an ideal world, plugins wouldn't ever panic. However, panics + // have been known to happen and cause issues. Therefore, we recover + // and do not crash the RPC server completely in the case of a panic + // during create. + defer trapPanic(&err) + + err = r.ActualDriver.Create() + + return err +} + +func (r *RPCServerDriver) DriverName(_ *struct{}, reply *string) error { + *reply = r.ActualDriver.DriverName() + return nil +} + +func (r *RPCServerDriver) GetIP(_ *struct{}, reply *string) error { + ip, err := r.ActualDriver.GetIP() + *reply = ip + return err +} + +func (r *RPCServerDriver) GetMachineName(_ *struct{}, reply *string) error { + *reply = r.ActualDriver.GetMachineName() + return nil +} + +func (r *RPCServerDriver) GetSSHHostname(_ *struct{}, reply *string) error { + hostname, err := r.ActualDriver.GetSSHHostname() + *reply = hostname + return err +} + +func (r *RPCServerDriver) GetSSHKeyPath(_ *struct{}, reply *string) error { + *reply = r.ActualDriver.GetSSHKeyPath() + return nil +} + +// GetSSHPort returns port for use with ssh +func (r *RPCServerDriver) GetSSHPort(_ *struct{}, reply *int) error { + port, err := r.ActualDriver.GetSSHPort() + *reply = port + return err +} + +func (r *RPCServerDriver) GetSSHUsername(_ *struct{}, reply *string) error { + *reply = r.ActualDriver.GetSSHUsername() + return nil +} + +func (r *RPCServerDriver) GetURL(_ *struct{}, reply *string) error { + info, err := r.ActualDriver.GetURL() + *reply = info + return err +} + +func (r *RPCServerDriver) GetState(_ *struct{}, reply *state.State) error { + s, err := r.ActualDriver.GetState() + *reply = s + return err +} + +func (r *RPCServerDriver) Kill(_ *struct{}, _ *struct{}) error { + return r.ActualDriver.Kill() +} + +func (r *RPCServerDriver) PreCreateCheck(_ *struct{}, _ *struct{}) error { + return r.ActualDriver.PreCreateCheck() +} + +func (r *RPCServerDriver) Remove(_ *struct{}, _ *struct{}) error { + return r.ActualDriver.Remove() +} + +func (r *RPCServerDriver) Restart(_ *struct{}, _ *struct{}) error { + return r.ActualDriver.Restart() +} + +func (r *RPCServerDriver) SetConfigFromFlags(flags *drivers.DriverOptions, _ *struct{}) error { + return r.ActualDriver.SetConfigFromFlags(*flags) +} + +func (r *RPCServerDriver) Start(_ *struct{}, _ *struct{}) error { + return r.ActualDriver.Start() +} + +func (r *RPCServerDriver) Stop(_ *struct{}, _ *struct{}) error { + return r.ActualDriver.Stop() +} + +func (r *RPCServerDriver) Heartbeat(_ *struct{}, _ *struct{}) error { + r.HeartbeatCh <- true + return nil +} diff --git a/pkg/libmachine/drivers/serial.go b/pkg/libmachine/drivers/serial.go new file mode 100644 index 000000000000..4b6cbabc1adb --- /dev/null +++ b/pkg/libmachine/drivers/serial.go @@ -0,0 +1,172 @@ +package drivers + +import ( + "sync" + + "encoding/json" + + "k8s.io/minikube/pkg/libmachine/mcnflag" + "k8s.io/minikube/pkg/libmachine/state" +) + +var stdLock = &sync.Mutex{} + +// SerialDriver is a wrapper struct which is used to ensure that RPC calls +// to a driver only occur one at a time. +// Some providers, e.g. virtualbox, should not run driver operations at the +// same time as other driver instances of the same type. Otherwise, we scrape +// up against VirtualBox's own locking mechanisms. +// +// It would be preferable to simply have a lock around, say, the VBoxManage +// command, but with our current one-server-process-per-machine model it is +// impossible to dictate this locking on the server side. +type SerialDriver struct { + Driver + sync.Locker +} + +func NewSerialDriver(innerDriver Driver) Driver { + return newSerialDriverWithLock(innerDriver, stdLock) +} + +func newSerialDriverWithLock(innerDriver Driver, lock sync.Locker) Driver { + return &SerialDriver{ + Driver: innerDriver, + Locker: lock, + } +} + +// Create a host using the driver's config +func (d *SerialDriver) Create() error { + d.Lock() + defer d.Unlock() + return d.Driver.Create() +} + +// DriverName returns the name of the driver as it is registered +func (d *SerialDriver) DriverName() string { + d.Lock() + defer d.Unlock() + return d.Driver.DriverName() +} + +// GetCreateFlags returns the mcnflag.Flag slice representing the flags +// that can be set, their descriptions and defaults. +func (d *SerialDriver) GetCreateFlags() []mcnflag.Flag { + d.Lock() + defer d.Unlock() + return d.Driver.GetCreateFlags() +} + +// GetIP returns an IP or hostname that this host is available at +// e.g. 1.2.3.4 or docker-host-d60b70a14d3a.cloudapp.net +func (d *SerialDriver) GetIP() (string, error) { + d.Lock() + defer d.Unlock() + return d.Driver.GetIP() +} + +// GetMachineName returns the name of the machine +func (d *SerialDriver) GetMachineName() string { + d.Lock() + defer d.Unlock() + return d.Driver.GetMachineName() +} + +// GetSSHHostname returns hostname for use with ssh +func (d *SerialDriver) GetSSHHostname() (string, error) { + d.Lock() + defer d.Unlock() + return d.Driver.GetSSHHostname() +} + +// GetSSHKeyPath returns key path for use with ssh +func (d *SerialDriver) GetSSHKeyPath() string { + d.Lock() + defer d.Unlock() + return d.Driver.GetSSHKeyPath() +} + +// GetSSHPort returns port for use with ssh +func (d *SerialDriver) GetSSHPort() (int, error) { + d.Lock() + defer d.Unlock() + return d.Driver.GetSSHPort() +} + +// GetSSHUsername returns username for use with ssh +func (d *SerialDriver) GetSSHUsername() string { + d.Lock() + defer d.Unlock() + return d.Driver.GetSSHUsername() +} + +// GetURL returns a Docker compatible host URL for connecting to this host +// e.g. tcp://1.2.3.4:2376 +func (d *SerialDriver) GetURL() (string, error) { + d.Lock() + defer d.Unlock() + return d.Driver.GetURL() +} + +// GetState returns the state that the host is in (running, stopped, etc) +func (d *SerialDriver) GetState() (state.State, error) { + d.Lock() + defer d.Unlock() + return d.Driver.GetState() +} + +// Kill stops a host forcefully +func (d *SerialDriver) Kill() error { + d.Lock() + defer d.Unlock() + return d.Driver.Kill() +} + +// PreCreateCheck allows for pre-create operations to make sure a driver is ready for creation +func (d *SerialDriver) PreCreateCheck() error { + d.Lock() + defer d.Unlock() + return d.Driver.PreCreateCheck() +} + +// Remove a host +func (d *SerialDriver) Remove() error { + d.Lock() + defer d.Unlock() + return d.Driver.Remove() +} + +// Restart a host. This may just call Stop(); Start() if the provider does not +// have any special restart behaviour. +func (d *SerialDriver) Restart() error { + d.Lock() + defer d.Unlock() + return d.Driver.Restart() +} + +// SetConfigFromFlags configures the driver with the object that was returned +// by RegisterCreateFlags +func (d *SerialDriver) SetConfigFromFlags(opts DriverOptions) error { + d.Lock() + defer d.Unlock() + return d.Driver.SetConfigFromFlags(opts) +} + +// Start a host +func (d *SerialDriver) Start() error { + d.Lock() + defer d.Unlock() + return d.Driver.Start() +} + +// Stop a host gracefully +func (d *SerialDriver) Stop() error { + d.Lock() + defer d.Unlock() + return d.Driver.Stop() +} + +func (d *SerialDriver) MarshalJSON() ([]byte, error) { + return json.Marshal(d.Driver) +} diff --git a/pkg/libmachine/drivers/utils.go b/pkg/libmachine/drivers/utils.go new file mode 100644 index 000000000000..56c6d838bd2d --- /dev/null +++ b/pkg/libmachine/drivers/utils.go @@ -0,0 +1,73 @@ +package drivers + +import ( + "fmt" + + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/ssh" +) + +func GetSSHClientFromDriver(d Driver) (ssh.Client, error) { + address, err := d.GetSSHHostname() + if err != nil { + return nil, err + } + + port, err := d.GetSSHPort() + if err != nil { + return nil, err + } + + var auth *ssh.Auth + if d.GetSSHKeyPath() == "" { + auth = &ssh.Auth{} + } else { + auth = &ssh.Auth{ + Keys: []string{d.GetSSHKeyPath()}, + } + } + + client, err := ssh.NewClient(d.GetSSHUsername(), address, port, auth) + return client, err + +} + +func RunSSHCommandFromDriver(d Driver, command string) (string, error) { + client, err := GetSSHClientFromDriver(d) + if err != nil { + return "", err + } + + log.Debugf("About to run SSH command:\n%s", command) + + output, err := client.Output(command) + log.Debugf("SSH cmd err, output: %v: %s", err, output) + if err != nil { + return "", fmt.Errorf(`ssh command error: +command : %s +err : %v +output : %s`, command, err, output) + } + + return output, nil +} + +func sshAvailableFunc(d Driver) func() bool { + return func() bool { + log.Debug("Getting to WaitForSSH function...") + if _, err := RunSSHCommandFromDriver(d, "exit 0"); err != nil { + log.Debugf("Error getting ssh command 'exit 0' : %s", err) + return false + } + return true + } +} + +func WaitForSSH(d Driver) error { + // Try to dial SSH for 30 seconds before timing out. + if err := mcnutils.WaitFor(sshAvailableFunc(d)); err != nil { + return fmt.Errorf("Too many retries waiting for SSH to be available. Last error: %s", err) + } + return nil +} diff --git a/pkg/libmachine/engine/engine.go b/pkg/libmachine/engine/engine.go new file mode 100644 index 000000000000..2910432c1947 --- /dev/null +++ b/pkg/libmachine/engine/engine.go @@ -0,0 +1,21 @@ +package engine + +const ( + DefaultPort = 2376 +) + +type Options struct { + ArbitraryFlags []string + DNS []string `json:"Dns"` + GraphDir string + Env []string + Ipv6 bool + InsecureRegistry []string + Labels []string + LogLevel string + StorageDriver string + SelinuxEnabled bool + TLSVerify bool `json:"TlsVerify"` + RegistryMirror []string + InstallURL string +} diff --git a/pkg/libmachine/host/host.go b/pkg/libmachine/host/host.go new file mode 100644 index 000000000000..a77535808cd4 --- /dev/null +++ b/pkg/libmachine/host/host.go @@ -0,0 +1,286 @@ +package host + +import ( + "regexp" + + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/cert" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcndockerclient" + "k8s.io/minikube/pkg/libmachine/mcnerror" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/provision" + "k8s.io/minikube/pkg/libmachine/provision/pkgaction" + "k8s.io/minikube/pkg/libmachine/provision/serviceaction" + "k8s.io/minikube/pkg/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/state" + "k8s.io/minikube/pkg/libmachine/swarm" + "k8s.io/minikube/pkg/libmachine/versioncmp" +) + +var ( + validHostNamePattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9\-\.]*$`) + stdSSHClientCreator SSHClientCreator = &StandardSSHClientCreator{} +) + +type SSHClientCreator interface { + CreateSSHClient(d drivers.Driver) (ssh.Client, error) +} + +type StandardSSHClientCreator struct { + drivers.Driver +} + +func SetSSHClientCreator(creator SSHClientCreator) { + stdSSHClientCreator = creator +} + +type Host struct { + ConfigVersion int + Driver drivers.Driver + DriverName string + HostOptions *Options + Name string + RawDriver []byte `json:"-"` +} + +type Options struct { + Driver string + Memory int + Disk int + EngineOptions *engine.Options + SwarmOptions *swarm.Options + AuthOptions *auth.Options +} + +type Metadata struct { + ConfigVersion int + DriverName string + HostOptions Options +} + +func ValidateHostName(name string) bool { + return validHostNamePattern.MatchString(name) +} + +func (h *Host) RunSSHCommand(command string) (string, error) { + return drivers.RunSSHCommandFromDriver(h.Driver, command) +} + +func (h *Host) CreateSSHClient() (ssh.Client, error) { + return stdSSHClientCreator.CreateSSHClient(h.Driver) +} + +func (creator *StandardSSHClientCreator) CreateSSHClient(d drivers.Driver) (ssh.Client, error) { + addr, err := d.GetSSHHostname() + if err != nil { + return &ssh.ExternalClient{}, err + } + + port, err := d.GetSSHPort() + if err != nil { + return &ssh.ExternalClient{}, err + } + + auth := &ssh.Auth{} + if d.GetSSHKeyPath() != "" { + auth.Keys = []string{d.GetSSHKeyPath()} + } + + return ssh.NewClient(d.GetSSHUsername(), addr, port, auth) +} + +func (h *Host) runActionForState(action func() error, desiredState state.State) error { + if drivers.MachineInState(h.Driver, desiredState)() { + return mcnerror.ErrHostAlreadyInState{ + Name: h.Name, + State: desiredState, + } + } + + if err := action(); err != nil { + return err + } + + return mcnutils.WaitFor(drivers.MachineInState(h.Driver, desiredState)) +} + +func (h *Host) WaitForDocker() error { + provisioner, err := provision.DetectProvisioner(h.Driver) + if err != nil { + return err + } + + return provision.WaitForDocker(provisioner, engine.DefaultPort) +} + +func (h *Host) Start() error { + log.Infof("Starting %q...", h.Name) + if err := h.runActionForState(h.Driver.Start, state.Running); err != nil { + return err + } + + log.Infof("Machine %q was started.", h.Name) + + return h.WaitForDocker() +} + +func (h *Host) Stop() error { + log.Infof("Stopping %q...", h.Name) + if err := h.runActionForState(h.Driver.Stop, state.Stopped); err != nil { + return err + } + + log.Infof("Machine %q was stopped.", h.Name) + return nil +} + +func (h *Host) Kill() error { + log.Infof("Killing %q...", h.Name) + if err := h.runActionForState(h.Driver.Kill, state.Stopped); err != nil { + return err + } + + log.Infof("Machine %q was killed.", h.Name) + return nil +} + +func (h *Host) Restart() error { + log.Infof("Restarting %q...", h.Name) + if drivers.MachineInState(h.Driver, state.Stopped)() { + if err := h.Start(); err != nil { + return err + } + } else if drivers.MachineInState(h.Driver, state.Running)() { + if err := h.Driver.Restart(); err != nil { + return err + } + if err := mcnutils.WaitFor(drivers.MachineInState(h.Driver, state.Running)); err != nil { + return err + } + } + + return h.WaitForDocker() +} + +func (h *Host) DockerVersion() (string, error) { + url, err := h.Driver.GetURL() + if err != nil { + return "", err + } + + dockerHost := &mcndockerclient.RemoteDocker{ + HostURL: url, + AuthOption: h.AuthOptions(), + } + dockerVersion, err := mcndockerclient.DockerVersion(dockerHost) + if err != nil { + return "", err + } + + return dockerVersion, nil +} + +func (h *Host) Upgrade() error { + machineState, err := h.Driver.GetState() + if err != nil { + return err + } + + if machineState != state.Running { + log.Info("Starting machine so machine can be upgraded...") + if err := h.Start(); err != nil { + return err + } + } + + provisioner, err := provision.DetectProvisioner(h.Driver) + if err != nil { + return err + } + + dockerVersion, err := h.DockerVersion() + if err != nil { + return err + } + + // If we're upgrading from a pre-CE (e.g., 1.13.1) release to a CE + // release (e.g., 17.03.0-ce), we should simply uninstall and + // re-install from scratch, since the official package names will + // change from 'docker-engine' to 'docker-ce'. + if versioncmp.LessThanOrEqualTo(dockerVersion, provision.LastReleaseBeforeCEVersioning) && + // RancherOS and boot2docker, being 'static ISO builds', have + // an upgrade process which simply grabs the latest if it's + // different, and so do not need to jump through this hoop to + // upgrade safely. + provisioner.String() != "rancheros" && + provisioner.String() != "boot2docker" { + + // Name of package 'docker-engine' will fall through in this + // case, so that we execute, e.g., + // + // 'sudo apt-get purge -y docker-engine' + if err := provisioner.Package("docker-engine", pkgaction.Purge); err != nil { + return err + } + + // Then we kick off the normal provisioning process which will + // go off and install Docker (get.docker.com script should work + // fine to install Docker from scratch after removing the old + // packages, and images/containers etc. should be preserved in + // /var/lib/docker) + return h.Provision() + } + + log.Info("Upgrading docker...") + if err := provisioner.Package("docker", pkgaction.Upgrade); err != nil { + return err + } + + log.Info("Restarting docker...") + return provisioner.Service("docker", serviceaction.Restart) +} + +func (h *Host) URL() (string, error) { + return h.Driver.GetURL() +} + +func (h *Host) AuthOptions() *auth.Options { + if h.HostOptions == nil { + return nil + } + return h.HostOptions.AuthOptions +} + +func (h *Host) ConfigureAuth() error { + provisioner, err := provision.DetectProvisioner(h.Driver) + if err != nil { + return err + } + + // TODO: This is kind of a hack (or is it? I'm not really sure until + // we have more clearly defined outlook on what the responsibilities + // and modularity of the provisioners should be). + // + // Call provision to re-provision the certs properly. + return provisioner.Provision(swarm.Options{}, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions) +} + +func (h *Host) ConfigureAllAuth() error { + log.Info("Regenerating local certificates") + if err := cert.BootstrapCertificates(h.AuthOptions()); err != nil { + return err + } + return h.ConfigureAuth() +} + +func (h *Host) Provision() error { + provisioner, err := provision.DetectProvisioner(h.Driver) + if err != nil { + return err + } + + return provisioner.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions) +} diff --git a/pkg/libmachine/libmachine.go b/pkg/libmachine/libmachine.go new file mode 100644 index 000000000000..ebaf7eb31f9b --- /dev/null +++ b/pkg/libmachine/libmachine.go @@ -0,0 +1,181 @@ +package libmachine + +import ( + "fmt" + "path/filepath" + + "io" + + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/cert" + "k8s.io/minikube/pkg/libmachine/check" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers/rpc" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnerror" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/persist" + "k8s.io/minikube/pkg/libmachine/provision" + "k8s.io/minikube/pkg/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/state" + "k8s.io/minikube/pkg/libmachine/swarm" + "k8s.io/minikube/pkg/libmachine/version" +) + +type API interface { + io.Closer + NewHost(driverName string, rawDriver []byte) (*host.Host, error) + Create(h *host.Host) error + persist.Store + GetMachinesDir() string +} + +type Client struct { + certsDir string + IsDebug bool + SSHClientType ssh.ClientType + GithubAPIToken string + *persist.Filestore + clientDriverFactory rpcdriver.RPCClientDriverFactory +} + +func NewClient(storePath, certsDir string) *Client { + return &Client{ + certsDir: certsDir, + IsDebug: false, + SSHClientType: ssh.External, + Filestore: persist.NewFilestore(storePath, certsDir, certsDir), + clientDriverFactory: rpcdriver.NewRPCClientDriverFactory(), + } +} + +func (api *Client) NewHost(driverName string, rawDriver []byte) (*host.Host, error) { + driver, err := api.clientDriverFactory.NewRPCClientDriver(driverName, rawDriver) + if err != nil { + return nil, err + } + + return &host.Host{ + ConfigVersion: version.ConfigVersion, + Name: driver.GetMachineName(), + Driver: driver, + DriverName: driver.DriverName(), + HostOptions: &host.Options{ + AuthOptions: &auth.Options{ + CertDir: api.certsDir, + CaCertPath: filepath.Join(api.certsDir, "ca.pem"), + CaPrivateKeyPath: filepath.Join(api.certsDir, "ca-key.pem"), + ClientCertPath: filepath.Join(api.certsDir, "cert.pem"), + ClientKeyPath: filepath.Join(api.certsDir, "key.pem"), + ServerCertPath: filepath.Join(api.GetMachinesDir(), "server.pem"), + ServerKeyPath: filepath.Join(api.GetMachinesDir(), "server-key.pem"), + }, + EngineOptions: &engine.Options{ + InstallURL: drivers.DefaultEngineInstallURL, + StorageDriver: "overlay2", + TLSVerify: true, + }, + SwarmOptions: &swarm.Options{ + Host: "tcp://0.0.0.0:3376", + Image: "swarm:latest", + Strategy: "spread", + }, + }, + }, nil +} + +func (api *Client) Load(name string) (*host.Host, error) { + h, err := api.Filestore.Load(name) + if err != nil { + return nil, err + } + + d, err := api.clientDriverFactory.NewRPCClientDriver(h.DriverName, h.RawDriver) + if err != nil { + return nil, err + } + + if h.DriverName == "virtualbox" { + h.Driver = drivers.NewSerialDriver(d) + } else { + h.Driver = d + } + + return h, nil +} + +// Create is the wrapper method which covers all of the boilerplate around +// actually creating, provisioning, and persisting an instance in the store. +func (api *Client) Create(h *host.Host) error { + if err := cert.BootstrapCertificates(h.AuthOptions()); err != nil { + return fmt.Errorf("Error generating certificates: %s", err) + } + + log.Info("Running pre-create checks...") + + if err := h.Driver.PreCreateCheck(); err != nil { + return mcnerror.ErrDuringPreCreate{ + Cause: err, + } + } + + if err := api.Save(h); err != nil { + return fmt.Errorf("Error saving host to store before attempting creation: %s", err) + } + + log.Info("Creating machine...") + + if err := api.performCreate(h); err != nil { + return fmt.Errorf("Error creating machine: %s", err) + } + + log.Debug("Reticulating splines...") + + return nil +} + +func (api *Client) performCreate(h *host.Host) error { + if err := h.Driver.Create(); err != nil { + return fmt.Errorf("Error in driver during machine creation: %s", err) + } + + if err := api.Save(h); err != nil { + return fmt.Errorf("Error saving host to store after attempting creation: %s", err) + } + + // TODO: Not really a fan of just checking "none" or "ci-test" here. + if h.Driver.DriverName() == "none" || h.Driver.DriverName() == "ci-test" { + return nil + } + + log.Info("Waiting for machine to be running, this may take a few minutes...") + if err := mcnutils.WaitFor(drivers.MachineInState(h.Driver, state.Running)); err != nil { + return fmt.Errorf("Error waiting for machine to be running: %s", err) + } + + log.Info("Detecting operating system of created instance...") + provisioner, err := provision.DetectProvisioner(h.Driver) + if err != nil { + return fmt.Errorf("Error detecting OS: %s", err) + } + + log.Infof("Provisioning with %s...", provisioner.String()) + if err := provisioner.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil { + return fmt.Errorf("Error running provisioning: %s", err) + } + + // We should check the connection to docker here + log.Info("Checking connection to Docker...") + if _, _, err = check.DefaultConnChecker.Check(h, false); err != nil { + return fmt.Errorf("Error checking the host: %s", err) + } + + log.Info("Docker is up and running!") + return nil +} + +func (api *Client) Close() error { + return api.clientDriverFactory.Close() +} diff --git a/pkg/libmachine/log/fmt_machine_logger.go b/pkg/libmachine/log/fmt_machine_logger.go new file mode 100644 index 000000000000..285e69949fa3 --- /dev/null +++ b/pkg/libmachine/log/fmt_machine_logger.go @@ -0,0 +1,84 @@ +package log + +import ( + "fmt" + "io" + "os" +) + +type FmtMachineLogger struct { + outWriter io.Writer + errWriter io.Writer + debug bool + history *HistoryRecorder +} + +// NewFmtMachineLogger creates a MachineLogger implementation used by the drivers +func NewFmtMachineLogger() MachineLogger { + return &FmtMachineLogger{ + outWriter: os.Stdout, + errWriter: os.Stderr, + debug: false, + history: NewHistoryRecorder(), + } +} + +func (ml *FmtMachineLogger) SetDebug(debug bool) { + ml.debug = debug +} + +func (ml *FmtMachineLogger) SetOutWriter(out io.Writer) { + ml.outWriter = out +} + +func (ml *FmtMachineLogger) SetErrWriter(err io.Writer) { + ml.errWriter = err +} + +func (ml *FmtMachineLogger) Debug(args ...interface{}) { + ml.history.Record(args...) + if ml.debug { + fmt.Fprintln(ml.errWriter, args...) + } +} + +func (ml *FmtMachineLogger) Debugf(fmtString string, args ...interface{}) { + ml.history.Recordf(fmtString, args...) + if ml.debug { + fmt.Fprintf(ml.errWriter, fmtString+"\n", args...) + } +} + +func (ml *FmtMachineLogger) Error(args ...interface{}) { + ml.history.Record(args...) + fmt.Fprintln(ml.errWriter, args...) +} + +func (ml *FmtMachineLogger) Errorf(fmtString string, args ...interface{}) { + ml.history.Recordf(fmtString, args...) + fmt.Fprintf(ml.errWriter, fmtString+"\n", args...) +} + +func (ml *FmtMachineLogger) Info(args ...interface{}) { + ml.history.Record(args...) + fmt.Fprintln(ml.outWriter, args...) +} + +func (ml *FmtMachineLogger) Infof(fmtString string, args ...interface{}) { + ml.history.Recordf(fmtString, args...) + fmt.Fprintf(ml.outWriter, fmtString+"\n", args...) +} + +func (ml *FmtMachineLogger) Warn(args ...interface{}) { + ml.history.Record(args...) + fmt.Fprintln(ml.outWriter, args...) +} + +func (ml *FmtMachineLogger) Warnf(fmtString string, args ...interface{}) { + ml.history.Recordf(fmtString, args...) + fmt.Fprintf(ml.outWriter, fmtString+"\n", args...) +} + +func (ml *FmtMachineLogger) History() []string { + return ml.history.records +} diff --git a/pkg/libmachine/log/history_recorder.go b/pkg/libmachine/log/history_recorder.go new file mode 100644 index 000000000000..defdbad69124 --- /dev/null +++ b/pkg/libmachine/log/history_recorder.go @@ -0,0 +1,34 @@ +package log + +import ( + "fmt" + "sync" +) + +type HistoryRecorder struct { + lock *sync.Mutex + records []string +} + +func NewHistoryRecorder() *HistoryRecorder { + return &HistoryRecorder{ + lock: &sync.Mutex{}, + records: []string{}, + } +} + +func (ml *HistoryRecorder) History() []string { + return ml.records +} + +func (ml *HistoryRecorder) Record(args ...interface{}) { + ml.lock.Lock() + defer ml.lock.Unlock() + ml.records = append(ml.records, fmt.Sprint(args...)) +} + +func (ml *HistoryRecorder) Recordf(fmtString string, args ...interface{}) { + ml.lock.Lock() + defer ml.lock.Unlock() + ml.records = append(ml.records, fmt.Sprintf(fmtString, args...)) +} diff --git a/pkg/libmachine/log/log.go b/pkg/libmachine/log/log.go new file mode 100644 index 000000000000..5a5823bc878c --- /dev/null +++ b/pkg/libmachine/log/log.go @@ -0,0 +1,74 @@ +package log + +import ( + "io" + "regexp" +) + +const redactedText = "" + +var ( + logger = NewFmtMachineLogger() + + // (?s) enables '.' to match '\n' -- see https://golang.org/pkg/regexp/syntax/ + certRegex = regexp.MustCompile("(?s)-----BEGIN CERTIFICATE-----.*-----END CERTIFICATE-----") + keyRegex = regexp.MustCompile("(?s)-----BEGIN RSA PRIVATE KEY-----.*-----END RSA PRIVATE KEY-----") +) + +func stripSecrets(original []string) []string { + stripped := []string{} + for _, line := range original { + line = certRegex.ReplaceAllString(line, redactedText) + line = keyRegex.ReplaceAllString(line, redactedText) + stripped = append(stripped, line) + } + return stripped +} + +func Debug(args ...interface{}) { + logger.Debug(args...) +} + +func Debugf(fmtString string, args ...interface{}) { + logger.Debugf(fmtString, args...) +} + +func Error(args ...interface{}) { + logger.Error(args...) +} + +func Errorf(fmtString string, args ...interface{}) { + logger.Errorf(fmtString, args...) +} + +func Info(args ...interface{}) { + logger.Info(args...) +} + +func Infof(fmtString string, args ...interface{}) { + logger.Infof(fmtString, args...) +} + +func Warn(args ...interface{}) { + logger.Warn(args...) +} + +func Warnf(fmtString string, args ...interface{}) { + logger.Warnf(fmtString, args...) +} + +func SetDebug(debug bool) { + logger.SetDebug(debug) +} + +func SetOutWriter(out io.Writer) { + logger.SetOutWriter(out) +} + +func SetErrWriter(err io.Writer) { + logger.SetErrWriter(err) +} + +func History() []string { + return stripSecrets(logger.History()) +} diff --git a/pkg/libmachine/log/machine_logger.go b/pkg/libmachine/log/machine_logger.go new file mode 100644 index 000000000000..19f613e230ca --- /dev/null +++ b/pkg/libmachine/log/machine_logger.go @@ -0,0 +1,24 @@ +package log + +import "io" + +type MachineLogger interface { + SetDebug(debug bool) + + SetOutWriter(io.Writer) + SetErrWriter(io.Writer) + + Debug(args ...interface{}) + Debugf(fmtString string, args ...interface{}) + + Error(args ...interface{}) + Errorf(fmtString string, args ...interface{}) + + Info(args ...interface{}) + Infof(fmtString string, args ...interface{}) + + Warn(args ...interface{}) + Warnf(fmtString string, args ...interface{}) + + History() []string +} diff --git a/pkg/libmachine/mcndockerclient/docker_client.go b/pkg/libmachine/mcndockerclient/docker_client.go new file mode 100644 index 000000000000..f772993aa7e2 --- /dev/null +++ b/pkg/libmachine/mcndockerclient/docker_client.go @@ -0,0 +1,47 @@ +package mcndockerclient + +import ( + "fmt" + + "k8s.io/minikube/pkg/libmachine/cert" + "github.com/sayboras/dockerclient" +) + +// DockerClient creates a docker client for a given host. +func DockerClient(dockerHost DockerHost) (*dockerclient.DockerClient, error) { + url, err := dockerHost.URL() + if err != nil { + return nil, err + } + + tlsConfig, err := cert.ReadTLSConfig(url, dockerHost.AuthOptions()) + if err != nil { + return nil, fmt.Errorf("Unable to read TLS config: %s", err) + } + + return dockerclient.NewDockerClient(url, tlsConfig) +} + +// CreateContainer creates a docker container. +func CreateContainer(dockerHost DockerHost, config *dockerclient.ContainerConfig, name string) error { + docker, err := DockerClient(dockerHost) + if err != nil { + return err + } + + if err = docker.PullImage(config.Image, nil); err != nil { + return fmt.Errorf("Unable to pull image: %s", err) + } + + var authConfig *dockerclient.AuthConfig + containerID, err := docker.CreateContainer(config, name, authConfig) + if err != nil { + return fmt.Errorf("Error while creating container: %s", err) + } + + if err = docker.StartContainer(containerID, &config.HostConfig); err != nil { + return fmt.Errorf("Error while starting container: %s", err) + } + + return nil +} diff --git a/pkg/libmachine/mcndockerclient/docker_host.go b/pkg/libmachine/mcndockerclient/docker_host.go new file mode 100644 index 000000000000..afac8fae7bba --- /dev/null +++ b/pkg/libmachine/mcndockerclient/docker_host.go @@ -0,0 +1,41 @@ +package mcndockerclient + +import ( + "fmt" + + "k8s.io/minikube/pkg/libmachine/auth" +) + +type URLer interface { + // URL returns the Docker host URL + URL() (string, error) +} + +type AuthOptionser interface { + // AuthOptions returns the authOptions + AuthOptions() *auth.Options +} + +type DockerHost interface { + URLer + AuthOptionser +} + +type RemoteDocker struct { + HostURL string + AuthOption *auth.Options +} + +// URL returns the Docker host URL +func (rd *RemoteDocker) URL() (string, error) { + if rd.HostURL == "" { + return "", fmt.Errorf("Docker Host URL not set") + } + + return rd.HostURL, nil +} + +// AuthOptions returns the authOptions +func (rd *RemoteDocker) AuthOptions() *auth.Options { + return rd.AuthOption +} diff --git a/pkg/libmachine/mcndockerclient/docker_versioner.go b/pkg/libmachine/mcndockerclient/docker_versioner.go new file mode 100644 index 000000000000..b8279c777e04 --- /dev/null +++ b/pkg/libmachine/mcndockerclient/docker_versioner.go @@ -0,0 +1,29 @@ +package mcndockerclient + +import "fmt" + +var CurrentDockerVersioner DockerVersioner = &defaultDockerVersioner{} + +type DockerVersioner interface { + DockerVersion(host DockerHost) (string, error) +} + +func DockerVersion(host DockerHost) (string, error) { + return CurrentDockerVersioner.DockerVersion(host) +} + +type defaultDockerVersioner struct{} + +func (dv *defaultDockerVersioner) DockerVersion(host DockerHost) (string, error) { + client, err := DockerClient(host) + if err != nil { + return "", fmt.Errorf("Unable to query docker version: %s", err) + } + + version, err := client.Version() + if err != nil { + return "", fmt.Errorf("Unable to query docker version: %s", err) + } + + return version.Version, nil +} diff --git a/pkg/libmachine/mcndockerclient/fake_docker_versioner.go b/pkg/libmachine/mcndockerclient/fake_docker_versioner.go new file mode 100644 index 000000000000..32c4bb4cdf82 --- /dev/null +++ b/pkg/libmachine/mcndockerclient/fake_docker_versioner.go @@ -0,0 +1,14 @@ +package mcndockerclient + +type FakeDockerVersioner struct { + Version string + Err error +} + +func (dv *FakeDockerVersioner) DockerVersion(host DockerHost) (string, error) { + if dv.Err != nil { + return "", dv.Err + } + + return dv.Version, nil +} diff --git a/pkg/libmachine/mcnerror/errors.go b/pkg/libmachine/mcnerror/errors.go new file mode 100644 index 000000000000..3ca71111f9f6 --- /dev/null +++ b/pkg/libmachine/mcnerror/errors.go @@ -0,0 +1,46 @@ +package mcnerror + +import ( + "errors" + "fmt" + "strings" + + "k8s.io/minikube/pkg/libmachine/state" +) + +var ( + ErrInvalidHostname = errors.New("Invalid hostname specified. Allowed hostname chars are: 0-9a-zA-Z . -") +) + +type ErrHostDoesNotExist struct { + Name string +} + +func (e ErrHostDoesNotExist) Error() string { + return fmt.Sprintf("Docker machine %q does not exist. Use \"docker-machine ls\" to list machines. Use \"docker-machine create\" to add a new one.", e.Name) +} + +type ErrHostAlreadyExists struct { + Name string +} + +func (e ErrHostAlreadyExists) Error() string { + return fmt.Sprintf("Docker machine %q already exists", e.Name) +} + +type ErrDuringPreCreate struct { + Cause error +} + +func (e ErrDuringPreCreate) Error() string { + return fmt.Sprintf("Error with pre-create check: %q", e.Cause) +} + +type ErrHostAlreadyInState struct { + Name string + State state.State +} + +func (e ErrHostAlreadyInState) Error() string { + return fmt.Sprintf("Machine %q is already %s.", e.Name, strings.ToLower(e.State.String())) +} diff --git a/pkg/libmachine/mcnflag/flag.go b/pkg/libmachine/mcnflag/flag.go new file mode 100644 index 000000000000..ec24d130aee1 --- /dev/null +++ b/pkg/libmachine/mcnflag/flag.go @@ -0,0 +1,71 @@ +package mcnflag + +import "fmt" + +type Flag interface { + fmt.Stringer + Default() interface{} +} + +type StringFlag struct { + Name string + Usage string + EnvVar string + Value string +} + +// TODO: Could this be done more succinctly using embedding? +func (f StringFlag) String() string { + return f.Name +} + +func (f StringFlag) Default() interface{} { + return f.Value +} + +type StringSliceFlag struct { + Name string + Usage string + EnvVar string + Value []string +} + +// TODO: Could this be done more succinctly using embedding? +func (f StringSliceFlag) String() string { + return f.Name +} + +func (f StringSliceFlag) Default() interface{} { + return f.Value +} + +type IntFlag struct { + Name string + Usage string + EnvVar string + Value int +} + +// TODO: Could this be done more succinctly using embedding? +func (f IntFlag) String() string { + return f.Name +} + +func (f IntFlag) Default() interface{} { + return f.Value +} + +type BoolFlag struct { + Name string + Usage string + EnvVar string +} + +// TODO: Could this be done more succinctly using embedding? +func (f BoolFlag) String() string { + return f.Name +} + +func (f BoolFlag) Default() interface{} { + return nil +} diff --git a/pkg/libmachine/mcnutils/b2d.go b/pkg/libmachine/mcnutils/b2d.go new file mode 100644 index 000000000000..94b4e4cd5fc4 --- /dev/null +++ b/pkg/libmachine/mcnutils/b2d.go @@ -0,0 +1,520 @@ +package mcnutils + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + + "k8s.io/minikube/pkg/libmachine/log" +) + +const ( + defaultURL = "https://api.github.com/repos/boot2docker/boot2docker/releases" + defaultISOFilename = "boot2docker.iso" + defaultVolumeIDOffset = int64(0x8028) + versionPrefix = "-v" + defaultVolumeIDLength = 32 +) + +var ( + GithubAPIToken string +) + +var ( + errGitHubAPIResponse = errors.New(`failure getting a version tag from the Github API response (are you getting rate limited by Github?)`) +) + +var ( + AUFSBugB2DVersions = map[string]string{ + "v1.9.1": "https://github.com/docker/docker/issues/18180", + } +) + +func defaultTimeout(network, addr string) (net.Conn, error) { + return net.Dial(network, addr) +} + +func getClient() *http.Client { + transport := http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + Dial: defaultTimeout, + } + + return &http.Client{ + Transport: &transport, + } +} + +func getRequest(apiURL string) (*http.Request, error) { + req, err := http.NewRequest("GET", apiURL, nil) + if err != nil { + return nil, err + } + + if GithubAPIToken != "" { + req.Header.Add("Authorization", fmt.Sprintf("token %s", GithubAPIToken)) + } + + return req, nil +} + +// releaseGetter is a client that gets release information of a product and downloads it. +type releaseGetter interface { + // filename returns filename of the product. + filename() string + // getReleaseTag gets a release tag from the given URL. + getReleaseTag(apiURL string) (string, error) + // getReleaseURL gets the latest release download URL from the given URL. + getReleaseURL(apiURL string) (string, error) + // download downloads a file from the given dlURL and saves it under dir. + download(dir, file, dlURL string) error +} + +// b2dReleaseGetter implements the releaseGetter interface for getting the release of Boot2Docker. +type b2dReleaseGetter struct { + isoFilename string +} + +func (b *b2dReleaseGetter) filename() string { + if b == nil { + return "" + } + return b.isoFilename +} + +// getReleaseTag gets the release tag of Boot2Docker from apiURL. +func (*b2dReleaseGetter) getReleaseTag(apiURL string) (string, error) { + if apiURL == "" { + apiURL = defaultURL + } + + client := getClient() + req, err := getRequest(apiURL) + if err != nil { + return "", err + } + rsp, err := client.Do(req) + if err != nil { + return "", err + } + defer rsp.Body.Close() + + // Otherwise, we get back just one release, which we can decode to get + // the tag. + var t struct { + TagName string `json:"tag_name"` + } + if err := json.NewDecoder(rsp.Body).Decode(&t); err != nil { + return "", err + } + if t.TagName == "" { + return "", errGitHubAPIResponse + } + return t.TagName, nil +} + +// getReleaseURL gets the latest release URL of Boot2Docker. +func (b *b2dReleaseGetter) getReleaseURL(apiURL string) (string, error) { + if apiURL == "" { + apiURL = defaultURL + } + + // match github (enterprise) release urls: + // https://api.github.com/repos/../../releases or + // https://some.github.enterprise/api/v3/repos/../../releases + re := regexp.MustCompile("(https?)://([^/]+)(/api/v3)?/repos/([^/]+)/([^/]+)/releases") + matches := re.FindStringSubmatch(apiURL) + if len(matches) != 6 { + // does not match a github releases api URL + return apiURL, nil + } + + scheme, host, org, repo := matches[1], matches[2], matches[4], matches[5] + if host == "api.github.com" { + host = "github.com" + } + + tag, err := b.getReleaseTag(apiURL) + if err != nil { + return "", err + } + + log.Infof("Latest release for %s/%s/%s is %s", host, org, repo, tag) + bugURL, ok := AUFSBugB2DVersions[tag] + if ok { + log.Warnf(` +Boot2Docker %s has a known issue with AUFS. +See here for more details: %s +Consider specifying another storage driver (e.g. 'overlay') using '--engine-storage-driver' instead. +`, tag, bugURL) + } + url := fmt.Sprintf("%s://%s/%s/%s/releases/download/%s/%s", scheme, host, org, repo, tag, b.isoFilename) + return url, nil +} + +func (*b2dReleaseGetter) download(dir, file, isoURL string) error { + u, err := url.Parse(isoURL) + + var src io.ReadCloser + if u.Scheme == "file" || u.Scheme == "" { + path := u.Path + if runtime.GOOS == "windows" { + path = u.Hostname() + ":/" + u.Path + } + s, err := os.Open(path) + if err != nil { + return err + } + + src = s + } else { + client := getClient() + s, err := client.Get(isoURL) + if err != nil { + return err + } + + src = &ReaderWithProgress{ + ReadCloser: s.Body, + out: os.Stdout, + expectedLength: s.ContentLength, + } + } + + defer src.Close() + + // Download to a temp file first then rename it to avoid partial download. + f, err := ioutil.TempFile(dir, file+".tmp") + if err != nil { + return err + } + + defer func() { + if err := removeFileIfExists(f.Name()); err != nil { + log.Warnf("Error removing file: %s", err) + } + }() + + if _, err := io.Copy(f, src); err != nil { + return err + } + + if err := f.Close(); err != nil { + return err + } + + // Dest is the final path of the boot2docker.iso file. + dest := filepath.Join(dir, file) + + // Windows can't rename in place, so remove the old file before + // renaming the temporary downloaded file. + if err := removeFileIfExists(dest); err != nil { + return err + } + + return os.Rename(f.Name(), dest) +} + +// iso is an ISO volume. +type iso interface { + // path returns the path of the ISO. + path() string + // exists reports whether the ISO exists. + exists() bool + // version returns version information of the ISO. + version() (string, error) +} + +// b2dISO represents a Boot2Docker ISO. It implements the ISO interface. +type b2dISO struct { + // path of Boot2Docker ISO + commonIsoPath string + + // offset and length of ISO volume ID + // cf. http://serverfault.com/questions/361474/is-there-a-way-to-change-a-iso-files-volume-id-from-the-command-line + volumeIDOffset int64 + volumeIDLength int +} + +func (b *b2dISO) path() string { + if b == nil { + return "" + } + return b.commonIsoPath +} + +func (b *b2dISO) exists() bool { + if b == nil { + return false + } + + _, err := os.Stat(b.commonIsoPath) + return !os.IsNotExist(err) +} + +// version scans the volume ID in b and returns its version tag. +func (b *b2dISO) version() (string, error) { + if b == nil { + return "", nil + } + + iso, err := os.Open(b.commonIsoPath) + if err != nil { + return "", err + } + defer iso.Close() + + isoMetadata := make([]byte, b.volumeIDLength) + _, err = iso.ReadAt(isoMetadata, b.volumeIDOffset) + if err != nil { + return "", err + } + + trimmedVersion := strings.TrimSpace(string(isoMetadata)) + + versionIndex := strings.Index(trimmedVersion, versionPrefix) + if versionIndex == -1 { + return "", fmt.Errorf("Did not find prefix %q in version string", versionPrefix) + } + + // Original magic file string looks similar to this: "Boot2Docker-v0.1.0 " + // This will return "v0.1.0" given the above string + vers := trimmedVersion[versionIndex+1:] + + log.Debug("local Boot2Docker ISO version: ", vers) + return vers, nil +} + +func removeFileIfExists(name string) error { + if _, err := os.Stat(name); err == nil { + if err := os.Remove(name); err != nil { + return fmt.Errorf("Error removing temporary download file: %s", err) + } + } + return nil +} + +type B2dUtils struct { + releaseGetter + iso + storePath string + imgCachePath string +} + +func NewB2dUtils(storePath string) *B2dUtils { + imgCachePath := filepath.Join(storePath, "cache") + + return &B2dUtils{ + releaseGetter: &b2dReleaseGetter{isoFilename: defaultISOFilename}, + iso: &b2dISO{ + commonIsoPath: filepath.Join(imgCachePath, defaultISOFilename), + volumeIDOffset: defaultVolumeIDOffset, + volumeIDLength: defaultVolumeIDLength, + }, + storePath: storePath, + imgCachePath: imgCachePath, + } +} + +// DownloadISO downloads boot2docker ISO image for the given tag and save it at dest. +func (b *B2dUtils) DownloadISO(dir, file, isoURL string) error { + log.Infof("Downloading %s from %s...", b.path(), isoURL) + return b.download(dir, file, isoURL) +} + +type ReaderWithProgress struct { + io.ReadCloser + out io.Writer + bytesTransferred int64 + expectedLength int64 + nextPercentToPrint int64 +} + +func (r *ReaderWithProgress) Read(p []byte) (int, error) { + n, err := r.ReadCloser.Read(p) + + if n > 0 { + r.bytesTransferred += int64(n) + percentage := r.bytesTransferred * 100 / r.expectedLength + + for percentage >= r.nextPercentToPrint { + if r.nextPercentToPrint%10 == 0 { + fmt.Fprintf(r.out, "%d%%", r.nextPercentToPrint) + } else if r.nextPercentToPrint%2 == 0 { + fmt.Fprint(r.out, ".") + } + r.nextPercentToPrint += 2 + } + } + + return n, err +} + +func (r *ReaderWithProgress) Close() error { + fmt.Fprintln(r.out) + return r.ReadCloser.Close() +} + +func (b *B2dUtils) DownloadLatestBoot2Docker(apiURL string) error { + latestReleaseURL, err := b.getReleaseURL(apiURL) + if err != nil { + return err + } + + return b.DownloadISOFromURL(latestReleaseURL) +} + +func (b *B2dUtils) DownloadISOFromURL(latestReleaseURL string) error { + return b.DownloadISO(b.imgCachePath, b.filename(), latestReleaseURL) +} + +func (b *B2dUtils) UpdateISOCache(isoURL string) error { + // recreate the cache dir if it has been manually deleted + if _, err := os.Stat(b.imgCachePath); os.IsNotExist(err) { + log.Infof("Image cache directory does not exist, creating it at %s...", b.imgCachePath) + if err := os.Mkdir(b.imgCachePath, 0700); err != nil { + return err + } + } + + exists := b.exists() + + if isoURL != "" { + if exists { + // Warn that the b2d iso won't be updated if isoURL is set + log.Warnf("Boot2Docker URL was explicitly set to %q at create time, so Docker Machine cannot upgrade this machine to the latest version.", isoURL) + } + // Non-default B2D are not cached + return nil + } + + if !exists { + log.Info("No default Boot2Docker ISO found locally, downloading the latest release...") + return b.DownloadLatestBoot2Docker("") + } + + latest := b.isLatest() + if !latest { + log.Info("Default Boot2Docker ISO is out-of-date, downloading the latest release...") + return b.DownloadLatestBoot2Docker("") + } + + return nil +} + +func (b *B2dUtils) CopyIsoToMachineDir(isoURL, machineName string) error { + if err := b.UpdateISOCache(isoURL); err != nil { + return err + } + + // TODO: This is a bit off-color. + machineDir := filepath.Join(b.storePath, "machines", machineName) + machineIsoPath := filepath.Join(machineDir, b.filename()) + + // By default just copy the existing "cached" iso to the machine's directory... + if isoURL == "" { + log.Infof("Copying %s to %s...", b.path(), machineIsoPath) + return CopyFile(b.path(), machineIsoPath) + } + + // if ISO is specified, check if it matches a github releases url or fallback to a direct download + downloadURL, err := b.getReleaseURL(isoURL) + if err != nil { + return err + } + + return b.DownloadISO(machineDir, b.filename(), downloadURL) +} + +// isLatest checks the latest release tag and +// reports whether the local ISO cache is the latest version. +// +// It returns false if failing to get the local ISO version +// and true if failing to fetch the latest release tag. +func (b *B2dUtils) isLatest() bool { + localVer, err := b.version() + if err != nil { + log.Warn("Unable to get the local Boot2Docker ISO version: ", err) + return false + } + + latestVer, err := b.getReleaseTag("") + if err != nil { + log.Warn("Unable to get the latest Boot2Docker ISO release version: ", err) + return true + } + + return localVer == latestVer +} + +// MakeDiskImage makes a boot2docker VM disk image. +// See https://github.com/boot2docker/boot2docker/blob/master/rootfs/rootfs/etc/rc.d/automount +func MakeDiskImage(publicSSHKeyPath string) (*bytes.Buffer, error) { + magicString := "boot2docker, please format-me" + + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + + // magicString first so the automount script knows to format the disk + file := &tar.Header{Name: magicString, Size: int64(len(magicString))} + + log.Debug("Writing magic tar header") + + if err := tw.WriteHeader(file); err != nil { + return nil, err + } + + if _, err := tw.Write([]byte(magicString)); err != nil { + return nil, err + } + + // .ssh/key.pub => authorized_keys + file = &tar.Header{Name: ".ssh", Typeflag: tar.TypeDir, Mode: 0700} + if err := tw.WriteHeader(file); err != nil { + return nil, err + } + + log.Debug("Writing SSH key tar header") + + pubKey, err := ioutil.ReadFile(publicSSHKeyPath) + if err != nil { + return nil, err + } + + file = &tar.Header{Name: ".ssh/authorized_keys", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return nil, err + } + + if _, err := tw.Write([]byte(pubKey)); err != nil { + return nil, err + } + + file = &tar.Header{Name: ".ssh/authorized_keys2", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return nil, err + } + + if _, err := tw.Write([]byte(pubKey)); err != nil { + return nil, err + } + + if err := tw.Close(); err != nil { + return nil, err + } + + return buf, nil +} diff --git a/pkg/libmachine/mcnutils/utils.go b/pkg/libmachine/mcnutils/utils.go new file mode 100644 index 000000000000..a3992ed1646e --- /dev/null +++ b/pkg/libmachine/mcnutils/utils.go @@ -0,0 +1,133 @@ +package mcnutils + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "os" + "runtime" + "strconv" + "time" +) + +type MultiError struct { + Errs []error +} + +func (e MultiError) Error() string { + aggregate := "" + for _, err := range e.Errs { + aggregate += err.Error() + "\n" + } + return aggregate +} + +// GetHomeDir returns the home directory +// TODO: Having this here just strikes me as dangerous, but some of the drivers +// depend on it ;_; +func GetHomeDir() string { + if runtime.GOOS == "windows" { + return os.Getenv("USERPROFILE") + } + return os.Getenv("HOME") +} + +func GetUsername() string { + u := "unknown" + osUser := "" + + switch runtime.GOOS { + case "darwin", "linux": + osUser = os.Getenv("USER") + case "windows": + osUser = os.Getenv("USERNAME") + } + + if osUser != "" { + u = osUser + } + + return u +} + +func CopyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return err + } + + defer out.Close() + + if _, err = io.Copy(out, in); err != nil { + return err + } + + fi, err := os.Stat(src) + if err != nil { + return err + } + + return os.Chmod(dst, fi.Mode()) +} + +func WaitForSpecificOrError(f func() (bool, error), maxAttempts int, waitInterval time.Duration) error { + for i := 0; i < maxAttempts; i++ { + stop, err := f() + if err != nil { + return err + } + if stop { + return nil + } + time.Sleep(waitInterval) + } + return fmt.Errorf("Maximum number of retries (%d) exceeded", maxAttempts) +} + +func WaitForSpecific(f func() bool, maxAttempts int, waitInterval time.Duration) error { + return WaitForSpecificOrError(func() (bool, error) { + return f(), nil + }, maxAttempts, waitInterval) +} + +func WaitFor(f func() bool) error { + return WaitForSpecific(f, 60, 3*time.Second) +} + +// TruncateID returns a shorten id +// Following two functions are from github.com/docker/docker/utils module. It +// was way overkill to include the whole module, so we just have these bits +// that we're using here. +func TruncateID(id string) string { + shortLen := 12 + if len(id) < shortLen { + shortLen = len(id) + } + return id[:shortLen] +} + +// GenerateRandomID returns an unique id +func GenerateRandomID() string { + for { + id := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, id); err != nil { + panic(err) // This shouldn't happen + } + value := hex.EncodeToString(id) + // if we try to parse the truncated for as an int and we don't have + // an error then the value is all numeric and causes issues when + // used as a hostname. ref #3869 + if _, err := strconv.ParseInt(TruncateID(value), 10, 64); err == nil { + continue + } + return value + } +} diff --git a/pkg/libmachine/persist/filestore.go b/pkg/libmachine/persist/filestore.go new file mode 100644 index 000000000000..15a358e7868c --- /dev/null +++ b/pkg/libmachine/persist/filestore.go @@ -0,0 +1,156 @@ +package persist + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/mcnerror" +) + +type Filestore struct { + Path string + CaCertPath string + CaPrivateKeyPath string +} + +func NewFilestore(path, caCertPath, caPrivateKeyPath string) *Filestore { + return &Filestore{ + Path: path, + CaCertPath: caCertPath, + CaPrivateKeyPath: caPrivateKeyPath, + } +} + +func (s Filestore) GetMachinesDir() string { + return filepath.Join(s.Path, "machines") +} + +func (s Filestore) saveToFile(data []byte, file string) error { + if _, err := os.Stat(file); os.IsNotExist(err) { + return ioutil.WriteFile(file, data, 0600) + } + + tmpfi, err := ioutil.TempFile(filepath.Dir(file), "config.json.tmp") + if err != nil { + return err + } + defer os.Remove(tmpfi.Name()) + + if err = ioutil.WriteFile(tmpfi.Name(), data, 0600); err != nil { + return err + } + + if err = tmpfi.Close(); err != nil { + return err + } + + if err = os.Remove(file); err != nil { + return err + } + + err = os.Rename(tmpfi.Name(), file) + return err +} + +func (s Filestore) Save(host *host.Host) error { + data, err := json.MarshalIndent(host, "", " ") + if err != nil { + return err + } + + hostPath := filepath.Join(s.GetMachinesDir(), host.Name) + + // Ensure that the directory we want to save to exists. + if err := os.MkdirAll(hostPath, 0700); err != nil { + return err + } + + return s.saveToFile(data, filepath.Join(hostPath, "config.json")) +} + +func (s Filestore) Remove(name string) error { + hostPath := filepath.Join(s.GetMachinesDir(), name) + return os.RemoveAll(hostPath) +} + +func (s Filestore) List() ([]string, error) { + dir, err := ioutil.ReadDir(s.GetMachinesDir()) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + + hostNames := []string{} + + for _, file := range dir { + if file.IsDir() && !strings.HasPrefix(file.Name(), ".") { + hostNames = append(hostNames, file.Name()) + } + } + + return hostNames, nil +} + +func (s Filestore) Exists(name string) (bool, error) { + _, err := os.Stat(filepath.Join(s.GetMachinesDir(), name)) + + if os.IsNotExist(err) { + return false, nil + } else if err == nil { + return true, nil + } + + return false, err +} + +func (s Filestore) loadConfig(h *host.Host) error { + data, err := ioutil.ReadFile(filepath.Join(s.GetMachinesDir(), h.Name, "config.json")) + if err != nil { + return err + } + + // Remember the machine name so we don't have to pass it through each + // struct in the migration. + name := h.Name + + migrationPerformed := false + + h.Name = name + + // If we end up performing a migration, we should save afterwards so we don't have to do it again on subsequent invocations. + if migrationPerformed { + if err := s.saveToFile(data, filepath.Join(s.GetMachinesDir(), h.Name, "config.json.bak")); err != nil { + return fmt.Errorf("Error attempting to save backup after migration: %s", err) + } + + if err := s.Save(h); err != nil { + return fmt.Errorf("Error saving config after migration was performed: %s", err) + } + } + + return nil +} + +func (s Filestore) Load(name string) (*host.Host, error) { + hostPath := filepath.Join(s.GetMachinesDir(), name) + + if _, err := os.Stat(hostPath); os.IsNotExist(err) { + return nil, mcnerror.ErrHostDoesNotExist{ + Name: name, + } + } + + host := &host.Host{ + Name: name, + } + + if err := s.loadConfig(host); err != nil { + return nil, err + } + + return host, nil +} diff --git a/pkg/libmachine/persist/store.go b/pkg/libmachine/persist/store.go new file mode 100644 index 000000000000..da2e542a95ea --- /dev/null +++ b/pkg/libmachine/persist/store.go @@ -0,0 +1,47 @@ +package persist + +import ( + "k8s.io/minikube/pkg/libmachine/host" +) + +type Store interface { + // Exists returns whether a machine exists or not + Exists(name string) (bool, error) + + // List returns a list of all hosts in the store + List() ([]string, error) + + // Load loads a host by name + Load(name string) (*host.Host, error) + + // Remove removes a machine from the store + Remove(name string) error + + // Save persists a machine in the store + Save(host *host.Host) error +} + +func LoadHosts(s Store, hostNames []string) ([]*host.Host, map[string]error) { + loadedHosts := []*host.Host{} + errors := map[string]error{} + + for _, hostName := range hostNames { + h, err := s.Load(hostName) + if err != nil { + errors[hostName] = err + } else { + loadedHosts = append(loadedHosts, h) + } + } + + return loadedHosts, errors +} + +func LoadAllHosts(s Store) ([]*host.Host, map[string]error, error) { + hostNames, err := s.List() + if err != nil { + return nil, nil, err + } + loadedHosts, hostInError := LoadHosts(s, hostNames) + return loadedHosts, hostInError, nil +} diff --git a/pkg/libmachine/provision/configure_swarm.go b/pkg/libmachine/provision/configure_swarm.go new file mode 100644 index 000000000000..e7fa7c302dfb --- /dev/null +++ b/pkg/libmachine/provision/configure_swarm.go @@ -0,0 +1,149 @@ +package provision + +import ( + "fmt" + "net/url" + "strconv" + "strings" + + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcndockerclient" + "k8s.io/minikube/pkg/libmachine/swarm" + "github.com/sayboras/dockerclient" +) + +func configureSwarm(p Provisioner, swarmOptions swarm.Options, authOptions auth.Options) error { + if !swarmOptions.IsSwarm { + return nil + } + + log.Info("Configuring swarm...") + + ip, err := p.GetDriver().GetIP() + if err != nil { + return err + } + + u, err := url.Parse(swarmOptions.Host) + if err != nil { + return err + } + + enginePort := engine.DefaultPort + engineURL, err := p.GetDriver().GetURL() + if err != nil { + return err + } + + parts := strings.Split(engineURL, ":") + if len(parts) == 3 { + dPort, err := strconv.Atoi(parts[2]) + if err != nil { + return err + } + enginePort = dPort + } + + parts = strings.Split(u.Host, ":") + port := parts[1] + + dockerDir := p.GetDockerOptionsDir() + dockerHost := &mcndockerclient.RemoteDocker{ + HostURL: fmt.Sprintf("tcp://%s:%d", ip, enginePort), + AuthOption: &authOptions, + } + advertiseInfo := fmt.Sprintf("%s:%d", ip, enginePort) + + if swarmOptions.Master { + advertiseMasterInfo := fmt.Sprintf("%s:%s", ip, "3376") + cmd := fmt.Sprintf("manage --tlsverify --tlscacert=%s --tlscert=%s --tlskey=%s -H %s --strategy %s --advertise %s", + authOptions.CaCertRemotePath, + authOptions.ServerCertRemotePath, + authOptions.ServerKeyRemotePath, + swarmOptions.Host, + swarmOptions.Strategy, + advertiseMasterInfo, + ) + if swarmOptions.IsExperimental { + cmd = "--experimental " + cmd + } + + cmdMaster := strings.Fields(cmd) + for _, option := range swarmOptions.ArbitraryFlags { + cmdMaster = append(cmdMaster, "--"+option) + } + + //Discovery must be at end of command + cmdMaster = append(cmdMaster, swarmOptions.Discovery) + + hostBind := fmt.Sprintf("%s:%s", dockerDir, dockerDir) + masterHostConfig := dockerclient.HostConfig{ + RestartPolicy: dockerclient.RestartPolicy{ + Name: "always", + MaximumRetryCount: 0, + }, + Binds: []string{hostBind}, + PortBindings: map[string][]dockerclient.PortBinding{ + fmt.Sprintf("%s/tcp", port): { + { + HostIp: "0.0.0.0", + HostPort: port, + }, + }, + }, + } + + swarmMasterConfig := &dockerclient.ContainerConfig{ + Image: swarmOptions.Image, + Env: swarmOptions.Env, + ExposedPorts: map[string]struct{}{ + "2375/tcp": {}, + fmt.Sprintf("%s/tcp", port): {}, + }, + Cmd: cmdMaster, + HostConfig: masterHostConfig, + } + + err = mcndockerclient.CreateContainer(dockerHost, swarmMasterConfig, "swarm-agent-master") + if err != nil { + return err + } + } + + if swarmOptions.Agent { + workerHostConfig := dockerclient.HostConfig{ + RestartPolicy: dockerclient.RestartPolicy{ + Name: "always", + MaximumRetryCount: 0, + }, + } + + cmdWorker := []string{ + "join", + "--advertise", + advertiseInfo, + } + for _, option := range swarmOptions.ArbitraryJoinFlags { + cmdWorker = append(cmdWorker, "--"+option) + } + cmdWorker = append(cmdWorker, swarmOptions.Discovery) + + swarmWorkerConfig := &dockerclient.ContainerConfig{ + Image: swarmOptions.Image, + Env: swarmOptions.Env, + Cmd: cmdWorker, + HostConfig: workerHostConfig, + } + if swarmOptions.IsExperimental { + swarmWorkerConfig.Cmd = append([]string{"--experimental"}, swarmWorkerConfig.Cmd...) + } + + err = mcndockerclient.CreateContainer(dockerHost, swarmWorkerConfig, "swarm-agent") + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/libmachine/provision/engine_config_context.go b/pkg/libmachine/provision/engine_config_context.go new file mode 100644 index 000000000000..0bf57cc75be0 --- /dev/null +++ b/pkg/libmachine/provision/engine_config_context.go @@ -0,0 +1,13 @@ +package provision + +import ( + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/engine" +) + +type EngineConfigContext struct { + DockerPort int + AuthOptions auth.Options + EngineOptions engine.Options + DockerOptionsDir string +} diff --git a/pkg/libmachine/provision/errors.go b/pkg/libmachine/provision/errors.go new file mode 100644 index 000000000000..46348b90ebb6 --- /dev/null +++ b/pkg/libmachine/provision/errors.go @@ -0,0 +1,24 @@ +package provision + +import ( + "errors" + "fmt" +) + +var ( + ErrDetectionFailed = errors.New("OS type not recognized") +) + +type ErrDaemonAvailable struct { + wrappedErr error +} + +func (e ErrDaemonAvailable) Error() string { + return fmt.Sprintf("Unable to verify the Docker daemon is listening: %s", e.wrappedErr) +} + +func NewErrDaemonAvailable(err error) ErrDaemonAvailable { + return ErrDaemonAvailable{ + wrappedErr: err, + } +} diff --git a/pkg/libmachine/provision/fake_provisioner.go b/pkg/libmachine/provision/fake_provisioner.go new file mode 100644 index 000000000000..9f2de788cbd0 --- /dev/null +++ b/pkg/libmachine/provision/fake_provisioner.go @@ -0,0 +1,110 @@ +package provision + +import ( + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/provision/pkgaction" + "k8s.io/minikube/pkg/libmachine/provision/serviceaction" + "k8s.io/minikube/pkg/libmachine/swarm" +) + +type FakeDetector struct { + Provisioner +} + +func (fd *FakeDetector) DetectProvisioner(d drivers.Driver) (Provisioner, error) { + return fd.Provisioner, nil +} + +type FakeProvisioner struct{} + +func NewFakeProvisioner(d drivers.Driver) Provisioner { + return &FakeProvisioner{} +} + +func (fp *FakeProvisioner) SSHCommand(args string) (string, error) { + return "", nil +} + +func (fp *FakeProvisioner) String() string { + return "fakeprovisioner" +} + +func (fp *FakeProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) { + return nil, nil +} + +func (fp *FakeProvisioner) GetDockerOptionsDir() string { + return "" +} + +func (fp *FakeProvisioner) GetAuthOptions() auth.Options { + return auth.Options{} +} + +func (fp *FakeProvisioner) GetSwarmOptions() swarm.Options { + return swarm.Options{} +} + +func (fp *FakeProvisioner) Package(name string, action pkgaction.PackageAction) error { + return nil +} + +func (fp *FakeProvisioner) Hostname() (string, error) { + return "", nil +} + +func (fp *FakeProvisioner) SetHostname(hostname string) error { + return nil +} + +func (fp *FakeProvisioner) CompatibleWithHost() bool { + return true +} + +func (fp *FakeProvisioner) Provision(swarmOptions swarm.Options, authOptions auth.Options, engineOptions engine.Options) error { + return nil +} + +func (fp *FakeProvisioner) Service(name string, action serviceaction.ServiceAction) error { + return nil +} + +func (fp *FakeProvisioner) GetDriver() drivers.Driver { + return nil +} + +func (fp *FakeProvisioner) SetOsReleaseInfo(info *OsRelease) {} + +func (fp *FakeProvisioner) GetOsReleaseInfo() (*OsRelease, error) { + return nil, nil +} + +type NetstatProvisioner struct { + *FakeProvisioner +} + +func (p *NetstatProvisioner) SSHCommand(args string) (string, error) { + return `Active Internet connections (servers and established) +Proto Recv-Q Send-Q Local Address Foreign Address State +tcp 0 0 0.0.0.0:ssh 0.0.0.0:* LISTEN +tcp 0 72 192.168.25.141:ssh 192.168.25.1:63235 ESTABLISHED +tcp 0 0 :::2376 :::* LISTEN +tcp 0 0 :::ssh :::* LISTEN +Active UNIX domain sockets (servers and established) +Proto RefCnt Flags Type State I-Node Path +unix 2 [ ACC ] STREAM LISTENING 17990 /var/run/acpid.socket +unix 2 [ ACC ] SEQPACKET LISTENING 14233 /run/udev/control +unix 2 [ ACC ] STREAM LISTENING 19365 /var/run/docker.sock +unix 3 [ ] STREAM CONNECTED 19774 +unix 3 [ ] STREAM CONNECTED 19775 +unix 3 [ ] DGRAM 14243 +unix 3 [ ] DGRAM 14242`, nil +} + +func NewNetstatProvisioner() Provisioner { + return &NetstatProvisioner{ + &FakeProvisioner{}, + } +} diff --git a/pkg/libmachine/provision/generic.go b/pkg/libmachine/provision/generic.go new file mode 100644 index 000000000000..80d3b47dddd6 --- /dev/null +++ b/pkg/libmachine/provision/generic.go @@ -0,0 +1,138 @@ +package provision + +import ( + "bytes" + "fmt" + "text/template" + + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/swarm" +) + +type GenericProvisioner struct { + SSHCommander + OsReleaseID string + DockerOptionsDir string + DaemonOptionsFile string + Packages []string + OsReleaseInfo *OsRelease + Driver drivers.Driver + AuthOptions auth.Options + EngineOptions engine.Options + SwarmOptions swarm.Options +} + +type GenericSSHCommander struct { + Driver drivers.Driver +} + +func (sshCmder GenericSSHCommander) SSHCommand(args string) (string, error) { + return drivers.RunSSHCommandFromDriver(sshCmder.Driver, args) +} + +func (provisioner *GenericProvisioner) Hostname() (string, error) { + return provisioner.SSHCommand("hostname") +} + +func (provisioner *GenericProvisioner) SetHostname(hostname string) error { + if _, err := provisioner.SSHCommand(fmt.Sprintf( + "sudo hostname %s && echo %q | sudo tee /etc/hostname", + hostname, + hostname, + )); err != nil { + return err + } + + // ubuntu/debian use 127.0.1.1 for non "localhost" loopback hostnames: https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution + if _, err := provisioner.SSHCommand(fmt.Sprintf(` + if ! grep -xq '.*\s%s' /etc/hosts; then + if grep -xq '127.0.1.1\s.*' /etc/hosts; then + sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 %s/g' /etc/hosts; + else + echo '127.0.1.1 %s' | sudo tee -a /etc/hosts; + fi + fi`, + hostname, + hostname, + hostname, + )); err != nil { + return err + } + + return nil +} + +func (provisioner *GenericProvisioner) GetDockerOptionsDir() string { + return provisioner.DockerOptionsDir +} + +func (provisioner *GenericProvisioner) CompatibleWithHost() bool { + return provisioner.OsReleaseInfo.ID == provisioner.OsReleaseID +} + +func (provisioner *GenericProvisioner) GetAuthOptions() auth.Options { + return provisioner.AuthOptions +} + +func (provisioner *GenericProvisioner) GetSwarmOptions() swarm.Options { + return provisioner.SwarmOptions +} + +func (provisioner *GenericProvisioner) SetOsReleaseInfo(info *OsRelease) { + provisioner.OsReleaseInfo = info +} + +func (provisioner *GenericProvisioner) GetOsReleaseInfo() (*OsRelease, error) { + return provisioner.OsReleaseInfo, nil +} + +func (provisioner *GenericProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) { + var ( + engineCfg bytes.Buffer + ) + + driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName()) + provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel) + + engineConfigTmpl := ` +DOCKER_OPTS=' +-H tcp://0.0.0.0:{{.DockerPort}} +-H unix:///var/run/docker.sock +--storage-driver {{.EngineOptions.StorageDriver}} +--tlsverify +--tlscacert {{.AuthOptions.CaCertRemotePath}} +--tlscert {{.AuthOptions.ServerCertRemotePath}} +--tlskey {{.AuthOptions.ServerKeyRemotePath}} +{{ range .EngineOptions.Labels }}--label {{.}} +{{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} +{{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} +{{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} +{{ end }} +' +{{range .EngineOptions.Env}}export \"{{ printf "%q" . }}\" +{{end}} +` + t, err := template.New("engineConfig").Parse(engineConfigTmpl) + if err != nil { + return nil, err + } + + engineConfigContext := EngineConfigContext{ + DockerPort: dockerPort, + AuthOptions: provisioner.AuthOptions, + EngineOptions: provisioner.EngineOptions, + } + + t.Execute(&engineCfg, engineConfigContext) + + return &DockerOptions{ + EngineOptions: engineCfg.String(), + EngineOptionsPath: provisioner.DaemonOptionsFile, + }, nil +} + +func (provisioner *GenericProvisioner) GetDriver() drivers.Driver { + return provisioner.Driver +} diff --git a/pkg/libmachine/provision/os_release.go b/pkg/libmachine/provision/os_release.go new file mode 100644 index 000000000000..a10a3f14053f --- /dev/null +++ b/pkg/libmachine/provision/os_release.go @@ -0,0 +1,91 @@ +package provision + +import ( + "bufio" + "bytes" + "fmt" + "reflect" + "strings" + + "k8s.io/minikube/pkg/libmachine/log" +) + +// The /etc/os-release file contains operating system identification data +// See http://www.freedesktop.org/software/systemd/man/os-release.html for more details + +// OsRelease reflects values in /etc/os-release +// Values in this struct must always be string +// or the reflection will not work properly. +type OsRelease struct { + AnsiColor string `osr:"ANSI_COLOR"` + Name string `osr:"NAME"` + Version string `osr:"VERSION"` + Variant string `osr:"VARIANT"` + VariantID string `osr:"VARIANT_ID"` + ID string `osr:"ID"` + IDLike string `osr:"ID_LIKE"` + PrettyName string `osr:"PRETTY_NAME"` + VersionID string `osr:"VERSION_ID"` + HomeURL string `osr:"HOME_URL"` + SupportURL string `osr:"SUPPORT_URL"` + BugReportURL string `osr:"BUG_REPORT_URL"` +} + +func stripQuotes(val string) string { + if len(val) > 0 && val[0] == '"' { + return val[1 : len(val)-1] + } + return val +} + +func (osr *OsRelease) setIfPossible(key, val string) error { + v := reflect.ValueOf(osr).Elem() + for i := 0; i < v.NumField(); i++ { + fieldValue := v.Field(i) + fieldType := v.Type().Field(i) + originalName := fieldType.Tag.Get("osr") + if key == originalName && fieldValue.Kind() == reflect.String { + fieldValue.SetString(val) + return nil + } + } + return fmt.Errorf("Couldn't set key %s, no corresponding struct field found", key) +} + +func parseLine(osrLine string) (string, string, error) { + if osrLine == "" { + return "", "", nil + } + + vals := strings.Split(osrLine, "=") + if len(vals) != 2 { + return "", "", fmt.Errorf("Expected %s to split by '=' char into two strings, instead got %d strings", osrLine, len(vals)) + } + key := vals[0] + val := stripQuotes(vals[1]) + return key, val, nil +} + +func (osr *OsRelease) ParseOsRelease(osReleaseContents []byte) error { + r := bytes.NewReader(osReleaseContents) + scanner := bufio.NewScanner(r) + for scanner.Scan() { + key, val, err := parseLine(scanner.Text()) + if err != nil { + log.Warnf("Warning: got an invalid line error parsing /etc/os-release: %s", err) + continue + } + if err := osr.setIfPossible(key, val); err != nil { + log.Debug(err) + } + } + return nil +} + +func NewOsRelease(contents []byte) (*OsRelease, error) { + osr := &OsRelease{} + if err := osr.ParseOsRelease(contents); err != nil { + return nil, err + } + return osr, nil +} diff --git a/pkg/libmachine/provision/pkgaction/pkg_action.go b/pkg/libmachine/provision/pkgaction/pkg_action.go new file mode 100644 index 000000000000..b7cf7e3279ac --- /dev/null +++ b/pkg/libmachine/provision/pkgaction/pkg_action.go @@ -0,0 +1,25 @@ +package pkgaction + +type PackageAction int + +const ( + Install PackageAction = iota + Remove + Upgrade + Purge +) + +var packageActions = []string{ + "install", + "remove", + "upgrade", + "purge", +} + +func (s PackageAction) String() string { + if int(s) >= 0 && int(s) < len(packageActions) { + return packageActions[s] + } + + return "" +} diff --git a/pkg/libmachine/provision/provisioner.go b/pkg/libmachine/provision/provisioner.go new file mode 100644 index 000000000000..c03b253149f6 --- /dev/null +++ b/pkg/libmachine/provision/provisioner.go @@ -0,0 +1,132 @@ +package provision + +import ( + "fmt" + + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/provision/pkgaction" + "k8s.io/minikube/pkg/libmachine/provision/serviceaction" + "k8s.io/minikube/pkg/libmachine/swarm" +) + +var ( + provisioners = make(map[string]*RegisteredProvisioner) + detector Detector = &StandardDetector{} +) + +const ( + LastReleaseBeforeCEVersioning = "1.13.1" +) + +type SSHCommander interface { + // Short-hand for accessing an SSH command from the driver. + SSHCommand(args string) (string, error) +} + +type Detector interface { + DetectProvisioner(d drivers.Driver) (Provisioner, error) +} + +type StandardDetector struct{} + +func SetDetector(newDetector Detector) { + detector = newDetector +} + +// Provisioner defines distribution specific actions +type Provisioner interface { + fmt.Stringer + SSHCommander + + // Create the files for the daemon to consume configuration settings (return struct of content and path) + GenerateDockerOptions(dockerPort int) (*DockerOptions, error) + + // Get the directory where the settings files for docker are to be found + GetDockerOptionsDir() string + + // Return the auth options used to configure remote connection for the daemon. + GetAuthOptions() auth.Options + + // Get the swarm options associated with this host. + GetSwarmOptions() swarm.Options + + // Run a package action e.g. install + Package(name string, action pkgaction.PackageAction) error + + // Get Hostname + Hostname() (string, error) + + // Set hostname + SetHostname(hostname string) error + + // Figure out if this is the right provisioner to use based on /etc/os-release info + CompatibleWithHost() bool + + // Do the actual provisioning piece: + // 1. Set the hostname on the instance. + // 2. Install Docker if it is not present. + // 3. Configure the daemon to accept connections over TLS. + // 4. Copy the needed certificates to the server and local config dir. + // 5. Configure / activate swarm if applicable. + Provision(swarmOptions swarm.Options, authOptions auth.Options, engineOptions engine.Options) error + + // Perform action on a named service e.g. stop + Service(name string, action serviceaction.ServiceAction) error + + // Get the driver which is contained in the provisioner. + GetDriver() drivers.Driver + + // Set the OS Release info depending on how it's represented + // internally + SetOsReleaseInfo(info *OsRelease) + + // Get the OS Release info for the current provisioner + GetOsReleaseInfo() (*OsRelease, error) +} + +// RegisteredProvisioner creates a new provisioner +type RegisteredProvisioner struct { + New func(d drivers.Driver) Provisioner +} + +func Register(name string, p *RegisteredProvisioner) { + provisioners[name] = p +} + +func DetectProvisioner(d drivers.Driver) (Provisioner, error) { + return detector.DetectProvisioner(d) +} + +func (detector StandardDetector) DetectProvisioner(d drivers.Driver) (Provisioner, error) { + log.Info("Waiting for SSH to be available...") + if err := drivers.WaitForSSH(d); err != nil { + return nil, err + } + + log.Info("Detecting the provisioner...") + + osReleaseOut, err := drivers.RunSSHCommandFromDriver(d, "cat /etc/os-release") + if err != nil { + return nil, fmt.Errorf("Error getting SSH command: %s", err) + } + + osReleaseInfo, err := NewOsRelease([]byte(osReleaseOut)) + if err != nil { + return nil, fmt.Errorf("Error parsing /etc/os-release file: %s", err) + } + + for _, p := range provisioners { + provisioner := p.New(d) + provisioner.SetOsReleaseInfo(osReleaseInfo) + + if provisioner.CompatibleWithHost() { + log.Debugf("found compatible host: %s", osReleaseInfo.ID) + return provisioner, nil + } + } + + return nil, ErrDetectionFailed +} diff --git a/pkg/libmachine/provision/serviceaction/service_action.go b/pkg/libmachine/provision/serviceaction/service_action.go new file mode 100644 index 000000000000..b2e22e5936bd --- /dev/null +++ b/pkg/libmachine/provision/serviceaction/service_action.go @@ -0,0 +1,29 @@ +package serviceaction + +type ServiceAction int + +const ( + Restart ServiceAction = iota + Start + Stop + Enable + Disable + DaemonReload +) + +var serviceActions = []string{ + "restart", + "start", + "stop", + "enable", + "disable", + "daemon-reload", +} + +func (s ServiceAction) String() string { + if int(s) >= 0 && int(s) < len(serviceActions) { + return serviceActions[s] + } + + return "" +} diff --git a/pkg/libmachine/provision/systemd.go b/pkg/libmachine/provision/systemd.go new file mode 100644 index 000000000000..11a1b187dfc6 --- /dev/null +++ b/pkg/libmachine/provision/systemd.go @@ -0,0 +1,101 @@ +package provision + +import ( + "bytes" + "fmt" + "text/template" + + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/provision/serviceaction" + "k8s.io/minikube/pkg/libmachine/versioncmp" +) + +type SystemdProvisioner struct { + GenericProvisioner +} + +func (p *SystemdProvisioner) String() string { + return "redhat" +} + +func NewSystemdProvisioner(osReleaseID string, d drivers.Driver) SystemdProvisioner { + return SystemdProvisioner{ + GenericProvisioner{ + SSHCommander: GenericSSHCommander{Driver: d}, + DockerOptionsDir: "/etc/docker", + DaemonOptionsFile: "/etc/systemd/system/docker.service.d/10-machine.conf", + OsReleaseID: osReleaseID, + Packages: []string{ + "curl", + }, + Driver: d, + }, + } +} + +func (p *SystemdProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) { + var ( + engineCfg bytes.Buffer + ) + + driverNameLabel := fmt.Sprintf("provider=%s", p.Driver.DriverName()) + p.EngineOptions.Labels = append(p.EngineOptions.Labels, driverNameLabel) + + dockerVersion, err := DockerClientVersion(p) + if err != nil { + return nil, err + } + + arg := "dockerd" + if versioncmp.LessThan(dockerVersion, "1.12.0") { + arg = "docker daemon" + } + + engineConfigTmpl := `[Service] +ExecStart= +ExecStart=/usr/bin/` + arg + ` -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --storage-driver {{.EngineOptions.StorageDriver}} --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }} +Environment={{range .EngineOptions.Env}}{{ printf "%q" . }} {{end}} +` + t, err := template.New("engineConfig").Parse(engineConfigTmpl) + if err != nil { + return nil, err + } + + engineConfigContext := EngineConfigContext{ + DockerPort: dockerPort, + AuthOptions: p.AuthOptions, + EngineOptions: p.EngineOptions, + } + + t.Execute(&engineCfg, engineConfigContext) + + return &DockerOptions{ + EngineOptions: engineCfg.String(), + EngineOptionsPath: p.DaemonOptionsFile, + }, nil +} + +func (p *SystemdProvisioner) Service(name string, action serviceaction.ServiceAction) error { + reloadDaemon := false + switch action { + case serviceaction.Start, serviceaction.Restart: + reloadDaemon = true + } + + // systemd needs reloaded when config changes on disk; we cannot + // be sure exactly when it changes from the provisioner so + // we call a reload on every restart to be safe + if reloadDaemon { + if _, err := p.SSHCommand("sudo systemctl daemon-reload"); err != nil { + return err + } + } + + command := fmt.Sprintf("sudo systemctl -f %s %s", action.String(), name) + + if _, err := p.SSHCommand(command); err != nil { + return err + } + + return nil +} diff --git a/pkg/libmachine/provision/utils.go b/pkg/libmachine/provision/utils.go new file mode 100644 index 000000000000..bb821397a4aa --- /dev/null +++ b/pkg/libmachine/provision/utils.go @@ -0,0 +1,322 @@ +package provision + +import ( + "fmt" + "io/ioutil" + "net/url" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/cert" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/provision/serviceaction" +) + +type DockerOptions struct { + EngineOptions string + EngineOptionsPath string +} + +func installDockerGeneric(p Provisioner, baseURL string) error { + // install docker - until cloudinit we use ubuntu everywhere so we + // just install it using the docker repos + if output, err := p.SSHCommand(fmt.Sprintf("if ! type docker; then curl -sSL %s | sh -; fi", baseURL)); err != nil { + return fmt.Errorf("error installing docker: %s", output) + } + + return nil +} + +func makeDockerOptionsDir(p Provisioner) error { + dockerDir := p.GetDockerOptionsDir() + if _, err := p.SSHCommand(fmt.Sprintf("sudo mkdir -p %s", dockerDir)); err != nil { + return err + } + + return nil +} + +func setRemoteAuthOptions(p Provisioner) auth.Options { + dockerDir := p.GetDockerOptionsDir() + authOptions := p.GetAuthOptions() + + // due to windows clients, we cannot use filepath.Join as the paths + // will be mucked on the linux hosts + authOptions.CaCertRemotePath = path.Join(dockerDir, "ca.pem") + authOptions.ServerCertRemotePath = path.Join(dockerDir, "server.pem") + authOptions.ServerKeyRemotePath = path.Join(dockerDir, "server-key.pem") + + return authOptions +} + +func ConfigureAuth(p Provisioner) error { + var ( + err error + ) + + driver := p.GetDriver() + machineName := driver.GetMachineName() + authOptions := p.GetAuthOptions() + swarmOptions := p.GetSwarmOptions() + org := mcnutils.GetUsername() + "." + machineName + bits := 2048 + + ip, err := driver.GetIP() + if err != nil { + return err + } + + log.Info("Copying certs to the local machine directory...") + + if err := mcnutils.CopyFile(authOptions.CaCertPath, filepath.Join(authOptions.StorePath, "ca.pem")); err != nil { + return fmt.Errorf("Copying ca.pem to machine dir failed: %s", err) + } + + if err := mcnutils.CopyFile(authOptions.ClientCertPath, filepath.Join(authOptions.StorePath, "cert.pem")); err != nil { + return fmt.Errorf("Copying cert.pem to machine dir failed: %s", err) + } + + if err := mcnutils.CopyFile(authOptions.ClientKeyPath, filepath.Join(authOptions.StorePath, "key.pem")); err != nil { + return fmt.Errorf("Copying key.pem to machine dir failed: %s", err) + } + + // The Host IP is always added to the certificate's SANs list + hosts := append(authOptions.ServerCertSANs, ip, "localhost", "127.0.0.1") + log.Debugf("generating server cert: %s ca-key=%s private-key=%s org=%s san=%s", + authOptions.ServerCertPath, + authOptions.CaCertPath, + authOptions.CaPrivateKeyPath, + org, + hosts, + ) + + // TODO: Switch to passing just authOptions to this func + // instead of all these individual fields + err = cert.GenerateCert(&cert.Options{ + Hosts: hosts, + CertFile: authOptions.ServerCertPath, + KeyFile: authOptions.ServerKeyPath, + CAFile: authOptions.CaCertPath, + CAKeyFile: authOptions.CaPrivateKeyPath, + Org: org, + Bits: bits, + SwarmMaster: swarmOptions.Master, + }) + + if err != nil { + return fmt.Errorf("error generating server cert: %s", err) + } + + if err := p.Service("docker", serviceaction.Stop); err != nil { + return err + } + + if _, err := p.SSHCommand(`if [ ! -z "$(ip link show docker0)" ]; then sudo ip link delete docker0; fi`); err != nil { + return err + } + + // upload certs and configure TLS auth + caCert, err := ioutil.ReadFile(authOptions.CaCertPath) + if err != nil { + return err + } + + serverCert, err := ioutil.ReadFile(authOptions.ServerCertPath) + if err != nil { + return err + } + serverKey, err := ioutil.ReadFile(authOptions.ServerKeyPath) + if err != nil { + return err + } + + log.Info("Copying certs to the remote machine...") + + // printf will choke if we don't pass a format string because of the + // dashes, so that's the reason for the '%%s' + certTransferCmdFmt := "printf '%%s' '%s' | sudo tee %s" + + // These ones are for Jessie and Mike <3 <3 <3 + if _, err := p.SSHCommand(fmt.Sprintf(certTransferCmdFmt, string(caCert), authOptions.CaCertRemotePath)); err != nil { + return err + } + + if _, err := p.SSHCommand(fmt.Sprintf(certTransferCmdFmt, string(serverCert), authOptions.ServerCertRemotePath)); err != nil { + return err + } + + if _, err := p.SSHCommand(fmt.Sprintf(certTransferCmdFmt, string(serverKey), authOptions.ServerKeyRemotePath)); err != nil { + return err + } + + dockerURL, err := driver.GetURL() + if err != nil { + return err + } + u, err := url.Parse(dockerURL) + if err != nil { + return err + } + dockerPort := engine.DefaultPort + parts := strings.Split(u.Host, ":") + if len(parts) == 2 { + dPort, err := strconv.Atoi(parts[1]) + if err != nil { + return err + } + dockerPort = dPort + } + + dkrcfg, err := p.GenerateDockerOptions(dockerPort) + if err != nil { + return err + } + + log.Info("Setting Docker configuration on the remote daemon...") + + if _, err = p.SSHCommand(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(dkrcfg.EngineOptionsPath), dkrcfg.EngineOptions, dkrcfg.EngineOptionsPath)); err != nil { + return err + } + + if err := p.Service("docker", serviceaction.Start); err != nil { + return err + } + + return WaitForDocker(p, dockerPort) +} + +func matchNetstatOut(reDaemonListening, netstatOut string) bool { + // TODO: I would really prefer this be a Scanner directly on + // the STDOUT of the executed command than to do all the string + // manipulation hokey-pokey. + // + // TODO: Unit test this matching. + for _, line := range strings.Split(netstatOut, "\n") { + match, err := regexp.MatchString(reDaemonListening, line) + if err != nil { + log.Warnf("Regex warning: %s", err) + } + if match && line != "" { + return true + } + } + + return false +} + +func decideStorageDriver(p Provisioner, defaultDriver, suppliedDriver string) (string, error) { + if suppliedDriver != "" { + return suppliedDriver, nil + } + bestSuitedDriver := "" + + defer func() { + if bestSuitedDriver != "" { + log.Debugf("No storagedriver specified, using %s\n", bestSuitedDriver) + } + }() + + if defaultDriver != "aufs" { + bestSuitedDriver = defaultDriver + } else { + remoteFilesystemType, err := getFilesystemType(p, "/var/lib") + if err != nil { + return "", err + } + if remoteFilesystemType == "btrfs" { + bestSuitedDriver = "btrfs" + } else { + bestSuitedDriver = defaultDriver + } + } + return bestSuitedDriver, nil + +} + +func getFilesystemType(p Provisioner, directory string) (string, error) { + statCommandOutput, err := p.SSHCommand("stat -f -c %T " + directory) + if err != nil { + err = fmt.Errorf("Error looking up filesystem type: %s", err) + return "", err + } + + fstype := strings.TrimSpace(statCommandOutput) + return fstype, nil +} + +func checkDaemonUp(p Provisioner, dockerPort int) func() bool { + reDaemonListening := fmt.Sprintf(":%d\\s+.*:.*", dockerPort) + return func() bool { + // HACK: Check netstat's output to see if anyone's listening on the Docker API port. + netstatOut, err := p.SSHCommand("if ! type netstat 1>/dev/null; then ss -tln; else netstat -tln; fi") + if err != nil { + log.Warnf("Error running SSH command: %s", err) + return false + } + + return matchNetstatOut(reDaemonListening, netstatOut) + } +} + +func WaitForDocker(p Provisioner, dockerPort int) error { + if err := mcnutils.WaitForSpecific(checkDaemonUp(p, dockerPort), 10, 3*time.Second); err != nil { + return NewErrDaemonAvailable(err) + } + + return nil +} + +// DockerClientVersion returns the version of the Docker client on the host +// that ssh is connected to, e.g. "1.12.1". +func DockerClientVersion(ssh SSHCommander) (string, error) { + // `docker version --format {{.Client.Version}}` would be preferable, but + // that fails if the server isn't running yet. + // + // output is expected to be something like + // + // Docker version 1.12.1, build 7a86f89 + output, err := ssh.SSHCommand("docker --version") + if err != nil { + return "", err + } + + words := strings.Fields(output) + if len(words) < 3 || words[0] != "Docker" || words[1] != "version" { + return "", fmt.Errorf("DockerClientVersion: cannot parse version string from %q", output) + } + + return strings.TrimRight(words[2], ","), nil +} + +func waitForLockAptGetUpdate(ssh SSHCommander) error { + return waitForLock(ssh, "sudo apt-get update") +} + +func waitForLock(ssh SSHCommander, cmd string) error { + var sshErr error + err := mcnutils.WaitFor(func() bool { + _, sshErr = ssh.SSHCommand(cmd) + if sshErr != nil { + if strings.Contains(sshErr.Error(), "Could not get lock") { + sshErr = nil + return false + } + return true + } + return true + }) + if sshErr != nil { + return fmt.Errorf("Error running %q: %s", cmd, sshErr) + } + if err != nil { + return fmt.Errorf("Failed to obtain lock: %s", err) + } + return nil +} diff --git a/pkg/libmachine/shell/shell.go b/pkg/libmachine/shell/shell.go new file mode 100644 index 000000000000..d0a271c9a9ac --- /dev/null +++ b/pkg/libmachine/shell/shell.go @@ -0,0 +1,26 @@ +// +build !windows + +package shell + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +var ( + ErrUnknownShell = errors.New("Error: Unknown shell") +) + +// Detect detects user's current shell. +func Detect() (string, error) { + shell := os.Getenv("SHELL") + + if shell == "" { + fmt.Printf("The default lines below are for a sh/bash shell, you can specify the shell you're using, with the --shell flag.\n\n") + return "", ErrUnknownShell + } + + return filepath.Base(shell), nil +} diff --git a/pkg/libmachine/shell/shell_windows.go b/pkg/libmachine/shell/shell_windows.go new file mode 100644 index 000000000000..eef11d43f6bb --- /dev/null +++ b/pkg/libmachine/shell/shell_windows.go @@ -0,0 +1,103 @@ +package shell + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "syscall" + "unsafe" +) + +// re-implementation of private function in https://github.com/golang/go/blob/master/src/syscall/syscall_windows.go#L945 +func getProcessEntry(pid int) (pe *syscall.ProcessEntry32, err error) { + snapshot, err := syscall.CreateToolhelp32Snapshot(syscall.TH32CS_SNAPPROCESS, 0) + if err != nil { + return nil, err + } + defer syscall.CloseHandle(syscall.Handle(snapshot)) + + var processEntry syscall.ProcessEntry32 + processEntry.Size = uint32(unsafe.Sizeof(processEntry)) + err = syscall.Process32First(snapshot, &processEntry) + if err != nil { + return nil, err + } + + for { + if processEntry.ProcessID == uint32(pid) { + pe = &processEntry + return + } + + err = syscall.Process32Next(snapshot, &processEntry) + if err != nil { + return nil, err + } + } +} + +// getNameAndItsPpid returns the exe file name its parent process id. +func getNameAndItsPpid(pid int) (exefile string, parentid int, err error) { + pe, err := getProcessEntry(pid) + if err != nil { + return "", 0, err + } + + name := syscall.UTF16ToString(pe.ExeFile[:]) + return name, int(pe.ParentProcessID), nil +} + +func Detect() (string, error) { + shell := os.Getenv("SHELL") + + // if you spawn a Powershell instance from CMD, sometimes the SHELL environment variable still points to CMD in the Powershell instance + // so if SHELL is pointing to CMD, let's do extra work to get the correct shell + if shell == "" || filepath.Base(shell) == "cmd.exe" { + shell, shellppid, err := getNameAndItsPpid(os.Getppid()) + if err != nil { + return "cmd", err // defaulting to cmd + } + shellMapping := mapShell(shell) + if shellMapping != "" { + return shellMapping, nil + } else { + shell, _, err := getNameAndItsPpid(shellppid) + if err != nil { + return "cmd", err // defaulting to cmd + } + shellMapping = mapShell(shell) + if shellMapping != "" { + return shellMapping, nil + } else { + fmt.Printf("You can further specify your shell with either 'cmd' or 'powershell' with the --shell flag.\n\n") + return "cmd", nil // this could be either powershell or cmd, defaulting to cmd + } + } + } + + if os.Getenv("__fish_bin_dir") != "" { + return "fish", nil + } + + if runtime.GOOS == "windows" { + shell = strings.TrimSuffix(shell, filepath.Ext(shell)) + } + + return filepath.Base(shell), nil +} + +func mapShell(shell string) string { + mappings := map[string]string{ + "cmd": "cmd", + "powershell": "powershell", + "pwsh": "powershell", + } + for k, v := range mappings { + if strings.Contains(strings.ToLower(shell), k) { + return v + } + } + return "" +} diff --git a/pkg/libmachine/ssh/client.go b/pkg/libmachine/ssh/client.go new file mode 100644 index 000000000000..fbd7142fac86 --- /dev/null +++ b/pkg/libmachine/ssh/client.go @@ -0,0 +1,447 @@ +package ssh + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "os" + "os/exec" + "runtime" + "strconv" + "strings" + + "k8s.io/minikube/pkg/libmachine/log" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "github.com/moby/term" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/terminal" +) + +type Client interface { + Output(command string) (string, error) + Shell(args ...string) error + + // Start starts the specified command without waiting for it to finish. You + // have to call the Wait function for that. + // + // The first two io.ReadCloser are the standard output and the standard + // error of the executing command respectively. The returned error follows + // the same logic as in the exec.Cmd.Start function. + Start(command string) (io.ReadCloser, io.ReadCloser, error) + + // Wait waits for the command started by the Start function to exit. The + // returned error follows the same logic as in the exec.Cmd.Wait function. + Wait() error +} + +type ExternalClient struct { + BaseArgs []string + BinaryPath string + cmd *exec.Cmd +} + +type NativeClient struct { + Config ssh.ClientConfig + Hostname string + Port int + openSession *ssh.Session + openClient *ssh.Client +} + +type Auth struct { + Passwords []string + Keys []string +} + +type ClientType string + +const ( + maxDialAttempts = 10 +) + +const ( + External ClientType = "external" + Native ClientType = "native" +) + +var ( + baseSSHArgs = []string{ + "-F", "/dev/null", + "-o", "ConnectionAttempts=3", // retry 3 times if SSH connection fails + "-o", "ConnectTimeout=10", // timeout after 10 seconds + "-o", "ControlMaster=no", // disable ssh multiplexing + "-o", "ControlPath=none", + "-o", "LogLevel=quiet", // suppress "Warning: Permanently added '[localhost]:2022' (ECDSA) to the list of known hosts." + "-o", "PasswordAuthentication=no", + "-o", "ServerAliveInterval=60", // prevents connection to be dropped if command takes too long + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + } + defaultClientType = External +) + +func SetDefaultClient(clientType ClientType) { + // Allow over-riding of default client type, so that even if ssh binary + // is found in PATH we can still use the Go native implementation if + // desired. + switch clientType { + case External: + defaultClientType = External + case Native: + defaultClientType = Native + } +} + +func NewClient(user string, host string, port int, auth *Auth) (Client, error) { + sshBinaryPath, err := exec.LookPath("ssh") + if err != nil { + log.Debug("SSH binary not found, using native Go implementation") + client, err := NewNativeClient(user, host, port, auth) + log.Debug(client) + return client, err + } + + if defaultClientType == Native { + log.Debug("Using SSH client type: native") + client, err := NewNativeClient(user, host, port, auth) + log.Debug(client) + return client, err + } + + log.Debug("Using SSH client type: external") + client, err := NewExternalClient(sshBinaryPath, user, host, port, auth) + log.Debug(client) + return client, err +} + +func NewNativeClient(user, host string, port int, auth *Auth) (Client, error) { + config, err := NewNativeConfig(user, auth) + if err != nil { + return nil, fmt.Errorf("Error getting config for native Go SSH: %s", err) + } + + return &NativeClient{ + Config: config, + Hostname: host, + Port: port, + }, nil +} + +func NewNativeConfig(user string, auth *Auth) (ssh.ClientConfig, error) { + var ( + authMethods []ssh.AuthMethod + ) + + for _, k := range auth.Keys { + key, err := ioutil.ReadFile(k) + if err != nil { + return ssh.ClientConfig{}, err + } + + privateKey, err := ssh.ParsePrivateKey(key) + if err != nil { + return ssh.ClientConfig{}, err + } + + authMethods = append(authMethods, ssh.PublicKeys(privateKey)) + } + + for _, p := range auth.Passwords { + authMethods = append(authMethods, ssh.Password(p)) + } + + return ssh.ClientConfig{ + User: user, + Auth: authMethods, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + }, nil +} + +func (client *NativeClient) dialSuccess() bool { + conn, err := ssh.Dial("tcp", net.JoinHostPort(client.Hostname, strconv.Itoa(client.Port)), &client.Config) + if err != nil { + log.Debugf("Error dialing TCP: %s", err) + return false + } + closeConn(conn) + return true +} + +func (client *NativeClient) session(command string) (*ssh.Client, *ssh.Session, error) { + if err := mcnutils.WaitFor(client.dialSuccess); err != nil { + return nil, nil, fmt.Errorf("Error attempting SSH client dial: %s", err) + } + + conn, err := ssh.Dial("tcp", net.JoinHostPort(client.Hostname, strconv.Itoa(client.Port)), &client.Config) + if err != nil { + return nil, nil, fmt.Errorf("Mysterious error dialing TCP for SSH (we already succeeded at least once) : %s", err) + } + session, err := conn.NewSession() + + return conn, session, err +} + +func (client *NativeClient) Output(command string) (string, error) { + conn, session, err := client.session(command) + if err != nil { + return "", nil + } + defer closeConn(conn) + defer session.Close() + + output, err := session.CombinedOutput(command) + + return string(output), err +} + +func (client *NativeClient) OutputWithPty(command string) (string, error) { + conn, session, err := client.session(command) + if err != nil { + return "", nil + } + defer closeConn(conn) + defer session.Close() + + fd := int(os.Stdout.Fd()) + + termWidth, termHeight, err := terminal.GetSize(fd) + if err != nil { + return "", err + } + + modes := ssh.TerminalModes{ + ssh.ECHO: 0, + ssh.TTY_OP_ISPEED: 14400, + ssh.TTY_OP_OSPEED: 14400, + } + + // request tty -- fixes error with hosts that use + // "Defaults requiretty" in /etc/sudoers - I'm looking at you RedHat + if err := session.RequestPty("xterm", termHeight, termWidth, modes); err != nil { + return "", err + } + + output, err := session.CombinedOutput(command) + + return string(output), err +} + +func (client *NativeClient) Start(command string) (io.ReadCloser, io.ReadCloser, error) { + conn, session, err := client.session(command) + if err != nil { + return nil, nil, err + } + + stdout, err := session.StdoutPipe() + if err != nil { + return nil, nil, err + } + stderr, err := session.StderrPipe() + if err != nil { + return nil, nil, err + } + if err := session.Start(command); err != nil { + return nil, nil, err + } + + client.openClient = conn + client.openSession = session + return ioutil.NopCloser(stdout), ioutil.NopCloser(stderr), nil +} + +func (client *NativeClient) Wait() error { + err := client.openSession.Wait() + if err != nil { + return err + } + + _ = client.openSession.Close() + + err = client.openClient.Close() + if err != nil { + return err + } + + client.openSession = nil + client.openClient = nil + return nil +} + +func (client *NativeClient) Shell(args ...string) error { + var ( + termWidth, termHeight int + ) + conn, err := ssh.Dial("tcp", net.JoinHostPort(client.Hostname, strconv.Itoa(client.Port)), &client.Config) + if err != nil { + return err + } + defer closeConn(conn) + + session, err := conn.NewSession() + if err != nil { + return err + } + + defer session.Close() + + session.Stdout = os.Stdout + session.Stderr = os.Stderr + session.Stdin = os.Stdin + + modes := ssh.TerminalModes{ + ssh.ECHO: 1, + } + + fd := os.Stdin.Fd() + + if term.IsTerminal(fd) { + oldState, err := term.MakeRaw(fd) + if err != nil { + return err + } + + defer term.RestoreTerminal(fd, oldState) + + winsize, err := term.GetWinsize(fd) + if err != nil { + termWidth = 80 + termHeight = 24 + } else { + termWidth = int(winsize.Width) + termHeight = int(winsize.Height) + } + } + + if err := session.RequestPty("xterm", termHeight, termWidth, modes); err != nil { + return err + } + + if len(args) == 0 { + if err := session.Shell(); err != nil { + return err + } + if err := session.Wait(); err != nil { + return err + } + } else { + if err := session.Run(strings.Join(args, " ")); err != nil { + return err + } + } + return nil +} + +func NewExternalClient(sshBinaryPath, user, host string, port int, auth *Auth) (*ExternalClient, error) { + client := &ExternalClient{ + BinaryPath: sshBinaryPath, + } + + args := append(baseSSHArgs, fmt.Sprintf("%s@%s", user, host)) + + // If no identities are explicitly provided, also look at the identities + // offered by ssh-agent + if len(auth.Keys) > 0 { + args = append(args, "-o", "IdentitiesOnly=yes") + } + + // Specify which private keys to use to authorize the SSH request. + for _, privateKeyPath := range auth.Keys { + if privateKeyPath != "" { + // Check each private key before use it + fi, err := os.Stat(privateKeyPath) + if err != nil { + // Abort if key not accessible + return nil, err + } + if runtime.GOOS != "windows" { + mode := fi.Mode() + log.Debugf("Using SSH private key: %s (%s)", privateKeyPath, mode) + // Private key file should have strict permissions + perm := mode.Perm() + if perm&0400 == 0 { + return nil, fmt.Errorf("'%s' is not readable", privateKeyPath) + } + if perm&0077 != 0 { + return nil, fmt.Errorf("permissions %#o for '%s' are too open", perm, privateKeyPath) + } + } + args = append(args, "-i", privateKeyPath) + } + } + + // Set which port to use for SSH. + args = append(args, "-p", fmt.Sprintf("%d", port)) + + client.BaseArgs = args + + return client, nil +} + +func getSSHCmd(binaryPath string, args ...string) *exec.Cmd { + return exec.Command(binaryPath, args...) +} + +func (client *ExternalClient) Output(command string) (string, error) { + args := append(client.BaseArgs, command) + cmd := getSSHCmd(client.BinaryPath, args...) + output, err := cmd.CombinedOutput() + return string(output), err +} + +func (client *ExternalClient) Shell(args ...string) error { + args = append(client.BaseArgs, args...) + cmd := getSSHCmd(client.BinaryPath, args...) + + log.Debug(cmd) + + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + return cmd.Run() +} + +func (client *ExternalClient) Start(command string) (io.ReadCloser, io.ReadCloser, error) { + args := append(client.BaseArgs, command) + cmd := getSSHCmd(client.BinaryPath, args...) + + log.Debug(cmd) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + stderr, err := cmd.StderrPipe() + if err != nil { + if closeErr := stdout.Close(); closeErr != nil { + return nil, nil, fmt.Errorf("%s, %s", err, closeErr) + } + return nil, nil, err + } + if err := cmd.Start(); err != nil { + stdOutCloseErr := stdout.Close() + stdErrCloseErr := stderr.Close() + if stdOutCloseErr != nil || stdErrCloseErr != nil { + return nil, nil, fmt.Errorf("%s, %s, %s", + err, stdOutCloseErr, stdErrCloseErr) + } + return nil, nil, err + } + + client.cmd = cmd + return stdout, stderr, nil +} + +func (client *ExternalClient) Wait() error { + err := client.cmd.Wait() + client.cmd = nil + return err +} + +func closeConn(c io.Closer) { + err := c.Close() + if err != nil { + log.Debugf("Error closing SSH Client: %s", err) + } +} diff --git a/pkg/libmachine/ssh/keys.go b/pkg/libmachine/ssh/keys.go new file mode 100644 index 000000000000..6016b6011a13 --- /dev/null +++ b/pkg/libmachine/ssh/keys.go @@ -0,0 +1,138 @@ +package ssh + +import ( + "crypto/md5" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "io" + "os" + "runtime" + + gossh "golang.org/x/crypto/ssh" + "github.com/hectane/go-acl" +) + +var ( + ErrKeyGeneration = errors.New("Unable to generate key") + ErrValidation = errors.New("Unable to validate key") + ErrPublicKey = errors.New("Unable to convert public key") + ErrUnableToWriteFile = errors.New("Unable to write file") +) + +type KeyPair struct { + PrivateKey []byte + PublicKey []byte +} + +// NewKeyPair generates a new SSH keypair +// This will return a private & public key encoded as DER. +func NewKeyPair() (keyPair *KeyPair, err error) { + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, ErrKeyGeneration + } + + if err := priv.Validate(); err != nil { + return nil, ErrValidation + } + + privDer := x509.MarshalPKCS1PrivateKey(priv) + + pubSSH, err := gossh.NewPublicKey(&priv.PublicKey) + if err != nil { + return nil, ErrPublicKey + } + + return &KeyPair{ + PrivateKey: privDer, + PublicKey: gossh.MarshalAuthorizedKey(pubSSH), + }, nil +} + +// WriteToFile writes keypair to files +func (kp *KeyPair) WriteToFile(privateKeyPath string, publicKeyPath string) error { + files := []struct { + File string + Type string + Value []byte + }{ + { + File: privateKeyPath, + Value: pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Headers: nil, Bytes: kp.PrivateKey}), + }, + { + File: publicKeyPath, + Value: kp.PublicKey, + }, + } + + for _, v := range files { + f, err := os.Create(v.File) + if err != nil { + return ErrUnableToWriteFile + } + + if _, err := f.Write(v.Value); err != nil { + return ErrUnableToWriteFile + } + + // windows does not support chmod + switch runtime.GOOS { + case "darwin", "freebsd", "linux", "openbsd": + if err := f.Chmod(0600); err != nil { + return err + } + case "windows": + if err = windowsChmod(v.File, 0600); err != nil { + return err + } + } + } + + return nil +} + +// Fingerprint calculates the fingerprint of the public key +func (kp *KeyPair) Fingerprint() string { + b, _ := base64.StdEncoding.DecodeString(string(kp.PublicKey)) + h := md5.New() + + io.WriteString(h, string(b)) + + return fmt.Sprintf("%x", h.Sum(nil)) +} + +// GenerateSSHKey generates SSH keypair based on path of the private key +// The public key would be generated to the same path with ".pub" added +func GenerateSSHKey(path string) error { + if _, err := os.Stat(path); err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("Desired directory for SSH keys does not exist: %s", err) + } + + kp, err := NewKeyPair() + if err != nil { + return fmt.Errorf("Error generating key pair: %s", err) + } + + if err := kp.WriteToFile(path, fmt.Sprintf("%s.pub", path)); err != nil { + return fmt.Errorf("Error writing keys to file(s): %s", err) + } + } + + return nil +} + +// change windows acl based permissions on file +func windowsChmod(filePath string, fileMode os.FileMode) error { + if err := acl.Chmod(filePath, fileMode); err != nil { + return err + } + return nil +} + diff --git a/pkg/libmachine/state/state.go b/pkg/libmachine/state/state.go new file mode 100644 index 000000000000..385a173be774 --- /dev/null +++ b/pkg/libmachine/state/state.go @@ -0,0 +1,36 @@ +package state + +// State represents the state of a host +type State int + +const ( + None State = iota + Running + Paused + Saved + Stopped + Stopping + Starting + Error + Timeout +) + +var states = []string{ + "", + "Running", + "Paused", + "Saved", + "Stopped", + "Stopping", + "Starting", + "Error", + "Timeout", +} + +// Given a State type, returns its string representation +func (s State) String() string { + if int(s) >= 0 && int(s) < len(states) { + return states[s] + } + return "" +} diff --git a/pkg/libmachine/swarm/swarm.go b/pkg/libmachine/swarm/swarm.go new file mode 100644 index 000000000000..ee45288390df --- /dev/null +++ b/pkg/libmachine/swarm/swarm.go @@ -0,0 +1,22 @@ +package swarm + +const ( + DiscoveryServiceEndpoint = "https://discovery-stage.hub.docker.com/v1" +) + +type Options struct { + IsSwarm bool + Address string + Discovery string + Agent bool + Master bool + Host string + Image string + Strategy string + Heartbeat int + Overcommit float64 + ArbitraryFlags []string + ArbitraryJoinFlags []string + Env []string + IsExperimental bool +} diff --git a/pkg/libmachine/version/version.go b/pkg/libmachine/version/version.go new file mode 100644 index 000000000000..3cacffdb0c0a --- /dev/null +++ b/pkg/libmachine/version/version.go @@ -0,0 +1,11 @@ +package version + +var ( + // APIVersion dictates which version of the libmachine API this is. + APIVersion = 1 + + // ConfigVersion dictates which version of the config.json format is + // used. It needs to be bumped if there is a breaking change, and + // therefore migration, introduced to the config file format. + ConfigVersion = 3 +) diff --git a/pkg/libmachine/versioncmp/compare.go b/pkg/libmachine/versioncmp/compare.go new file mode 100644 index 000000000000..2df82cef6eb4 --- /dev/null +++ b/pkg/libmachine/versioncmp/compare.go @@ -0,0 +1,121 @@ +// Package versioncmp provides functions for comparing version strings. +// +// Version strings are dot-separated integers with an optional +// pre-release suffix. A pre-release suffix is an arbitrary string with a +// leading dash character. All functions ignore these suffixes, so "1.2" and +// "1.2-rc" are considered equivalent. +package versioncmp + +import ( + "strconv" + "strings" +) + +const ( + rcString = "-rc" + ceEdition = "-ce" +) + +// compare compares two versions of Docker to decipher which came first. +// +// compare returns -1 if v1 < v2, 1 if v1 > v2, 0 otherwise. +func compare(v1, v2 string) int { + // Replace RC string with "." to make the RC number appear as simply + // another sub-version. + v1 = strings.Replace(v1, rcString, ".", -1) + v2 = strings.Replace(v2, rcString, ".", -1) + + // All releases before the community edition (differentiated by + // presence of the "ce" string in the version string) are "less than" + // any community edition release (first occurring in March 2017). + if strings.Contains(v1, ceEdition) && !strings.Contains(v2, ceEdition) { + return 1 + } + if !strings.Contains(v1, ceEdition) && strings.Contains(v2, ceEdition) { + return -1 + } + + // Without this tag, both are pre-CE versions. + if !strings.Contains(v1, ceEdition) && !strings.Contains(v2, ceEdition) { + return compareNumeric(v1, v2) + } + + return compareCE(v1, v2) +} + +// compareCE ("Community Edition") will differentiate between versions of +// Docker that use the versioning scheme +// {{release-year}}.{{release-month}}-{{ce|ee}}-{{rcnum|""}} +// +// This will be every release after 1.13.1. +func compareCE(v1, v2 string) int { + return compareNumeric( + strings.Replace(v1, ceEdition, "", -1), + strings.Replace(v2, ceEdition, "", -1), + ) +} + +// compareNumeric compares two version that use pre-17.03 Docker. +// +// Non-numeric segments in either argument are considered equal, so +// compare("1.a", "1.b") == 0, but compare("2.a", "1.b") == 1. +func compareNumeric(v1, v2 string) int { + if n := strings.IndexByte(v1, '-'); n != -1 { + v1 = v1[:n] + } + if n := strings.IndexByte(v2, '-'); n != -1 { + v2 = v2[:n] + } + var ( + currTab = strings.Split(v1, ".") + otherTab = strings.Split(v2, ".") + ) + + max := len(currTab) + if len(otherTab) > max { + max = len(otherTab) + } + for i := 0; i < max; i++ { + var currInt, otherInt int + + if len(currTab) > i { + currInt, _ = strconv.Atoi(currTab[i]) + } + if len(otherTab) > i { + otherInt, _ = strconv.Atoi(otherTab[i]) + } + if currInt > otherInt { + return 1 + } + if otherInt > currInt { + return -1 + } + } + return 0 +} + +// LessThan checks if a version is less than another. +func LessThan(v, other string) bool { + return compare(v, other) == -1 +} + +// LessThanOrEqualTo checks if a version is less than or equal to another. +func LessThanOrEqualTo(v, other string) bool { + return compare(v, other) <= 0 +} + +// GreaterThan checks if a version is greater than another. +func GreaterThan(v, other string) bool { + return compare(v, other) == 1 +} + +// GreaterThanOrEqualTo checks if a version is greater than or equal to +// another. +func GreaterThanOrEqualTo(v, other string) bool { + return compare(v, other) >= 0 +} + +// Equal checks if a version is equal to another. +func Equal(v, other string) bool { + return compare(v, other) == 0 +} diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go index f06722f5a8c7..fa5fa72a2da6 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go @@ -32,7 +32,7 @@ import ( "strings" "time" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/version" diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/system_svc.go b/pkg/minikube/bootstrapper/bsutil/kverify/system_svc.go index 5322693a1bd9..fc2a471aff0b 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/system_svc.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/system_svc.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/state" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/sysinit" diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 65500a332b94..a1fbeeb95b76 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -34,8 +34,8 @@ import ( // WARNING: Do not use path/filepath in this package unless you want bizarre Windows paths "github.com/blang/semver/v4" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index c8a3cd39b7a4..939886db231b 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -19,8 +19,8 @@ package cluster import ( "fmt" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/ssh" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/bootstrapper" diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go index ca4d048a986d..aa5bb89ad1b3 100644 --- a/pkg/minikube/cluster/ip.go +++ b/pkg/minikube/cluster/ip.go @@ -24,8 +24,8 @@ import ( "regexp" "strings" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic/oci" @@ -104,41 +104,11 @@ func HostIP(hostInfo *host.Host, clusterName string) (net.IP, error) { re = regexp.MustCompile(`(?sm)Name:\s*` + iface + `\s*$.+?IPAddress:\s*(\S+)`) ip := re.FindStringSubmatch(string(ipList))[1] - return net.ParseIP(ip), nil - case driver.Parallels: - bin := "prlsrvctl" - var binPath string - if fullPath, err := exec.LookPath(bin); err != nil { - binPath = fullPath - } else { - binPath = bin - } - out, err := exec.Command(binPath, "net", "info", "Shared").Output() - if err != nil { - return []byte{}, errors.Wrap(err, "Error reading the info of Parallels Shared network interface") - } - re := regexp.MustCompile(`IPv4 address: (.*)`) - ipMatch := re.FindStringSubmatch(string(out)) - if len(ipMatch) < 2 { - return []byte{}, errors.Wrap(err, "Error getting the IP address of Parallels Shared network interface") - } - ip := ipMatch[1] - return net.ParseIP(ip), nil case driver.HyperKit: vmIPString, _ := hostInfo.Driver.GetIP() gatewayIPString := vmIPString[:strings.LastIndex(vmIPString, ".")+1] + "1" return net.ParseIP(gatewayIPString), nil - case driver.VMware: - vmIPString, err := hostInfo.Driver.GetIP() - if err != nil { - return []byte{}, errors.Wrap(err, "Error getting VM IP address") - } - vmIP := net.ParseIP(vmIPString).To4() - if vmIP == nil { - return []byte{}, errors.Wrap(err, "Error converting VM IP address to IPv4 address") - } - return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil case driver.VFKit, driver.Krunkit: // TODO: check why we need this and test with: // - vfkkit+nat diff --git a/pkg/minikube/cluster/status.go b/pkg/minikube/cluster/status.go index 3963cbde4f17..1ce29333f394 100644 --- a/pkg/minikube/cluster/status.go +++ b/pkg/minikube/cluster/status.go @@ -26,8 +26,8 @@ import ( "time" cloudevents "github.com/cloudevents/sdk-go/v2" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" diff --git a/pkg/minikube/command/ssh_runner.go b/pkg/minikube/command/ssh_runner.go index 20356b172bdf..fa9bcde23e71 100644 --- a/pkg/minikube/command/ssh_runner.go +++ b/pkg/minikube/command/ssh_runner.go @@ -28,7 +28,7 @@ import ( "sync" "time" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "github.com/kballard/go-shellquote" "github.com/pkg/errors" "golang.org/x/crypto/ssh" diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 476b85c0c597..aa8159fce99d 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -55,12 +55,8 @@ const ( VirtualBox = "virtualbox" // HyperKit driver HyperKit = "hyperkit" - // VMware driver - VMware = "vmware" // HyperV driver HyperV = "hyperv" - // Parallels driver - Parallels = "parallels" // VFKit driver VFKit = "vfkit" // Krunkit driver @@ -208,11 +204,6 @@ func IsVirtualBox(name string) bool { return name == VirtualBox } -// IsVMware checks if the driver is VMware -func IsVMware(name string) bool { - return name == VMware -} - // IsHyperV check if the driver is Hyper-V func IsHyperV(name string) bool { return name == HyperV diff --git a/pkg/minikube/driver/driver_darwin.go b/pkg/minikube/driver/driver_darwin.go index a0faf66a4f8d..709fea84c54e 100644 --- a/pkg/minikube/driver/driver_darwin.go +++ b/pkg/minikube/driver/driver_darwin.go @@ -30,7 +30,6 @@ var supportedDrivers = func() []string { QEMU2, VFKit, Krunkit, - Parallels, Docker, Podman, SSH, @@ -40,18 +39,14 @@ var supportedDrivers = func() []string { if strings.HasPrefix(runtime.GOARCH, "ppc") { return []string{ VirtualBox, - Parallels, HyperKit, - VMware, Docker, SSH, } } return []string{ VirtualBox, - Parallels, HyperKit, - VMware, QEMU2, VFKit, Docker, diff --git a/pkg/minikube/driver/driver_linux.go b/pkg/minikube/driver/driver_linux.go index 613effa2aa02..026e3dd9de89 100644 --- a/pkg/minikube/driver/driver_linux.go +++ b/pkg/minikube/driver/driver_linux.go @@ -26,7 +26,6 @@ var supportedDrivers = []string{ KVM2, QEMU2, QEMU, - VMware, None, Docker, Podman, diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go index 42d7691ed7c0..a391dc290db6 100644 --- a/pkg/minikube/driver/driver_test.go +++ b/pkg/minikube/driver/driver_test.go @@ -72,9 +72,7 @@ func TestMachineType(t *testing.T) { Krunkit: "VM", VirtualBox: "VM", HyperKit: "VM", - VMware: "VM", HyperV: "VM", - Parallels: "VM", } drivers := SupportedDrivers() diff --git a/pkg/minikube/driver/driver_windows.go b/pkg/minikube/driver/driver_windows.go index 255d28a0f3f6..c2a2d8dfd786 100644 --- a/pkg/minikube/driver/driver_windows.go +++ b/pkg/minikube/driver/driver_windows.go @@ -30,7 +30,6 @@ import ( var supportedDrivers = []string{ VirtualBox, HyperV, - VMware, QEMU2, Docker, Podman, diff --git a/pkg/minikube/machine/build_images.go b/pkg/minikube/machine/build_images.go index 5366c2e710bc..35e27579ec14 100644 --- a/pkg/minikube/machine/build_images.go +++ b/pkg/minikube/machine/build_images.go @@ -27,7 +27,7 @@ import ( dockerref "github.com/distribution/reference" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/assets" diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index f09d1a2c151e..4df28a6d3b32 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -31,7 +31,7 @@ import ( "github.com/docker/docker/client" "github.com/docker/go-units" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/state" "github.com/olekukonko/tablewriter" "github.com/olekukonko/tablewriter/tw" "github.com/pkg/errors" diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go index 67f370f649e9..7c33f3901055 100644 --- a/pkg/minikube/machine/client.go +++ b/pkg/minikube/machine/client.go @@ -25,21 +25,21 @@ import ( "path/filepath" "time" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/auth" - "github.com/docker/machine/libmachine/cert" - "github.com/docker/machine/libmachine/check" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/drivers/plugin" - "github.com/docker/machine/libmachine/drivers/plugin/localbinary" - "github.com/docker/machine/libmachine/engine" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/mcnutils" - "github.com/docker/machine/libmachine/persist" - lmssh "github.com/docker/machine/libmachine/ssh" - "github.com/docker/machine/libmachine/state" - "github.com/docker/machine/libmachine/swarm" - "github.com/docker/machine/libmachine/version" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/cert" + "k8s.io/minikube/pkg/libmachine/check" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers/plugin" + "k8s.io/minikube/pkg/libmachine/drivers/plugin/localbinary" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/persist" + lmssh "k8s.io/minikube/pkg/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/state" + "k8s.io/minikube/pkg/libmachine/swarm" + "k8s.io/minikube/pkg/libmachine/version" "github.com/juju/fslock" "github.com/pkg/errors" "k8s.io/klog/v2" diff --git a/pkg/minikube/machine/client_test.go b/pkg/minikube/machine/client_test.go index 7ddd379babe2..f262d3d7184a 100644 --- a/pkg/minikube/machine/client_test.go +++ b/pkg/minikube/machine/client_test.go @@ -23,7 +23,7 @@ import ( "os" "testing" - "github.com/docker/machine/libmachine/drivers/plugin/localbinary" + "k8s.io/minikube/pkg/libmachine/drivers/plugin/localbinary" "k8s.io/minikube/pkg/minikube/driver" _ "k8s.io/minikube/pkg/minikube/registry/drvs/virtualbox" diff --git a/pkg/minikube/machine/cluster_test.go b/pkg/minikube/machine/cluster_test.go index a4cb3347c906..baa63660178b 100644 --- a/pkg/minikube/machine/cluster_test.go +++ b/pkg/minikube/machine/cluster_test.go @@ -27,10 +27,10 @@ import ( "k8s.io/minikube/pkg/minikube/download" _ "k8s.io/minikube/pkg/minikube/registry/drvs/virtualbox" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/provision" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/provision" + "k8s.io/minikube/pkg/libmachine/state" "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/driver" diff --git a/pkg/minikube/machine/delete.go b/pkg/minikube/machine/delete.go index a883f1028a6c..1f89ee6085cc 100644 --- a/pkg/minikube/machine/delete.go +++ b/pkg/minikube/machine/delete.go @@ -21,10 +21,10 @@ import ( "os/exec" "time" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/mcnerror" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/mcnerror" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic/oci" diff --git a/pkg/minikube/machine/fix.go b/pkg/minikube/machine/fix.go index 082e624d422c..e4cbdb005fea 100644 --- a/pkg/minikube/machine/fix.go +++ b/pkg/minikube/machine/fix.go @@ -24,9 +24,9 @@ import ( "strings" "time" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/config" @@ -280,12 +280,8 @@ func machineExists(d string, s state.State, err error) (bool, error) { return machineExistsState(s, err) case driver.None: return machineExistsState(s, err) - case driver.Parallels: - return machineExistsMessage(s, err, "connection is shut down") case driver.VirtualBox: return machineExistsMessage(s, err, "machine does not exist") - case driver.VMware: - return machineExistsState(s, err) case driver.Docker: return machineExistsDocker(s, err) case driver.Mock: diff --git a/pkg/minikube/machine/host.go b/pkg/minikube/machine/host.go index bf9c33251010..6c7e452dbdfe 100644 --- a/pkg/minikube/machine/host.go +++ b/pkg/minikube/machine/host.go @@ -17,9 +17,9 @@ limitations under the License. package machine import ( - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" diff --git a/pkg/minikube/machine/info.go b/pkg/minikube/machine/info.go index 968a1e1440a2..d54ad980d336 100644 --- a/pkg/minikube/machine/info.go +++ b/pkg/minikube/machine/info.go @@ -23,7 +23,7 @@ import ( "strconv" "strings" - "github.com/docker/machine/libmachine/provision" + "k8s.io/minikube/pkg/libmachine/provision" "github.com/shirou/gopsutil/v3/cpu" "github.com/shirou/gopsutil/v3/disk" "github.com/shirou/gopsutil/v3/mem" diff --git a/pkg/minikube/machine/machine.go b/pkg/minikube/machine/machine.go index 62a24d4bae65..ade5d163ace6 100644 --- a/pkg/minikube/machine/machine.go +++ b/pkg/minikube/machine/machine.go @@ -22,9 +22,9 @@ import ( "strings" "time" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" - libprovision "github.com/docker/machine/libmachine/provision" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" + libprovision "k8s.io/minikube/pkg/libmachine/provision" "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/config" diff --git a/pkg/minikube/machine/ssh.go b/pkg/minikube/machine/ssh.go index a0206a567870..a192a64ffdc6 100644 --- a/pkg/minikube/machine/ssh.go +++ b/pkg/minikube/machine/ssh.go @@ -20,10 +20,10 @@ import ( "fmt" "os/exec" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/ssh" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/config" ) diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 12ca77f21c9c..81f974e6a58e 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -29,10 +29,10 @@ import ( "time" "github.com/blang/semver/v4" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/engine" - "github.com/docker/machine/libmachine/host" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/host" "github.com/juju/mutex/v2" "github.com/pkg/errors" "github.com/spf13/viper" diff --git a/pkg/minikube/machine/stop.go b/pkg/minikube/machine/stop.go index e6a83029b299..51bfee988783 100644 --- a/pkg/minikube/machine/stop.go +++ b/pkg/minikube/machine/stop.go @@ -19,10 +19,10 @@ package machine import ( "time" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/mcnerror" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/mcnerror" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic/oci" diff --git a/pkg/minikube/mustload/mustload.go b/pkg/minikube/mustload/mustload.go index 7d61b81e6da2..6dbc342786ad 100644 --- a/pkg/minikube/mustload/mustload.go +++ b/pkg/minikube/mustload/mustload.go @@ -21,9 +21,9 @@ import ( "fmt" "net" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/state" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/command" diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 3325049811ec..9c52d3e31417 100755 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -30,8 +30,8 @@ import ( "time" "github.com/blang/semver/v4" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" "github.com/pkg/errors" "github.com/spf13/viper" "golang.org/x/sync/errgroup" diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index 89f5da5617fa..d62d853212d6 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -26,7 +26,7 @@ import ( "time" "github.com/blang/semver/v4" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "github.com/pkg/errors" "github.com/spf13/viper" "k8s.io/klog/v2" diff --git a/pkg/minikube/registry/drvs/hyperkit/hyperkit.go b/pkg/minikube/registry/drvs/hyperkit/hyperkit.go index 1461bea7034a..69f66a1674e7 100644 --- a/pkg/minikube/registry/drvs/hyperkit/hyperkit.go +++ b/pkg/minikube/registry/drvs/hyperkit/hyperkit.go @@ -27,7 +27,7 @@ import ( "github.com/pkg/errors" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "github.com/google/uuid" "k8s.io/minikube/pkg/drivers/hyperkit" diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go index 7a3836c0f865..166f39df32b3 100644 --- a/pkg/minikube/registry/drvs/hyperv/hyperv.go +++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go @@ -25,9 +25,9 @@ import ( "strings" "time" - "github.com/docker/machine/drivers/hyperv" - "github.com/docker/machine/libmachine/drivers" "github.com/pkg/errors" + "k8s.io/minikube/pkg/drivers/hyperv" + "k8s.io/minikube/pkg/libmachine/drivers" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/download" diff --git a/pkg/minikube/registry/drvs/init.go b/pkg/minikube/registry/drvs/init.go index c38ed7d1714c..2f193f86c4df 100644 --- a/pkg/minikube/registry/drvs/init.go +++ b/pkg/minikube/registry/drvs/init.go @@ -24,11 +24,9 @@ import ( _ "k8s.io/minikube/pkg/minikube/registry/drvs/krunkit" _ "k8s.io/minikube/pkg/minikube/registry/drvs/kvm2" _ "k8s.io/minikube/pkg/minikube/registry/drvs/none" - _ "k8s.io/minikube/pkg/minikube/registry/drvs/parallels" _ "k8s.io/minikube/pkg/minikube/registry/drvs/podman" _ "k8s.io/minikube/pkg/minikube/registry/drvs/qemu2" _ "k8s.io/minikube/pkg/minikube/registry/drvs/ssh" _ "k8s.io/minikube/pkg/minikube/registry/drvs/vfkit" _ "k8s.io/minikube/pkg/minikube/registry/drvs/virtualbox" - _ "k8s.io/minikube/pkg/minikube/registry/drvs/vmware" ) diff --git a/pkg/minikube/registry/drvs/krunkit/krunkit.go b/pkg/minikube/registry/drvs/krunkit/krunkit.go index 2faf138dba5f..0b382157e939 100644 --- a/pkg/minikube/registry/drvs/krunkit/krunkit.go +++ b/pkg/minikube/registry/drvs/krunkit/krunkit.go @@ -25,7 +25,7 @@ import ( "path/filepath" "runtime" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "github.com/google/uuid" "k8s.io/minikube/pkg/drivers/common/virtiofs" diff --git a/pkg/minikube/registry/drvs/kvm2/kvm2.go b/pkg/minikube/registry/drvs/kvm2/kvm2.go index 492d5319c602..48f2d3a29a8b 100644 --- a/pkg/minikube/registry/drvs/kvm2/kvm2.go +++ b/pkg/minikube/registry/drvs/kvm2/kvm2.go @@ -29,7 +29,7 @@ import ( "strings" "time" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/download" diff --git a/pkg/minikube/registry/drvs/none/none.go b/pkg/minikube/registry/drvs/none/none.go index 22a183281fb8..219b7fdcd861 100644 --- a/pkg/minikube/registry/drvs/none/none.go +++ b/pkg/minikube/registry/drvs/none/none.go @@ -23,7 +23,7 @@ import ( "os/exec" "os/user" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "k8s.io/minikube/pkg/drivers/none" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/driver" diff --git a/pkg/minikube/registry/drvs/parallels/doc.go b/pkg/minikube/registry/drvs/parallels/doc.go deleted file mode 100644 index e216ec23bf61..000000000000 --- a/pkg/minikube/registry/drvs/parallels/doc.go +++ /dev/null @@ -1,17 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parallels diff --git a/pkg/minikube/registry/drvs/parallels/parallels.go b/pkg/minikube/registry/drvs/parallels/parallels.go deleted file mode 100644 index d03143d7c00f..000000000000 --- a/pkg/minikube/registry/drvs/parallels/parallels.go +++ /dev/null @@ -1,64 +0,0 @@ -//go:build darwin - -/* -Copyright 2018 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parallels - -import ( - "fmt" - "os/exec" - - parallels "github.com/Parallels/docker-machine-parallels/v2" - "github.com/docker/machine/libmachine/drivers" - "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/download" - "k8s.io/minikube/pkg/minikube/driver" - "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/minikube/registry" -) - -func init() { - err := registry.Register(registry.DriverDef{ - Name: driver.Parallels, - Config: configure, - Status: status, - Default: true, - Priority: registry.Default, - Init: func() drivers.Driver { return parallels.NewDriver("", "") }, - }) - if err != nil { - panic(fmt.Sprintf("unable to register: %v", err)) - } - -} - -func configure(cfg config.ClusterConfig, n config.Node) (interface{}, error) { - d := parallels.NewDriver(config.MachineName(cfg, n), localpath.MiniPath()).(*parallels.Driver) - d.Boot2DockerURL = download.LocalISOResource(cfg.MinikubeISO) - d.Memory = cfg.Memory - d.CPU = cfg.CPUs - d.DiskSize = cfg.DiskSize - return d, nil -} - -func status() registry.State { - _, err := exec.LookPath("prlctl") - if err != nil { - return registry.State{Error: err, Fix: "Install Parallels Desktop for Mac", Doc: "https://minikube.sigs.k8s.io/docs/drivers/parallels/"} - } - return registry.State{Installed: true, Healthy: true} -} diff --git a/pkg/minikube/registry/drvs/podman/podman.go b/pkg/minikube/registry/drvs/podman/podman.go index 17fd054bbadd..e90255dcb667 100644 --- a/pkg/minikube/registry/drvs/podman/podman.go +++ b/pkg/minikube/registry/drvs/podman/podman.go @@ -27,7 +27,7 @@ import ( "time" "github.com/blang/semver/v4" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/drivers/kic/oci" diff --git a/pkg/minikube/registry/drvs/qemu2/qemu2.go b/pkg/minikube/registry/drvs/qemu2/qemu2.go index 4b848694abcc..975cc2bc8aa5 100644 --- a/pkg/minikube/registry/drvs/qemu2/qemu2.go +++ b/pkg/minikube/registry/drvs/qemu2/qemu2.go @@ -26,7 +26,7 @@ import ( "strings" "github.com/blang/semver/v4" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "github.com/spf13/viper" "k8s.io/minikube/pkg/drivers/qemu" diff --git a/pkg/minikube/registry/drvs/ssh/ssh.go b/pkg/minikube/registry/drvs/ssh/ssh.go index 8308cf90053a..0d9e457846d2 100644 --- a/pkg/minikube/registry/drvs/ssh/ssh.go +++ b/pkg/minikube/registry/drvs/ssh/ssh.go @@ -22,7 +22,7 @@ import ( "path/filepath" "strings" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "github.com/pkg/errors" "k8s.io/minikube/pkg/drivers/ssh" diff --git a/pkg/minikube/registry/drvs/vfkit/vfkit.go b/pkg/minikube/registry/drvs/vfkit/vfkit.go index 970a47246e86..4963a29de62c 100644 --- a/pkg/minikube/registry/drvs/vfkit/vfkit.go +++ b/pkg/minikube/registry/drvs/vfkit/vfkit.go @@ -25,7 +25,7 @@ import ( "path/filepath" "runtime" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "github.com/google/uuid" "k8s.io/minikube/pkg/drivers/common/virtiofs" diff --git a/pkg/minikube/registry/drvs/virtualbox/virtualbox.go b/pkg/minikube/registry/drvs/virtualbox/virtualbox.go index edff7849d71d..b34ed4451183 100644 --- a/pkg/minikube/registry/drvs/virtualbox/virtualbox.go +++ b/pkg/minikube/registry/drvs/virtualbox/virtualbox.go @@ -24,8 +24,8 @@ import ( "strings" "time" - "github.com/docker/machine/drivers/virtualbox" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/drivers/virtualbox" + "k8s.io/minikube/pkg/libmachine/drivers" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/config" diff --git a/pkg/minikube/registry/drvs/vmware/doc.go b/pkg/minikube/registry/drvs/vmware/doc.go deleted file mode 100644 index 9661626293a6..000000000000 --- a/pkg/minikube/registry/drvs/vmware/doc.go +++ /dev/null @@ -1,17 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vmware diff --git a/pkg/minikube/registry/drvs/vmware/vmware.go b/pkg/minikube/registry/drvs/vmware/vmware.go deleted file mode 100644 index efc17a78733b..000000000000 --- a/pkg/minikube/registry/drvs/vmware/vmware.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vmware - -import ( - "fmt" - "os/exec" - - "github.com/docker/machine/libmachine/drivers" - vmware "github.com/machine-drivers/docker-machine-driver-vmware/pkg/drivers/vmware" - "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/download" - "k8s.io/minikube/pkg/minikube/driver" - "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/minikube/registry" -) - -func init() { - err := registry.Register(registry.DriverDef{ - Name: driver.VMware, - Config: configure, - Default: false, - Priority: registry.Deprecated, - Init: func() drivers.Driver { return vmware.NewDriver("", "") }, - Status: status, - }) - if err != nil { - panic(fmt.Sprintf("unable to register: %v", err)) - } -} - -func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { - d := vmware.NewDriver(config.MachineName(cc, n), localpath.MiniPath()).(*vmware.Driver) - d.Boot2DockerURL = download.LocalISOResource(cc.MinikubeISO) - d.Memory = cc.Memory - d.CPU = cc.CPUs - d.DiskSize = cc.DiskSize - - // TODO(frapposelli): push these defaults upstream to fixup this driver - d.SSHPort = 22 - d.ISO = d.ResolveStorePath("boot2docker.iso") - return d, nil -} - -func status() registry.State { - _, err := exec.LookPath("vmrun") - if err != nil { - return registry.State{Error: err, Fix: "Install vmrun", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/"} - } - return registry.State{Installed: true, Healthy: true} -} diff --git a/pkg/minikube/registry/registry.go b/pkg/minikube/registry/registry.go index ef1d961315f4..720c9bf1ba43 100644 --- a/pkg/minikube/registry/registry.go +++ b/pkg/minikube/registry/registry.go @@ -22,7 +22,7 @@ import ( "slices" "sync" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" "k8s.io/minikube/pkg/minikube/config" ) diff --git a/pkg/minikube/service/service.go b/pkg/minikube/service/service.go index 0172267b0be5..3531c6cd7367 100644 --- a/pkg/minikube/service/service.go +++ b/pkg/minikube/service/service.go @@ -28,7 +28,7 @@ import ( "text/template" "time" - "github.com/docker/machine/libmachine" + "k8s.io/minikube/pkg/libmachine" "github.com/olekukonko/tablewriter" "github.com/olekukonko/tablewriter/tw" "github.com/pkg/errors" diff --git a/pkg/minikube/service/service_test.go b/pkg/minikube/service/service_test.go index 5d589939cab1..eb36c652ebae 100644 --- a/pkg/minikube/service/service_test.go +++ b/pkg/minikube/service/service_test.go @@ -27,8 +27,8 @@ import ( "text/template" "time" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" "github.com/pkg/errors" "github.com/spf13/viper" core "k8s.io/api/core/v1" diff --git a/pkg/minikube/shell/shell.go b/pkg/minikube/shell/shell.go index d3ff9957d779..4a4e275f37c0 100644 --- a/pkg/minikube/shell/shell.go +++ b/pkg/minikube/shell/shell.go @@ -23,7 +23,7 @@ import ( "runtime" "text/template" - "github.com/docker/machine/libmachine/shell" + "k8s.io/minikube/pkg/libmachine/shell" "k8s.io/minikube/pkg/minikube/constants" ) diff --git a/pkg/minikube/sshutil/sshutil.go b/pkg/minikube/sshutil/sshutil.go index 2ef0454a101f..ee6f272ab901 100644 --- a/pkg/minikube/sshutil/sshutil.go +++ b/pkg/minikube/sshutil/sshutil.go @@ -24,8 +24,8 @@ import ( "strconv" "time" - "github.com/docker/machine/libmachine/drivers" - machinessh "github.com/docker/machine/libmachine/ssh" + "k8s.io/minikube/pkg/libmachine/drivers" + machinessh "k8s.io/minikube/pkg/libmachine/ssh" "github.com/pkg/errors" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" diff --git a/pkg/minikube/tests/api_mock.go b/pkg/minikube/tests/api_mock.go index 4f553b027f6d..66fc844fd5df 100644 --- a/pkg/minikube/tests/api_mock.go +++ b/pkg/minikube/tests/api_mock.go @@ -22,9 +22,9 @@ import ( "math/rand" "testing" - "github.com/docker/machine/libmachine/auth" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/swarm" + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/swarm" "github.com/pkg/errors" "github.com/spf13/viper" diff --git a/pkg/minikube/tests/driver_mock.go b/pkg/minikube/tests/driver_mock.go index 14b54b90c94a..c1fedcf47f0c 100644 --- a/pkg/minikube/tests/driver_mock.go +++ b/pkg/minikube/tests/driver_mock.go @@ -20,9 +20,9 @@ import ( "runtime" "testing" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/mcnflag" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/mcnflag" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/constants" diff --git a/pkg/minikube/tests/fake_store.go b/pkg/minikube/tests/fake_store.go index d5881f67f4cd..e13690fdf154 100644 --- a/pkg/minikube/tests/fake_store.go +++ b/pkg/minikube/tests/fake_store.go @@ -19,8 +19,8 @@ package tests import ( "testing" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/mcnerror" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/mcnerror" ) // FakeStore implements persist.Store from libmachine diff --git a/pkg/minikube/tests/host_mock.go b/pkg/minikube/tests/host_mock.go index 377af318114b..a099f18fb9a1 100644 --- a/pkg/minikube/tests/host_mock.go +++ b/pkg/minikube/tests/host_mock.go @@ -19,7 +19,7 @@ package tests import ( "errors" - "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/drivers" ) // MockHost used for testing. When commands are run, the output from CommandOutput diff --git a/pkg/minikube/tests/provision_mock.go b/pkg/minikube/tests/provision_mock.go index 1810b5d13f86..b70e411897fc 100644 --- a/pkg/minikube/tests/provision_mock.go +++ b/pkg/minikube/tests/provision_mock.go @@ -17,13 +17,13 @@ limitations under the License. package tests import ( - "github.com/docker/machine/libmachine/auth" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/engine" - "github.com/docker/machine/libmachine/provision" - "github.com/docker/machine/libmachine/provision/pkgaction" - "github.com/docker/machine/libmachine/provision/serviceaction" - "github.com/docker/machine/libmachine/swarm" + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/provision" + "k8s.io/minikube/pkg/libmachine/provision/pkgaction" + "k8s.io/minikube/pkg/libmachine/provision/serviceaction" + "k8s.io/minikube/pkg/libmachine/swarm" ) // MockProvisioner defines distribution specific actions diff --git a/pkg/minikube/tunnel/cluster_inspector.go b/pkg/minikube/tunnel/cluster_inspector.go index 658c60a92ad3..143c40953753 100644 --- a/pkg/minikube/tunnel/cluster_inspector.go +++ b/pkg/minikube/tunnel/cluster_inspector.go @@ -20,9 +20,9 @@ import ( "fmt" "net" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/state" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/machine" diff --git a/pkg/minikube/tunnel/cluster_inspector_test.go b/pkg/minikube/tunnel/cluster_inspector_test.go index 813a743305f9..52c5e6f49f5a 100644 --- a/pkg/minikube/tunnel/cluster_inspector_test.go +++ b/pkg/minikube/tunnel/cluster_inspector_test.go @@ -25,8 +25,8 @@ import ( "reflect" "strings" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/state" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/tests" ) diff --git a/pkg/minikube/tunnel/tunnel.go b/pkg/minikube/tunnel/tunnel.go index 9ac3747b8865..e0160ff754a1 100644 --- a/pkg/minikube/tunnel/tunnel.go +++ b/pkg/minikube/tunnel/tunnel.go @@ -23,8 +23,8 @@ import ( "os/exec" "regexp" - "github.com/docker/machine/libmachine" - "github.com/docker/machine/libmachine/host" + "k8s.io/minikube/pkg/libmachine" + "k8s.io/minikube/pkg/libmachine/host" "github.com/pkg/errors" typed_core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/klog/v2" diff --git a/pkg/minikube/tunnel/tunnel_manager.go b/pkg/minikube/tunnel/tunnel_manager.go index 3a6cf8eb2f02..018e47bc076b 100644 --- a/pkg/minikube/tunnel/tunnel_manager.go +++ b/pkg/minikube/tunnel/tunnel_manager.go @@ -23,7 +23,7 @@ import ( "context" "fmt" - "github.com/docker/machine/libmachine" + "k8s.io/minikube/pkg/libmachine" typed_core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/config" diff --git a/pkg/minikube/tunnel/tunnel_test.go b/pkg/minikube/tunnel/tunnel_test.go index b3f72a2bc335..c88346c53d62 100644 --- a/pkg/minikube/tunnel/tunnel_test.go +++ b/pkg/minikube/tunnel/tunnel_test.go @@ -19,8 +19,8 @@ package tunnel import ( "errors" - "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/host" + "k8s.io/minikube/pkg/libmachine/state" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/tests" diff --git a/pkg/provision/buildroot.go b/pkg/provision/buildroot.go index d063e30d0eaf..44ad9540a27e 100644 --- a/pkg/provision/buildroot.go +++ b/pkg/provision/buildroot.go @@ -22,12 +22,12 @@ import ( "text/template" "time" - "github.com/docker/machine/libmachine/auth" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/engine" - "github.com/docker/machine/libmachine/provision" - "github.com/docker/machine/libmachine/provision/pkgaction" - "github.com/docker/machine/libmachine/swarm" + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/provision" + "k8s.io/minikube/pkg/libmachine/provision/pkgaction" + "k8s.io/minikube/pkg/libmachine/swarm" "github.com/spf13/viper" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/config" diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go index 661988c47db5..68a76ed4c9d1 100644 --- a/pkg/provision/provision.go +++ b/pkg/provision/provision.go @@ -28,13 +28,13 @@ import ( "text/template" "time" - "github.com/docker/machine/libmachine/auth" - "github.com/docker/machine/libmachine/cert" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/engine" - "github.com/docker/machine/libmachine/mcnutils" - "github.com/docker/machine/libmachine/provision" - "github.com/docker/machine/libmachine/swarm" + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/cert" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/mcnutils" + "k8s.io/minikube/pkg/libmachine/provision" + "k8s.io/minikube/pkg/libmachine/swarm" "github.com/pkg/errors" "k8s.io/klog/v2" diff --git a/pkg/provision/ubuntu.go b/pkg/provision/ubuntu.go index ea18c7bd7f6e..3c1f9a055c26 100644 --- a/pkg/provision/ubuntu.go +++ b/pkg/provision/ubuntu.go @@ -22,12 +22,12 @@ import ( "text/template" "time" - "github.com/docker/machine/libmachine/auth" - "github.com/docker/machine/libmachine/drivers" - "github.com/docker/machine/libmachine/engine" - "github.com/docker/machine/libmachine/provision" - "github.com/docker/machine/libmachine/provision/pkgaction" - "github.com/docker/machine/libmachine/swarm" + "k8s.io/minikube/pkg/libmachine/auth" + "k8s.io/minikube/pkg/libmachine/drivers" + "k8s.io/minikube/pkg/libmachine/engine" + "k8s.io/minikube/pkg/libmachine/provision" + "k8s.io/minikube/pkg/libmachine/provision/pkgaction" + "k8s.io/minikube/pkg/libmachine/swarm" "github.com/spf13/viper" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/config" diff --git a/site/content/en/docs/commands/ssh.md b/site/content/en/docs/commands/ssh.md index 0b304cca8000..da621c4df495 100644 --- a/site/content/en/docs/commands/ssh.md +++ b/site/content/en/docs/commands/ssh.md @@ -11,7 +11,7 @@ Log into the minikube environment (for debugging) ### Synopsis -Log into or run a command on a machine with SSH; similar to 'docker-machine ssh'. +Log into or run a command on a machine with SSH. ```shell minikube ssh [flags] diff --git a/site/content/en/docs/contrib/drivers.en.md b/site/content/en/docs/contrib/drivers.en.md index 0f472ce0f083..d656c75fa790 100644 --- a/site/content/en/docs/contrib/drivers.en.md +++ b/site/content/en/docs/contrib/drivers.en.md @@ -8,11 +8,7 @@ description: > This document is written for contributors who are familiar with minikube, who would like to add support for a new VM driver. -minikube relies on docker-machine drivers to manage machines. This document discusses how to modify minikube, so that this driver may be used by `minikube start --driver=`. - -## Creating a new driver - -See [machine-drivers](https://github.com/machine-drivers) , the fork where all new docker-machine drivers are located. +This document discusses how to modify minikube, so that this driver may be used by `minikube start --driver=`. ## Builtin vs External Drivers diff --git a/site/content/en/docs/drivers/_index.md b/site/content/en/docs/drivers/_index.md index 65460b5219b3..3d9e7c943d3a 100644 --- a/site/content/en/docs/drivers/_index.md +++ b/site/content/en/docs/drivers/_index.md @@ -10,7 +10,7 @@ aliases: --- minikube can be deployed as a VM, a container, or bare-metal. -To do so, we use the [Docker Machine](https://github.com/docker/machine) library to provide a consistent way to interact with different environments. Here is what's supported: +Here is what's supported: ## Linux @@ -28,8 +28,6 @@ To do so, we use the [Docker Machine](https://github.com/docker/machine) library * [Docker]({{}}) - VM + Container (preferred) * [Hyperkit]({{}}) - VM * [VirtualBox]({{}}) - VM -* [Parallels]({{}}) - VM -* [VMware Fusion]({{}}) - VM * [QEMU]({{}}) - VM * [Podman]({{}}) - VM + Container (experimental) * [VFkit]({{}}) - VM (preferred) @@ -41,7 +39,6 @@ To do so, we use the [Docker Machine](https://github.com/docker/machine) library * [Hyper-V]({{}}) - VM (preferred) * [Docker]({{}}) - VM + Container (preferred) * [VirtualBox]({{}}) - VM -* [VMware Workstation]({{}}) - VM * [QEMU]({{}}) - VM (experimental) * [Podman]({{}}) - VM + Container (experimental) * [SSH]({{}}) - remote ssh diff --git a/site/content/en/docs/drivers/includes/parallels_usage.inc b/site/content/en/docs/drivers/includes/parallels_usage.inc deleted file mode 100644 index 40671413b0c8..000000000000 --- a/site/content/en/docs/drivers/includes/parallels_usage.inc +++ /dev/null @@ -1,17 +0,0 @@ -## Requirements - -* Parallels Desktop 11.0.0+ Pro or Business edition - -## Usage - -Start a cluster using the parallels driver: - -```shell -minikube start --driver=parallels -``` - -To make parallels the default driver: - -```shell -minikube config set driver parallels -``` diff --git a/site/content/en/docs/drivers/includes/vmware_macos_usage.inc b/site/content/en/docs/drivers/includes/vmware_macos_usage.inc deleted file mode 100644 index 8d13cb2dcedc..000000000000 --- a/site/content/en/docs/drivers/includes/vmware_macos_usage.inc +++ /dev/null @@ -1,36 +0,0 @@ -## Requirements - -* VMware Fusion - -## Driver Installation - -If the [Brew Package Manager](https://brew.sh/) is installed, run: - -```shell -brew install docker-machine-driver-vmware -``` - -Otherwise: - -```shell -r=https://api.github.com/repos/machine-drivers/docker-machine-driver-vmware -d=docker-machine-driver-vmware_darwin_amd64 -u=$(curl -s $r/releases/latest | grep -o 'http.*Darwin_amd64.tar.gz' | head -n1) -mkdir $d \ - && (cd $d && curl -L $u > $d.tar.gz && tar -xf $d.tar.gz) \ - && install $d/docker-machine-driver-vmware /usr/local/bin/docker-machine-driver-vmware \ - && rm -rf $d -``` - -## Usage - -Start a cluster using the vmware driver: - -```shell -minikube start --driver=vmware -``` -To make vmware the default driver: - -```shell -minikube config set driver vmware -``` diff --git a/site/content/en/docs/drivers/includes/vmware_windows_usage.inc b/site/content/en/docs/drivers/includes/vmware_windows_usage.inc deleted file mode 100644 index 46245257a1d2..000000000000 --- a/site/content/en/docs/drivers/includes/vmware_windows_usage.inc +++ /dev/null @@ -1,13 +0,0 @@ -## Requirements - -* VMware Workstation - -## Driver Installation - -First install the [VMware Workstation](https://www.vmware.com/in/products/workstation-pro/workstation-pro-evaluation.html) - -After installation, start a cluster using the vmware driver: - -```shell -minikube start --driver vmware -``` diff --git a/site/content/en/docs/drivers/parallels.md b/site/content/en/docs/drivers/parallels.md deleted file mode 100644 index 7e0973ca05e5..000000000000 --- a/site/content/en/docs/drivers/parallels.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: "parallels" -weight: 4 -aliases: - - /docs/reference/drivers/parallels ---- - -## Overview - -The Parallels driver is particularly useful for users who own Parallels Desktop for Mac, as it does not require VT-x hardware support. - -{{% readfile file="/docs/drivers/includes/parallels_usage.inc" %}} - -## Issues - -* [Full list of open 'parallels' driver issues](https://github.com/kubernetes/minikube/labels/co%2Fparallels-driver) - -## Troubleshooting - -* Run `minikube start --alsologtostderr -v=7` to debug crashes diff --git a/site/content/en/docs/drivers/vmware.md b/site/content/en/docs/drivers/vmware.md deleted file mode 100644 index 12d7a60d9705..000000000000 --- a/site/content/en/docs/drivers/vmware.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: "vmware" -weight: 6 -aliases: - - /docs/reference/drivers/vmware ---- - -## Overview - -The vmware driver supports virtualization across all VMware based hypervisors. - -{{% tabs %}} -{{% mactab %}} -{{% readfile file="/docs/drivers/includes/vmware_macos_usage.inc" %}} -{{% /mactab %}} -{{% linuxtab %}} -No documentation is available yet. -{{% /linuxtab %}} -{{% windowstab %}} -{{% readfile file="/docs/drivers/includes/vmware_windows_usage.inc" %}} -{{% /windowstab %}} -{{% /tabs %}} - -## Issues - -* [Full list of open 'vmware-driver' issues](https://github.com/kubernetes/minikube/labels/co%2Fvmware-driver) - -## Troubleshooting - -* Run `minikube start --alsologtostderr -v=7` to debug crashes diff --git a/site/content/en/docs/handbook/mount.md b/site/content/en/docs/handbook/mount.md index a88f5e59fc0d..cf3c94fd1386 100644 --- a/site/content/en/docs/handbook/mount.md +++ b/site/content/en/docs/handbook/mount.md @@ -72,7 +72,6 @@ Some hypervisors, have built-in host folder sharing. Driver mounts are reliable | VirtualBox | Linux | /home | /hosthome | | VirtualBox | macOS | /Users | /Users | | VirtualBox | Windows | C://Users | /c/Users | -| VMware Fusion | macOS | /Users | /mnt/hgfs/Users | | KVM | Linux | Unsupported | | | HyperKit | macOS | Supported | | diff --git a/site/content/en/docs/start/_index.md b/site/content/en/docs/start/_index.md index b23ef1952ca3..8e391e7d0904 100644 --- a/site/content/en/docs/start/_index.md +++ b/site/content/en/docs/start/_index.md @@ -16,7 +16,7 @@ All you need is Docker (or similarly compatible) container or a Virtual Machine * 2GB of free memory * 20GB of free disk space * Internet connection -* Container or virtual machine manager, such as: [Docker]({{}}), [QEMU]({{}}), [Hyperkit]({{}}), [Hyper-V]({{}}), [KVM]({{}}), [Parallels]({{}}), [Podman]({{}}), [VirtualBox]({{}}), or [VMware Fusion/Workstation]({{}}) +* Container or virtual machine manager, such as: [Docker]({{}}), [QEMU]({{}}), [Hyperkit]({{}}), [Hyper-V]({{}}), [KVM]({{}}), [Podman]({{}}), or [VirtualBox]({{}})

1Installation

diff --git a/test/integration/helpers_test.go b/test/integration/helpers_test.go index 5c14aa9d07ff..0f5ab8631e94 100644 --- a/test/integration/helpers_test.go +++ b/test/integration/helpers_test.go @@ -37,7 +37,7 @@ import ( "testing" "time" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/state" "github.com/google/go-cmp/cmp" "github.com/shirou/gopsutil/v3/process" core "k8s.io/api/core/v1" diff --git a/test/integration/scheduled_stop_test.go b/test/integration/scheduled_stop_test.go index 2b0181be2ea4..f39dd9c6f202 100644 --- a/test/integration/scheduled_stop_test.go +++ b/test/integration/scheduled_stop_test.go @@ -30,7 +30,7 @@ import ( "testing" "time" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/state" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/util/retry" diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 716bb1e42b0a..bcbcfd5310c5 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -30,7 +30,7 @@ import ( "testing" "github.com/blang/semver/v4" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/state" "github.com/google/go-cmp/cmp" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/constants" diff --git a/test/integration/version_upgrade_test.go b/test/integration/version_upgrade_test.go index 7cd231ac595e..8953771965ae 100644 --- a/test/integration/version_upgrade_test.go +++ b/test/integration/version_upgrade_test.go @@ -30,7 +30,7 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/util/retry" - "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/libmachine/state" "github.com/hashicorp/go-getter" pkgutil "k8s.io/minikube/pkg/util" )