From 6319af242bddb5da933b95bcfb583270a6c09769 Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Fri, 30 Aug 2024 06:46:09 +0530 Subject: [PATCH 01/28] Initial Commit. Initial Commit. --- pkg/addons/validations.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/addons/validations.go b/pkg/addons/validations.go index 593e27991756..18639607aeec 100644 --- a/pkg/addons/validations.go +++ b/pkg/addons/validations.go @@ -12,6 +12,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +TODO: Remove this line before PR is merged. */ package addons From cd8327c11651bb36b8e4bbbca25c0079ad9623af Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Fri, 30 Aug 2024 06:50:55 +0530 Subject: [PATCH 02/28] Update validations.go --- pkg/addons/validations.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/addons/validations.go b/pkg/addons/validations.go index 18639607aeec..593e27991756 100644 --- a/pkg/addons/validations.go +++ b/pkg/addons/validations.go @@ -12,7 +12,6 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -TODO: Remove this line before PR is merged. */ package addons From 938f6f0eecd4a012fe5b75ba1745172301f5db61 Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Sat, 31 Aug 2024 16:50:10 +0530 Subject: [PATCH 03/28] Removing isKVMDriverForNVIDIA from validations for nvidia-gpu-device-plugin. Removing isKVMDriverForNVIDIA from validations for nvidia-gpu-device-plugin. This is being done to allow merger of functionalities of nvidia-gpu-device-plugin and nvidia-device-plugin into nvidia-gpu-device-plugin. This basically means nvidia-gpu-device-plugin will work for both KVM & non-KVM (including docker) drivers. --- pkg/addons/config.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/addons/config.go b/pkg/addons/config.go index a849538eb815..dd30e09ecc74 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -131,10 +131,9 @@ var Addons = []*Addon{ callbacks: []setFn{EnableOrDisableAddon}, }, { - name: "nvidia-gpu-device-plugin", - set: SetBool, - validations: []setFn{isKVMDriverForNVIDIA}, - callbacks: []setFn{EnableOrDisableAddon}, + name: "nvidia-gpu-device-plugin", + set: SetBool, + callbacks: []setFn{EnableOrDisableAddon}, }, { name: "olm", From b282913a9110fe82a9f91511801b78af93e33950 Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Sat, 31 Aug 2024 17:48:55 +0530 Subject: [PATCH 04/28] Messages, Warnings and In-line Code Comments w.r.t the merger. Messages, Warnings and In-line Code Comments w.r.t the merger. --- pkg/addons/addons.go | 4 ++++ pkg/addons/config.go | 1 + site/content/en/docs/tutorials/nvidia.md | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 4029555bd7ed..bb19a8b6d70b 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -104,6 +104,8 @@ func preStartMessages(name, value string) { out.Styled(style.Warning, "The ambassador addon has stopped working as of v1.23.0, for more details visit: https://github.com/datawire/ambassador-operator/issues/73") case "olm": out.Styled(style.Warning, "The OLM addon has stopped working, for more details visit: https://github.com/operator-framework/operator-lifecycle-manager/issues/2534") + case "nvidia-device-plugin": + out.Styled(style.Warning, "The nvidia-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-gpu-device-plugin addon. It will be removed in a future release. Please use the nvidia-gpu-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114") } } @@ -163,6 +165,8 @@ func Deprecations(name string) (bool, string, string) { return true, "metrics-server", "using metrics-server addon, heapster is deprecated" case "efk": return true, "", "The current images used in the efk addon contain Log4j vulnerabilities, the addon will be disabled until images are updated, see: https://github.com/kubernetes/minikube/issues/15280" + case "nvidia-device-plugin": + return true, "nvidia-gpu-device-plugin", "The nvidia-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-gpu-device-plugin addon. It will be removed in a future release. Please use the nvidia-gpu-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114" } return false, "", "" } diff --git a/pkg/addons/config.go b/pkg/addons/config.go index dd30e09ecc74..b2916a94f188 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -234,6 +234,7 @@ var Addons = []*Addon{ callbacks: []setFn{EnableOrDisableAddon}, }, { + // The nvidia-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-gpu-device-plugin addon. name: "nvidia-device-plugin", set: SetBool, callbacks: []setFn{EnableOrDisableAddon}, diff --git a/site/content/en/docs/tutorials/nvidia.md b/site/content/en/docs/tutorials/nvidia.md index 58f5d78442ae..b5a80dd44b62 100644 --- a/site/content/en/docs/tutorials/nvidia.md +++ b/site/content/en/docs/tutorials/nvidia.md @@ -59,8 +59,9 @@ to expose GPUs with `--driver=kvm`. Please don't mix these instructions. - Install NVIDIA's device plugin: ```shell - minikube addons enable nvidia-device-plugin + minikube addons enable nvidia-gpu-device-plugin ``` + NOTE: `nvidia-device-plugin` addon has been deprecated and it's functionality is merged inside of `nvidia-gpu-device-plugin` addon. {{% /tab %}} {{% tab kvm %}} ## Using the kvm driver From 9491f8db62944a2521fddd62349b95f304e2ab07 Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Sat, 31 Aug 2024 18:13:13 +0530 Subject: [PATCH 05/28] Update addons.go --- pkg/addons/addons.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index bb19a8b6d70b..b2717c8ea017 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -105,7 +105,7 @@ func preStartMessages(name, value string) { case "olm": out.Styled(style.Warning, "The OLM addon has stopped working, for more details visit: https://github.com/operator-framework/operator-lifecycle-manager/issues/2534") case "nvidia-device-plugin": - out.Styled(style.Warning, "The nvidia-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-gpu-device-plugin addon. It will be removed in a future release. Please use the nvidia-gpu-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114") + out.Styled(style.Warning, "The nvidia-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-gpu-device-plugin addon. It will be removed in a future release. Please use the nvidia-gpu-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114.") } } @@ -166,7 +166,7 @@ func Deprecations(name string) (bool, string, string) { case "efk": return true, "", "The current images used in the efk addon contain Log4j vulnerabilities, the addon will be disabled until images are updated, see: https://github.com/kubernetes/minikube/issues/15280" case "nvidia-device-plugin": - return true, "nvidia-gpu-device-plugin", "The nvidia-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-gpu-device-plugin addon. It will be removed in a future release. Please use the nvidia-gpu-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114" + return true, "nvidia-gpu-device-plugin", "The nvidia-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-gpu-device-plugin addon. It will be removed in a future release. Please use the nvidia-gpu-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114." } return false, "", "" } From 7ee5901bd36fd1bdf581e9104db79d21f6233f16 Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Fri, 6 Sep 2024 05:55:11 +0530 Subject: [PATCH 06/28] Merged nvidia-device-plugin addon into nvidia-gpu-device-plugin addon. Merged nvidia-device-plugin addon into nvidia-gpu-device-plugin addon. --- deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl | 3 +++ pkg/minikube/cruntime/docker.go | 2 +- test/integration/addons_test.go | 8 ++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl b/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl index 0ec4cd4fff99..6976fe77cc49 100644 --- a/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl +++ b/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl @@ -47,6 +47,9 @@ spec: - image: {{.CustomRegistries.NvidiaDevicePlugin | default .ImageRepository | default .Registries.NvidiaDevicePlugin }}{{.Images.NvidiaDevicePlugin}} command: ["/usr/bin/nvidia-gpu-device-plugin", "-logtostderr"] name: nvidia-gpu-device-plugin + env: + - name: FAIL_ON_INIT_ERROR + value: "false" resources: requests: cpu: 50m diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 4c27c11dcfb0..54b3df586f49 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -581,7 +581,7 @@ func (r *Docker) configureDocker(driver string) error { StorageDriver: "overlay2", } if r.GPUs { - assets.Addons["nvidia-device-plugin"].EnableByDefault() + assets.Addons["nvidia-gpu-device-plugin"].EnableByDefault() daemonConfig.DefaultRuntime = "nvidia" runtimes := &dockerDaemonRuntimes{} runtimes.Nvidia.Path = "/usr/bin/nvidia-container-runtime" diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 3b0888c78678..901aa0f04718 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -100,7 +100,7 @@ func TestAddons(t *testing.T) { // so we override that here to let minikube auto-detect appropriate cgroup driver os.Setenv(constants.MinikubeForceSystemdEnv, "") - args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner", "--addons=inspektor-gadget", "--addons=storage-provisioner-rancher", "--addons=nvidia-device-plugin", "--addons=yakd", "--addons=volcano"}, StartArgs()...) + args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner", "--addons=inspektor-gadget", "--addons=storage-provisioner-rancher", "--addons=nvidia-gpu-device-plugin", "--addons=yakd", "--addons=volcano"}, StartArgs()...) if !NoneDriver() { // none driver does not support ingress args = append(args, "--addons=ingress", "--addons=ingress-dns") } @@ -1054,15 +1054,15 @@ func validateDisablingAddonOnNonExistingCluster(ctx context.Context, t *testing. } } -// validateNvidiaDevicePlugin tests the nvidia-device-plugin addon by ensuring the pod comes up and the addon disables +// validateNvidiaDevicePlugin tests the nvidia-gpu-device-plugin addon by ensuring the pod comes up and the addon disables func validateNvidiaDevicePlugin(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) if _, err := PodWait(ctx, t, profile, "kube-system", "name=nvidia-device-plugin-ds", Minutes(6)); err != nil { t.Fatalf("failed waiting for nvidia-device-plugin-ds pod: %v", err) } - if rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "disable", "nvidia-device-plugin", "-p", profile)); err != nil { - t.Errorf("failed to disable nvidia-device-plugin: args %q : %v", rr.Command(), err) + if rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "disable", "nvidia-gpu-device-plugin", "-p", profile)); err != nil { + t.Errorf("failed to disable nvidia-gpu-device-plugin: args %q : %v", rr.Command(), err) } } From 1a0ef3c88fd1c73778fa2bf49e7f5849140bc7bd Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Fri, 4 Oct 2024 19:56:16 +0530 Subject: [PATCH 07/28] Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. --- test/integration/addons_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 7677f606d477..759a15d61be4 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -951,15 +951,15 @@ func validateDisablingAddonOnNonExistingCluster(ctx context.Context, t *testing. } } -// validateNvidiaDevicePlugin tests the nvidia-gpu-device-plugin addon by ensuring the pod comes up and the addon disables +// validateNvidiaDevicePlugin tests the nvidia-device-plugin addon by ensuring the pod comes up and the addon disables func validateNvidiaDevicePlugin(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) if _, err := PodWait(ctx, t, profile, "kube-system", "name=nvidia-device-plugin-ds", Minutes(6)); err != nil { t.Fatalf("failed waiting for nvidia-device-plugin-ds pod: %v", err) } - if rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "disable", "nvidia-gpu-device-plugin", "-p", profile)); err != nil { - t.Errorf("failed to disable nvidia-gpu-device-plugin: args %q : %v", rr.Command(), err) + if rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "disable", "nvidia-device-plugin", "-p", profile)); err != nil { + t.Errorf("failed to disable nvidia-device-plugin: args %q : %v", rr.Command(), err) } } From a98eefef98ff041f926ac413192efa8de1a9c036 Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Fri, 4 Oct 2024 19:59:41 +0530 Subject: [PATCH 08/28] Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. --- site/content/en/docs/tutorials/nvidia.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/content/en/docs/tutorials/nvidia.md b/site/content/en/docs/tutorials/nvidia.md index 5117abeb7bb3..968dff68d188 100644 --- a/site/content/en/docs/tutorials/nvidia.md +++ b/site/content/en/docs/tutorials/nvidia.md @@ -70,9 +70,8 @@ to expose GPUs with `--driver=kvm`. Please don't mix these instructions. - Install NVIDIA's device plugin: ```shell - minikube addons enable nvidia-gpu-device-plugin + minikube addons enable nvidia-device-plugin ``` - NOTE: `nvidia-device-plugin` addon has been deprecated and it's functionality is merged inside of `nvidia-gpu-device-plugin` addon. {{% /tab %}} {{% tab kvm %}} ## Using the kvm driver @@ -114,9 +113,10 @@ host to the minikube VM. Doing so has a few prerequisites: If this succeeded, run the following commands: ```shell - minikube addons enable nvidia-gpu-device-plugin + minikube addons enable nvidia-device-plugin minikube addons enable nvidia-driver-installer ``` + NOTE: `nvidia-gpu-device-plugin` addon has been deprecated and it's functionality is merged inside of `nvidia-device-plugin` addon. This will install the NVIDIA driver (that works for GeForce/Quadro cards) on the VM. From f5c192e05742ba1a0ebcdd6bae2eb414e993953c Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Fri, 4 Oct 2024 20:01:44 +0530 Subject: [PATCH 09/28] Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. --- pkg/minikube/cruntime/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 54b3df586f49..4c27c11dcfb0 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -581,7 +581,7 @@ func (r *Docker) configureDocker(driver string) error { StorageDriver: "overlay2", } if r.GPUs { - assets.Addons["nvidia-gpu-device-plugin"].EnableByDefault() + assets.Addons["nvidia-device-plugin"].EnableByDefault() daemonConfig.DefaultRuntime = "nvidia" runtimes := &dockerDaemonRuntimes{} runtimes.Nvidia.Path = "/usr/bin/nvidia-container-runtime" From d6e44196d727885d23108f3f6394d1311b1019ca Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Fri, 4 Oct 2024 20:04:34 +0530 Subject: [PATCH 10/28] Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. --- pkg/addons/config.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/addons/config.go b/pkg/addons/config.go index d677d640f8c6..c4385dfd018d 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -126,9 +126,11 @@ var Addons = []*Addon{ callbacks: []setFn{EnableOrDisableAddon}, }, { - name: "nvidia-gpu-device-plugin", - set: SetBool, - callbacks: []setFn{EnableOrDisableAddon}, + // The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. + name: "nvidia-gpu-device-plugin", + set: SetBool, + validations: []setFn{isKVMDriverForNVIDIA}, + callbacks: []setFn{EnableOrDisableAddon}, }, { name: "olm", @@ -229,7 +231,6 @@ var Addons = []*Addon{ callbacks: []setFn{EnableOrDisableAddon}, }, { - // The nvidia-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-gpu-device-plugin addon. name: "nvidia-device-plugin", set: SetBool, callbacks: []setFn{EnableOrDisableAddon}, From d4d2a4678271fab672d87e9dfdbc2b441c932534 Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Fri, 4 Oct 2024 20:09:36 +0530 Subject: [PATCH 11/28] Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. --- pkg/addons/addons.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index b2717c8ea017..8ab6bed63ee9 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -104,8 +104,8 @@ func preStartMessages(name, value string) { out.Styled(style.Warning, "The ambassador addon has stopped working as of v1.23.0, for more details visit: https://github.com/datawire/ambassador-operator/issues/73") case "olm": out.Styled(style.Warning, "The OLM addon has stopped working, for more details visit: https://github.com/operator-framework/operator-lifecycle-manager/issues/2534") - case "nvidia-device-plugin": - out.Styled(style.Warning, "The nvidia-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-gpu-device-plugin addon. It will be removed in a future release. Please use the nvidia-gpu-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114.") + case "nvidia-gpu-device-plugin": + out.Styled(style.Warning, "The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. It will be removed in a future release. Please use the nvidia-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114.") } } @@ -165,8 +165,8 @@ func Deprecations(name string) (bool, string, string) { return true, "metrics-server", "using metrics-server addon, heapster is deprecated" case "efk": return true, "", "The current images used in the efk addon contain Log4j vulnerabilities, the addon will be disabled until images are updated, see: https://github.com/kubernetes/minikube/issues/15280" - case "nvidia-device-plugin": - return true, "nvidia-gpu-device-plugin", "The nvidia-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-gpu-device-plugin addon. It will be removed in a future release. Please use the nvidia-gpu-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114." + case "nvidia-gpu-device-plugin": + return true, "nvidia-device-plugin", "The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. It will be removed in a future release. Please use the nvidia-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114." } return false, "", "" } From b1ab86bdeb852493b690b30ba17d70d5bee41663 Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Fri, 4 Oct 2024 20:12:10 +0530 Subject: [PATCH 12/28] Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. Reverting nvidia-gpu-device-plugin to nvidia-device-plugin. --- deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl | 3 --- 1 file changed, 3 deletions(-) diff --git a/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl b/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl index 6976fe77cc49..0ec4cd4fff99 100644 --- a/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl +++ b/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl @@ -47,9 +47,6 @@ spec: - image: {{.CustomRegistries.NvidiaDevicePlugin | default .ImageRepository | default .Registries.NvidiaDevicePlugin }}{{.Images.NvidiaDevicePlugin}} command: ["/usr/bin/nvidia-gpu-device-plugin", "-logtostderr"] name: nvidia-gpu-device-plugin - env: - - name: FAIL_ON_INIT_ERROR - value: "false" resources: requests: cpu: 50m From a0f915eda8a31049eb26e6eaa3dd0133e507bce0 Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Fri, 4 Oct 2024 20:42:19 +0530 Subject: [PATCH 13/28] Merging deploy\addons\gpu\nvidia-gpu-device-plugin.yaml.tmpl into deploy\addons\nvidia-device-plugin\nvidia-device-plugin.yaml.tmpl. Merging deploy\addons\gpu\nvidia-gpu-device-plugin.yaml.tmpl into deploy\addons\nvidia-device-plugin\nvidia-device-plugin.yaml.tmpl. --- .../nvidia-device-plugin.yaml.tmpl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl index c05c586edcbe..e7555f65d41c 100644 --- a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl +++ b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl @@ -29,6 +29,10 @@ spec: name: nvidia-device-plugin-ds spec: tolerations: + - operator: Exists + effect: NoExecute + - operator: Exists + effect: NoSchedule - key: nvidia.com/gpu operator: Exists effect: NoSchedule @@ -43,6 +47,10 @@ spec: env: - name: FAIL_ON_INIT_ERROR value: "false" + resources: + requests: + cpu: 50m + memory: 10Mi securityContext: allowPrivilegeEscalation: false capabilities: @@ -50,7 +58,11 @@ spec: volumeMounts: - name: device-plugin mountPath: /var/lib/kubelet/device-plugins + - name: dev + mountPath: /dev volumes: - name: device-plugin hostPath: path: /var/lib/kubelet/device-plugins + - name: dev + mountPath: /dev \ No newline at end of file From 4be65dcc671604c96a861fd34295283726a48c2d Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Fri, 4 Oct 2024 21:09:14 +0530 Subject: [PATCH 14/28] Update nvidia-device-plugin.yaml.tmpl --- .../nvidia-device-plugin.yaml.tmpl | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl index e7555f65d41c..fcb6012b1333 100644 --- a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl +++ b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl @@ -29,10 +29,6 @@ spec: name: nvidia-device-plugin-ds spec: tolerations: - - operator: Exists - effect: NoExecute - - operator: Exists - effect: NoSchedule - key: nvidia.com/gpu operator: Exists effect: NoSchedule @@ -47,10 +43,6 @@ spec: env: - name: FAIL_ON_INIT_ERROR value: "false" - resources: - requests: - cpu: 50m - memory: 10Mi securityContext: allowPrivilegeEscalation: false capabilities: @@ -58,11 +50,7 @@ spec: volumeMounts: - name: device-plugin mountPath: /var/lib/kubelet/device-plugins - - name: dev - mountPath: /dev volumes: - name: device-plugin hostPath: - path: /var/lib/kubelet/device-plugins - - name: dev - mountPath: /dev \ No newline at end of file + path: /var/lib/kubelet/device-plugins \ No newline at end of file From 8db684b8b1d5d813822c6ef6e8aab7f63ed09533 Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Sat, 5 Oct 2024 04:14:48 +0530 Subject: [PATCH 15/28] Revert "Update nvidia-device-plugin.yaml.tmpl" This reverts commit 4be65dcc671604c96a861fd34295283726a48c2d. --- .../nvidia-device-plugin.yaml.tmpl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl index fcb6012b1333..e7555f65d41c 100644 --- a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl +++ b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl @@ -29,6 +29,10 @@ spec: name: nvidia-device-plugin-ds spec: tolerations: + - operator: Exists + effect: NoExecute + - operator: Exists + effect: NoSchedule - key: nvidia.com/gpu operator: Exists effect: NoSchedule @@ -43,6 +47,10 @@ spec: env: - name: FAIL_ON_INIT_ERROR value: "false" + resources: + requests: + cpu: 50m + memory: 10Mi securityContext: allowPrivilegeEscalation: false capabilities: @@ -50,7 +58,11 @@ spec: volumeMounts: - name: device-plugin mountPath: /var/lib/kubelet/device-plugins + - name: dev + mountPath: /dev volumes: - name: device-plugin hostPath: - path: /var/lib/kubelet/device-plugins \ No newline at end of file + path: /var/lib/kubelet/device-plugins + - name: dev + mountPath: /dev \ No newline at end of file From e7141bcc73573a70aff2ad563579afd6e9c61825 Mon Sep 17 00:00:00 2001 From: Ramachandran C Date: Sat, 5 Oct 2024 04:48:05 +0530 Subject: [PATCH 16/28] dev volume added. dev volume added. --- .../nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl index e7555f65d41c..d5f9347634c6 100644 --- a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl +++ b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl @@ -64,5 +64,6 @@ spec: - name: device-plugin hostPath: path: /var/lib/kubelet/device-plugins - - name: dev - mountPath: /dev \ No newline at end of file + - name: dev + hostPath: + path: /dev From 69bab32b875b756d49903925ec3e0c5948a1ccd2 Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Mon, 16 Dec 2024 06:10:03 +0530 Subject: [PATCH 17/28] Update nvidia-device-plugin.yaml.tmpl These values are likely no longer needed when using the updated image. --- .../nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl index d5f9347634c6..5950c3946907 100644 --- a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl +++ b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl @@ -29,10 +29,6 @@ spec: name: nvidia-device-plugin-ds spec: tolerations: - - operator: Exists - effect: NoExecute - - operator: Exists - effect: NoSchedule - key: nvidia.com/gpu operator: Exists effect: NoSchedule From 6aec15cfa9de763d65021ef8cf2999a75664ec17 Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Thu, 19 Dec 2024 04:24:17 +0530 Subject: [PATCH 18/28] Update nvidia-device-plugin.yaml.tmpl --- .../nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl index 5950c3946907..c05c586edcbe 100644 --- a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl +++ b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl @@ -43,10 +43,6 @@ spec: env: - name: FAIL_ON_INIT_ERROR value: "false" - resources: - requests: - cpu: 50m - memory: 10Mi securityContext: allowPrivilegeEscalation: false capabilities: @@ -54,12 +50,7 @@ spec: volumeMounts: - name: device-plugin mountPath: /var/lib/kubelet/device-plugins - - name: dev - mountPath: /dev volumes: - name: device-plugin hostPath: path: /var/lib/kubelet/device-plugins - - name: dev - hostPath: - path: /dev From 9ade6fecb3c2a911f592e6af6c87a040a30dd7df Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Thu, 19 Dec 2024 04:57:57 +0530 Subject: [PATCH 19/28] Update nvidia-device-plugin.yaml.tmpl --- .../nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl index c05c586edcbe..5950c3946907 100644 --- a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl +++ b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl @@ -43,6 +43,10 @@ spec: env: - name: FAIL_ON_INIT_ERROR value: "false" + resources: + requests: + cpu: 50m + memory: 10Mi securityContext: allowPrivilegeEscalation: false capabilities: @@ -50,7 +54,12 @@ spec: volumeMounts: - name: device-plugin mountPath: /var/lib/kubelet/device-plugins + - name: dev + mountPath: /dev volumes: - name: device-plugin hostPath: path: /var/lib/kubelet/device-plugins + - name: dev + hostPath: + path: /dev From 72b1672ee9f0d6f527af5fd8acc2073b63870923 Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Thu, 19 Dec 2024 05:04:03 +0530 Subject: [PATCH 20/28] Update nvidia-device-plugin.yaml.tmpl --- .../nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl index 5950c3946907..c05c586edcbe 100644 --- a/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl +++ b/deploy/addons/nvidia-device-plugin/nvidia-device-plugin.yaml.tmpl @@ -43,10 +43,6 @@ spec: env: - name: FAIL_ON_INIT_ERROR value: "false" - resources: - requests: - cpu: 50m - memory: 10Mi securityContext: allowPrivilegeEscalation: false capabilities: @@ -54,12 +50,7 @@ spec: volumeMounts: - name: device-plugin mountPath: /var/lib/kubelet/device-plugins - - name: dev - mountPath: /dev volumes: - name: device-plugin hostPath: path: /var/lib/kubelet/device-plugins - - name: dev - hostPath: - path: /dev From 51859e4fce01884c20d0e728a9c3c19dcbbb860f Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Thu, 19 Dec 2024 05:05:47 +0530 Subject: [PATCH 21/28] Update addons.go --- pkg/addons/addons.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 8ab6bed63ee9..e28f1929d8f0 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -104,8 +104,8 @@ func preStartMessages(name, value string) { out.Styled(style.Warning, "The ambassador addon has stopped working as of v1.23.0, for more details visit: https://github.com/datawire/ambassador-operator/issues/73") case "olm": out.Styled(style.Warning, "The OLM addon has stopped working, for more details visit: https://github.com/operator-framework/operator-lifecycle-manager/issues/2534") - case "nvidia-gpu-device-plugin": - out.Styled(style.Warning, "The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. It will be removed in a future release. Please use the nvidia-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114.") + // case "nvidia-gpu-device-plugin": + // out.Styled(style.Warning, "The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. It will be removed in a future release. Please use the nvidia-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114.") } } @@ -165,8 +165,8 @@ func Deprecations(name string) (bool, string, string) { return true, "metrics-server", "using metrics-server addon, heapster is deprecated" case "efk": return true, "", "The current images used in the efk addon contain Log4j vulnerabilities, the addon will be disabled until images are updated, see: https://github.com/kubernetes/minikube/issues/15280" - case "nvidia-gpu-device-plugin": - return true, "nvidia-device-plugin", "The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. It will be removed in a future release. Please use the nvidia-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114." + // case "nvidia-gpu-device-plugin": + // return true, "nvidia-device-plugin", "The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. It will be removed in a future release. Please use the nvidia-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114." } return false, "", "" } From 2741016fbabce64751b7894093bf30e3aef3c940 Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Thu, 19 Dec 2024 05:08:34 +0530 Subject: [PATCH 22/28] Update addons.go --- pkg/addons/addons.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index e28f1929d8f0..8ab6bed63ee9 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -104,8 +104,8 @@ func preStartMessages(name, value string) { out.Styled(style.Warning, "The ambassador addon has stopped working as of v1.23.0, for more details visit: https://github.com/datawire/ambassador-operator/issues/73") case "olm": out.Styled(style.Warning, "The OLM addon has stopped working, for more details visit: https://github.com/operator-framework/operator-lifecycle-manager/issues/2534") - // case "nvidia-gpu-device-plugin": - // out.Styled(style.Warning, "The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. It will be removed in a future release. Please use the nvidia-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114.") + case "nvidia-gpu-device-plugin": + out.Styled(style.Warning, "The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. It will be removed in a future release. Please use the nvidia-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114.") } } @@ -165,8 +165,8 @@ func Deprecations(name string) (bool, string, string) { return true, "metrics-server", "using metrics-server addon, heapster is deprecated" case "efk": return true, "", "The current images used in the efk addon contain Log4j vulnerabilities, the addon will be disabled until images are updated, see: https://github.com/kubernetes/minikube/issues/15280" - // case "nvidia-gpu-device-plugin": - // return true, "nvidia-device-plugin", "The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. It will be removed in a future release. Please use the nvidia-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114." + case "nvidia-gpu-device-plugin": + return true, "nvidia-device-plugin", "The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. It will be removed in a future release. Please use the nvidia-device-plugin addon instead. For more details, visit: https://github.com/kubernetes/minikube/issues/19114." } return false, "", "" } From a88b4b40a3809e4a8d9b436efa812f1138662c5b Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Thu, 19 Dec 2024 05:15:00 +0530 Subject: [PATCH 23/28] Update config.go --- pkg/addons/config.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/addons/config.go b/pkg/addons/config.go index c4385dfd018d..dbceeb385ffa 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -125,13 +125,13 @@ var Addons = []*Addon{ validations: []setFn{isKVMDriverForNVIDIA}, callbacks: []setFn{EnableOrDisableAddon}, }, - { - // The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. - name: "nvidia-gpu-device-plugin", - set: SetBool, - validations: []setFn{isKVMDriverForNVIDIA}, - callbacks: []setFn{EnableOrDisableAddon}, - }, + // { + // // The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. + // name: "nvidia-gpu-device-plugin", + // set: SetBool, + // validations: []setFn{isKVMDriverForNVIDIA}, + // callbacks: []setFn{EnableOrDisableAddon}, + // }, { name: "olm", set: SetBool, From aced0197fd3729387e2ea080ea95c99c7a5be9f0 Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Thu, 19 Dec 2024 05:18:23 +0530 Subject: [PATCH 24/28] Update config.go --- pkg/addons/config.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/addons/config.go b/pkg/addons/config.go index dbceeb385ffa..97b5862b4745 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -125,13 +125,13 @@ var Addons = []*Addon{ validations: []setFn{isKVMDriverForNVIDIA}, callbacks: []setFn{EnableOrDisableAddon}, }, - // { - // // The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. - // name: "nvidia-gpu-device-plugin", - // set: SetBool, - // validations: []setFn{isKVMDriverForNVIDIA}, - // callbacks: []setFn{EnableOrDisableAddon}, - // }, + { + // The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. + name: "nvidia-gpu-device-plugin", + set: SetBool, + // validations: []setFn{isKVMDriverForNVIDIA}, + // callbacks: []setFn{EnableOrDisableAddon}, + }, { name: "olm", set: SetBool, From d77948195e719c6c646841950e37518d28bf0442 Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Thu, 19 Dec 2024 05:22:14 +0530 Subject: [PATCH 25/28] Update config.go --- pkg/addons/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/addons/config.go b/pkg/addons/config.go index 97b5862b4745..525ccbf80c64 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -130,7 +130,7 @@ var Addons = []*Addon{ name: "nvidia-gpu-device-plugin", set: SetBool, // validations: []setFn{isKVMDriverForNVIDIA}, - // callbacks: []setFn{EnableOrDisableAddon}, + callbacks: []setFn{EnableOrDisableAddon}, }, { name: "olm", From cf3c03fa7872eb77fa6442fd078eadf26dff30a2 Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Thu, 19 Dec 2024 05:24:34 +0530 Subject: [PATCH 26/28] Update config.go --- pkg/addons/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/addons/config.go b/pkg/addons/config.go index 525ccbf80c64..c4385dfd018d 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -129,7 +129,7 @@ var Addons = []*Addon{ // The nvidia-gpu-device-plugin addon is deprecated and it's functionality is merged inside of nvidia-device-plugin addon. name: "nvidia-gpu-device-plugin", set: SetBool, - // validations: []setFn{isKVMDriverForNVIDIA}, + validations: []setFn{isKVMDriverForNVIDIA}, callbacks: []setFn{EnableOrDisableAddon}, }, { From 7debb54f293092757ecbcb6aa99e0e638eb64ab0 Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Thu, 19 Dec 2024 05:33:54 +0530 Subject: [PATCH 27/28] Update Makefile --- Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 15d916f703bc..e41c359c76d6 100644 --- a/Makefile +++ b/Makefile @@ -190,9 +190,11 @@ define DOCKER docker run --rm -e GOCACHE=/app/.cache -e IN_DOCKER=1 --user $(shell id -u):$(shell id -g) -w /app -v $(PWD):/app -v $(GOPATH):/go --init $(1) /bin/bash -c '$(2)' endef -ifeq ($(BUILD_IN_DOCKER),y) - MINIKUBE_BUILD_IN_DOCKER=y -endif +# ifeq ($(BUILD_IN_DOCKER),y) +# MINIKUBE_BUILD_IN_DOCKER=y +# endif + +MINIKUBE_BUILD_IN_DOCKER=y # If we are already running in docker, # prevent recursion by unsetting the BUILD_IN_DOCKER directives. From 1db9a8723b8e1fab63096e048147105823567e1e Mon Sep 17 00:00:00 2001 From: Ramachandran C <61575596+RamBITS-AI@users.noreply.github.com> Date: Thu, 19 Dec 2024 05:39:52 +0530 Subject: [PATCH 28/28] Update Makefile --- Makefile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index e41c359c76d6..15d916f703bc 100644 --- a/Makefile +++ b/Makefile @@ -190,11 +190,9 @@ define DOCKER docker run --rm -e GOCACHE=/app/.cache -e IN_DOCKER=1 --user $(shell id -u):$(shell id -g) -w /app -v $(PWD):/app -v $(GOPATH):/go --init $(1) /bin/bash -c '$(2)' endef -# ifeq ($(BUILD_IN_DOCKER),y) -# MINIKUBE_BUILD_IN_DOCKER=y -# endif - -MINIKUBE_BUILD_IN_DOCKER=y +ifeq ($(BUILD_IN_DOCKER),y) + MINIKUBE_BUILD_IN_DOCKER=y +endif # If we are already running in docker, # prevent recursion by unsetting the BUILD_IN_DOCKER directives.