diff --git a/demos/demo-kubernetes-minikube/app/Main.hs b/demos/demo-kubernetes-minikube/app/Main.hs index c582e6ad..00a97d41 100644 --- a/demos/demo-kubernetes-minikube/app/Main.hs +++ b/demos/demo-kubernetes-minikube/app/Main.hs @@ -4,7 +4,6 @@ module Main where -import Control.Concurrent import Control.Monad import Control.Monad.IO.Class import Control.Monad.Logger @@ -24,6 +23,7 @@ import Test.Sandwich.Contexts.Kubernetes.MinioS3Server import Test.Sandwich.Contexts.Kubernetes.Namespace import Test.Sandwich.Contexts.Nix import Test.Sandwich.Contexts.Waits +import UnliftIO.Concurrent import UnliftIO.Environment import UnliftIO.Process @@ -42,8 +42,7 @@ spec = describe "Introducing a Kubernetes cluster" $ do forM_ images $ \image -> info [i|Image: #{image}|] introduceBinaryViaNixPackage @"kubectl" "kubectl" $ - introduceBinaryViaNixDerivation @"kubectl-minio" kubectlMinioDerivation $ - introduceMinioOperator $ do + introduceMinioOperator defaultMinioOperatorOptions $ do it "Has a MinIO operator" $ do moc <- getContext minioOperator info [i|Got MinIO operator: #{moc}|] @@ -61,29 +60,13 @@ spec = describe "Introducing a Kubernetes cluster" $ do p <- createProcessWithLogging ((proc kubectlBinary args) { env = Just env, delegate_ctlc = True }) waitForProcess p >>= (`shouldBe` ExitSuccess) - introduceK8SMinioS3Server "foo" $ do + introduceK8SMinioS3Server (defaultMinioS3ServerOptions "foo") $ do it "has a MinIO S3 server" $ do serv <- getContext testS3Server info [i|Got test S3 server: #{serv}|] -kubectlMinioDerivation :: Text -kubectlMinioDerivation = [i| -{ fetchurl -}: - -fetchurl { - url = "https://github.com/minio/operator/releases/download/v5.0.6/kubectl-minio_5.0.6_linux_amd64"; - hash = "sha256-j3mpgV1HLmFwYRdxfPXT1XzDWeiyQC2Ye8aeZt511bc="; - - downloadToTemp = true; - executable = true; - - postFetch = '' - mkdir -p $out/bin - mv "$downloadedFile" $out/bin/kubectl-minio - ''; -} -|] + it "pauses" $ do + threadDelay 9999999999999 main :: IO () diff --git a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/FindImages.hs b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/FindImages.hs new file mode 100644 index 00000000..8a0a5abc --- /dev/null +++ b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/FindImages.hs @@ -0,0 +1,47 @@ +{-# LANGUAGE ConstraintKinds #-} +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE TypeOperators #-} + +module Test.Sandwich.Contexts.Kubernetes.FindImages ( + findAllImages + , findAllImages' + ) where + +import Control.Lens +import Data.Aeson (FromJSON) +import Data.Text as T +import qualified Data.Yaml as Yaml +import Kubernetes.OpenAPI.Model as Kubernetes +import Kubernetes.OpenAPI.ModelLens as Kubernetes +import Relude + + +-- | Find all image references in a chunk of YAML containing multiple sections +findAllImages :: Text -> [Text] +findAllImages = Relude.concatMap findAllImages' . T.splitOn "---\n" + +-- | Find all image references in a single chunk of YAML +findAllImages' :: Text -> [Text] +findAllImages' (decode -> Right x@(V1Pod {})) = maybe [] imagesFromPodSpec (v1PodSpec x) +findAllImages' (decode -> Right x@(V1Deployment {})) = maybe [] imagesFromPodSpec maybePodSpec + where + maybePodSpec :: Maybe V1PodSpec + maybePodSpec = x ^? (v1DeploymentSpecL . _Just . v1DeploymentSpecTemplateL . v1PodTemplateSpecSpecL . _Just) +findAllImages' (decode -> Right x@(V1StatefulSet {})) = maybe [] imagesFromPodSpec maybePodSpec + where + maybePodSpec :: Maybe V1PodSpec + maybePodSpec = x ^? (v1StatefulSetSpecL . _Just . v1StatefulSetSpecTemplateL . v1PodTemplateSpecSpecL . _Just) +findAllImages' (decode -> Right x@(V1DaemonSet {})) = maybe [] imagesFromPodSpec maybePodSpec + where + maybePodSpec :: Maybe V1PodSpec + maybePodSpec = x ^? (v1DaemonSetSpecL . _Just . v1DaemonSetSpecTemplateL . v1PodTemplateSpecSpecL . _Just) +findAllImages' _ = [] + +imagesFromPodSpec :: V1PodSpec -> [Text] +imagesFromPodSpec x = mapMaybe v1ContainerImage allContainers + where + allContainers = x ^. v1PodSpecContainersL <> fromMaybe [] (x ^. v1PodSpecInitContainersL) + +decode :: FromJSON a => Text -> Either Yaml.ParseException a +decode = Yaml.decodeEither' . encodeUtf8 diff --git a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Images.hs b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Images.hs index f3b31558..4c9cee29 100644 --- a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Images.hs +++ b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Images.hs @@ -18,6 +18,9 @@ module Test.Sandwich.Contexts.Kubernetes.Images ( , introduceImages + , findAllImages + , findAllImages' + , ImageLoadSpec(..) , ImagePullPolicy(..) ) where @@ -28,6 +31,7 @@ import Data.String.Interpolate import Data.Text as T import Relude import Test.Sandwich +import Test.Sandwich.Contexts.Kubernetes.FindImages import qualified Test.Sandwich.Contexts.Kubernetes.KindCluster.Images as Kind import qualified Test.Sandwich.Contexts.Kubernetes.MinikubeCluster.Images as Minikube import Test.Sandwich.Contexts.Kubernetes.Types diff --git a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Kubectl.hs b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Kubectl.hs index 7311c04d..3a865244 100644 --- a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Kubectl.hs +++ b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Kubectl.hs @@ -22,27 +22,24 @@ runWithKubectl :: ( MonadLoggerIO m , HasBaseContextMonad context m, HasFile context "kubectl", HasKubernetesClusterContext context ) - -- | Callback receiving the kubectl binary and env. - => (FilePath -> [(String, String)] -> m a) - -> m a -runWithKubectl cb = do + -- | Return the kubectl binary and env. + => m (FilePath, [(String, String)]) +runWithKubectl = do kcc <- getContext kubernetesCluster kubectlBinary <- askFile @"kubectl" - runWithKubectl' kcc kubectlBinary cb + runWithKubectl' kcc kubectlBinary runWithKubectl' :: ( MonadLoggerIO m - , HasBaseContextMonad context m ) -- | Kubernetes cluster context => KubernetesClusterContext - -- | Binary path for kubectl + -- | Path to kubectl binary -> FilePath - -- | Callback receiving the kubectl binary and env. - -> (FilePath -> [(String, String)] -> m a) - -> m a -runWithKubectl' (KubernetesClusterContext {..}) kubectlBinary cb = do + -- | Return the kubectl binary and env. + -> m (FilePath, [(String, String)]) +runWithKubectl' (KubernetesClusterContext {..}) kubectlBinary = do baseEnv <- getEnvironment let env = L.nubBy (\x y -> fst x == fst y) (("KUBECONFIG", kubernetesClusterKubeConfigPath) : baseEnv) - cb kubectlBinary env + return (kubectlBinary, env) diff --git a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioOperator.hs b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioOperator.hs index 90296533..0c3d4bda 100644 --- a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioOperator.hs +++ b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioOperator.hs @@ -15,69 +15,80 @@ module Test.Sandwich.Contexts.Kubernetes.MinioOperator ( -- * Types , minioOperator , MinioOperatorContext(..) + , MinioOperatorOptions(..) + , defaultMinioOperatorOptions , HasMinioOperatorContext ) where import Control.Monad import Control.Monad.IO.Unlift import Control.Monad.Logger -import qualified Data.List as L import Relude import System.Exit -import System.FilePath import Test.Sandwich import Test.Sandwich.Contexts.Files +import Test.Sandwich.Contexts.Kubernetes.FindImages +import Test.Sandwich.Contexts.Kubernetes.Images +import Test.Sandwich.Contexts.Kubernetes.Kubectl import Test.Sandwich.Contexts.Kubernetes.Types -import UnliftIO.Environment import UnliftIO.Exception import UnliftIO.Process +data MinioOperatorOptions = MinioOperatorOptions { + minioOperatorPreloadImages :: Bool + } +defaultMinioOperatorOptions :: MinioOperatorOptions +defaultMinioOperatorOptions = MinioOperatorOptions { + minioOperatorPreloadImages = True + } + -- | Install the MinIO Kubernetes plugin onto a Kubernetes cluster. -- See the docs [here](https://min.io/docs/minio/kubernetes/upstream/reference/kubectl-minio-plugin.html). introduceMinioOperator :: ( - MonadUnliftIO m, HasKubernetesClusterContext context, HasFile context "kubectl", HasFile context "kubectl-minio" - ) => SpecFree (LabelValue "minioOperator" MinioOperatorContext :> context) m () -> SpecFree context m () -introduceMinioOperator = introduceWith "introduce MinIO operator" minioOperator $ \action -> do + MonadUnliftIO m, HasBaseContext context, HasKubernetesClusterContext context, HasFile context "kubectl" + ) => MinioOperatorOptions -> SpecFree (LabelValue "minioOperator" MinioOperatorContext :> context) m () -> SpecFree context m () +introduceMinioOperator options = introduceWith "introduce MinIO operator" minioOperator $ \action -> do kcc <- getContext kubernetesCluster - void $ withMinioOperator kcc action + void $ withMinioOperator options kcc action --- | Same as 'introduceMinioOperator', but allows you to pass in the "kubectl-minio" binary path. +-- | Same as 'introduceMinioOperator', but allows you to pass in the "kubectl" binary path. introduceMinioOperator' :: ( - MonadUnliftIO m, HasKubernetesClusterContext context - ) => FilePath -> FilePath -> SpecFree (LabelValue "minioOperator" MinioOperatorContext :> context) m () -> SpecFree context m () -introduceMinioOperator' kubectlBinary kubectlMinioBinary = introduceWith "introduce MinIO operator" minioOperator $ \action -> do + MonadUnliftIO m, MonadFail m, HasKubernetesClusterContext context, HasBaseContext context + ) => MinioOperatorOptions -> FilePath -> SpecFree (LabelValue "minioOperator" MinioOperatorContext :> context) m () -> SpecFree context m () +introduceMinioOperator' options kubectlBinary = introduceWith "introduce MinIO operator" minioOperator $ \action -> do kcc <- getContext kubernetesCluster - void $ withMinioOperator' kubectlBinary kubectlMinioBinary kcc action + void $ withMinioOperator' options kubectlBinary kcc action -- | Bracket-style variant of 'introduceMinioOperator'. withMinioOperator :: ( - MonadLoggerIO m, MonadUnliftIO m, MonadReader context m, HasFile context "kubectl", HasFile context "kubectl-minio" - ) => KubernetesClusterContext -> (MinioOperatorContext -> m a) -> m a -withMinioOperator kcc action = do + MonadLoggerIO m, MonadUnliftIO m, MonadFail m + , HasBaseContextMonad context m, HasFile context "kubectl" + ) => MinioOperatorOptions -> KubernetesClusterContext -> (MinioOperatorContext -> m a) -> m a +withMinioOperator options kcc action = do kubectlBinary <- askFile @"kubectl" - kubectlMinioBinary <- askFile @"kubectl-minio" - withMinioOperator' kubectlBinary kubectlMinioBinary kcc action + withMinioOperator' options kubectlBinary kcc action --- | Same as 'withMinioOperator', but allows you to pass in the "kubectl-minio" binary path. +-- | Same as 'withMinioOperator', but allows you to pass in the "kubectl" binary path. withMinioOperator' :: ( - MonadLoggerIO m, MonadUnliftIO m - ) => FilePath -> FilePath -> KubernetesClusterContext -> (MinioOperatorContext -> m a) -> m a -withMinioOperator' kubectlBinary kubectlMinioBinary (KubernetesClusterContext {..}) action = do - baseEnv <- getEnvironment + MonadLoggerIO m, MonadUnliftIO m, MonadFail m + , HasBaseContextMonad context m + ) => MinioOperatorOptions -> FilePath -> KubernetesClusterContext -> (MinioOperatorContext -> m a) -> m a +withMinioOperator' (MinioOperatorOptions {..}) kubectlBinary kcc action = do + (_, env) <- runWithKubectl' kcc kubectlBinary + + allYaml <- readCreateProcessWithLogging ((proc kubectlBinary ["kustomize", "github.com/minio/operator?ref=v6.0.1"]) { env = Just env }) "" - let basePathParts = maybe [] splitSearchPath (L.lookup "PATH" baseEnv) + when minioOperatorPreloadImages $ do + let images = findAllImages (toText allYaml) - let newPath = L.intercalate [searchPathSeparator] ((takeDirectory kubectlBinary) : basePathParts) + forM_ images $ \image -> + loadImageIfNecessary' kcc (ImageLoadSpecDocker image IfNotPresent) - let env = L.nubBy (\x y -> fst x == fst y) (("PATH", newPath) : ("KUBECONFIG", kubernetesClusterKubeConfigPath) : baseEnv) + let create = createProcessWithLoggingAndStdin ((proc kubectlBinary ["apply", "-f", "-"]) { env = Just env }) allYaml + >>= waitForProcess >>= (`shouldBe` ExitSuccess) - let runWithKubeConfig exe args = do - p <- createProcessWithLogging ((proc exe args) { env = Just env, delegate_ctlc = True }) - code <- waitForProcess p - code `shouldBe` ExitSuccess + let destroy = createProcessWithLoggingAndStdin ((proc kubectlBinary ["delete", "-f", "-"]) { env = Just env }) allYaml + >>= waitForProcess >>= (`shouldBe` ExitSuccess) - bracket_ (runWithKubeConfig kubectlMinioBinary ["init"]) - -- Can't delete -f yet; see https://github.com/minio/operator/issues/1683 - (return ()) -- (runWithKubeConfig kubectlMinioBinary ["delete"]) - (action MinioOperatorContext) + bracket_ create destroy (action MinioOperatorContext) diff --git a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioS3Server.hs b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioS3Server.hs index cae9c270..cb46d9e0 100644 --- a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioS3Server.hs +++ b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioS3Server.hs @@ -8,6 +8,9 @@ module Test.Sandwich.Contexts.Kubernetes.MinioS3Server ( , withK8SMinioS3Server , withK8SMinioS3Server' + , MinioS3ServerOptions(..) + , defaultMinioS3ServerOptions + -- * Re-exports , testS3Server , TestS3Server(..) @@ -18,8 +21,6 @@ import Control.Monad import Control.Monad.Catch (MonadMask) import Control.Monad.IO.Unlift import Control.Monad.Logger -import qualified Data.ByteString.Base64 as B64 -import qualified Data.List as L import Data.String.Interpolate import Data.Text as T import Network.Minio @@ -28,152 +29,171 @@ import System.Exit import Test.Sandwich import Test.Sandwich.Contexts.Files import Test.Sandwich.Contexts.Kubernetes.Cluster +import Test.Sandwich.Contexts.Kubernetes.FindImages +import Test.Sandwich.Contexts.Kubernetes.Images import Test.Sandwich.Contexts.Kubernetes.MinioS3Server.Parsing import Test.Sandwich.Contexts.Kubernetes.Types import Test.Sandwich.Contexts.Kubernetes.Util.UUID import Test.Sandwich.Contexts.MinIO +import Test.Sandwich.Contexts.Nix import Test.Sandwich.Contexts.Waits -import UnliftIO.Environment +import UnliftIO.Concurrent import UnliftIO.Exception import UnliftIO.Process import UnliftIO.Timeout +data MinioS3ServerOptions = MinioS3ServerOptions { + minioS3ServerNamespace :: Text + , minioS3ServerKustomizationDir :: KustomizationDir + , minioS3ServerPreloadImages :: Bool + } +defaultMinioS3ServerOptions :: Text -> MinioS3ServerOptions +defaultMinioS3ServerOptions namespace = MinioS3ServerOptions { + minioS3ServerNamespace = namespace + , minioS3ServerKustomizationDir = KustomizationDirUrl "https://github.com/minio/operator/examples/kustomization/base?ref=v6.0.1" + , minioS3ServerPreloadImages = True + } + +data KustomizationDir = + -- | URL Kustomize dir to be downloaded + KustomizationDirUrl Text + -- | Local Kustomize dir + | KustomizationDirLocal FilePath + -- | A Nix callPackage-style derivation to produce the Kustomize dir + | KustomizationDirNixDerivation Text + deriving (Show, Eq) + -- | Introduce a MinIO server on a Kubernetes cluster. -- Must have a 'minioOperator' context. introduceK8SMinioS3Server :: ( - MonadMask m, MonadUnliftIO m + MonadMask m, MonadUnliftIO m, Typeable context , HasBaseContext context, HasMinioOperatorContext context, HasKubernetesClusterContext context - , HasFile context "kubectl", HasFile context "kubectl-minio" + , HasFile context "kubectl" ) - -- | Namespace - => Text + -- | Options + => MinioS3ServerOptions -> SpecFree (LabelValue "testS3Server" TestS3Server :> context) m () -> SpecFree context m () -introduceK8SMinioS3Server namespace = do +introduceK8SMinioS3Server options = do introduceWith "minio S3 server" testS3Server $ \action -> do kcc <- getContext kubernetesCluster moc <- getContext minioOperator - withK8SMinioS3Server kcc moc namespace action + withK8SMinioS3Server kcc moc options action -- | Same as 'introduceK8SMinioS3Server', but allows you to pass in the 'KubernetesClusterContext'. introduceK8SMinioS3Server' :: ( - MonadMask m, MonadUnliftIO m - , HasBaseContext context, HasMinioOperatorContext context, HasFile context "kubectl", HasFile context "kubectl-minio" + MonadMask m, MonadUnliftIO m, Typeable context + , HasBaseContext context, HasMinioOperatorContext context, HasFile context "kubectl" ) => KubernetesClusterContext - -- | Namespace - -> Text + -- | Options + -> MinioS3ServerOptions -> SpecFree (LabelValue "testS3Server" TestS3Server :> context) m () -> SpecFree context m () -introduceK8SMinioS3Server' kubernetesClusterContext namespace = +introduceK8SMinioS3Server' kubernetesClusterContext options = introduceWith "minio S3 server" testS3Server $ \action -> do moc <- getContext minioOperator - withK8SMinioS3Server kubernetesClusterContext moc namespace action + withK8SMinioS3Server kubernetesClusterContext moc options action -- | Bracket-style variant of 'introduceK8SMinioS3Server'. withK8SMinioS3Server :: ( MonadLoggerIO m, MonadMask m, MonadUnliftIO m, MonadFail m - , HasBaseContextMonad context m, HasFile context "kubectl", HasFile context "kubectl-minio" + , HasBaseContextMonad context m, HasFile context "kubectl", Typeable context ) => KubernetesClusterContext -> MinioOperatorContext - -- | Namespace - -> Text + -- | Options + -> MinioS3ServerOptions -> (TestS3Server -> m [Result]) -> m () -withK8SMinioS3Server kcc moc namespace action = do +withK8SMinioS3Server kcc moc options action = do kubectlBinary <- askFile @"kubectl" - kubectlMinioBinary <- askFile @"kubectl-minio" - withK8SMinioS3Server' kubectlBinary kubectlMinioBinary kcc moc namespace action + withK8SMinioS3Server' kubectlBinary kcc moc options action -- | Same as 'withK8SMinioS3Server', but allows you to pass in the kubectl and kubectl-minio binaries. withK8SMinioS3Server' :: ( MonadLoggerIO m, MonadMask m, MonadUnliftIO m, MonadFail m - , HasBaseContextMonad context m + , HasBaseContextMonad context m, Typeable context ) -- | Path to kubectl binary => FilePath - -- | Path to kubectl-minio binary - -> FilePath -> KubernetesClusterContext -> MinioOperatorContext - -- | Namespace - -> Text + -- | Options + -> MinioS3ServerOptions -> (TestS3Server -> m [Result]) -> m () -withK8SMinioS3Server' kubectlBinary kubectlMinioBinary (KubernetesClusterContext {..}) MinioOperatorContext namespace action = do - baseEnv <- getEnvironment - let env = L.nubBy (\x y -> fst x == fst y) (("KUBECONFIG", kubernetesClusterKubeConfigPath) : baseEnv) +withK8SMinioS3Server' kubectlBinary kcc@(KubernetesClusterContext {..}) MinioOperatorContext (MinioS3ServerOptions {..}) action = do + (_, env) <- runWithKubectl' kcc kubectlBinary let runWithKubeConfig prog args = do - p <- createProcessWithLogging ((proc prog args) { env = Just env, delegate_ctlc = True }) - waitForProcess p >>= (`shouldBe` ExitSuccess) - let runWithKubeConfig' prog args input = do - p <- createProcessWithLoggingAndStdin ((proc prog args) { env = Just env, delegate_ctlc = True }) input - waitForProcess p >>= (`shouldBe` ExitSuccess) + createProcessWithLogging ((proc prog args) { env = Just env, delegate_ctlc = True }) + >>= waitForProcess >>= (`shouldBe` ExitSuccess) deploymentName <- ("minio-" <>) <$> makeUUID' 5 - let pool = "pool1" + -- let pool = "pool1" let port = 80 + kustomizationDir <- case minioS3ServerKustomizationDir of + KustomizationDirLocal p -> pure p + KustomizationDirUrl u -> pure (toString u) + KustomizationDirNixDerivation d -> do + getContextMaybe nixContext >>= \case + Nothing -> expectationFailure [i|Couldn't find a Nix context to use with KustomizationDirNixDerivation|] + Just nc -> buildNixCallPackageDerivation' nc d + + let busyboxImage = "busybox:1.36.1-musl" + let create = do - runWithKubeConfig kubectlMinioBinary [ - "tenant", "create", toString deploymentName - , "--namespace", toString namespace - , "--servers", "1" - , "--volumes", "1" - , "--capacity", "10G" - , "--pool", pool - , "--disable-tls" - ] - - let destroy = do - runWithKubeConfig kubectlMinioBinary ["tenant", "delete", toString deploymentName - , "--namespace", toString namespace - , "-f" - ] + allYaml <- readCreateProcessWithLogging ((proc kubectlBinary ["kustomize", kustomizationDir]) { env = Just env, delegate_ctlc = True }) "" + + when minioS3ServerPreloadImages $ do + let images = findAllImages (toText allYaml) + + forM_ images $ \image -> + loadImageIfNecessary' kcc (ImageLoadSpecDocker image IfNotPresent) + + loadImageIfNecessary' kcc (ImageLoadSpecDocker busyboxImage IfNotPresent) + + (userAndPassword@(username, password), finalYaml) <- case transformKustomizeChunks (toString minioS3ServerNamespace) (T.splitOn "---\n" (toText allYaml)) of + Left err -> expectationFailure [i|Couldn't transform kustomize chunks: #{err}|] + Right x -> pure x + + info [i|Got username and password: #{(username, password)}|] + + createProcessWithLoggingAndStdin ((proc kubectlBinary ["apply", "-f", "-"]) { env = Just env }) (toString finalYaml) + >>= waitForProcess >>= (`shouldBe` ExitSuccess) + + return userAndPassword + + let destroy _ = + runWithKubeConfig kubectlBinary ["delete", "-k", kustomizationDir + , "--namespace", toString minioS3ServerNamespace + , "-f" + ] let createNetworkPolicy = do let (policyName, discoverPodPolicyName, yaml) = networkPolicy deploymentName - runWithKubeConfig' kubectlBinary ["create", "--namespace", toString namespace, "-f", "-"] yaml + createProcessWithLoggingAndStdin ((proc kubectlBinary ["create", "--namespace", toString minioS3ServerNamespace, "-f", "-"]) { env = Just env, delegate_ctlc = True }) yaml + >>= waitForProcess >>= (`shouldBe` ExitSuccess) pure (policyName, discoverPodPolicyName) let destroyNetworkPolicy (policyName, discoverPodPolicyName) = do - runWithKubeConfig kubectlBinary ["delete", "NetworkPolicy", policyName, "--namespace", toString namespace] - runWithKubeConfig kubectlBinary ["delete", "NetworkPolicy", discoverPodPolicyName, "--namespace", toString namespace] + runWithKubeConfig kubectlBinary ["delete", "NetworkPolicy", policyName, "--namespace", toString minioS3ServerNamespace] + runWithKubeConfig kubectlBinary ["delete", "NetworkPolicy", discoverPodPolicyName, "--namespace", toString minioS3ServerNamespace] -- TODO: create network policy allowing ingress/egress for v1.min.io/tenant = deploymentName - bracket createNetworkPolicy destroyNetworkPolicy $ \_ -> bracket_ create destroy $ do - envConfig <- ((B64.decodeLenient . encodeUtf8 . T.strip . toText) <$>) $ do - let getSecretArgs = ["get", "secret", [i|#{deploymentName}-env-configuration|] - , "--namespace", toString namespace - , "-o", [i|jsonpath="{.data.config\\.env}"|] - ] - debug [i|export KUBECONFIG='#{kubernetesClusterKubeConfigPath}'|] - debug [i|#{kubectlBinary} #{T.unwords $ fmap T.pack getSecretArgs}|] - ret <- readCreateProcessWithLogging ((proc kubectlBinary getSecretArgs) { env = Just env }) "" - debug [i|Got ret: #{ret}|] - return ret - - -- envConfig <- case eitherEnvConfig of - -- Right x -> pure x - -- Left err -> expectationFailure [i|Failed to decode MinIO environment config: #{err}|] - - -- info [i|Got envConfig: #{envConfig}|] - - Just (username, password) <- return (parseMinioUserAndPassword (decodeUtf8 envConfig)) - info [i|Got username and password: #{(username, password)}|] - + bracket createNetworkPolicy destroyNetworkPolicy $ \_ -> bracket create destroy $ \(username, password) -> do do uuid <- makeUUID p <- createProcessWithLogging ((proc kubectlBinary [ "run", "discoverer-" <> toString uuid , "--rm", "-i" , "--attach" - , "--image=busybox:1.36.1-musl" + , [i|--image=#{busyboxImage}|] , "--restart=Never" , "--command" - , "--namespace", toString namespace + , "--namespace", toString minioS3ServerNamespace , "--labels=app=discover-pod" , "--" , "sh", "-c", [i|until nc -vz minio 80; do echo "Waiting for minio..."; sleep 3; done;|] @@ -184,9 +204,9 @@ withK8SMinioS3Server' kubectlBinary kubectlMinioBinary (KubernetesClusterContext info [__i|Ready to try port-forward: export KUBECONFIG=#{kubernetesClusterKubeConfigPath} - kubectl --namespace #{namespace} port-forward "service/minio" 8080:#{port}|] + kubectl --namespace #{minioS3ServerNamespace} port-forward "service/minio" 8080:#{port}|] - withKubectlPortForward' kubectlBinary kubernetesClusterKubeConfigPath namespace (const True) Nothing "service/minio" port $ \(KubectlPortForwardContext {..}) -> do + withKubectlPortForward' kubectlBinary kubernetesClusterKubeConfigPath minioS3ServerNamespace (const True) Nothing "service/minio" port $ \(KubectlPortForwardContext {..}) -> do info [i|Did forward to localhost:#{kubectlPortForwardPort}|] -- liftIO $ threadDelay 999999999999 diff --git a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioS3Server/Parsing.hs b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioS3Server/Parsing.hs index ee7cd21d..60506496 100644 --- a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioS3Server/Parsing.hs +++ b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/MinioS3Server/Parsing.hs @@ -2,11 +2,19 @@ module Test.Sandwich.Contexts.Kubernetes.MinioS3Server.Parsing ( parseMinioUserAndPassword + , transformKustomizeChunks ) where +import Data.Aeson (FromJSON) +import qualified Data.Aeson as A +import qualified Data.Map as M import Data.String.Interpolate -import Data.Text +import Data.Text as T +import qualified Data.Yaml as Yaml +import Kubernetes.OpenAPI.Model as Kubernetes import Relude +import Safe +import Test.Sandwich.Contexts.Kubernetes.Util.Aeson import Text.Regex.TDFA @@ -21,3 +29,49 @@ parseMinioUserAndPassword txt = case (userValues, passwordValues) of -- testInput :: Text -- testInput = [__i|export MINIO_ROOT_USER="WXSTFUWIRS04LMGIMJGV" -- export MINIO_ROOT_PASSWORD="NCDCfTaiXcGHq8QRfSaXMAWOXgdrhpGwPSkoYMWf"|] + +transformKustomizeChunks :: String -> [Text] -> Either String ((Text, Text), Text) +transformKustomizeChunks namespace initialChunks = do + userAndPassword <- getUserAndPassword initialChunks + + return (userAndPassword, finalYaml) + + where + finalYaml = initialChunks + -- Don't include a kind: Namespace + & Relude.filter (not . isNamespace) + + & fmap (setMetaNamespace namespace) + + -- Combine everything into multi-document Yaml + & T.intercalate "---\n" + +getUserAndPassword :: [Text] -> Either String (Text, Text) +getUserAndPassword chunks = case headMay (mapMaybe getUserAndPassword' chunks) of + Nothing -> Left "Couldn't find user/password YAML." + Just x -> Right x + where + getUserAndPassword' :: Text -> Maybe (Text, Text) + getUserAndPassword' (decode -> Right (V1Secret {v1SecretMetadata=(Just (V1ObjectMeta {v1ObjectMetaName=(Just "storage-configuration")})) + , v1SecretStringData=(Just (M.lookup "config.env" -> Just t)) + })) + = parseMinioUserAndPassword t + getUserAndPassword' _ = Nothing + +isNamespace :: Text -> Bool +isNamespace (decode -> Right (A.Object (aesonLookup "kind" -> Just (A.String "Namespace")))) = True +isNamespace _ = False + +setMetaNamespace :: String -> Text -> Text +setMetaNamespace namespace (decode -> Right (A.Object obj1@(aesonLookup "metadata" -> Just (A.Object obj2@(aesonLookup "namespace" -> Just (A.String _)))))) = + decodeUtf8 (Yaml.encode obj1') + where + obj1' :: A.Value + obj1' = A.Object (aesonInsert "metadata" obj2' obj1) + + obj2' :: A.Value + obj2' = A.Object (aesonInsert "namespace" (A.String (toText namespace)) obj2) +setMetaNamespace _ t = t + +decode :: FromJSON a => Text -> Either Yaml.ParseException a +decode = Yaml.decodeEither' . encodeUtf8 diff --git a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Namespace.hs b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Namespace.hs index 3b225555..05e80f2e 100644 --- a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Namespace.hs +++ b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Namespace.hs @@ -60,9 +60,9 @@ create :: ( ) => Text -> m () create namespace = do let args = ["create", "namespace", toString namespace] - runWithKubectl $ \kubectl env -> - createProcessWithLogging ((proc kubectl args) { env = Just env, delegate_ctlc = True }) - >>= waitForProcess >>= (`shouldBe` ExitSuccess) + (kubectl, env) <- runWithKubectl + createProcessWithLogging ((proc kubectl args) { env = Just env, delegate_ctlc = True }) + >>= waitForProcess >>= (`shouldBe` ExitSuccess) destroy :: ( MonadUnliftIO m, MonadLoggerIO m @@ -73,6 +73,6 @@ destroy :: ( ) => Text -> m () destroy namespace = do let args = ["delete", "namespace", toString namespace] - runWithKubectl $ \kubectl env -> - createProcessWithLogging ((proc kubectl args) { env = Just env, delegate_ctlc = True }) - >>= waitForProcess >>= (`shouldBe` ExitSuccess) + (kubectl, env) <- runWithKubectl + createProcessWithLogging ((proc kubectl args) { env = Just env, delegate_ctlc = True }) + >>= waitForProcess >>= (`shouldBe` ExitSuccess) diff --git a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Types.hs b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Types.hs index 2e62b4c9..064c9c4c 100644 --- a/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Types.hs +++ b/sandwich-contexts-kubernetes/lib/Test/Sandwich/Contexts/Kubernetes/Types.hs @@ -59,10 +59,10 @@ data ImageLoadSpec = ImageLoadSpecTarball FilePath -- | An image pulled via Docker | ImageLoadSpecDocker { imageName :: Text - , pullPolicy :: ImagePullPolicy } + , pullPolicy :: ImagePullPolicy } -- | An image pulled via Podman | ImageLoadSpecPodman { imageName :: Text - , pullPolicy :: ImagePullPolicy } + , pullPolicy :: ImagePullPolicy } deriving (Show, Eq) -- * MinIO Operator diff --git a/sandwich-contexts-kubernetes/package.yaml b/sandwich-contexts-kubernetes/package.yaml index f707978c..6e81bf30 100644 --- a/sandwich-contexts-kubernetes/package.yaml +++ b/sandwich-contexts-kubernetes/package.yaml @@ -47,7 +47,6 @@ library: - Test.Sandwich.Contexts.Kubernetes.Types dependencies: - aeson - - base64-bytestring - bytestring - sandwich-contexts - sandwich-contexts-minio @@ -58,6 +57,7 @@ library: - http-client - kubernetes-client - kubernetes-client-core + - lens - minio-hs - monad-logger - network diff --git a/sandwich-contexts-kubernetes/sandwich-contexts-kubernetes.cabal b/sandwich-contexts-kubernetes/sandwich-contexts-kubernetes.cabal index 03cf2e7b..6d7a4ce9 100644 --- a/sandwich-contexts-kubernetes/sandwich-contexts-kubernetes.cabal +++ b/sandwich-contexts-kubernetes/sandwich-contexts-kubernetes.cabal @@ -29,6 +29,7 @@ library Test.Sandwich.Contexts.Kubernetes.SeaweedFS Test.Sandwich.Contexts.Kubernetes.Types other-modules: + Test.Sandwich.Contexts.Kubernetes.FindImages Test.Sandwich.Contexts.Kubernetes.KindCluster.Config Test.Sandwich.Contexts.Kubernetes.KindCluster.Network Test.Sandwich.Contexts.Kubernetes.KindCluster.ServiceForwardIngress @@ -69,7 +70,6 @@ library build-depends: aeson , base - , base64-bytestring , bytestring , containers , exceptions @@ -78,6 +78,7 @@ library , http-client , kubernetes-client , kubernetes-client-core + , lens , minio-hs , monad-logger , network diff --git a/sandwich-contexts-kubernetes/test/Spec.hs b/sandwich-contexts-kubernetes/test/Spec.hs index dbd23a06..86a3ed49 100644 --- a/sandwich-contexts-kubernetes/test/Spec.hs +++ b/sandwich-contexts-kubernetes/test/Spec.hs @@ -101,35 +101,36 @@ loadImageTests' = do -- withKubernetesNamespace' (toText namespace) $ let namespace = "default" - runWithKubectl $ \kubectlBinary env -> do - -- Wait for service account to exist; see - -- https://github.com/kubernetes/kubernetes/issues/66689 - waitUntil 60 $ - createProcessWithLogging ((proc kubectlBinary ["--namespace", namespace - , "get", "serviceaccount", "default" - , "-o", "name"]) { env = Just env }) - >>= waitForProcess >>= (`shouldBe` ExitSuccess) - - let deletePod = createProcessWithLogging ((proc kubectlBinary ["--namespace", namespace - , "delete", "pod", podName]) { env = Just env }) - >>= waitForProcess >>= (`shouldBe` ExitSuccess) - - flip finally deletePod $ do - createProcessWithLogging ((proc kubectlBinary ["--namespace", namespace - , "run", podName - , "--image", toString image - , "--image-pull-policy=IfNotPresent" - , "--command", "--", "/bin/sh", "-c", "sleep infinity" - ]) { env = Just env }) - >>= waitForProcess >>= (`shouldBe` ExitSuccess) - - waitUntil 300 $ do - events <- readCreateProcessWithLogging ((proc kubectlBinary ["--namespace", namespace - , "get", "events" - , "--field-selector", [i|involvedObject.kind=Pod,involvedObject.name=#{podName}|] - ]) { env = Just env }) "" - info [i|events: #{events}|] - events `shouldContain` "already present on machine" + (kubectlBinary, env) <- runWithKubectl + + -- Wait for service account to exist; see + -- https://github.com/kubernetes/kubernetes/issues/66689 + waitUntil 60 $ + createProcessWithLogging ((proc kubectlBinary ["--namespace", namespace + , "get", "serviceaccount", "default" + , "-o", "name"]) { env = Just env }) + >>= waitForProcess >>= (`shouldBe` ExitSuccess) + + let deletePod = createProcessWithLogging ((proc kubectlBinary ["--namespace", namespace + , "delete", "pod", podName]) { env = Just env }) + >>= waitForProcess >>= (`shouldBe` ExitSuccess) + + flip finally deletePod $ do + createProcessWithLogging ((proc kubectlBinary ["--namespace", namespace + , "run", podName + , "--image", toString image + , "--image-pull-policy=IfNotPresent" + , "--command", "--", "/bin/sh", "-c", "sleep infinity" + ]) { env = Just env }) + >>= waitForProcess >>= (`shouldBe` ExitSuccess) + + waitUntil 300 $ do + events <- readCreateProcessWithLogging ((proc kubectlBinary ["--namespace", namespace + , "get", "events" + , "--field-selector", [i|involvedObject.kind=Pod,involvedObject.name=#{podName}|] + ]) { env = Just env }) "" + info [i|events: #{events}|] + events `shouldContain` "already present on machine"