From c0611683e5532b27a225a5e47ff9f1fcdca30828 Mon Sep 17 00:00:00 2001 From: doodgeMatvey Date: Thu, 13 Jul 2023 00:14:27 +0300 Subject: [PATCH] issue-467, use the latest clusters version for packaged provisioning was implemented --- apis/clusters/v1beta1/opensearch_types.go | 2 +- ...clusters.instaclustr.com_opensearches.yaml | 2 - config/samples/clusters_v1beta1_cadence.yaml | 42 ++++++------- controllers/clusters/cadence_controller.go | 61 ++++++++++++++++--- controllers/clusters/helpers.go | 21 +++++++ go.mod | 1 + go.sum | 2 + pkg/instaclustr/interfaces.go | 1 + pkg/models/operator.go | 7 +-- 9 files changed, 101 insertions(+), 38 deletions(-) diff --git a/apis/clusters/v1beta1/opensearch_types.go b/apis/clusters/v1beta1/opensearch_types.go index 141084dc1..9c79c8b30 100644 --- a/apis/clusters/v1beta1/opensearch_types.go +++ b/apis/clusters/v1beta1/opensearch_types.go @@ -48,7 +48,7 @@ type OpenSearchSpec struct { NotificationsPlugin bool `json:"notificationsPlugin,omitempty"` AnomalyDetectionPlugin bool `json:"anomalyDetectionPlugin,omitempty"` LoadBalancer bool `json:"loadBalancer,omitempty"` - ClusterManagerNodes []*ClusterManagerNodes `json:"clusterManagerNodes"` + ClusterManagerNodes []*ClusterManagerNodes `json:"clusterManagerNodes,omitempty"` IndexManagementPlugin bool `json:"indexManagementPlugin,omitempty"` AlertingPlugin bool `json:"alertingPlugin,omitempty"` BundledUseOnly bool `json:"bundleUseOnly,omitempty"` diff --git a/config/crd/bases/clusters.instaclustr.com_opensearches.yaml b/config/crd/bases/clusters.instaclustr.com_opensearches.yaml index 63214ffd1..0e1715068 100644 --- a/config/crd/bases/clusters.instaclustr.com_opensearches.yaml +++ b/config/crd/bases/clusters.instaclustr.com_opensearches.yaml @@ -224,8 +224,6 @@ spec: type: object version: type: string - required: - - clusterManagerNodes type: object status: description: OpenSearchStatus defines the observed state of OpenSearch diff --git a/config/samples/clusters_v1beta1_cadence.yaml b/config/samples/clusters_v1beta1_cadence.yaml index 323a03a7a..adec194c5 100644 --- a/config/samples/clusters_v1beta1_cadence.yaml +++ b/config/samples/clusters_v1beta1_cadence.yaml @@ -12,31 +12,31 @@ metadata: name: cadence-sample spec: name: "operatorCadence" - version: "0.24.0" + version: "1.0.0" standardProvisioning: - targetCassandra: dependencyCdcId: "9d43ac54-7317-4ce5-859a-e9d0443508a4" dependencyVpcType: "VPC_PEERED" - # packagedProvisioning: - # - bundledCassandraSpec: - # nodeSize: "CAS-DEV-t4g.small-5" - # network: "10.2.0.0/16" - # replicationFactor: 3 - # nodesNumber: 3 - # privateIPBroadcastForDiscovery: false - # passwordAndUserAuth: true - # useAdvancedVisibility: true - # bundledKafkaSpec: - # nodeSize: "KFK-DEV-t4g.small-5" - # nodesNumber: 3 - # network: "10.3.0.0/16" - # replicationFactor: 3 - # partitionsNumber: 3 - # bundledOpenSearchSpec: - # nodeSize: "SRH-DEV-t4g.small-5" - # replicationFactor: 3 - # nodesPerReplicationFactor: 1 - # network: "10.4.0.0/16" +# packagedProvisioning: +# - bundledCassandraSpec: +# nodeSize: "CAS-DEV-t4g.small-5" +# network: "10.2.0.0/16" +# replicationFactor: 3 +# nodesNumber: 3 +# privateIPBroadcastForDiscovery: false +# passwordAndUserAuth: true +# useAdvancedVisibility: true +# bundledKafkaSpec: +# nodeSize: "KFK-DEV-t4g.small-5" +# nodesNumber: 3 +# network: "10.3.0.0/16" +# replicationFactor: 3 +# partitionsNumber: 3 +# bundledOpenSearchSpec: +# nodeSize: "SRH-DEV-t4g.small-5" +# replicationFactor: 3 +# nodesPerReplicationFactor: 3 +# network: "10.4.0.0/16" # twoFactorDelete: # - email: "rostyslp@netapp.com" privateNetworkCluster: false diff --git a/controllers/clusters/cadence_controller.go b/controllers/clusters/cadence_controller.go index 9f7ff4197..239702d99 100644 --- a/controllers/clusters/cadence_controller.go +++ b/controllers/clusters/cadence_controller.go @@ -527,7 +527,19 @@ func (r *CadenceReconciler) preparePackagedSolution( } if len(cassandraList.Items) == 0 { - cassandraSpec, err := r.newCassandraSpec(cluster) + appVersions, err := r.API.ListAppVersions(models.CassandraAppKind) + if err != nil { + return false, fmt.Errorf("cannot list versions for kind: %v, err: %w", + models.CassandraAppKind, err) + } + + cassandraVersions := getSortedAppVersions(appVersions, models.CassandraAppType) + if len(cassandraVersions) == 0 { + return false, fmt.Errorf("there are no versions for %v kind", + models.CassandraAppKind) + } + + cassandraSpec, err := r.newCassandraSpec(cluster, cassandraVersions[len(cassandraVersions)-1].String()) if err != nil { return false, err } @@ -553,7 +565,19 @@ func (r *CadenceReconciler) preparePackagedSolution( return false, err } if len(kafkaList.Items) == 0 { - kafkaSpec, err := r.newKafkaSpec(cluster) + appVersions, err := r.API.ListAppVersions(models.KafkaAppKind) + if err != nil { + return false, fmt.Errorf("cannot list versions for kind: %v, err: %w", + models.KafkaAppKind, err) + } + + kafkaVersions := getSortedAppVersions(appVersions, models.KafkaAppType) + if len(kafkaVersions) == 0 { + return false, fmt.Errorf("there are no versions for %v kind", + models.KafkaAppType) + } + + kafkaSpec, err := r.newKafkaSpec(cluster, kafkaVersions[len(kafkaVersions)-1].String()) if err != nil { return false, err } @@ -578,7 +602,20 @@ func (r *CadenceReconciler) preparePackagedSolution( return false, err } if len(osList.Items) == 0 { - osSpec, err := r.newOpenSearchSpec(cluster) + appVersions, err := r.API.ListAppVersions(models.OpenSearchAppKind) + if err != nil { + return false, fmt.Errorf("cannot list versions for kind: %v, err: %w", + models.OpenSearchAppKind, err) + } + + openSearchVersions := getSortedAppVersions(appVersions, models.OpenSearchAppType) + if len(openSearchVersions) == 0 { + return false, fmt.Errorf("there are no versions for %v kind", + models.OpenSearchAppType) + } + + // For OpenSearch we cannot use the latest version because is not supported by Cadence. So we use the oldest one. + osSpec, err := r.newOpenSearchSpec(cluster, openSearchVersions[0].String()) if err != nil { return false, err } @@ -615,7 +652,7 @@ func (r *CadenceReconciler) preparePackagedSolution( return false, nil } -func (r *CadenceReconciler) newCassandraSpec(cadence *v1beta1.Cadence) (*v1beta1.Cassandra, error) { +func (r *CadenceReconciler) newCassandraSpec(cadence *v1beta1.Cadence, latestCassandraVersion string) (*v1beta1.Cassandra, error) { typeMeta := v1.TypeMeta{ Kind: models.CassandraKind, APIVersion: models.ClustersV1beta1APIVersion, @@ -691,7 +728,7 @@ func (r *CadenceReconciler) newCassandraSpec(cadence *v1beta1.Cadence) (*v1beta1 spec := v1beta1.CassandraSpec{ Cluster: v1beta1.Cluster{ Name: models.CassandraChildPrefix + cadence.Name, - Version: models.CassandraV3_11_13, + Version: latestCassandraVersion, SLATier: slaTier, PrivateNetworkCluster: privateClusterNetwork, TwoFactorDelete: twoFactorDelete, @@ -884,7 +921,7 @@ func (r *CadenceReconciler) newWatchStatusJob(cadence *v1beta1.Cadence) schedule } } -func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence) (*v1beta1.Kafka, error) { +func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence, latestKafkaVersion string) (*v1beta1.Kafka, error) { typeMeta := v1.TypeMeta{ Kind: models.KafkaKind, APIVersion: models.ClustersV1beta1APIVersion, @@ -950,7 +987,7 @@ func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence) (*v1beta1.Kaf spec := v1beta1.KafkaSpec{ Cluster: v1beta1.Cluster{ Name: models.KafkaChildPrefix + cadence.Name, - Version: models.KafkaV3_1_2, + Version: latestKafkaVersion, SLATier: slaTier, PrivateNetworkCluster: privateClusterNetwork, TwoFactorDelete: kafkaTFD, @@ -971,7 +1008,7 @@ func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence) (*v1beta1.Kaf }, nil } -func (r *CadenceReconciler) newOpenSearchSpec(cadence *v1beta1.Cadence) (*v1beta1.OpenSearch, error) { +func (r *CadenceReconciler) newOpenSearchSpec(cadence *v1beta1.Cadence, oldestOpenSearchVersion string) (*v1beta1.OpenSearch, error) { typeMeta := v1.TypeMeta{ Kind: models.OpenSearchKind, APIVersion: models.ClustersV1beta1APIVersion, @@ -1039,12 +1076,18 @@ func (r *CadenceReconciler) newOpenSearchSpec(cadence *v1beta1.Cadence) (*v1beta spec := v1beta1.OpenSearchSpec{ Cluster: v1beta1.Cluster{ Name: models.OpenSearchChildPrefix + cadence.Name, - Version: models.OpenSearchV1_3_7, + Version: oldestOpenSearchVersion, SLATier: slaTier, PrivateNetworkCluster: privateClusterNetwork, TwoFactorDelete: twoFactorDelete, PCICompliance: pciCompliance, }, + ClusterManagerNodes: []*v1beta1.ClusterManagerNodes{ + { + NodeSize: osNodeSize, + DedicatedManager: false, + }, + }, DataCentres: osDataCentres, } diff --git a/controllers/clusters/helpers.go b/controllers/clusters/helpers.go index b9ae2e2b8..92e2b55e9 100644 --- a/controllers/clusters/helpers.go +++ b/controllers/clusters/helpers.go @@ -17,6 +17,9 @@ limitations under the License. package clusters import ( + "sort" + + "github.com/hashicorp/go-version" "k8s.io/utils/strings/slices" "sigs.k8s.io/controller-runtime/pkg/client" @@ -134,6 +137,24 @@ func isClusterActive(clusterID string, activeClusters []*models.ActiveClusters) return false } +func getSortedAppVersions(versions []*models.AppVersions, appType string) []*version.Version { + for _, apps := range versions { + if apps.Application == appType { + newVersions := make([]*version.Version, len(apps.Versions)) + for i, versionStr := range apps.Versions { + v, _ := version.NewVersion(versionStr) + newVersions[i] = v + } + + sort.Sort(version.Collection(newVersions)) + + return newVersions + } + } + + return nil +} + var msgDeleteClusterWithTwoFactorDelete = "Please confirm cluster deletion via email or phone. " + "If you have canceled a cluster deletion and want to put the cluster on deletion again, " + "remove \"triggered\" from Instaclustr.com/clusterDeletion annotation." diff --git a/go.mod b/go.mod index c106dc1fa..22ee4f0fa 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( github.com/go-logr/logr v1.2.0 github.com/gorilla/mux v1.8.0 + github.com/hashicorp/go-version v1.6.0 github.com/onsi/ginkgo/v2 v2.0.0 github.com/onsi/gomega v1.18.1 k8s.io/api v0.24.2 diff --git a/go.sum b/go.sum index 50934d6eb..0af4ef4f3 100644 --- a/go.sum +++ b/go.sum @@ -277,6 +277,8 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= diff --git a/pkg/instaclustr/interfaces.go b/pkg/instaclustr/interfaces.go index 637036a18..7a41eaea1 100644 --- a/pkg/instaclustr/interfaces.go +++ b/pkg/instaclustr/interfaces.go @@ -94,4 +94,5 @@ type API interface { DeleteEncryptionKey(encryptionKeyID string) error CreateUser(userSpec any, clusterID, app string) error DeleteUser(username, clusterID, app string) error + ListAppVersions(app string) ([]*models.AppVersions, error) } diff --git a/pkg/models/operator.go b/pkg/models/operator.go index b55abf4cd..45b4c5017 100644 --- a/pkg/models/operator.go +++ b/pkg/models/operator.go @@ -53,11 +53,8 @@ const ( OpenSearchChildPrefix = "opensearch-" OpenSearchChildDCName = "opensearch-cadence-dc" - CassandraV3_11_13 = "3.11.13" - KafkaV3_1_2 = "3.1.2" - OpenSearchV1_3_7 = "1.3.7" - K8sAPIVersionV1 = "v1" - VPCPeered = "VPC_PEERED" + K8sAPIVersionV1 = "v1" + VPCPeered = "VPC_PEERED" True = "true" False = "false"