From 029e1e8c7cb1eab15a5eb09620619710d79a1b02 Mon Sep 17 00:00:00 2001 From: Aleksandr Tuliakov Date: Wed, 24 Apr 2024 10:16:44 +0000 Subject: [PATCH] Merge pull request #1199 in CLOUD/terraform-provider-yandex-mirror from feature/opensearch/host-list to master Squashed commit of the following: commit e8359cb1c6cd9fc5b53a80a9697d147f429d7ede Author: Aleksandr Tuliakov Date: Fri Apr 5 15:52:17 2024 +0200 add compute field "hosts" for resource and data_source --- .../ENHANCEMENTS-20240418-211916.yaml | 3 + ...ource_mdb_opensearch_cluster.html.markdown | 17 +++ .../r/mdb_opensearch_cluster.html.markdown | 18 +++ ...ta_source_yandex_mdb_opensearch_cluster.go | 41 +++++- ...urce_yandex_mdb_opensearch_cluster_test.go | 5 +- yandex/mdb_opensearch_structures.go | 76 +++++----- .../resource_yandex_mdb_opensearch_cluster.go | 135 +++++++++++++++++- ...urce_yandex_mdb_opensearch_cluster_test.go | 15 +- 8 files changed, 271 insertions(+), 39 deletions(-) create mode 100644 .changes/unreleased/ENHANCEMENTS-20240418-211916.yaml diff --git a/.changes/unreleased/ENHANCEMENTS-20240418-211916.yaml b/.changes/unreleased/ENHANCEMENTS-20240418-211916.yaml new file mode 100644 index 00000000..8c79d809 --- /dev/null +++ b/.changes/unreleased/ENHANCEMENTS-20240418-211916.yaml @@ -0,0 +1,3 @@ +kind: ENHANCEMENTS +body: 'opensearch: add `hosts` computed attribute' +time: 2024-04-18T21:19:16.797354+02:00 diff --git a/website/docs/d/datasource_mdb_opensearch_cluster.html.markdown b/website/docs/d/datasource_mdb_opensearch_cluster.html.markdown index 7b52b531..d9f8c18e 100644 --- a/website/docs/d/datasource_mdb_opensearch_cluster.html.markdown +++ b/website/docs/d/datasource_mdb_opensearch_cluster.html.markdown @@ -48,6 +48,7 @@ exported: * `health` - Aggregated health of the cluster. * `status` - Status of the cluster. * `config` - Configuration of the OpenSearch cluster. The structure is documented below. +* `hosts` - A hosts of the OpenSearch cluster. The structure is documented below. * `security_group_ids` - A set of ids of security groups assigned to hosts of the cluster. * `service_account_id` - ID of the service account authorized for this cluster. @@ -102,6 +103,22 @@ The `resources` block supports: * `disk_size` - Volume of the storage available to a OpenSearch host, in gigabytes. * `disk_type_id` - Type of the storage of OpenSearch hosts. +The `hosts` block supports: + +* `fqdn` - The fully qualified domain name of the host. + +* `zone` - The availability zone where the OpenSearch host was created. + For more information see [the official documentation](https://cloud.yandex.com/docs/overview/concepts/geo-scope). + +* `type` - The type of the deployed host. Can be either `OPENSEARCH` or `DASHBOARDS`. + +* `roles` - The roles of the deployed host. Can contain `DATA` and/or `MANAGER` roles. Will be empty for `DASHBOARDS` type. + +* `subnet_id` - The ID of the subnet, to which the host belongs. The subnet must + be a part of the network to which the cluster belongs. + +* `assign_public_ip` - Sets whether the host should get a public IP address. Can be either `true` or `false`. + The `maintenance_window` block supports: * `type` - Type of a maintenance window. Can be either `ANYTIME` or `WEEKLY`. A day and hour need to be specified with the weekly window. diff --git a/website/docs/r/mdb_opensearch_cluster.html.markdown b/website/docs/r/mdb_opensearch_cluster.html.markdown index 1d6189bd..0eb2cc54 100644 --- a/website/docs/r/mdb_opensearch_cluster.html.markdown +++ b/website/docs/r/mdb_opensearch_cluster.html.markdown @@ -264,6 +264,24 @@ In addition to the arguments listed above, the following computed attributes are * `status` - Status of the cluster. Can be either `CREATING`, `STARTING`, `RUNNING`, `UPDATING`, `STOPPING`, `STOPPED`, `ERROR` or `STATUS_UNKNOWN`. For more information see `status` field of JSON representation in [the official documentation](https://cloud.yandex.com/docs/managed-opensearch/api-ref/Cluster/). +* `hosts` - A hosts of the OpenSearch cluster. The structure is documented below. + +The `hosts` block supports: + +* `fqdn` - The fully qualified domain name of the host. + +* `zone` - The availability zone where the OpenSearch host will be created. + For more information see [the official documentation](https://cloud.yandex.com/docs/overview/concepts/geo-scope). + +* `type` - The type of the deployed host. Can be either `OPENSEARCH` or `DASHBOARDS`. + +* `roles` - The roles of the deployed host. Can contain `DATA` and/or `MANAGER` roles. Will be empty for `DASHBOARDS` type. + +* `subnet_id` - The ID of the subnet, to which the host belongs. The subnet must + be a part of the network to which the cluster belongs. + +* `assign_public_ip` - Sets whether the host should get a public IP address. Can be either `true` or `false`. + ## Import A cluster can be imported using the `id` of the resource, e.g. diff --git a/yandex/data_source_yandex_mdb_opensearch_cluster.go b/yandex/data_source_yandex_mdb_opensearch_cluster.go index 0427a863..4a646299 100644 --- a/yandex/data_source_yandex_mdb_opensearch_cluster.go +++ b/yandex/data_source_yandex_mdb_opensearch_cluster.go @@ -3,9 +3,10 @@ package yandex import ( "context" "fmt" - "github.com/yandex-cloud/go-genproto/yandex/cloud/mdb/opensearch/v1" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/yandex-cloud/go-genproto/yandex/cloud/mdb/opensearch/v1" + "github.com/yandex-cloud/go-sdk/sdkresolvers" ) @@ -211,6 +212,44 @@ func dataSourceYandexMDBOpenSearchCluster() *schema.Resource { }, }, + "hosts": { + Type: schema.TypeSet, + Computed: true, + Set: opensearchHostFQDNHash, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "fqdn": { + Type: schema.TypeString, + Computed: true, + }, + "zone": { + Type: schema.TypeString, + Computed: true, + }, + "type": { + Type: schema.TypeString, + Computed: true, + }, + "roles": { + Type: schema.TypeSet, + Computed: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + }, + "assign_public_ip": { + Type: schema.TypeBool, + Optional: true, + Computed: true, + }, + "subnet_id": { + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + }, + }, + }, + "network_id": { Type: schema.TypeString, Computed: true, diff --git a/yandex/data_source_yandex_mdb_opensearch_cluster_test.go b/yandex/data_source_yandex_mdb_opensearch_cluster_test.go index 79cd937e..ba1bd94a 100644 --- a/yandex/data_source_yandex_mdb_opensearch_cluster_test.go +++ b/yandex/data_source_yandex_mdb_opensearch_cluster_test.go @@ -80,7 +80,7 @@ func testAccDataSourceMDBOpenSearchClusterAttributesCheck(datasourceName string, "description", "labels", "environment", - "host", + "hosts", "config", "security_group_ids", "service_account_id", @@ -120,6 +120,9 @@ func testAccDataSourceMDBOpenSearchClusterCheck(datasourceName string, resourceN resource.TestCheckResourceAttr(datasourceName, "config.#", "1"), resource.TestCheckResourceAttrSet(datasourceName, "service_account_id"), resource.TestCheckResourceAttr(datasourceName, "deletion_protection", "false"), + resource.TestCheckResourceAttr(openSearchResource, "hosts.#", "2"), + resource.TestCheckResourceAttrSet(openSearchResource, "hosts.0.fqdn"), + resource.TestCheckResourceAttrSet(openSearchResource, "hosts.1.fqdn"), testAccCheckCreatedAtAttr(datasourceName), resource.TestCheckResourceAttr(datasourceName, "maintenance_window.0.type", "WEEKLY"), resource.TestCheckResourceAttr(datasourceName, "maintenance_window.0.day", "FRI"), diff --git a/yandex/mdb_opensearch_structures.go b/yandex/mdb_opensearch_structures.go index c90de485..edfff2a7 100644 --- a/yandex/mdb_opensearch_structures.go +++ b/yandex/mdb_opensearch_structures.go @@ -1,7 +1,6 @@ package yandex import ( - "context" "fmt" "reflect" "strings" @@ -633,27 +632,6 @@ func copyDashboardsNodeGroupsData(oldGroups []*opensearch.DashboardsCreateSpec_N return modified } -func opensearchNodeGroupsDiffCustomize(ctx context.Context, rdiff *schema.ResourceDiff, _ interface{}) error { - oc, nc := rdiff.GetChange("config") - if oc == nil { - if nc == nil { - return fmt.Errorf("Missing required option: config") - } - } - - var ( - oldConfig = expandOpenSearchConfigCreateSpec(oc) - newConfig = expandOpenSearchConfigCreateSpec(nc) - ) - - if modifyConfig(oldConfig, newConfig) { - flattened := flattenOpenSearchConfigCreateSpec(newConfig) - return rdiff.SetNew("config", flattened) - } - - return nil -} - func modifyConfig(oldConfig, newConfig *opensearch.ConfigCreateSpec) bool { var modified bool @@ -664,19 +642,51 @@ func modifyConfig(oldConfig, newConfig *opensearch.ConfigCreateSpec) bool { return false } - if oldConfig != nil { - if copyOpenSearchNodeGroupsData(oldConfig.GetOpensearchSpec().GetNodeGroups(), newConfig.GetOpensearchSpec().GetNodeGroups()) { - modified = true - } - if copyDashboardsNodeGroupsData(oldConfig.GetDashboardsSpec().GetNodeGroups(), newConfig.GetDashboardsSpec().GetNodeGroups()) { - modified = true - } + if copyOpenSearchNodeGroupsData(oldConfig.GetOpensearchSpec().GetNodeGroups(), newConfig.GetOpensearchSpec().GetNodeGroups()) { + modified = true + } + if copyDashboardsNodeGroupsData(oldConfig.GetDashboardsSpec().GetNodeGroups(), newConfig.GetDashboardsSpec().GetNodeGroups()) { + modified = true + } - if newConfig.GetOpensearchSpec().GetPlugins() == nil || len(oldConfig.GetOpensearchSpec().GetPlugins()) == 0 { - newConfig.OpensearchSpec.Plugins = oldConfig.GetOpensearchSpec().Plugins - modified = true - } + if newConfig.GetOpensearchSpec().GetPlugins() == nil || len(oldConfig.GetOpensearchSpec().GetPlugins()) == 0 { + newConfig.OpensearchSpec.Plugins = oldConfig.GetOpensearchSpec().Plugins + modified = true } return modified } + +func opensearchHostFQDNHash(v interface{}) int { + m := v.(map[string]interface{}) + + if n, ok := m["fqdn"]; ok { + return hashcode.String(n.(string)) + } + return 0 +} + +func flattenOpensearchHosts(hosts []*opensearch.Host) []interface{} { + res := []interface{}{} + + for _, h := range hosts { + res = append(res, map[string]interface{}{ + "type": h.Type.String(), + "roles": mapRoles(h.Roles), + "zone": h.ZoneId, + "subnet_id": h.SubnetId, + "assign_public_ip": h.AssignPublicIp, + "fqdn": h.Name, + }) + } + + return res +} + +func mapRoles(roles []opensearch.OpenSearch_GroupRole) []string { + res := make([]string, 0, len(roles)) + for _, role := range roles { + res = append(res, role.String()) + } + return res +} diff --git a/yandex/resource_yandex_mdb_opensearch_cluster.go b/yandex/resource_yandex_mdb_opensearch_cluster.go index 96fcc3c8..d62b41dd 100644 --- a/yandex/resource_yandex_mdb_opensearch_cluster.go +++ b/yandex/resource_yandex_mdb_opensearch_cluster.go @@ -3,13 +3,15 @@ package yandex import ( "context" "fmt" + "log" + "reflect" + "time" + "github.com/yandex-cloud/go-genproto/yandex/cloud/mdb/opensearch/v1" "github.com/yandex-cloud/go-genproto/yandex/cloud/operation" sdkoperation "github.com/yandex-cloud/go-sdk/operation" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "log" - "time" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" @@ -271,6 +273,45 @@ func resourceYandexMDBOpenSearchCluster() *schema.Resource { Computed: true, }, + // Current nodes in the cluster + "hosts": { + Type: schema.TypeSet, + Computed: true, + Set: opensearchHostFQDNHash, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "fqdn": { + Type: schema.TypeString, + Computed: true, + }, + "zone": { + Type: schema.TypeString, + Computed: true, + }, + "type": { + Type: schema.TypeString, + Computed: true, + }, + "roles": { + Type: schema.TypeSet, + Computed: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + }, + "assign_public_ip": { + Type: schema.TypeBool, + Optional: true, + Computed: true, + }, + "subnet_id": { + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + }, + }, + }, + // User security groups "security_group_ids": { Type: schema.TypeSet, @@ -387,6 +428,17 @@ func resourceYandexMDBOpenSearchClusterReadEx(d *schema.ResourceData, meta inter return err } + actualHosts, err := listOpensearchHosts(ctx, config, d.Id()) + if err != nil { + return err + } + + result := flattenOpensearchHosts(actualHosts) + + if err := d.Set("hosts", result); err != nil { + return err + } + d.Set("service_account_id", cluster.GetServiceAccountId()) d.Set("deletion_protection", cluster.GetDeletionProtection()) @@ -949,3 +1001,82 @@ func makeDeleteDashboardsNodeGroupRequest(req *opensearch.DeleteDashboardsNodeGr } return nil } + +func listOpensearchHosts(ctx context.Context, config *Config, clusterID string) ([]*opensearch.Host, error) { + hosts := []*opensearch.Host{} + pageToken := "" + for { + resp, err := config.sdk.MDB().OpenSearch().Cluster().ListHosts(ctx, &opensearch.ListClusterHostsRequest{ + ClusterId: clusterID, + PageSize: defaultMDBPageSize, + PageToken: pageToken, + }) + if err != nil { + return nil, fmt.Errorf("error while getting list of hosts for '%s': %s", clusterID, err) + } + hosts = append(hosts, resp.Hosts...) + if resp.NextPageToken == "" || resp.NextPageToken == "0" { + break + } + pageToken = resp.NextPageToken + } + return hosts, nil +} + +func opensearchNodeGroupsDiffCustomize(ctx context.Context, rdiff *schema.ResourceDiff, meta interface{}) error { + oc, nc := rdiff.GetChange("config") + if oc == nil { + if nc == nil { + return fmt.Errorf("Missing required option: config") + } + } + + var ( + oldConfig = expandOpenSearchConfigCreateSpec(oc) + newConfig = expandOpenSearchConfigCreateSpec(nc) + ) + + if isNodeGroupsChanged(oldConfig, newConfig) { + err := rdiff.SetNewComputed("hosts") + if err != nil { + return err + } + } + + if modifyConfig(oldConfig, newConfig) { + flattened := flattenOpenSearchConfigCreateSpec(newConfig) + err := rdiff.SetNew("config", flattened) + if err != nil { + return err + } + } + + return nil +} + +func isNodeGroupsChanged(oldConfig, newConfig *opensearch.ConfigCreateSpec) bool { + if (oldConfig == nil || newConfig == nil) || + (oldConfig.OpensearchSpec == nil || newConfig.OpensearchSpec == nil) || + (oldConfig.DashboardsSpec == nil || newConfig.DashboardsSpec == nil) { + + return false + } + + if len(oldConfig.OpensearchSpec.NodeGroups) != len(newConfig.OpensearchSpec.NodeGroups) { + return true + } + + if len(oldConfig.DashboardsSpec.NodeGroups) != len(newConfig.DashboardsSpec.NodeGroups) { + return true + } + + if !reflect.DeepEqual(oldConfig.OpensearchSpec.NodeGroups, newConfig.OpensearchSpec.NodeGroups) { + return true + } + + if !reflect.DeepEqual(oldConfig.DashboardsSpec.NodeGroups, newConfig.DashboardsSpec.NodeGroups) { + return true + } + + return false +} diff --git a/yandex/resource_yandex_mdb_opensearch_cluster_test.go b/yandex/resource_yandex_mdb_opensearch_cluster_test.go index 99cbf63a..a27d1bc4 100644 --- a/yandex/resource_yandex_mdb_opensearch_cluster_test.go +++ b/yandex/resource_yandex_mdb_opensearch_cluster_test.go @@ -117,7 +117,9 @@ func TestAccMDBOpenSearchCluster_basic(t *testing.T) { resource.TestCheckResourceAttr(openSearchResource, "config.0.admin_password", "password"), resource.TestCheckResourceAttrSet(openSearchResource, "service_account_id"), resource.TestCheckResourceAttr(openSearchResource, "deletion_protection", "true"), - //$resource.TestCheckResourceAttrSet(openSearchResource, "host.0.fqdn"), + resource.TestCheckResourceAttr(openSearchResource, "hosts.#", "2"), + resource.TestCheckResourceAttrSet(openSearchResource, "hosts.0.fqdn"), + resource.TestCheckResourceAttrSet(openSearchResource, "hosts.1.fqdn"), testAccCheckCreatedAtAttr(openSearchResource), testAccCheckMDBOpenSearchClusterContainsLabel(&r, "test_key", "test_value"), testAccCheckMDBOpenSearchClusterDataNodeHasResources(&r, "s2.micro", "network-ssd", 10*1024*1024*1024), @@ -174,6 +176,12 @@ func TestAccMDBOpenSearchCluster_basic(t *testing.T) { resource.TestCheckResourceAttr(openSearchResource, "folder_id", folderID), resource.TestCheckResourceAttr(openSearchResource, "description", openSearchDesc2), resource.TestCheckResourceAttr(openSearchResource, "service_account_id", ""), + resource.TestCheckResourceAttr(openSearchResource, "hosts.#", "5"), + resource.TestCheckResourceAttrSet(openSearchResource, "hosts.0.fqdn"), + resource.TestCheckResourceAttrSet(openSearchResource, "hosts.1.fqdn"), + resource.TestCheckResourceAttrSet(openSearchResource, "hosts.2.fqdn"), + resource.TestCheckResourceAttrSet(openSearchResource, "hosts.3.fqdn"), + resource.TestCheckResourceAttrSet(openSearchResource, "hosts.4.fqdn"), testAccCheckCreatedAtAttr(openSearchResource), testAccCheckMDBOpenSearchClusterContainsLabel(&r, "test_key2", "test_value2"), testAccCheckMDBOpenSearchClusterDataNodeHasResources(&r, "s2.small", "network-ssd", 11*1024*1024*1024), @@ -186,12 +194,14 @@ func TestAccMDBOpenSearchCluster_basic(t *testing.T) { resource.TestCheckResourceAttr(openSearchResource, "maintenance_window.0.type", "ANYTIME"), ), }, + //TODO: add step with changing roles after fix/implement https://st.yandex-team.ru/MDB-28703 mdbOpenSearchClusterImportStep(openSearchResource), //Add nodegroups { Config: testAccMDBOpenSearchClusterConfigWithManagerGroup(openSearchName, openSearchDesc2, randInt), Check: resource.ComposeTestCheckFunc( testAccCheckMDBOpenSearchClusterExists(openSearchResource, &r, 12), + resource.TestCheckResourceAttr(openSearchResource, "hosts.#", "12"), testAccCheckCreatedAtAttr(openSearchResource), func(s *terraform.State) error { time.Sleep(time.Minute * 5) @@ -205,6 +215,7 @@ func TestAccMDBOpenSearchCluster_basic(t *testing.T) { Config: testAccMDBOpenSearchClusterConfigRemoveGroup(openSearchName, openSearchDesc2, randInt), Check: resource.ComposeTestCheckFunc( testAccCheckMDBOpenSearchClusterExists(openSearchResource, &r, 11), + resource.TestCheckResourceAttr(openSearchResource, "hosts.#", "11"), testAccCheckCreatedAtAttr(openSearchResource), func(s *terraform.State) error { time.Sleep(time.Minute * 5) @@ -670,7 +681,7 @@ resource "yandex_mdb_opensearch_cluster" "foo" { "${yandex_vpc_subnet.mdb-opensearch-test-subnet-b.id}", "${yandex_vpc_subnet.mdb-opensearch-test-subnet-d.id}", ] - roles = ["DATA"] + roles = ["DATA", "MANAGER"] resources { resource_preset_id = "s2.small" disk_size = 11811160064