Skip to content

Commit

Permalink
Merge pull request hashicorp#40199 from ryanmiville/f-aws_glue_catalo…
Browse files Browse the repository at this point in the history 
…g_table_optimizer-support-all-types

[Enhancement]: `r/aws_glue_catalog_table_optimizer` Add configuration support for `retention` and `orphan_file_deletion` types
  • Loading branch information
@johnsonaj
johnsonaj authored Dec 6, 2024
2 parents 2b9b162 + ced517a commit f382471
Show file tree
Hide file tree
Showing 5 changed files with 317 additions and 10 deletions.
3 changes: 3 additions & 0 deletions .changelog/40199.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
resource/aws_glue_catalog_table_optimizer: Add `configuration.retention_configuration` and `configuration.orphan_file_deletion_configuration` attributes.
```
82 changes: 80 additions & 2 deletions internal/service/glue/catalog_table_optimizer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,63 @@ func (r *resourceCatalogTableOptimizer) Schema(ctx context.Context, _ resource.S
Required: true,
},
},
Blocks: map[string]schema.Block{
"retention_configuration": schema.ListNestedBlock{
CustomType: fwtypes.NewListNestedObjectTypeOf[retentionConfigurationData](ctx),
Validators: []validator.List{
listvalidator.SizeAtMost(1),
},
NestedObject: schema.NestedBlockObject{
Blocks: map[string]schema.Block{
"iceberg_configuration": schema.ListNestedBlock{
CustomType: fwtypes.NewListNestedObjectTypeOf[icebergRetentionConfigurationData](ctx),
Validators: []validator.List{
listvalidator.SizeAtMost(1),
},
NestedObject: schema.NestedBlockObject{
Attributes: map[string]schema.Attribute{
"snapshot_retention_period_in_days": schema.Int32Attribute{
Optional: true,
},
"number_of_snapshots_to_retain": schema.Int32Attribute{
Optional: true,
},
"clean_expired_files": schema.BoolAttribute{
Optional: true,
},
},
},
},
},
},
},
"orphan_file_deletion_configuration": schema.ListNestedBlock{
CustomType: fwtypes.NewListNestedObjectTypeOf[orphanFileDeletionConfigurationData](ctx),
Validators: []validator.List{
listvalidator.SizeAtMost(1),
},
NestedObject: schema.NestedBlockObject{
Blocks: map[string]schema.Block{
"iceberg_configuration": schema.ListNestedBlock{
CustomType: fwtypes.NewListNestedObjectTypeOf[icebergOrphanFileDeletionConfigurationData](ctx),
Validators: []validator.List{
listvalidator.SizeAtMost(1),
},
NestedObject: schema.NestedBlockObject{
Attributes: map[string]schema.Attribute{
"orphan_file_retention_period_in_days": schema.Int32Attribute{
Optional: true,
},
names.AttrLocation: schema.StringAttribute{
Optional: true,
},
},
},
},
},
},
},
},
},
},
},
Expand Down Expand Up @@ -315,8 +372,29 @@ type resourceCatalogTableOptimizerData struct {
}

type configurationData struct {
Enabled types.Bool `tfsdk:"enabled"`
RoleARN fwtypes.ARN `tfsdk:"role_arn"`
Enabled types.Bool `tfsdk:"enabled"`
RoleARN fwtypes.ARN `tfsdk:"role_arn"`
RetentionConfiguration fwtypes.ListNestedObjectValueOf[retentionConfigurationData] `tfsdk:"retention_configuration"`
OrphanFileDeletionConfiguration fwtypes.ListNestedObjectValueOf[orphanFileDeletionConfigurationData] `tfsdk:"orphan_file_deletion_configuration"`
}

type retentionConfigurationData struct {
IcebergConfiguration fwtypes.ListNestedObjectValueOf[icebergRetentionConfigurationData] `tfsdk:"iceberg_configuration"`
}

type icebergRetentionConfigurationData struct {
SnapshotRetentionPeriodInDays types.Int32 `tfsdk:"snapshot_retention_period_in_days"`
NumberOfSnapshotsToRetain types.Int32 `tfsdk:"number_of_snapshots_to_retain"`
CleanExpiredFiles types.Bool `tfsdk:"clean_expired_files"`
}

type orphanFileDeletionConfigurationData struct {
IcebergConfiguration fwtypes.ListNestedObjectValueOf[icebergOrphanFileDeletionConfigurationData] `tfsdk:"iceberg_configuration"`
}

type icebergOrphanFileDeletionConfigurationData struct {
OrphanFileRetentionPeriodInDays types.Int32 `tfsdk:"orphan_file_retention_period_in_days"`
Location types.String `tfsdk:"location"`
}

func findCatalogTableOptimizer(ctx context.Context, conn *glue.Client, catalogID, dbName, tableName, optimizerType string) (*glue.GetTableOptimizerOutput, error) {
Expand Down
155 changes: 155 additions & 0 deletions internal/service/glue/catalog_table_optimizer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,110 @@ func testAccCatalogTableOptimizer_disappears(t *testing.T) {
})
}

func testAccCatalogTableOptimizer_RetentionConfiguration(t *testing.T) {
ctx := acctest.Context(t)
var catalogTableOptimizer glue.GetTableOptimizerOutput

resourceName := "aws_glue_catalog_table_optimizer.test"

rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix)

resource.Test(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(ctx, t) },
ErrorCheck: acctest.ErrorCheck(t, names.GlueServiceID),
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories,
CheckDestroy: testAccCheckCatalogTableOptimizerDestroy(ctx),
Steps: []resource.TestStep{
{
Config: testAccCatalogTableOptimizerConfig_retentionConfiguration(rName, 7),
Check: resource.ComposeTestCheckFunc(
testAccCheckCatalogTableOptimizerExists(ctx, resourceName, &catalogTableOptimizer),
acctest.CheckResourceAttrAccountID(ctx, resourceName, names.AttrCatalogID),
resource.TestCheckResourceAttr(resourceName, names.AttrDatabaseName, rName),
resource.TestCheckResourceAttr(resourceName, names.AttrTableName, rName),
resource.TestCheckResourceAttr(resourceName, names.AttrType, "retention"),
resource.TestCheckResourceAttr(resourceName, "configuration.0.enabled", acctest.CtTrue),
resource.TestCheckResourceAttr(resourceName, "configuration.0.retention_configuration.0.iceberg_configuration.0.snapshot_retention_period_in_days", "7"),
resource.TestCheckResourceAttr(resourceName, "configuration.0.retention_configuration.0.iceberg_configuration.0.number_of_snapshots_to_retain", "3"),
resource.TestCheckResourceAttr(resourceName, "configuration.0.retention_configuration.0.iceberg_configuration.0.clean_expired_files", acctest.CtTrue),
),
},
{
ResourceName: resourceName,
ImportStateIdFunc: testAccCatalogTableOptimizerStateIDFunc(resourceName),
ImportStateVerifyIdentifierAttribute: names.AttrTableName,
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccCatalogTableOptimizerConfig_retentionConfiguration(rName, 6),
Check: resource.ComposeTestCheckFunc(
testAccCheckCatalogTableOptimizerExists(ctx, resourceName, &catalogTableOptimizer),
acctest.CheckResourceAttrAccountID(ctx, resourceName, names.AttrCatalogID),
resource.TestCheckResourceAttr(resourceName, names.AttrDatabaseName, rName),
resource.TestCheckResourceAttr(resourceName, names.AttrTableName, rName),
resource.TestCheckResourceAttr(resourceName, names.AttrType, "retention"),
resource.TestCheckResourceAttr(resourceName, "configuration.0.enabled", acctest.CtTrue),
resource.TestCheckResourceAttr(resourceName, "configuration.0.retention_configuration.0.iceberg_configuration.0.snapshot_retention_period_in_days", "6"),
resource.TestCheckResourceAttr(resourceName, "configuration.0.retention_configuration.0.iceberg_configuration.0.number_of_snapshots_to_retain", "3"),
resource.TestCheckResourceAttr(resourceName, "configuration.0.retention_configuration.0.iceberg_configuration.0.clean_expired_files", acctest.CtTrue),
),
},
},
})
}

func testAccCatalogTableOptimizer_DeleteOrphanFileConfiguration(t *testing.T) {
ctx := acctest.Context(t)
var catalogTableOptimizer glue.GetTableOptimizerOutput

resourceName := "aws_glue_catalog_table_optimizer.test"

rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix)

resource.Test(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(ctx, t) },
ErrorCheck: acctest.ErrorCheck(t, names.GlueServiceID),
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories,
CheckDestroy: testAccCheckCatalogTableOptimizerDestroy(ctx),
Steps: []resource.TestStep{
{
Config: testAccCatalogTableOptimizerConfig_orphanFileDeletionConfiguration(rName, 7),
Check: resource.ComposeTestCheckFunc(
testAccCheckCatalogTableOptimizerExists(ctx, resourceName, &catalogTableOptimizer),
acctest.CheckResourceAttrAccountID(ctx, resourceName, names.AttrCatalogID),
resource.TestCheckResourceAttr(resourceName, names.AttrDatabaseName, rName),
resource.TestCheckResourceAttr(resourceName, names.AttrTableName, rName),
resource.TestCheckResourceAttr(resourceName, names.AttrType, "orphan_file_deletion"),
resource.TestCheckResourceAttr(resourceName, "configuration.0.enabled", acctest.CtTrue),
resource.TestCheckResourceAttr(resourceName, "configuration.0.orphan_file_deletion_configuration.0.iceberg_configuration.0.orphan_file_retention_period_in_days", "7"),
resource.TestCheckResourceAttr(resourceName, "configuration.0.orphan_file_deletion_configuration.0.iceberg_configuration.0.location", fmt.Sprintf("s3://%s/files/", rName)),
),
},
{
ResourceName: resourceName,
ImportStateIdFunc: testAccCatalogTableOptimizerStateIDFunc(resourceName),
ImportStateVerifyIdentifierAttribute: names.AttrTableName,
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccCatalogTableOptimizerConfig_orphanFileDeletionConfiguration(rName, 6),
Check: resource.ComposeTestCheckFunc(
testAccCheckCatalogTableOptimizerExists(ctx, resourceName, &catalogTableOptimizer),
acctest.CheckResourceAttrAccountID(ctx, resourceName, names.AttrCatalogID),
resource.TestCheckResourceAttr(resourceName, names.AttrDatabaseName, rName),
resource.TestCheckResourceAttr(resourceName, names.AttrTableName, rName),
resource.TestCheckResourceAttr(resourceName, names.AttrType, "orphan_file_deletion"),
resource.TestCheckResourceAttr(resourceName, "configuration.0.enabled", acctest.CtTrue),
resource.TestCheckResourceAttr(resourceName, "configuration.0.orphan_file_deletion_configuration.0.iceberg_configuration.0.orphan_file_retention_period_in_days", "6"),
resource.TestCheckResourceAttr(resourceName, "configuration.0.orphan_file_deletion_configuration.0.iceberg_configuration.0.location", fmt.Sprintf("s3://%s/files/", rName)),
),
},
},
})
}

func testAccCatalogTableOptimizerStateIDFunc(resourceName string) resource.ImportStateIdFunc {
return func(s *terraform.State) (string, error) {
rs, ok := s.RootModule().Resources[resourceName]
Expand Down Expand Up @@ -345,3 +449,54 @@ resource "aws_glue_catalog_table_optimizer" "test" {
}
`, enabled))
}

func testAccCatalogTableOptimizerConfig_retentionConfiguration(rName string, retentionPeriod int) string {
return acctest.ConfigCompose(
testAccCatalogTableOptimizerConfig_baseConfig(rName),
fmt.Sprintf(`
resource "aws_glue_catalog_table_optimizer" "test" {
catalog_id = data.aws_caller_identity.current.account_id
database_name = aws_glue_catalog_database.test.name
table_name = aws_glue_catalog_table.test.name
type = "retention"
configuration {
role_arn = aws_iam_role.test.arn
enabled = true
retention_configuration {
iceberg_configuration {
snapshot_retention_period_in_days = %[1]d
number_of_snapshots_to_retain = 3
clean_expired_files = true
}
}
}
}
`, retentionPeriod))
}

func testAccCatalogTableOptimizerConfig_orphanFileDeletionConfiguration(rName string, retentionPeriod int) string {
return acctest.ConfigCompose(
testAccCatalogTableOptimizerConfig_baseConfig(rName),
fmt.Sprintf(`
resource "aws_glue_catalog_table_optimizer" "test" {
catalog_id = data.aws_caller_identity.current.account_id
database_name = aws_glue_catalog_database.test.name
table_name = aws_glue_catalog_table.test.name
type = "orphan_file_deletion"
configuration {
role_arn = aws_iam_role.test.arn
enabled = true
orphan_file_deletion_configuration {
iceberg_configuration {
orphan_file_retention_period_in_days = %[1]d
location = "s3://${aws_s3_bucket.bucket.bucket}/files/"
}
}
}
}
`, retentionPeriod))
}
8 changes: 5 additions & 3 deletions internal/service/glue/glue_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,11 @@ func TestAccGlue_serial(t *testing.T) {

testCases := map[string]map[string]func(t *testing.T){
"CatalogTableOptimizer": {
acctest.CtBasic: testAccCatalogTableOptimizer_basic,
acctest.CtDisappears: testAccCatalogTableOptimizer_disappears,
"update": testAccCatalogTableOptimizer_update,
acctest.CtBasic: testAccCatalogTableOptimizer_basic,
"deleteOrphanFileConfiguration": testAccCatalogTableOptimizer_DeleteOrphanFileConfiguration,
acctest.CtDisappears: testAccCatalogTableOptimizer_disappears,
"retentionConfiguration": testAccCatalogTableOptimizer_RetentionConfiguration,
"update": testAccCatalogTableOptimizer_update,
},
"DataCatalogEncryptionSettings": {
acctest.CtBasic: testAccDataCatalogEncryptionSettings_basic,
Expand Down
79 changes: 74 additions & 5 deletions website/docs/r/glue_catalog_table_optimizer.html.markdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ Terraform resource for managing an AWS Glue Catalog Table Optimizer.

## Example Usage

### Basic Usage
### Compaction Optimizer

```terraform
resource "aws_glue_catalog_table_optimizer" "example" {
Expand All @@ -29,17 +29,86 @@ resource "aws_glue_catalog_table_optimizer" "example" {
}
```

### Snapshot Retention Optimizer

```terraform
resource "aws_glue_catalog_table_optimizer" "example" {
catalog_id = "123456789012"
database_name = "example_database"
table_name = "example_table"
configuration {
role_arn = "arn:aws:iam::123456789012:role/example-role"
enabled = true
retention_configuration {
iceberg_configuration {
snapshot_retention_period_in_days = 7
number_of_snapshots_to_retain = 3
clean_expired_files = true
}
}
}
type = "retention"
}
```

### Orphan File Deletion Optimizer

```terraform
resource "aws_glue_catalog_table_optimizer" "example" {
catalog_id = "123456789012"
database_name = "example_database"
table_name = "example_table"
configuration {
role_arn = "arn:aws:iam::123456789012:role/example-role"
enabled = true
orphan_file_deletion_configuration {
iceberg_configuration {
orphan_file_retention_period_in_days = 7
location = "s3://example-bucket/example_table/"
}
}
}
type = "orphan_file_deletion"
}
```

## Argument Reference

The following arguments are required:

* `catalog_id` - (Required) The Catalog ID of the table.
* `configuration` - (Required) A configuration block that defines the table optimizer settings. See [Configuration](#configuration) for additional details.
* `database_name` - (Required) The name of the database in the catalog in which the table resides.
* `table_name` - (Required) The name of the table.
* `type` - (Required) The type of table optimizer. Currently, the only valid value is compaction.
* `configuration` - (Required) A configuration block that defines the table optimizer settings. The block contains:
* `role_arn` - (Required) The ARN of the IAM role to use for the table optimizer.
* `enabled` - (Required) Indicates whether the table optimizer is enabled.
* `type` - (Required) The type of table optimizer. Valid values are `compaction`, `retention`, and `orphan_file_deletion`.

### Configuration

* `enabled` - (Required) Indicates whether the table optimizer is enabled.
* `orphan_file_deletion_configuration` (Optional) - The configuration block for an orphan file deletion optimizer. See [Orphan File Deletion Configuration](#orphan-file-deletion-configuration) for additional details.
* `retention_configuration` (Optional) - The configuration block for a snapshot retention optimizer. See [Retention Configuration](#retention-configuration) for additional details.
* `role_arn` - (Required) The ARN of the IAM role to use for the table optimizer.

### Orphan File Deletion Configuration

* `iceberg_configuration` (Optional) - The configuration for an Iceberg orphan file deletion optimizer.
* `orphan_file_retention_period_in_days` (Optional) - The number of days that orphan files should be retained before file deletion. Defaults to `3`.
* `location` (Optional) - Specifies a directory in which to look for files. You may choose a sub-directory rather than the top-level table location. Defaults to the table's location.

### Retention Configuration

* `iceberg_configuration` (Optional) - The configuration for an Iceberg snapshot retention optimizer.
* `snapshot_retention_period_in_days` (Optional) - The number of days to retain the Iceberg snapshots. Defaults to `5`, or the corresponding Iceberg table configuration field if it exists.
* `number_of_snapshots_to_retain` (Optional) - The number of Iceberg snapshots to retain within the retention period. Defaults to `1` or the corresponding Iceberg table configuration field if it exists.
* `clean_expired_files` (Optional) - If set to `false`, snapshots are only deleted from table metadata, and the underlying data and metadata files are not deleted. Defaults to `false`.

## Attribute Reference

Expand Down

0 comments on commit f382471

Please sign in to comment.