diff --git a/client-dynamodb-v2/build.gradle.kts b/client-dynamodb-v2/build.gradle.kts index 5c3c7f259..8a24b5527 100644 --- a/client-dynamodb-v2/build.gradle.kts +++ b/client-dynamodb-v2/build.gradle.kts @@ -39,6 +39,12 @@ dependencies { testImplementation(project(":backfila-embedded")) testImplementation(project(":client-testing")) + if (org.apache.tools.ant.taskdefs.condition.Os.isArch("aarch64")) { + // Without this, we can't compile on Apple Silicon currently. This is likely not necessary to + // have longterm, so we should remove it when platform fixes things across Square. + testImplementation("io.github.ganadist.sqlite4java:libsqlite4java-osx-aarch64:1.0.392") + } + // **************************************** // For TESTING purposes only. We only want Misk for easy testing. // DO NOT turn these into regular dependencies. diff --git a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbBackfill.kt b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbBackfill.kt index 9483ae038..378aab7f7 100644 --- a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbBackfill.kt +++ b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbBackfill.kt @@ -87,4 +87,7 @@ abstract class DynamoDbBackfill : Backfill { /** See [ScanRequest.setExpressionAttributeNames]. */ open fun expressionAttributeNames(config: BackfillConfig

): Map? = null + + /** See [ScanRequest.setIndexName]. */ + open fun indexName(config: BackfillConfig

): String? = null } diff --git a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/UpdateInPlaceDynamoDbBackfill.kt b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/UpdateInPlaceDynamoDbBackfill.kt index 96d2f8e29..f5416d143 100644 --- a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/UpdateInPlaceDynamoDbBackfill.kt +++ b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/UpdateInPlaceDynamoDbBackfill.kt @@ -31,7 +31,7 @@ abstract class UpdateInPlaceDynamoDbBackfill( dynamoDbTable.tableName() to itemsToSave.map { WriteRequest.builder().putRequest( PutRequest.builder().item( - dynamoDbTable.tableSchema().itemToMap(it, false), + dynamoDbTable.tableSchema().itemToMap(it, true), ).build(), ).build() }, diff --git a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/internal/DynamoDbBackfillOperator.kt b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/internal/DynamoDbBackfillOperator.kt index 80895b017..2886e5273 100644 --- a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/internal/DynamoDbBackfillOperator.kt +++ b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/internal/DynamoDbBackfillOperator.kt @@ -133,6 +133,7 @@ class DynamoDbBackfillOperator( .filterExpression(backfill.filterExpression(config)) .expressionAttributeValues(backfill.expressionAttributeValues(config)) .expressionAttributeNames(backfill.expressionAttributeNames(config)) + .indexName(backfill.indexName(config)) .build() val result = dynamoDbClient.scan(scanRequest) diff --git a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/BackfillsModule.kt b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/BackfillsModule.kt index 2ceb57e62..bf17a3769 100644 --- a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/BackfillsModule.kt +++ b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/BackfillsModule.kt @@ -29,5 +29,6 @@ class BackfillsModule : KAbstractModule() { install(DynamoDbBackfillModule.create()) install(DynamoDbBackfillModule.create()) install(DynamoDbBackfillModule.create()) + install(DynamoDbBackfillModule.create()) } } diff --git a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbIndexTest.kt b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbIndexTest.kt new file mode 100644 index 000000000..246e3a18c --- /dev/null +++ b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbIndexTest.kt @@ -0,0 +1,94 @@ +package app.cash.backfila.client.dynamodbv2 + +import app.cash.backfila.client.BackfillConfig +import app.cash.backfila.client.misk.TestingModule +import app.cash.backfila.embedded.Backfila +import app.cash.backfila.embedded.createWetRun +import javax.inject.Inject +import misk.testing.MiskTest +import misk.testing.MiskTestModule +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import software.amazon.awssdk.enhanced.dynamodb.DynamoDbEnhancedClient +import software.amazon.awssdk.enhanced.dynamodb.DynamoDbTable +import software.amazon.awssdk.enhanced.dynamodb.Key +import software.amazon.awssdk.enhanced.dynamodb.TableSchema +import software.amazon.awssdk.enhanced.dynamodb.model.ScanEnhancedRequest +import software.amazon.awssdk.services.dynamodb.DynamoDbClient + +@MiskTest(startService = true) +class DynamoDbIndexTest { + @Suppress("unused") + @MiskTestModule + val module = TestingModule() + + @Inject + lateinit var dynamoDb: DynamoDbTable + + @Inject + lateinit var backfila: Backfila + + @Inject + lateinit var testData: TrackData + + @Test + fun `only items in the specified index are processed`() { + val trackItem = TrackItem().apply { + this.album_token = "ALBUM_2" + this.sort_key = "TRACK_03" + this.track_title = "Thriller" + this.album_title = "MJ" + } + dynamoDb.putItem(trackItem) + + val trackItem2 = TrackItem().apply { + this.album_token = "ALBUM_3" + this.sort_key = "TRACK_01" + this.track_title = "Anon" + // No album title so its excluded from the trackTitleIndex sparse GSI. + } + dynamoDb.putItem(trackItem2) + + assertThat(rowsInIndex().size).isEqualTo(1) + assertThat(testData.getRowsDump().size).isEqualTo(2) + + val run = backfila.createWetRun() + run.execute() + + // Only rows from the index were updated. + assertThat(dynamoDb.getItem(Key.builder().partitionValue("ALBUM_2").sortValue("TRACK_03").build()).track_title) + .isEqualTo("Thriller (Single)") + assertThat(dynamoDb.getItem(Key.builder().partitionValue("ALBUM_3").sortValue("TRACK_01").build()).track_title) + .isEqualTo("Anon") + } + + private fun rowsInIndex(): List { + val scanRequest = ScanEnhancedRequest.builder() + .limit(10000) + .build() + return dynamoDb.index("trackTitleIndex").scan(scanRequest).stream().flatMap { it.items().stream() }.toList() + } + + class MakeTracksAsSinglesBackfill @Inject constructor( + dynamoDb: DynamoDbClient, + private val dynamoDbEnhancedClient: DynamoDbEnhancedClient, + ) : UpdateInPlaceDynamoDbBackfill(dynamoDb) { + + override fun runOne(item: TrackItem, config: BackfillConfig): Boolean { + val trackTitle = item.track_title ?: return false + if (trackTitle.endsWith(" (Single)")) return true + item.track_title = "$trackTitle (Single)" + return true + } + + data class SingleParameters(val validate: Boolean = true) + + override fun indexName(config: BackfillConfig): String? = "trackTitleIndex" + override fun dynamoDbTable(): DynamoDbTable { + return dynamoDbEnhancedClient.table( + TrackItem.TABLE_NAME, + TableSchema.fromClass(TrackItem::class.java), + ) + } + } +} diff --git a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TestingModule.kt b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TestingModule.kt index 97a8a6535..94cabcee5 100644 --- a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TestingModule.kt +++ b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TestingModule.kt @@ -13,7 +13,10 @@ import misk.inject.KAbstractModule import misk.logging.LogCollectorModule import software.amazon.awssdk.enhanced.dynamodb.DynamoDbEnhancedClient import software.amazon.awssdk.enhanced.dynamodb.TableSchema +import software.amazon.awssdk.enhanced.dynamodb.model.EnhancedGlobalSecondaryIndex import software.amazon.awssdk.services.dynamodb.DynamoDbClient +import software.amazon.awssdk.services.dynamodb.model.ProjectionType +import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput /** * Simulates a specific service implementation module @@ -27,7 +30,24 @@ class TestingModule : KAbstractModule() { install(EmbeddedBackfilaModule()) - install(InProcessDynamoDbModule(DynamoDbTable(TrackItem.TABLE_NAME, TrackItem::class))) + install( + InProcessDynamoDbModule( + DynamoDbTable(TrackItem.TABLE_NAME, TrackItem::class) { table -> + val provisionedThroughput = ProvisionedThroughput.builder() + .readCapacityUnits(100) + .writeCapacityUnits(100) + .build() + table.globalSecondaryIndices( + EnhancedGlobalSecondaryIndex.builder() + .indexName("trackTitleIndex") + .projection { it.projectionType(ProjectionType.KEYS_ONLY) } + .provisionedThroughput(provisionedThroughput) + .build(), + ) + table + }, + ), + ) } @Provides diff --git a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TrackItem.kt b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TrackItem.kt index e3ee29c13..d5bf5444a 100644 --- a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TrackItem.kt +++ b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TrackItem.kt @@ -2,6 +2,8 @@ package app.cash.backfila.client.dynamodbv2 import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbBean import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbPartitionKey +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbSecondaryPartitionKey +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbSecondarySortKey import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbSortKey @DynamoDbBean() @@ -17,8 +19,10 @@ class TrackItem { @get:DynamoDbSortKey var sort_key: String? = null + @get:DynamoDbSecondaryPartitionKey(indexNames = ["trackTitleIndex"]) var track_title: String? = null + @get:DynamoDbSecondarySortKey(indexNames = ["trackTitleIndex"]) var album_title: String? = null var artist_name: String? = null