diff --git a/client-dynamodb-v2/build.gradle.kts b/client-dynamodb-v2/build.gradle.kts
index 5c3c7f259..8a24b5527 100644
--- a/client-dynamodb-v2/build.gradle.kts
+++ b/client-dynamodb-v2/build.gradle.kts
@@ -39,6 +39,12 @@ dependencies {
testImplementation(project(":backfila-embedded"))
testImplementation(project(":client-testing"))
+ if (org.apache.tools.ant.taskdefs.condition.Os.isArch("aarch64")) {
+ // Without this, we can't compile on Apple Silicon currently. This is likely not necessary to
+ // have longterm, so we should remove it when platform fixes things across Square.
+ testImplementation("io.github.ganadist.sqlite4java:libsqlite4java-osx-aarch64:1.0.392")
+ }
+
// ****************************************
// For TESTING purposes only. We only want Misk for easy testing.
// DO NOT turn these into regular dependencies.
diff --git a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbBackfill.kt b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbBackfill.kt
index 9483ae038..378aab7f7 100644
--- a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbBackfill.kt
+++ b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbBackfill.kt
@@ -87,4 +87,7 @@ abstract class DynamoDbBackfill : Backfill {
/** See [ScanRequest.setExpressionAttributeNames]. */
open fun expressionAttributeNames(config: BackfillConfig
): Map? = null
+
+ /** See [ScanRequest.setIndexName]. */
+ open fun indexName(config: BackfillConfig): String? = null
}
diff --git a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/UpdateInPlaceDynamoDbBackfill.kt b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/UpdateInPlaceDynamoDbBackfill.kt
index 96d2f8e29..f5416d143 100644
--- a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/UpdateInPlaceDynamoDbBackfill.kt
+++ b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/UpdateInPlaceDynamoDbBackfill.kt
@@ -31,7 +31,7 @@ abstract class UpdateInPlaceDynamoDbBackfill(
dynamoDbTable.tableName() to itemsToSave.map {
WriteRequest.builder().putRequest(
PutRequest.builder().item(
- dynamoDbTable.tableSchema().itemToMap(it, false),
+ dynamoDbTable.tableSchema().itemToMap(it, true),
).build(),
).build()
},
diff --git a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/internal/DynamoDbBackfillOperator.kt b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/internal/DynamoDbBackfillOperator.kt
index 80895b017..2886e5273 100644
--- a/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/internal/DynamoDbBackfillOperator.kt
+++ b/client-dynamodb-v2/src/main/kotlin/app/cash/backfila/client/dynamodbv2/internal/DynamoDbBackfillOperator.kt
@@ -133,6 +133,7 @@ class DynamoDbBackfillOperator(
.filterExpression(backfill.filterExpression(config))
.expressionAttributeValues(backfill.expressionAttributeValues(config))
.expressionAttributeNames(backfill.expressionAttributeNames(config))
+ .indexName(backfill.indexName(config))
.build()
val result = dynamoDbClient.scan(scanRequest)
diff --git a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/BackfillsModule.kt b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/BackfillsModule.kt
index 2ceb57e62..bf17a3769 100644
--- a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/BackfillsModule.kt
+++ b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/BackfillsModule.kt
@@ -29,5 +29,6 @@ class BackfillsModule : KAbstractModule() {
install(DynamoDbBackfillModule.create())
install(DynamoDbBackfillModule.create())
install(DynamoDbBackfillModule.create())
+ install(DynamoDbBackfillModule.create())
}
}
diff --git a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbIndexTest.kt b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbIndexTest.kt
new file mode 100644
index 000000000..246e3a18c
--- /dev/null
+++ b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/DynamoDbIndexTest.kt
@@ -0,0 +1,94 @@
+package app.cash.backfila.client.dynamodbv2
+
+import app.cash.backfila.client.BackfillConfig
+import app.cash.backfila.client.misk.TestingModule
+import app.cash.backfila.embedded.Backfila
+import app.cash.backfila.embedded.createWetRun
+import javax.inject.Inject
+import misk.testing.MiskTest
+import misk.testing.MiskTestModule
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import software.amazon.awssdk.enhanced.dynamodb.DynamoDbEnhancedClient
+import software.amazon.awssdk.enhanced.dynamodb.DynamoDbTable
+import software.amazon.awssdk.enhanced.dynamodb.Key
+import software.amazon.awssdk.enhanced.dynamodb.TableSchema
+import software.amazon.awssdk.enhanced.dynamodb.model.ScanEnhancedRequest
+import software.amazon.awssdk.services.dynamodb.DynamoDbClient
+
+@MiskTest(startService = true)
+class DynamoDbIndexTest {
+ @Suppress("unused")
+ @MiskTestModule
+ val module = TestingModule()
+
+ @Inject
+ lateinit var dynamoDb: DynamoDbTable
+
+ @Inject
+ lateinit var backfila: Backfila
+
+ @Inject
+ lateinit var testData: TrackData
+
+ @Test
+ fun `only items in the specified index are processed`() {
+ val trackItem = TrackItem().apply {
+ this.album_token = "ALBUM_2"
+ this.sort_key = "TRACK_03"
+ this.track_title = "Thriller"
+ this.album_title = "MJ"
+ }
+ dynamoDb.putItem(trackItem)
+
+ val trackItem2 = TrackItem().apply {
+ this.album_token = "ALBUM_3"
+ this.sort_key = "TRACK_01"
+ this.track_title = "Anon"
+ // No album title so its excluded from the trackTitleIndex sparse GSI.
+ }
+ dynamoDb.putItem(trackItem2)
+
+ assertThat(rowsInIndex().size).isEqualTo(1)
+ assertThat(testData.getRowsDump().size).isEqualTo(2)
+
+ val run = backfila.createWetRun()
+ run.execute()
+
+ // Only rows from the index were updated.
+ assertThat(dynamoDb.getItem(Key.builder().partitionValue("ALBUM_2").sortValue("TRACK_03").build()).track_title)
+ .isEqualTo("Thriller (Single)")
+ assertThat(dynamoDb.getItem(Key.builder().partitionValue("ALBUM_3").sortValue("TRACK_01").build()).track_title)
+ .isEqualTo("Anon")
+ }
+
+ private fun rowsInIndex(): List {
+ val scanRequest = ScanEnhancedRequest.builder()
+ .limit(10000)
+ .build()
+ return dynamoDb.index("trackTitleIndex").scan(scanRequest).stream().flatMap { it.items().stream() }.toList()
+ }
+
+ class MakeTracksAsSinglesBackfill @Inject constructor(
+ dynamoDb: DynamoDbClient,
+ private val dynamoDbEnhancedClient: DynamoDbEnhancedClient,
+ ) : UpdateInPlaceDynamoDbBackfill(dynamoDb) {
+
+ override fun runOne(item: TrackItem, config: BackfillConfig): Boolean {
+ val trackTitle = item.track_title ?: return false
+ if (trackTitle.endsWith(" (Single)")) return true
+ item.track_title = "$trackTitle (Single)"
+ return true
+ }
+
+ data class SingleParameters(val validate: Boolean = true)
+
+ override fun indexName(config: BackfillConfig): String? = "trackTitleIndex"
+ override fun dynamoDbTable(): DynamoDbTable {
+ return dynamoDbEnhancedClient.table(
+ TrackItem.TABLE_NAME,
+ TableSchema.fromClass(TrackItem::class.java),
+ )
+ }
+ }
+}
diff --git a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TestingModule.kt b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TestingModule.kt
index 97a8a6535..94cabcee5 100644
--- a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TestingModule.kt
+++ b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TestingModule.kt
@@ -13,7 +13,10 @@ import misk.inject.KAbstractModule
import misk.logging.LogCollectorModule
import software.amazon.awssdk.enhanced.dynamodb.DynamoDbEnhancedClient
import software.amazon.awssdk.enhanced.dynamodb.TableSchema
+import software.amazon.awssdk.enhanced.dynamodb.model.EnhancedGlobalSecondaryIndex
import software.amazon.awssdk.services.dynamodb.DynamoDbClient
+import software.amazon.awssdk.services.dynamodb.model.ProjectionType
+import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput
/**
* Simulates a specific service implementation module
@@ -27,7 +30,24 @@ class TestingModule : KAbstractModule() {
install(EmbeddedBackfilaModule())
- install(InProcessDynamoDbModule(DynamoDbTable(TrackItem.TABLE_NAME, TrackItem::class)))
+ install(
+ InProcessDynamoDbModule(
+ DynamoDbTable(TrackItem.TABLE_NAME, TrackItem::class) { table ->
+ val provisionedThroughput = ProvisionedThroughput.builder()
+ .readCapacityUnits(100)
+ .writeCapacityUnits(100)
+ .build()
+ table.globalSecondaryIndices(
+ EnhancedGlobalSecondaryIndex.builder()
+ .indexName("trackTitleIndex")
+ .projection { it.projectionType(ProjectionType.KEYS_ONLY) }
+ .provisionedThroughput(provisionedThroughput)
+ .build(),
+ )
+ table
+ },
+ ),
+ )
}
@Provides
diff --git a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TrackItem.kt b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TrackItem.kt
index e3ee29c13..d5bf5444a 100644
--- a/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TrackItem.kt
+++ b/client-dynamodb-v2/src/test/kotlin/app/cash/backfila/client/dynamodbv2/TrackItem.kt
@@ -2,6 +2,8 @@ package app.cash.backfila.client.dynamodbv2
import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbBean
import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbPartitionKey
+import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbSecondaryPartitionKey
+import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbSecondarySortKey
import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbSortKey
@DynamoDbBean()
@@ -17,8 +19,10 @@ class TrackItem {
@get:DynamoDbSortKey
var sort_key: String? = null
+ @get:DynamoDbSecondaryPartitionKey(indexNames = ["trackTitleIndex"])
var track_title: String? = null
+ @get:DynamoDbSecondarySortKey(indexNames = ["trackTitleIndex"])
var album_title: String? = null
var artist_name: String? = null