Describe the bug
I am in the process of moving from the Java ddb mapper to the kotlin one and found when querying on a GSI the ExclusiveStartKey is missing the primary table's PK and SK but correctly includes the GSI's PK and SK. As a result queries with non-null ExclusiveStartKey fail with Exclusive Start Key must have same size as table's key schema
Feels related to the following:
#1594
#1596
Regression Issue
Expected behavior
The correct ExclusiveStartKey is generated.
Current behavior
The ExclusiveStartKey is missing the primary table's PK and SK
Steps to Reproduce
| Kotlin - DDB Bean |
Java - DDB Bean |
@DynamoDbItem
data class Invitation(
@DynamoDbPartitionKey
@DynamoDbAttribute(PRIMARY_KEY)
var invitationId: String = "",
@DynamoDbSortKey
@DynamoDbAttribute(SORT_KEY)
var sk: String = invitationId, // Same id as invitationId for direct access
@DynamoDbAttribute(INVITATION_INDEX_PARTITION_KEY)
var type: Type = Type.INVITATION,
) {
companion object {
const val PRIMARY_KEY = "PK"
const val SORT_KEY = "SK"
// The invitation index and registration Index have the same SecondaryPartitionKey and SecondarySortKey
// so they share the index. See `config.ts`
const val INVITATION_INDEX = "registrationIndex"
const val INVITATION_INDEX_PARTITION_KEY = "type"
const val INVITATION_INDEX_SECONDARY_SORT_KEY = SORT_KEY
}
}
|
@DynamoDbBean
data class Invitation(
@get:DynamoDbPartitionKey
@get:DynamoDbAttribute(PRIMARY_KEY)
var invitationId: String = "",
@get:DynamoDbSortKey
@get:DynamoDbAttribute(SORT_KEY)
@get:DynamoDbSecondarySortKey(indexNames = [INVITATION_INDEX])
var sk: String = invitationId, // Same id as invitationId for direct access
@get:DynamoDbSecondaryPartitionKey(indexNames = [INVITATION_INDEX])
@get:DynamoDbAttribute(INVITATION_INDEX_PARTITION_KEY)
var type: Type = Type.INVITATION,
...
) {
companion object {
const val PRIMARY_KEY = "PK"
const val SORT_KEY = "SK"
// The invitation index and registration Index have the same SecondaryPartitionKey and SecondarySortKey
// so they share the index. See `config.ts`
const val INVITATION_INDEX = "registrationIndex"
const val INVITATION_INDEX_PARTITION_KEY = "type"
const val INVITATION_INDEX_SECONDARY_SORT_KEY = SORT_KEY
}
}
|
| Kotlin - Dao |
Java - Dao |
@OptIn(ExperimentalApi::class)
class InvitationDao(
private val ddbMapper: DynamoDbMapper,
private val tableName: String
) {
private val invitationTable: Table.CompositeKey = ddbMapper.getTable(tableName, InvitationSchema)
private val typeToInvitationIdIndex: Index.CompositeKey
init {
val typeToInvitationIdIndexSchema = ItemSchema(
converter = InvitationSchema.converter,
partitionKey = KeySpec.String(Invitation.INVITATION_INDEX_PARTITION_KEY),
sortKey = KeySpec.String(Invitation.INVITATION_INDEX_SECONDARY_SORT_KEY)
)
typeToInvitationIdIndex = invitationTable.getIndex(Invitation.INVITATION_INDEX, typeToInvitationIdIndexSchema)
}
fun listInvitation(nextToken: String?, pageSize: Int?): Pair<List<Invitation>, String?> = runBlocking {
try {
@OptIn(ManualPagination::class)
val page = typeToInvitationIdIndex.query {
keyCondition = KeyFilter(Type.INVITATION.name)
limit = pageSize
exclusiveStartKey = decodeNextToken(nextToken)
}
return@runBlocking Pair(page.items ?: emptyList(), encodeNextToken(page.lastEvaluatedKey))
} catch (e: Exception) {
throw InternalServerError(e.stackTraceToString())
}
}
private fun decodeNextToken(nextToken: String?): Invitation? {
return nextToken?.let {
val token = try {
val json = Base64.UrlSafe.withPadding(Base64.PaddingOption.PRESENT_OPTIONAL).decode(it)
.toString(Charsets.UTF_8)
Json.decodeFromString<InvitationToken>(json)
} catch (e: Exception) {
throw ValidationError("Invalid nextToken")
}
Invitation(
invitationId = token.invitationId,
sk = token.invitationId,
type = Type.INVITATION,
)
}
}
private fun encodeNextToken(lastEvalKey: Invitation?): String? {
return lastEvalKey?.let {
val json = Json.encodeToString(
InvitationToken(
invitationId = lastEvalKey.invitationId,
)
)
Base64.UrlSafe.withPadding(Base64.PaddingOption.ABSENT).encode(json.toByteArray(Charsets.UTF_8))
}
}
}
|
class InvitationDao(
private val invitationTable: DynamoDbTable,
private val typeToInvitationIdIndex: DynamoDbIndex
) {
fun listInvitation(nextToken: String?, pageSize: Int?): Pair<List<Invitation>, String?> {
try {
val query = QueryEnhancedRequest
.builder()
.queryConditional(
QueryConditional.keyEqualTo(
Key.builder()
.partitionValue(Type.INVITATION.name)
.build()
)
)
.exclusiveStartKey(decodeNextToken(nextToken))
.limit(pageSize)
.build()
val result = typeToInvitationIdIndex.query(query).iterator().next()
return Pair(result.items(), encodeNextToken(result.lastEvaluatedKey()))
} catch (e: Exception) {
throw InternalServerError(e.stackTraceToString())
}
}
private fun decodeNextToken(nextToken: String?): Map<String, AttributeValue>? {
return nextToken?.let {
val token = try {
val json = Base64.UrlSafe.withPadding(Base64.PaddingOption.PRESENT_OPTIONAL).decode(it)
.toString(Charsets.UTF_8)
Json.decodeFromString<InvitationToken>(json)
} catch (e: Exception) {
throw ValidationError("Invalid nextToken")
}
mapOf(
Invitation.INVITATION_INDEX_PARTITION_KEY to
AttributeValue.builder().s(Type.INVITATION.name).build(),
Invitation.INVITATION_INDEX_SECONDARY_SORT_KEY to
AttributeValue.builder().s(token.invitationId).build(),
Invitation.PRIMARY_KEY to
AttributeValue.builder().s(token.invitationId).build(),
Invitation.SORT_KEY to
AttributeValue.builder().s(token.invitationId).build()
)
}
}
private fun encodeNextToken(lastEvalKey: Map<String, AttributeValue>?): String? {
return lastEvalKey?.let {
val json = Json.encodeToString(
InvitationToken(
invitationId = lastEvalKey[Invitation.PRIMARY_KEY]!!.s(),
)
)
Base64.UrlSafe.withPadding(Base64.PaddingOption.ABSENT).encode(json.toByteArray(Charsets.UTF_8))
}
}
}
|
Kotlin Query Request
{
"ExclusiveStartKey": {
"SK": {
"S": "C-2"
},
"type": {
"S": "INVITATION"
}
},
"ExpressionAttributeNames": {
"#k0": "type"
},
"ExpressionAttributeValues": {
":v0": {
"S": "INVITATION"
}
},
"IndexName": "registrationIndex",
"KeyConditionExpression": "#k0 = :v0",
"Limit": 3,
"TableName": "ApplicationTable"
}
Java Query Request
{
"TableName": "ApplicationTable",
"IndexName": "registrationIndex",
"Limit": 3,
"ExclusiveStartKey": {
"type": {
"S": "INVITATION"
},
"SK": {
"S": "C-2"
},
"PK": {
"S": "C-2"
}
},
"KeyConditionExpression": "#AMZN_MAPPED_type = :AMZN_MAPPED_type",
"ExpressionAttributeNames": {
"#AMZN_MAPPED_type": "type"
},
"ExpressionAttributeValues": {
":AMZN_MAPPED_type": {
"S": "INVITATION"
}
}
}
Possible Solution
It is possible this is user error, I have a GSI and didn't want to redefine all the types as you guys did in the cars vs model example instead I did:
private val invitationTable: Table.CompositeKey = ddbMapper.getTable(tableName, InvitationSchema)
private val typeToInvitationIdIndex: Index.CompositeKey
init {
val typeToInvitationIdIndexSchema = ItemSchema(
converter = InvitationSchema.converter,
partitionKey = KeySpec.String(Invitation.INVITATION_INDEX_PARTITION_KEY),
sortKey = KeySpec.String(Invitation.INVITATION_INDEX_SECONDARY_SORT_KEY)
)
typeToInvitationIdIndex = invitationTable.getIndex(Invitation.INVITATION_INDEX, typeToInvitationIdIndexSchema)
}
The other thing I considered was that there was some bug due to the fact that my PK / SK overlap between my GSI and primary table. But I did do a sanity check and even with separate keys the generated start key is missing values.
Context
I did all my testing using LocalDDB in docker.
services:
dynamodb-local:
command: "-jar DynamoDBLocal.jar -sharedDb -inMemory"
image: "amazon/dynamodb-local:latest"
container_name: dynamodb-local
ports:
- "8000:8000"
AWS SDK for Kotlin version
1.5.33-beta
Platform (JVM/JS/Native)
JVM
Operating system and version
OSX
