Skip to content

[v14] Fix DynamoDB BatchWriteItemInput items limit #38762

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
smallinsky merged 1 commit into from
Feb 28, 2024
Merged

[v14] Fix DynamoDB BatchWriteItemInput items limit #38762

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion lib/backend/dynamo/dynamodbbk.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,14 @@ func (b *Backend) getAllRecords(ctx context.Context, startKey []byte, endKey []b
return nil, trace.BadParameter("backend entered endless loop")
}

const (
// batchOperationItemsLimit is the maximum number of items that can be put or deleted in a single batch operation.
// From https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html:
// A single call to BatchWriteItem can transmit up to 16MB of data over the network,
// consisting of up to 25 item put or delete operations.
batchOperationItemsLimit = 25
)

// DeleteRange deletes range of items with keys between startKey and endKey
func (b *Backend) DeleteRange(ctx context.Context, startKey, endKey []byte) error {
if len(startKey) == 0 {
Expand All @@ -447,7 +455,7 @@ func (b *Backend) DeleteRange(ctx context.Context, startKey, endKey []byte) erro
// keep the very large limit, just in case if someone else
// keeps adding records
for i := 0; i < backend.DefaultRangeLimit/100; i++ {
result, err := b.getRecords(ctx, prependPrefix(startKey), prependPrefix(endKey), 100, nil)
result, err := b.getRecords(ctx, prependPrefix(startKey), prependPrefix(endKey), batchOperationItemsLimit, nil)
if err != nil {
return trace.Wrap(err)
}
Expand Down
10 changes: 10 additions & 0 deletions lib/backend/test/suite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,16 @@ func testDeleteRange(t *testing.T, newBackend Constructor) {
require.NoError(t, err, "Failed creating value: %q => %q", item.Key, item.Value)
}

// Some Backends (e.g. DynamoDB) have a limit on the number of items that can
// be deleted in a single operation. This test is designed to be run with
// a backend that has a limit of 25 items per delete operation.
for i := 0; i < 100; i++ {
item := &backend.Item{Key: prefix(fmt.Sprintf("/prefix/c/cn%d", i)), Value: []byte(fmt.Sprintf("val cn%d", i))}
lease, err := uut.Create(ctx, *item)
require.NoError(t, err, "Failed creating value: %q => %q", item.Key, item.Value)
item.Revision = lease.Revision
}

err = uut.DeleteRange(ctx, prefix("/prefix/c"), backend.RangeEnd(prefix("/prefix/c")))
require.NoError(t, err)

Expand Down