Skip to content

Commit

Permalink
Fix tests for new Algorithm CRC64NVME
Browse files Browse the repository at this point in the history
AWS decided to refactor their ChecksumAlgorithms, but it works only
partly right now.
  • Loading branch information
afranken committed Mar 7, 2025
1 parent 8144635 commit 735f7a5
Show file tree
Hide file tree
Showing 6 changed files with 35 additions and 8 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017-2024 Adobe.
* Copyright 2017-2025 Adobe.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -591,6 +591,7 @@ internal abstract class S3TestBase {
ChecksumAlgorithm.SHA256 -> Algorithm.SHA256
ChecksumAlgorithm.CRC32 -> Algorithm.CRC32
ChecksumAlgorithm.CRC32_C -> Algorithm.CRC32C
ChecksumAlgorithm.CRC64_NVME -> Algorithm.CRC64NVME
else -> throw IllegalArgumentException("Unknown checksum algorithm")
}

Expand Down Expand Up @@ -639,11 +640,23 @@ internal abstract class S3TestBase {
}
}

fun S3Response.checksumCRC64NVME(): String? {
return when (this) {
is GetObjectResponse -> this.checksumCRC64NVME()
is PutObjectResponse -> this.checksumCRC64NVME()
is HeadObjectResponse -> this.checksumCRC64NVME()
is UploadPartResponse -> this.checksumCRC64NVME()
is GetObjectAttributesResponse -> this.checksum().checksumCRC64NVME()
else -> throw RuntimeException("Unexpected response type ${this::class.java}")
}
}

return when (checksumAlgorithm) {
ChecksumAlgorithm.SHA1 -> this.checksumSHA1()
ChecksumAlgorithm.SHA256 -> this.checksumSHA256()
ChecksumAlgorithm.CRC32 -> this.checksumCRC32()
ChecksumAlgorithm.CRC32_C -> this.checksumCRC32C()
ChecksumAlgorithm.CRC64_NVME -> this.checksumCRC64NVME()
ChecksumAlgorithm.UNKNOWN_TO_SDK_VERSION -> "UNKNOWN_TO_SDK_VERSION"
}
}
Expand Down Expand Up @@ -696,6 +709,7 @@ internal abstract class S3TestBase {
protected fun checksumAlgorithms(): Stream<ChecksumAlgorithm> {
return ChecksumAlgorithm
.entries
.filter { it != ChecksumAlgorithm.CRC64_NVME }
.filter { it != ChecksumAlgorithm.UNKNOWN_TO_SDK_VERSION }
.map { it }
.stream()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017-2024 Adobe.
* Copyright 2017-2025 Adobe.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -23,6 +23,7 @@
public enum ChecksumAlgorithm {
CRC32("CRC32"),
CRC32C("CRC32C"),
CRC64NVME("CRC64NVME"),
SHA1("SHA1"),
SHA256("SHA256");

Expand All @@ -39,6 +40,7 @@ public static ChecksumAlgorithm fromString(String value) {
case "sha1", "SHA1" -> SHA1;
case "crc32", "CRC32" -> CRC32;
case "crc32c", "CRC32C" -> CRC32C;
case "crc64nvme", "CRC64NVME" -> CRC64NVME;
default -> null;
};
}
Expand All @@ -49,6 +51,7 @@ public static ChecksumAlgorithm fromHeader(String value) {
case "x-amz-checksum-sha1" -> SHA1;
case "x-amz-checksum-crc32" -> CRC32;
case "x-amz-checksum-crc32c" -> CRC32C;
case "x-amz-checksum-crc64nvme" -> CRC64NVME;
default -> null;
};
}
Expand All @@ -57,6 +60,7 @@ public Algorithm toAlgorithm() {
return switch (this) {
case CRC32 -> Algorithm.CRC32;
case CRC32C -> Algorithm.CRC32C;
case CRC64NVME -> Algorithm.CRC64NVME;
case SHA1 -> Algorithm.SHA1;
case SHA256 -> Algorithm.SHA256;
};
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017-2024 Adobe.
* Copyright 2017-2025 Adobe.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -60,6 +60,7 @@ public final class AwsHttpHeaders {
public static final String X_AMZ_CHECKSUM = "x-amz-checksum";
public static final String X_AMZ_CHECKSUM_CRC32 = "x-amz-checksum-crc32";
public static final String X_AMZ_CHECKSUM_CRC32C = "x-amz-checksum-crc32c";
public static final String X_AMZ_CHECKSUM_CRC64NVME = "x-amz-checksum-crc64nvme";
public static final String X_AMZ_CHECKSUM_SHA1 = "x-amz-checksum-sha1";
public static final String X_AMZ_CHECKSUM_SHA256 = "x-amz-checksum-sha256";
public static final String X_AMZ_STORAGE_CLASS = "x-amz-storage-class";
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017-2024 Adobe.
* Copyright 2017-2025 Adobe.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -20,6 +20,7 @@
import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_ALGORITHM;
import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_CRC32;
import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_CRC32C;
import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_CRC64NVME;
import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_SHA1;
import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_SHA256;
import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CONTENT_SHA256;
Expand Down Expand Up @@ -239,6 +240,8 @@ public static String checksumFrom(HttpHeaders headers) {
return headers.getFirst(X_AMZ_CHECKSUM_CRC32);
} else if (headers.containsKey(X_AMZ_CHECKSUM_CRC32C)) {
return headers.getFirst(X_AMZ_CHECKSUM_CRC32C);
} else if (headers.containsKey(X_AMZ_CHECKSUM_CRC64NVME)) {
return headers.getFirst(X_AMZ_CHECKSUM_CRC64NVME);
}
return null;
}
Expand All @@ -249,6 +252,7 @@ public static String mapChecksumToHeader(ChecksumAlgorithm checksumAlgorithm) {
case SHA1 -> X_AMZ_CHECKSUM_SHA1;
case CRC32 -> X_AMZ_CHECKSUM_CRC32;
case CRC32C -> X_AMZ_CHECKSUM_CRC32C;
case CRC64NVME -> X_AMZ_CHECKSUM_CRC64NVME;
};
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017-2024 Adobe.
* Copyright 2017-2025 Adobe.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -184,7 +184,9 @@ internal class AwsChunkedDecodingChecksumInputStreamTest {
companion object {
@JvmStatic
private fun algorithms(): Stream<Algorithm> {
return Algorithm.entries.toList().stream()
//ignore new CRC64NVME for now, looks like AWS is moving checksums from core to a new checksums module, but not all
//types are currently compatible with the new types.
return Algorithm.entries.toList().stream().filter { it != Algorithm.CRC64NVME }
}
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017-2024 Adobe.
* Copyright 2017-2025 Adobe.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -170,7 +170,9 @@ internal class AwsUnsignedChunkedDecodingChecksumInputStreamTest {
companion object {
@JvmStatic
private fun algorithms(): Stream<Algorithm> {
return stream(Algorithm.entries.toTypedArray())
//ignore new CRC64NVME for now, looks like AWS is moving checksums from core to a new checksums module, but not all
//types are currently compatible with the new types.
return stream(Algorithm.entries.toTypedArray()).filter { it != Algorithm.CRC64NVME }
}
}
}

0 comments on commit 735f7a5

Please sign in to comment.