From 5ebbd1553b6036b19d48f6556552c84c0adef924 Mon Sep 17 00:00:00 2001 From: ndr_brt Date: Fri, 21 Jun 2024 10:02:14 +0200 Subject: [PATCH] refactor: extract validator-data-address-s3 module --- .github/workflows/verify.yaml | 70 +--------- DEPENDENCIES | 35 ++--- .../common/aws/aws-s3-core/build.gradle.kts | 1 + .../S3DataAddressCredentialsValidator.java | 4 +- ...rg.eclipse.edc.spi.system.ServiceExtension | 1 - ...S3DataAddressCredentialsValidatorTest.java | 6 +- extensions/common/aws/aws-s3-test/README.md | 2 +- .../aws/s3/testfixtures/AbstractS3Test.java | 9 +- .../edc/aws/s3/testfixtures/S3TestClient.java | 6 +- .../annotations/AwsS3IntegrationTest.java | 3 + .../build.gradle.kts | 26 ++++ .../S3DataAddressValidatorExtension.java | 7 +- .../S3DestinationDataAddressValidator.java | 6 +- .../S3SourceDataAddressValidator.java | 14 +- ...rg.eclipse.edc.spi.system.ServiceExtension | 15 ++ .../S3DataAddressValidatorExtensionTest.java} | 7 +- ...S3DestinationDataAddressValidatorTest.java | 8 +- .../S3SourceDataAddressValidatorTest.java | 12 +- .../aws/s3/S3BucketProvisionedResource.java | 6 +- ...S3ConsumerResourceDefinitionGenerator.java | 2 +- ...nsumerResourceDefinitionGeneratorTest.java | 2 +- .../data-plane-aws-s3/build.gradle.kts | 1 + .../aws/s3/DataPlaneS3Extension.java | 16 ++- .../dataplane/aws/s3/S3DataSinkFactory.java | 26 ++-- .../dataplane/aws/s3/S3DataSourceFactory.java | 27 ++-- .../dataplane/aws/s3/MinioContainer.java | 29 ++++ .../aws/s3/S3DataPlaneIntegrationTest.java | 99 ++++++++----- .../aws/s3/S3DataSinkFactoryTest.java | 91 +++++------- .../dataplane/aws/s3/S3DataSinkTest.java | 2 +- .../aws/s3/S3DataSourceFactoryTest.java | 102 ++++++-------- .../dataplane/aws/s3/TestFunctions.java | 2 +- gradle/libs.versions.toml | 2 + resources/ci/discord_webhook.sh | 131 ------------------ settings.gradle.kts | 7 +- spi/common/aws-spi/build.gradle.kts | 24 ++++ .../edc/aws/s3/spi}/S3BucketSchema.java | 2 +- 36 files changed, 349 insertions(+), 454 deletions(-) create mode 100644 extensions/common/validator/validator-data-address-s3/build.gradle.kts rename extensions/common/{aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3 => validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator}/S3DataAddressValidatorExtension.java (85%) rename extensions/common/{aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation => validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator}/S3DestinationDataAddressValidator.java (89%) rename extensions/common/{aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation => validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator}/S3SourceDataAddressValidator.java (80%) create mode 100644 extensions/common/validator/validator-data-address-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension rename extensions/common/{aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/S3DestinationDataAddressValidatorExtensionTest.java => validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator/S3DataAddressValidatorExtensionTest.java} (87%) rename extensions/common/{aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation => validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator}/S3DestinationDataAddressValidatorTest.java (88%) rename extensions/common/{aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation => validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator}/S3SourceDataAddressValidatorTest.java (88%) create mode 100644 extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/MinioContainer.java delete mode 100755 resources/ci/discord_webhook.sh create mode 100644 spi/common/aws-spi/build.gradle.kts rename {extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3 => spi/common/aws-spi/src/main/java/org/eclipse/edc/aws/s3/spi}/S3BucketSchema.java (96%) diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 27958135..5794c2a1 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -17,11 +17,8 @@ concurrency: cancel-in-progress: true jobs: + Checkstyle: - permissions: - id-token: write - checks: write - pull-requests: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -43,81 +40,23 @@ jobs: with: command: ./gradlew test jacocoTestReport - Component-Tests: - env: - JACOCO: true - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: eclipse-edc/.github/.github/actions/setup-build@main - - - name: Component Tests - uses: eclipse-edc/.github/.github/actions/run-tests@main - with: - command: ./gradlew jacocoTestReport -DincludeTags="ComponentTest" - End-To-End-Tests: runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: eclipse-edc/.github/.github/actions/setup-build@main - - - name: End to End Integration Tests - uses: eclipse-edc/.github/.github/actions/run-tests@main - with: - command: ./gradlew test -DincludeTags="EndToEndTest" - - API-Tests: env: JACOCO: true - runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: eclipse-edc/.github/.github/actions/setup-build@main - - name: Component Tests - uses: eclipse-edc/.github/.github/actions/run-tests@main - with: - command: ./gradlew test jacocoTestReport -DincludeTags="ApiTest" - - Aws-Integration-Tests: - runs-on: ubuntu-latest - - env: - S3_ACCESS_KEY_ID: root - S3_SECRET_ACCESS_KEY: password - - services: - minio-source: - image: bitnami/minio:latest - ports: - - 9000:9000 - env: - MINIO_ROOT_USER: root - MINIO_ROOT_PASSWORD: password - minio-destination: - image: bitnami/minio:latest - ports: - - 9002:9000 - env: - MINIO_ROOT_USER: root - MINIO_ROOT_PASSWORD: password - - steps: - - uses: actions/checkout@v4 - - uses: eclipse-edc/.github/.github/actions/setup-build@main - - - name: AWS Tests + - name: End to End Integration Tests uses: eclipse-edc/.github/.github/actions/run-tests@main with: - command: ./gradlew -p extensions test -DincludeTags="AwsS3IntegrationTest" + command: ./gradlew test -DincludeTags="EndToEndTest" Upload-Test-Report: needs: - Unit-Tests - - Aws-Integration-Tests - End-To-End-Tests - - Component-Tests permissions: checks: write @@ -139,8 +78,7 @@ jobs: Upload-Coverage-Report-To-Codecov: needs: - Unit-Tests - - Component-Tests - - API-Tests + - End-To-End-Tests runs-on: ubuntu-latest if: always() steps: diff --git a/DEPENDENCIES b/DEPENDENCIES index bd08bc9e..8dced534 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -3,7 +3,7 @@ maven/mavencentral/com.apicatalog/copper-multibase/0.5.0, Apache-2.0, approved, maven/mavencentral/com.apicatalog/copper-multicodec/0.1.1, Apache-2.0, approved, #14500 maven/mavencentral/com.apicatalog/iron-verifiable-credentials/0.14.0, Apache-2.0, approved, clearlydefined maven/mavencentral/com.apicatalog/titanium-json-ld/1.0.0, Apache-2.0, approved, clearlydefined -maven/mavencentral/com.apicatalog/titanium-json-ld/1.4.0, Apache-2.0, approved, #13683 +maven/mavencentral/com.apicatalog/titanium-json-ld/1.4.0, Apache-2.0, approved, #15200 maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.10.3, Apache-2.0, approved, CQ21280 maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.17.0, Apache-2.0, approved, #13672 maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.17.1, Apache-2.0, approved, #13672 @@ -15,9 +15,9 @@ maven/mavencentral/com.fasterxml.jackson.module/jackson-module-jakarta-xmlbind-a maven/mavencentral/com.fasterxml.jackson.module/jackson-module-jakarta-xmlbind-annotations/2.17.1, Apache-2.0, approved, #13668 maven/mavencentral/com.fasterxml.jackson/jackson-bom/2.17.1, Apache-2.0, approved, #14162 maven/mavencentral/com.github.docker-java/docker-java-api/3.3.6, Apache-2.0, approved, #10346 -maven/mavencentral/com.github.docker-java/docker-java-transport-zerodep/3.3.6, Apache-2.0 AND (Apache-2.0 AND BSD-3-Clause), approved, #7946 +maven/mavencentral/com.github.docker-java/docker-java-transport-zerodep/3.3.6, Apache-2.0 AND (Apache-2.0 AND BSD-3-Clause), approved, #15251 maven/mavencentral/com.github.docker-java/docker-java-transport/3.3.6, Apache-2.0, approved, #7942 -maven/mavencentral/com.google.code.findbugs/jsr305/3.0.2, Apache-2.0, approved, #20 +maven/mavencentral/com.google.code.findbugs/jsr305/3.0.2, CC-BY-2.5, approved, #15220 maven/mavencentral/com.google.code.gson/gson/2.10.1, Apache-2.0, approved, #6159 maven/mavencentral/com.google.crypto.tink/tink/1.13.0, Apache-2.0, approved, #14502 maven/mavencentral/com.google.errorprone/error_prone_annotations/2.22.0, Apache-2.0, approved, #10661 @@ -26,19 +26,19 @@ maven/mavencentral/com.google.guava/failureaccess/1.0.2, Apache-2.0, approved, C maven/mavencentral/com.google.guava/guava/33.2.0-jre, Apache-2.0 AND CC0-1.0 AND (Apache-2.0 AND CC-PDDC), approved, #14607 maven/mavencentral/com.google.guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava, Apache-2.0, approved, CQ22657 maven/mavencentral/com.google.protobuf/protobuf-java/3.25.1, BSD-3-Clause, approved, clearlydefined -maven/mavencentral/com.nimbusds/nimbus-jose-jwt/9.39.3, Apache-2.0, approved, #14830 +maven/mavencentral/com.nimbusds/nimbus-jose-jwt/9.40, Apache-2.0, approved, #15156 maven/mavencentral/com.puppycrawl.tools/checkstyle/10.17.0, LGPL-2.1-or-later AND (Apache-2.0 AND LGPL-2.1-or-later) AND Apache-2.0, approved, #15077 maven/mavencentral/com.squareup.okhttp3/okhttp-dnsoverhttps/4.12.0, Apache-2.0, approved, #11159 -maven/mavencentral/com.squareup.okhttp3/okhttp/4.12.0, Apache-2.0, approved, #11156 +maven/mavencentral/com.squareup.okhttp3/okhttp/4.12.0, Apache-2.0, approved, #15227 maven/mavencentral/com.squareup.okhttp3/okhttp/4.9.3, Apache-2.0 AND MPL-2.0, approved, #3225 maven/mavencentral/com.squareup.okio/okio-jvm/3.6.0, Apache-2.0, approved, #11158 maven/mavencentral/com.squareup.okio/okio/3.6.0, Apache-2.0, approved, #11155 maven/mavencentral/commons-beanutils/commons-beanutils/1.9.4, Apache-2.0, approved, CQ12654 maven/mavencentral/commons-codec/commons-codec/1.11, Apache-2.0 AND BSD-3-Clause, approved, CQ15971 maven/mavencentral/commons-codec/commons-codec/1.15, Apache-2.0 AND BSD-3-Clause AND LicenseRef-Public-Domain, approved, CQ22641 -maven/mavencentral/commons-collections/commons-collections/3.2.2, Apache-2.0, approved, CQ10385 +maven/mavencentral/commons-collections/commons-collections/3.2.2, Apache-2.0, approved, #15185 maven/mavencentral/commons-logging/commons-logging/1.2, Apache-2.0, approved, CQ10162 -maven/mavencentral/dev.failsafe/failsafe-okhttp/3.3.2, Apache-2.0, approved, #9178 +maven/mavencentral/dev.failsafe/failsafe-okhttp/3.3.2, Apache-2.0, approved, #15208 maven/mavencentral/dev.failsafe/failsafe/3.3.1, Apache-2.0, approved, #9268 maven/mavencentral/dev.failsafe/failsafe/3.3.2, Apache-2.0, approved, #9268 maven/mavencentral/info.picocli/picocli/4.7.6, Apache-2.0, approved, #4365 @@ -59,6 +59,7 @@ maven/mavencentral/io.setl/rdf-urdna/1.1, Apache-2.0, approved, clearlydefined maven/mavencentral/jakarta.activation/jakarta.activation-api/2.1.3, EPL-2.0 OR BSD-3-Clause OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.jaf maven/mavencentral/jakarta.annotation/jakarta.annotation-api/2.1.1, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.ca maven/mavencentral/jakarta.inject/jakarta.inject-api/2.0.1, Apache-2.0, approved, ee4j.cdi +maven/mavencentral/jakarta.json/jakarta.json-api/2.1.3, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.jsonp maven/mavencentral/jakarta.transaction/jakarta.transaction-api/2.0.0, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.jta maven/mavencentral/jakarta.validation/jakarta.validation-api/3.0.2, Apache-2.0, approved, ee4j.validation maven/mavencentral/jakarta.ws.rs/jakarta.ws.rs-api/3.1.0, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.rest @@ -70,7 +71,7 @@ maven/mavencentral/net.bytebuddy/byte-buddy-agent/1.14.15, Apache-2.0, approved, maven/mavencentral/net.bytebuddy/byte-buddy/1.14.1, Apache-2.0 AND BSD-3-Clause, approved, #7163 maven/mavencentral/net.bytebuddy/byte-buddy/1.14.15, Apache-2.0 AND BSD-3-Clause, approved, #7163 maven/mavencentral/net.bytebuddy/byte-buddy/1.14.16, Apache-2.0 AND BSD-3-Clause, approved, #7163 -maven/mavencentral/net.java.dev.jna/jna/5.13.0, Apache-2.0 AND LGPL-2.1-or-later, approved, #6709 +maven/mavencentral/net.java.dev.jna/jna/5.13.0, Apache-2.0 AND LGPL-2.1-or-later, approved, #15196 maven/mavencentral/net.sf.saxon/Saxon-HE/12.4, MPL-2.0 AND (MPL-2.0 AND Apache-2.0) AND (MPL-2.0 AND LicenseRef-X11-style) AND MPL-1.0 AND W3C, approved, #12716 maven/mavencentral/org.antlr/antlr4-runtime/4.13.1, BSD-3-Clause, approved, #10767 maven/mavencentral/org.apache.commons/commons-compress/1.24.0, Apache-2.0 AND BSD-3-Clause AND bzip2-1.0.6 AND LicenseRef-Public-Domain, approved, #10368 @@ -80,7 +81,7 @@ maven/mavencentral/org.apache.commons/commons-text/1.3, Apache-2.0, approved, cl maven/mavencentral/org.apache.httpcomponents.client5/httpclient5/5.1.3, Apache-2.0, approved, #6276 maven/mavencentral/org.apache.httpcomponents.core5/httpcore5-h2/5.1.3, Apache-2.0, approved, clearlydefined maven/mavencentral/org.apache.httpcomponents.core5/httpcore5/5.1.3, Apache-2.0, approved, clearlydefined -maven/mavencentral/org.apache.httpcomponents/httpclient/4.5.13, Apache-2.0 AND LicenseRef-Public-Domain, approved, CQ23527 +maven/mavencentral/org.apache.httpcomponents/httpclient/4.5.13, Apache-2.0, approved, #15248 maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.13, Apache-2.0, approved, CQ23528 maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.14, Apache-2.0, approved, CQ23528 maven/mavencentral/org.apache.maven.doxia/doxia-core/1.12.0, Apache-2.0, approved, clearlydefined @@ -101,6 +102,7 @@ maven/mavencentral/org.codehaus.plexus/plexus-container-default/2.1.0, Apache-2. maven/mavencentral/org.codehaus.plexus/plexus-utils/3.1.1, , approved, CQ16492 maven/mavencentral/org.codehaus.plexus/plexus-utils/3.3.0, , approved, CQ21066 maven/mavencentral/org.eclipse.edc/asset-spi/0.7.1-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/auth-spi/0.7.1-SNAPSHOT, Apache-2.0, approved, technology.edc maven/mavencentral/org.eclipse.edc/autodoc-processor/0.7.1-SNAPSHOT, Apache-2.0, approved, technology.edc maven/mavencentral/org.eclipse.edc/boot-lib/0.7.1-SNAPSHOT, Apache-2.0, approved, technology.edc maven/mavencentral/org.eclipse.edc/boot-spi/0.7.1-SNAPSHOT, Apache-2.0, approved, technology.edc @@ -164,6 +166,7 @@ maven/mavencentral/org.eclipse.jetty/jetty-servlet/11.0.21, EPL-2.0 OR Apache-2. maven/mavencentral/org.eclipse.jetty/jetty-util/11.0.21, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-webapp/11.0.21, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-xml/11.0.21, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.parsson/parsson/1.1.6, EPL-2.0, approved, ee4j.parsson maven/mavencentral/org.glassfish.hk2.external/aopalliance-repackaged/3.0.6, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.glassfish maven/mavencentral/org.glassfish.hk2/hk2-api/3.0.6, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.glassfish maven/mavencentral/org.glassfish.hk2/hk2-locator/3.0.6, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.glassfish @@ -200,13 +203,13 @@ maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.10.1, EPL-2.0, approved maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.10.2, EPL-2.0, approved, #9714 maven/mavencentral/org.junit.jupiter/junit-jupiter-engine/5.10.1, EPL-2.0, approved, #9711 maven/mavencentral/org.junit.jupiter/junit-jupiter-engine/5.10.2, EPL-2.0, approved, #9711 -maven/mavencentral/org.junit.jupiter/junit-jupiter-params/5.10.1, EPL-2.0, approved, #9708 -maven/mavencentral/org.junit.jupiter/junit-jupiter-params/5.10.2, EPL-2.0, approved, #9708 +maven/mavencentral/org.junit.jupiter/junit-jupiter-params/5.10.1, EPL-2.0, approved, #15304 +maven/mavencentral/org.junit.jupiter/junit-jupiter-params/5.10.2, EPL-2.0, approved, #15250 maven/mavencentral/org.junit.platform/junit-platform-commons/1.10.1, EPL-2.0, approved, #9715 maven/mavencentral/org.junit.platform/junit-platform-commons/1.10.2, EPL-2.0, approved, #9715 maven/mavencentral/org.junit.platform/junit-platform-engine/1.10.1, EPL-2.0, approved, #9709 maven/mavencentral/org.junit.platform/junit-platform-engine/1.10.2, EPL-2.0, approved, #9709 -maven/mavencentral/org.junit.platform/junit-platform-launcher/1.10.2, EPL-2.0, approved, #9704 +maven/mavencentral/org.junit.platform/junit-platform-launcher/1.10.2, EPL-2.0, approved, #15216 maven/mavencentral/org.junit/junit-bom/5.10.1, EPL-2.0, approved, #9844 maven/mavencentral/org.junit/junit-bom/5.10.2, EPL-2.0, approved, #9844 maven/mavencentral/org.junit/junit-bom/5.9.2, EPL-2.0, approved, #4711 @@ -236,7 +239,7 @@ maven/mavencentral/software.amazon.awssdk/apache-client/2.25.66, Apache-2.0, app maven/mavencentral/software.amazon.awssdk/arns/2.25.66, Apache-2.0, approved, #13695 maven/mavencentral/software.amazon.awssdk/auth/2.25.66, Apache-2.0, approved, #13692 maven/mavencentral/software.amazon.awssdk/aws-core/2.25.66, Apache-2.0, approved, #13702 -maven/mavencentral/software.amazon.awssdk/aws-json-protocol/2.25.66, , restricted, clearlydefined +maven/mavencentral/software.amazon.awssdk/aws-json-protocol/2.25.66, Apache-2.0, approved, clearlydefined maven/mavencentral/software.amazon.awssdk/aws-query-protocol/2.25.66, Apache-2.0, approved, #13701 maven/mavencentral/software.amazon.awssdk/aws-xml-protocol/2.25.66, Apache-2.0, approved, #13684 maven/mavencentral/software.amazon.awssdk/checksums-spi/2.25.66, Apache-2.0, approved, #13686 @@ -247,7 +250,7 @@ maven/mavencentral/software.amazon.awssdk/http-auth-aws/2.25.66, Apache-2.0, app maven/mavencentral/software.amazon.awssdk/http-auth-spi/2.25.66, Apache-2.0, approved, #13704 maven/mavencentral/software.amazon.awssdk/http-auth/2.25.66, Apache-2.0, approved, #13682 maven/mavencentral/software.amazon.awssdk/http-client-spi/2.25.66, Apache-2.0, approved, #13706 -maven/mavencentral/software.amazon.awssdk/iam/2.25.66, , restricted, clearlydefined +maven/mavencentral/software.amazon.awssdk/iam/2.25.66, Apache-2.0, approved, clearlydefined maven/mavencentral/software.amazon.awssdk/identity-spi/2.25.66, Apache-2.0, approved, #13685 maven/mavencentral/software.amazon.awssdk/json-utils/2.25.66, Apache-2.0, approved, #13698 maven/mavencentral/software.amazon.awssdk/metrics-spi/2.25.66, Apache-2.0, approved, #13680 @@ -257,8 +260,8 @@ maven/mavencentral/software.amazon.awssdk/protocol-core/2.25.66, Apache-2.0, app maven/mavencentral/software.amazon.awssdk/regions/2.25.66, Apache-2.0, approved, #13694 maven/mavencentral/software.amazon.awssdk/s3/2.25.66, Apache-2.0, approved, #13688 maven/mavencentral/software.amazon.awssdk/sdk-core/2.25.66, Apache-2.0, approved, #13700 -maven/mavencentral/software.amazon.awssdk/secretsmanager/2.25.66, , restricted, clearlydefined -maven/mavencentral/software.amazon.awssdk/sts/2.25.66, , restricted, clearlydefined +maven/mavencentral/software.amazon.awssdk/secretsmanager/2.25.66, Apache-2.0, approved, clearlydefined +maven/mavencentral/software.amazon.awssdk/sts/2.25.66, Apache-2.0, approved, clearlydefined maven/mavencentral/software.amazon.awssdk/third-party-jackson-core/2.25.66, Apache-2.0, approved, #13703 maven/mavencentral/software.amazon.awssdk/utils/2.25.66, Apache-2.0, approved, #13689 maven/mavencentral/software.amazon.eventstream/eventstream/1.0.1, Apache-2.0, approved, clearlydefined diff --git a/extensions/common/aws/aws-s3-core/build.gradle.kts b/extensions/common/aws/aws-s3-core/build.gradle.kts index e48ea3af..527fe8c6 100644 --- a/extensions/common/aws/aws-s3-core/build.gradle.kts +++ b/extensions/common/aws/aws-s3-core/build.gradle.kts @@ -17,6 +17,7 @@ plugins { } dependencies { + api(project(":spi:common:aws-spi")) api(libs.edc.spi.transfer) api(libs.edc.spi.validation) diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation/S3DataAddressCredentialsValidator.java b/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation/S3DataAddressCredentialsValidator.java index 1082e3f0..c9da3546 100644 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation/S3DataAddressCredentialsValidator.java +++ b/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation/S3DataAddressCredentialsValidator.java @@ -21,8 +21,8 @@ import java.util.Objects; import java.util.stream.Stream; -import static org.eclipse.edc.aws.s3.S3BucketSchema.ACCESS_KEY_ID; -import static org.eclipse.edc.aws.s3.S3BucketSchema.SECRET_ACCESS_KEY; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.ACCESS_KEY_ID; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.SECRET_ACCESS_KEY; import static org.eclipse.edc.validator.spi.Violation.violation; /** diff --git a/extensions/common/aws/aws-s3-core/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/common/aws/aws-s3-core/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension index 5bea16c6..4fb76b32 100644 --- a/extensions/common/aws/aws-s3-core/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ b/extensions/common/aws/aws-s3-core/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -13,4 +13,3 @@ # org.eclipse.edc.aws.s3.S3CoreExtension -org.eclipse.edc.aws.s3.S3DataAddressValidatorExtension diff --git a/extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation/S3DataAddressCredentialsValidatorTest.java b/extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation/S3DataAddressCredentialsValidatorTest.java index 6712f21d..59df3ada 100644 --- a/extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation/S3DataAddressCredentialsValidatorTest.java +++ b/extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation/S3DataAddressCredentialsValidatorTest.java @@ -20,9 +20,9 @@ import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.eclipse.edc.aws.s3.S3BucketSchema.ACCESS_KEY_ID; -import static org.eclipse.edc.aws.s3.S3BucketSchema.SECRET_ACCESS_KEY; -import static org.eclipse.edc.aws.s3.S3BucketSchema.TYPE; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.ACCESS_KEY_ID; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.SECRET_ACCESS_KEY; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.TYPE; import static org.eclipse.edc.junit.assertions.AbstractResultAssert.assertThat; public class S3DataAddressCredentialsValidatorTest { diff --git a/extensions/common/aws/aws-s3-test/README.md b/extensions/common/aws/aws-s3-test/README.md index e1155546..8c0bb5ce 100644 --- a/extensions/common/aws/aws-s3-test/README.md +++ b/extensions/common/aws/aws-s3-test/README.md @@ -25,7 +25,7 @@ for running integration tests against AWS S3 by environment variable: $ IT_AWS_ENDPOINT=https://s3.us-east-1.amazonaws.com/ \ IT_AWS_REGION=us-east-1 \ IT_AWS_PROFILE=myprofie \ - ./gradlew clean test -DincludeTags="AwsS3IntegrationTest" + ./gradlew clean test -DincludeTags="EndToEndTest" ``` `IT_AWS_REGION` must be set to your region code in order to avoid diff --git a/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/AbstractS3Test.java b/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/AbstractS3Test.java index d3b03120..e4c49bcf 100644 --- a/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/AbstractS3Test.java +++ b/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/AbstractS3Test.java @@ -16,7 +16,6 @@ package org.eclipse.edc.aws.s3.testfixtures; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import software.amazon.awssdk.regions.Region; @@ -30,7 +29,10 @@ /** * Base class for tests that need an S3 bucket created and deleted on every test run. + * + * @deprecated it is not used anymore in the Technology-Aws repository, so it will be removed. */ +@Deprecated(since = "0.7.1") public abstract class AbstractS3Test { protected static final String REGION = propOrEnv("it.aws.region", Region.US_EAST_1.id()); @@ -46,16 +48,13 @@ public abstract class AbstractS3Test { protected String bucketName = "test-bucket-" + UUID.randomUUID() + "-" + REGION; protected S3TestClient sourceClient = S3TestClient.create(SOURCE_MINIO_ENDPOINT, REGION); - protected S3TestClient destinationClient = S3TestClient.create(DESTINATION_MINIO_ENDPOINT, REGION); protected static final String OBJECT_PREFIX = "object-prefix/"; - protected static final String OBJECT_NAME = "text-document.txt"; - protected static final String KEY_NAME = "key-name"; - @BeforeAll + @BeforeEach void prepareAll() { await().atLeast(Duration.ofSeconds(2)) .atMost(Duration.ofSeconds(15)) diff --git a/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/S3TestClient.java b/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/S3TestClient.java index 8fb31895..f9bcc001 100644 --- a/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/S3TestClient.java +++ b/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/S3TestClient.java @@ -62,7 +62,7 @@ public class S3TestClient { private S3TestClient(String url, String region) { this.url = url; this.s3Endpoint = URI.create(propOrEnv("it.aws.endpoint", url)); - AwsClientProviderConfiguration configuration = AwsClientProviderConfiguration.Builder.newInstance() + var configuration = AwsClientProviderConfiguration.Builder.newInstance() .credentialsProvider(this::getCredentials) .endpointOverride(this.s3Endpoint) .build(); @@ -93,7 +93,7 @@ public static S3TestClient create(String url, String region) { * * @return true if HTTP status [200..300[ */ - protected boolean isAvailable() throws IOException { + public boolean isAvailable() throws IOException { var httpClient = testHttpClient(); var healthRq = new Request.Builder().url(s3Endpoint + "/minio/health/live").get().build(); try (var response = httpClient.execute(healthRq)) { @@ -178,7 +178,7 @@ public CompletableFuture> getObject(String buck .getObject(GetObjectRequest.builder().bucket(bucketName).key(key).build(), new ByteArrayAsyncResponseTransformer<>()); } - protected boolean isMinio() { + public boolean isMinio() { return url.equals(s3Endpoint.toString()); } diff --git a/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/annotations/AwsS3IntegrationTest.java b/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/annotations/AwsS3IntegrationTest.java index 9a56e253..a624341f 100644 --- a/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/annotations/AwsS3IntegrationTest.java +++ b/extensions/common/aws/aws-s3-test/src/testFixtures/java/org/eclipse/edc/aws/s3/testfixtures/annotations/AwsS3IntegrationTest.java @@ -24,10 +24,13 @@ /** * Annotation for AWS S3 integration testing. It applies specific Junit Tag. + * + * @deprecated not used anymore, please use {@link org.eclipse.edc.junit.annotations.EndToEndTest}. */ @Target({ ElementType.TYPE }) @Retention(RetentionPolicy.RUNTIME) @IntegrationTest @Tag("AwsS3IntegrationTest") +@Deprecated(since = "0.7.1") public @interface AwsS3IntegrationTest { } diff --git a/extensions/common/validator/validator-data-address-s3/build.gradle.kts b/extensions/common/validator/validator-data-address-s3/build.gradle.kts new file mode 100644 index 00000000..308e19df --- /dev/null +++ b/extensions/common/validator/validator-data-address-s3/build.gradle.kts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +plugins { + `java-library` +} + +dependencies { + api(project(":spi:common:aws-spi")) + api(libs.edc.spi.validation) + + testImplementation(libs.edc.junit) +} + + diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3DataAddressValidatorExtension.java b/extensions/common/validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator/S3DataAddressValidatorExtension.java similarity index 85% rename from extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3DataAddressValidatorExtension.java rename to extensions/common/validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator/S3DataAddressValidatorExtension.java index d69bc2e3..7597806b 100644 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3DataAddressValidatorExtension.java +++ b/extensions/common/validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator/S3DataAddressValidatorExtension.java @@ -12,17 +12,16 @@ * */ -package org.eclipse.edc.aws.s3; +package org.eclipse.edc.aws.s3.validator; -import org.eclipse.edc.aws.s3.validation.S3DestinationDataAddressValidator; -import org.eclipse.edc.aws.s3.validation.S3SourceDataAddressValidator; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.validator.spi.DataAddressValidatorRegistry; -import static org.eclipse.edc.aws.s3.S3DataAddressValidatorExtension.NAME; +import static org.eclipse.edc.aws.s3.validator.S3DataAddressValidatorExtension.NAME; @Extension(NAME) public class S3DataAddressValidatorExtension implements ServiceExtension { diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation/S3DestinationDataAddressValidator.java b/extensions/common/validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator/S3DestinationDataAddressValidator.java similarity index 89% rename from extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation/S3DestinationDataAddressValidator.java rename to extensions/common/validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator/S3DestinationDataAddressValidator.java index 70f7922d..7080198b 100644 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation/S3DestinationDataAddressValidator.java +++ b/extensions/common/validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator/S3DestinationDataAddressValidator.java @@ -12,7 +12,7 @@ * */ -package org.eclipse.edc.aws.s3.validation; +package org.eclipse.edc.aws.s3.validator; import org.eclipse.edc.spi.types.domain.DataAddress; import org.eclipse.edc.validator.spi.ValidationResult; @@ -21,8 +21,8 @@ import java.util.Objects; import java.util.stream.Stream; -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.BUCKET_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.REGION; import static org.eclipse.edc.validator.spi.Violation.violation; /** diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation/S3SourceDataAddressValidator.java b/extensions/common/validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator/S3SourceDataAddressValidator.java similarity index 80% rename from extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation/S3SourceDataAddressValidator.java rename to extensions/common/validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator/S3SourceDataAddressValidator.java index 245fe76c..6bc5921e 100644 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/validation/S3SourceDataAddressValidator.java +++ b/extensions/common/validator/validator-data-address-s3/src/main/java/org/eclipse/edc/aws/s3/validator/S3SourceDataAddressValidator.java @@ -12,7 +12,7 @@ * */ -package org.eclipse.edc.aws.s3.validation; +package org.eclipse.edc.aws.s3.validator; import org.eclipse.edc.spi.types.domain.DataAddress; import org.eclipse.edc.validator.spi.ValidationResult; @@ -22,12 +22,12 @@ import java.util.ArrayList; import java.util.stream.Stream; -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.KEY_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.KEY_PREFIX; -import static org.eclipse.edc.aws.s3.S3BucketSchema.OBJECT_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.OBJECT_PREFIX; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.BUCKET_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.KEY_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.KEY_PREFIX; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.OBJECT_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.OBJECT_PREFIX; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.REGION; import static org.eclipse.edc.validator.spi.Violation.violation; /** diff --git a/extensions/common/validator/validator-data-address-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/common/validator/validator-data-address-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 00000000..a00e4ed9 --- /dev/null +++ b/extensions/common/validator/validator-data-address-s3/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1,15 @@ +# +# Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# +# Contributors: +# Bayerische Motoren Werke Aktiengesellschaft (BMW AG) +# +# + +org.eclipse.edc.aws.s3.validator.S3DataAddressValidatorExtension diff --git a/extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/S3DestinationDataAddressValidatorExtensionTest.java b/extensions/common/validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator/S3DataAddressValidatorExtensionTest.java similarity index 87% rename from extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/S3DestinationDataAddressValidatorExtensionTest.java rename to extensions/common/validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator/S3DataAddressValidatorExtensionTest.java index 88e6b7f5..85d8de94 100644 --- a/extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/S3DestinationDataAddressValidatorExtensionTest.java +++ b/extensions/common/validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator/S3DataAddressValidatorExtensionTest.java @@ -12,11 +12,10 @@ * */ -package org.eclipse.edc.aws.s3; +package org.eclipse.edc.aws.s3.validator; -import org.eclipse.edc.aws.s3.validation.S3DestinationDataAddressValidator; -import org.eclipse.edc.aws.s3.validation.S3SourceDataAddressValidator; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; import org.eclipse.edc.junit.extensions.DependencyInjectionExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.validator.spi.DataAddressValidatorRegistry; @@ -31,7 +30,7 @@ import static org.mockito.Mockito.verify; @ExtendWith(DependencyInjectionExtension.class) -public class S3DestinationDataAddressValidatorExtensionTest { +public class S3DataAddressValidatorExtensionTest { private final DataAddressValidatorRegistry registry = mock(); diff --git a/extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation/S3DestinationDataAddressValidatorTest.java b/extensions/common/validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator/S3DestinationDataAddressValidatorTest.java similarity index 88% rename from extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation/S3DestinationDataAddressValidatorTest.java rename to extensions/common/validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator/S3DestinationDataAddressValidatorTest.java index 80564a72..fb0b7dc9 100644 --- a/extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation/S3DestinationDataAddressValidatorTest.java +++ b/extensions/common/validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator/S3DestinationDataAddressValidatorTest.java @@ -12,7 +12,7 @@ * */ -package org.eclipse.edc.aws.s3.validation; +package org.eclipse.edc.aws.s3.validator; import org.eclipse.edc.spi.types.domain.DataAddress; import org.eclipse.edc.validator.spi.ValidationFailure; @@ -20,9 +20,9 @@ import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; -import static org.eclipse.edc.aws.s3.S3BucketSchema.TYPE; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.BUCKET_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.REGION; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.TYPE; import static org.eclipse.edc.junit.assertions.AbstractResultAssert.assertThat; public class S3DestinationDataAddressValidatorTest { diff --git a/extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation/S3SourceDataAddressValidatorTest.java b/extensions/common/validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator/S3SourceDataAddressValidatorTest.java similarity index 88% rename from extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation/S3SourceDataAddressValidatorTest.java rename to extensions/common/validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator/S3SourceDataAddressValidatorTest.java index 3e2430f1..83151a1d 100644 --- a/extensions/common/aws/aws-s3-core/src/test/java/org/eclipse/edc/aws/s3/validation/S3SourceDataAddressValidatorTest.java +++ b/extensions/common/validator/validator-data-address-s3/src/test/java/org/eclipse/edc/aws/s3/validator/S3SourceDataAddressValidatorTest.java @@ -12,7 +12,7 @@ * */ -package org.eclipse.edc.aws.s3.validation; +package org.eclipse.edc.aws.s3.validator; import org.eclipse.edc.spi.types.domain.DataAddress; import org.eclipse.edc.validator.spi.ValidationFailure; @@ -20,11 +20,11 @@ import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.OBJECT_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.OBJECT_PREFIX; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; -import static org.eclipse.edc.aws.s3.S3BucketSchema.TYPE; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.BUCKET_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.OBJECT_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.OBJECT_PREFIX; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.REGION; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.TYPE; import static org.eclipse.edc.junit.assertions.AbstractResultAssert.assertThat; public class S3SourceDataAddressValidatorTest { diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionedResource.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionedResource.java index 1e5a4a36..9b442946 100644 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionedResource.java +++ b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3BucketProvisionedResource.java @@ -18,11 +18,11 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; -import org.eclipse.edc.aws.s3.S3BucketSchema; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; import org.eclipse.edc.connector.controlplane.transfer.spi.types.ProvisionedDataDestinationResource; -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.BUCKET_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.REGION; /** diff --git a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGenerator.java b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGenerator.java index 129b5b76..7512eb7d 100644 --- a/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGenerator.java +++ b/extensions/control-plane/provision/provision-aws-s3/src/main/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGenerator.java @@ -16,7 +16,7 @@ package org.eclipse.edc.connector.provision.aws.s3; -import org.eclipse.edc.aws.s3.S3BucketSchema; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ConsumerResourceDefinitionGenerator; import org.eclipse.edc.connector.controlplane.transfer.spi.types.ResourceDefinition; import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcess; diff --git a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGeneratorTest.java b/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGeneratorTest.java index 7bb3dc10..fc501d92 100644 --- a/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGeneratorTest.java +++ b/extensions/control-plane/provision/provision-aws-s3/src/test/java/org/eclipse/edc/connector/provision/aws/s3/S3ConsumerResourceDefinitionGeneratorTest.java @@ -15,7 +15,7 @@ package org.eclipse.edc.connector.provision.aws.s3; -import org.eclipse.edc.aws.s3.S3BucketSchema; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcess; import org.eclipse.edc.policy.model.Policy; diff --git a/extensions/data-plane/data-plane-aws-s3/build.gradle.kts b/extensions/data-plane/data-plane-aws-s3/build.gradle.kts index ebbe2a48..368257eb 100644 --- a/extensions/data-plane/data-plane-aws-s3/build.gradle.kts +++ b/extensions/data-plane/data-plane-aws-s3/build.gradle.kts @@ -28,6 +28,7 @@ dependencies { testImplementation(libs.edc.core.dataplane) testImplementation(testFixtures(project(":extensions:common:aws:aws-s3-test"))) testImplementation(libs.edc.junit) + testImplementation(libs.testcontainers.junit.jupiter) } diff --git a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/DataPlaneS3Extension.java b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/DataPlaneS3Extension.java index 624850be..71e0b3df 100644 --- a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/DataPlaneS3Extension.java +++ b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/DataPlaneS3Extension.java @@ -23,19 +23,18 @@ import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.validator.spi.DataAddressValidatorRegistry; import java.util.concurrent.Executors; -import static java.lang.Integer.parseInt; - @Extension(value = DataPlaneS3Extension.NAME) public class DataPlaneS3Extension implements ServiceExtension { public static final String NAME = "Data Plane S3 Storage"; - static final String DEFAULT_CHUNK_SIZE_IN_MB = "500"; // 500MB chunk size + private static final int DEFAULT_CHUNK_SIZE_IN_MB = 500; // 500MB chunk size - @Setting(value = "The maximum chunk of stream to be read, in mb", defaultValue = DEFAULT_CHUNK_SIZE_IN_MB, type = "int") + @Setting(value = "The maximum chunk of stream to be read, in mb", defaultValue = DEFAULT_CHUNK_SIZE_IN_MB + "", type = "int") private static final String EDC_DATAPLANE_S3_SINK_CHUNK_SIZE_MB = "edc.dataplane.aws.sink.chunk.size.mb"; @Inject @@ -50,6 +49,9 @@ public class DataPlaneS3Extension implements ServiceExtension { @Inject private TypeManager typeManager; + @Inject + private DataAddressValidatorRegistry validator; + @Override public String name() { return NAME; @@ -59,17 +61,17 @@ public String name() { public void initialize(ServiceExtensionContext context) { var executorService = Executors.newFixedThreadPool(10); // TODO make configurable - var chunkSizeInMb = context.getSetting(EDC_DATAPLANE_S3_SINK_CHUNK_SIZE_MB, parseInt(DEFAULT_CHUNK_SIZE_IN_MB)); + var chunkSizeInMb = context.getSetting(EDC_DATAPLANE_S3_SINK_CHUNK_SIZE_MB, DEFAULT_CHUNK_SIZE_IN_MB); var chunkSizeInBytes = 1024 * 1024 * chunkSizeInMb; if (chunkSizeInBytes < 1) { throw new IllegalArgumentException("Chunk size must be greater than zero! Actual value is: " + chunkSizeInBytes); } var monitor = context.getMonitor(); - var sourceFactory = new S3DataSourceFactory(awsClientProvider, monitor, vault, typeManager); + var sourceFactory = new S3DataSourceFactory(awsClientProvider, monitor, vault, typeManager, validator); pipelineService.registerFactory(sourceFactory); - var sinkFactory = new S3DataSinkFactory(awsClientProvider, executorService, monitor, vault, typeManager, chunkSizeInBytes); + var sinkFactory = new S3DataSinkFactory(awsClientProvider, executorService, monitor, vault, typeManager, chunkSizeInBytes, validator); pipelineService.registerFactory(sinkFactory); } diff --git a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactory.java b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactory.java index 20e3266f..a28441e0 100644 --- a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactory.java +++ b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactory.java @@ -18,10 +18,9 @@ import org.eclipse.edc.aws.s3.AwsClientProvider; import org.eclipse.edc.aws.s3.AwsSecretToken; import org.eclipse.edc.aws.s3.AwsTemporarySecretToken; -import org.eclipse.edc.aws.s3.S3BucketSchema; import org.eclipse.edc.aws.s3.S3ClientRequest; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; import org.eclipse.edc.aws.s3.validation.S3DataAddressCredentialsValidator; -import org.eclipse.edc.aws.s3.validation.S3DestinationDataAddressValidator; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSink; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSinkFactory; import org.eclipse.edc.spi.EdcException; @@ -32,6 +31,7 @@ import org.eclipse.edc.spi.types.domain.DataAddress; import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage; import org.eclipse.edc.util.string.StringUtils; +import org.eclipse.edc.validator.spi.DataAddressValidatorRegistry; import org.eclipse.edc.validator.spi.ValidationResult; import org.eclipse.edc.validator.spi.Validator; import org.jetbrains.annotations.NotNull; @@ -39,17 +39,16 @@ import java.util.concurrent.ExecutorService; import static java.util.Optional.ofNullable; -import static org.eclipse.edc.aws.s3.S3BucketSchema.ACCESS_KEY_ID; -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.ENDPOINT_OVERRIDE; -import static org.eclipse.edc.aws.s3.S3BucketSchema.FOLDER_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.OBJECT_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; -import static org.eclipse.edc.aws.s3.S3BucketSchema.SECRET_ACCESS_KEY; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.ACCESS_KEY_ID; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.BUCKET_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.ENDPOINT_OVERRIDE; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.FOLDER_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.OBJECT_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.REGION; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.SECRET_ACCESS_KEY; public class S3DataSinkFactory implements DataSinkFactory { - private final Validator validation = new S3DestinationDataAddressValidator(); private final Validator credentialsValidation = new S3DataAddressCredentialsValidator(); private final AwsClientProvider clientProvider; private final ExecutorService executorService; @@ -57,14 +56,17 @@ public class S3DataSinkFactory implements DataSinkFactory { private final Vault vault; private final TypeManager typeManager; private final int chunkSizeInBytes; + private final DataAddressValidatorRegistry dataAddressValidator; - public S3DataSinkFactory(AwsClientProvider clientProvider, ExecutorService executorService, Monitor monitor, Vault vault, TypeManager typeManager, int chunkSizeInBytes) { + public S3DataSinkFactory(AwsClientProvider clientProvider, ExecutorService executorService, Monitor monitor, Vault vault, + TypeManager typeManager, int chunkSizeInBytes, DataAddressValidatorRegistry dataAddressValidator) { this.clientProvider = clientProvider; this.executorService = executorService; this.monitor = monitor; this.vault = vault; this.typeManager = typeManager; this.chunkSizeInBytes = chunkSizeInBytes; + this.dataAddressValidator = dataAddressValidator; } @Override @@ -104,7 +106,7 @@ public DataSink createSink(DataFlowStartMessage request) { public @NotNull Result validateRequest(DataFlowStartMessage request) { var destination = request.getDestinationDataAddress(); - return validation.validate(destination).flatMap(ValidationResult::toResult); + return dataAddressValidator.validateDestination(destination).flatMap(ValidationResult::toResult); } private S3ClientRequest createS3ClientRequest(DataAddress address) { diff --git a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactory.java b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactory.java index 0cd05819..5c412cf3 100644 --- a/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactory.java +++ b/extensions/data-plane/data-plane-aws-s3/src/main/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactory.java @@ -18,10 +18,9 @@ import org.eclipse.edc.aws.s3.AwsClientProvider; import org.eclipse.edc.aws.s3.AwsSecretToken; import org.eclipse.edc.aws.s3.AwsTemporarySecretToken; -import org.eclipse.edc.aws.s3.S3BucketSchema; import org.eclipse.edc.aws.s3.S3ClientRequest; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; import org.eclipse.edc.aws.s3.validation.S3DataAddressCredentialsValidator; -import org.eclipse.edc.aws.s3.validation.S3SourceDataAddressValidator; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSourceFactory; import org.eclipse.edc.spi.EdcException; @@ -32,34 +31,36 @@ import org.eclipse.edc.spi.types.domain.DataAddress; import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage; import org.eclipse.edc.util.string.StringUtils; +import org.eclipse.edc.validator.spi.DataAddressValidatorRegistry; import org.eclipse.edc.validator.spi.ValidationResult; import org.eclipse.edc.validator.spi.Validator; import org.jetbrains.annotations.NotNull; import static java.util.Optional.ofNullable; -import static org.eclipse.edc.aws.s3.S3BucketSchema.ACCESS_KEY_ID; -import static org.eclipse.edc.aws.s3.S3BucketSchema.BUCKET_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.ENDPOINT_OVERRIDE; -import static org.eclipse.edc.aws.s3.S3BucketSchema.KEY_PREFIX; -import static org.eclipse.edc.aws.s3.S3BucketSchema.OBJECT_NAME; -import static org.eclipse.edc.aws.s3.S3BucketSchema.OBJECT_PREFIX; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; -import static org.eclipse.edc.aws.s3.S3BucketSchema.SECRET_ACCESS_KEY; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.ACCESS_KEY_ID; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.BUCKET_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.ENDPOINT_OVERRIDE; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.KEY_PREFIX; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.OBJECT_NAME; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.OBJECT_PREFIX; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.REGION; +import static org.eclipse.edc.aws.s3.spi.S3BucketSchema.SECRET_ACCESS_KEY; public class S3DataSourceFactory implements DataSourceFactory { - private final Validator validation = new S3SourceDataAddressValidator(); private final Validator credentialsValidation = new S3DataAddressCredentialsValidator(); private final AwsClientProvider clientProvider; private final Monitor monitor; private final Vault vault; private final TypeManager typeManager; + private final DataAddressValidatorRegistry validator; - public S3DataSourceFactory(AwsClientProvider clientProvider, Monitor monitor, Vault vault, TypeManager typeManager) { + public S3DataSourceFactory(AwsClientProvider clientProvider, Monitor monitor, Vault vault, TypeManager typeManager, DataAddressValidatorRegistry validator) { this.clientProvider = clientProvider; this.monitor = monitor; this.vault = vault; this.typeManager = typeManager; + this.validator = validator; } @Override @@ -96,7 +97,7 @@ public DataSource createSource(DataFlowStartMessage request) { @Override public @NotNull Result validateRequest(DataFlowStartMessage request) { var source = request.getSourceDataAddress(); - return validation.validate(source).flatMap(ValidationResult::toResult); + return validator.validateSource(source).flatMap(ValidationResult::toResult); } private S3ClientRequest createS3ClientRequest(DataAddress address) { diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/MinioContainer.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/MinioContainer.java new file mode 100644 index 00000000..fe437640 --- /dev/null +++ b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/MinioContainer.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.connector.dataplane.aws.s3; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +import java.util.List; + +public class MinioContainer extends GenericContainer { + + public MinioContainer() { + super(DockerImageName.parse("bitnami/minio:latest")); + setExposedPorts(List.of(9000)); + setEnv(List.of("MINIO_ROOT_USER=root", "MINIO_ROOT_PASSWORD=password")); + } +} diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataPlaneIntegrationTest.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataPlaneIntegrationTest.java index 4bff4ceb..31dc80a4 100644 --- a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataPlaneIntegrationTest.java +++ b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataPlaneIntegrationTest.java @@ -15,47 +15,78 @@ package org.eclipse.edc.connector.dataplane.aws.s3; -import org.eclipse.edc.aws.s3.S3BucketSchema; -import org.eclipse.edc.aws.s3.testfixtures.AbstractS3Test; -import org.eclipse.edc.aws.s3.testfixtures.annotations.AwsS3IntegrationTest; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; +import org.eclipse.edc.aws.s3.testfixtures.S3TestClient; import org.eclipse.edc.json.JacksonTypeManager; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.security.Vault; +import org.eclipse.edc.junit.annotations.EndToEndTest; import org.eclipse.edc.spi.types.domain.DataAddress; import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage; +import org.eclipse.edc.validator.spi.DataAddressValidatorRegistry; +import org.eclipse.edc.validator.spi.ValidationResult; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import java.util.List; import java.util.UUID; import java.util.concurrent.Executors; import java.util.stream.Stream; -import static java.lang.Integer.parseInt; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; -import static org.eclipse.edc.connector.dataplane.aws.s3.DataPlaneS3Extension.DEFAULT_CHUNK_SIZE_IN_MB; +import static org.eclipse.edc.util.configuration.ConfigurationFunctions.propOrEnv; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; - -@TestInstance(Lifecycle.PER_CLASS) -@AwsS3IntegrationTest -public class S3DataPlaneIntegrationTest extends AbstractS3Test { +import static org.mockito.Mockito.when; + +@Testcontainers +@EndToEndTest +public class S3DataPlaneIntegrationTest { + + // Adding REGION to bucket prevents errors of + // "A conflicting conditional operation is currently in progress against this resource." + // when bucket is rapidly added/deleted and consistency propagation causes this error. + // (Should not be necessary if REGION remains static, but added to prevent future frustration.) + // [see http://stackoverflow.com/questions/13898057/aws-error-message-a-conflicting-conditional-operation-is-currently-in-progress] + private static final String REGION = propOrEnv("it.aws.region", Region.US_EAST_1.id()); + private static final String OBJECT_PREFIX = "object-prefix/"; + private static final String KEY_NAME = "key-name"; + + @Container + private final MinioContainer sourceMinio = new MinioContainer(); + @Container + private final MinioContainer destinationMinio = new MinioContainer(); private final String sourceBucketName = "source-" + UUID.randomUUID(); private final String destinationBucketName = "destination-" + UUID.randomUUID(); - private final int defaultChunkSizeInBytes = 1024 * 1024 * parseInt(DEFAULT_CHUNK_SIZE_IN_MB); - + private S3TestClient sourceClient; + private S3TestClient destinationClient; + private S3DataSinkFactory sinkFactory; + private S3DataSourceFactory sourceFactory; @BeforeEach void setup() { + DataAddressValidatorRegistry validator = mock(); + when(validator.validateSource(any())).thenReturn(ValidationResult.success()); + when(validator.validateDestination(any())).thenReturn(ValidationResult.success()); + + sourceClient = S3TestClient.create("http://localhost:" + sourceMinio.getFirstMappedPort(), REGION); + destinationClient = S3TestClient.create("http://localhost:" + destinationMinio.getFirstMappedPort(), REGION); + + var typeManager = new JacksonTypeManager(); + var chunkSizeInBytes = 1024 * 1024 * 20; + sourceFactory = new S3DataSourceFactory(sourceClient.getClientProvider(), mock(), mock(), typeManager, validator); + sinkFactory = new S3DataSinkFactory(destinationClient.getClientProvider(), Executors.newSingleThreadExecutor(), mock(), mock(), typeManager, chunkSizeInBytes, validator); + sourceClient.createBucket(sourceBucketName); destinationClient.createBucket(destinationBucketName); } @@ -68,9 +99,9 @@ void tearDown() { @ParameterizedTest @ArgumentsSource(ObjectNamesToTransferProvider.class) - void should_copy_using_destination_object_name_case_single_transfer(String[] objectNames) { + void should_copy_using_destination_object_name_case_single_transfer(List objectNames) { - var isSingleObject = objectNames.length == 1; + var isSingleObject = objectNames.size() == 1; var objectNameInDestination = "object-name-in-destination"; var objectContent = UUID.randomUUID().toString(); @@ -78,11 +109,6 @@ void should_copy_using_destination_object_name_case_single_transfer(String[] obj sourceClient.putStringOnBucket(sourceBucketName, objectName, objectContent); } - var vault = mock(Vault.class); - var typeManager = new JacksonTypeManager(); - - var sinkFactory = new S3DataSinkFactory(destinationClient.getClientProvider(), Executors.newSingleThreadExecutor(), mock(Monitor.class), vault, typeManager, defaultChunkSizeInBytes); - var sourceFactory = new S3DataSourceFactory(sourceClient.getClientProvider(), mock(Monitor.class), vault, typeManager); var sourceAddress = createDataAddress(objectNames, isSingleObject); var destinationAddress = DataAddress.Builder.newInstance() @@ -93,7 +119,7 @@ void should_copy_using_destination_object_name_case_single_transfer(String[] obj .property(S3BucketSchema.OBJECT_NAME, objectNameInDestination) .property(S3BucketSchema.ACCESS_KEY_ID, destinationClient.getCredentials().accessKeyId()) .property(S3BucketSchema.SECRET_ACCESS_KEY, destinationClient.getCredentials().secretAccessKey()) - .property(S3BucketSchema.ENDPOINT_OVERRIDE, DESTINATION_MINIO_ENDPOINT) + .property(S3BucketSchema.ENDPOINT_OVERRIDE, "http://localhost:" + destinationMinio.getFirstMappedPort()) .build(); var request = DataFlowStartMessage.Builder.newInstance() @@ -131,9 +157,9 @@ void should_copy_using_destination_object_name_case_single_transfer(String[] obj @ParameterizedTest @ArgumentsSource(ObjectNamesToTransferProvider.class) - void should_copy_to_folder_case_property_is_present(String[] objectNames) { + void should_copy_to_folder_case_property_is_present(List objectNames) { - var isSingleObject = objectNames.length == 1; + var isSingleObject = objectNames.size() == 1; var objectNameInDestination = "object-name-in-destination"; var folderNameInDestination = "folder-name-in-destination/"; var objectBody = UUID.randomUUID().toString(); @@ -142,11 +168,6 @@ void should_copy_to_folder_case_property_is_present(String[] objectNames) { sourceClient.putStringOnBucket(sourceBucketName, objectToTransfer, objectBody); } - var vault = mock(Vault.class); - var typeManager = new JacksonTypeManager(); - - var sinkFactory = new S3DataSinkFactory(destinationClient.getClientProvider(), Executors.newSingleThreadExecutor(), mock(Monitor.class), vault, typeManager, defaultChunkSizeInBytes); - var sourceFactory = new S3DataSourceFactory(sourceClient.getClientProvider(), mock(Monitor.class), vault, typeManager); var sourceAddress = createDataAddress(objectNames, isSingleObject); var destinationAddress = DataAddress.Builder.newInstance() @@ -158,7 +179,7 @@ void should_copy_to_folder_case_property_is_present(String[] objectNames) { .property(S3BucketSchema.FOLDER_NAME, folderNameInDestination) .property(S3BucketSchema.ACCESS_KEY_ID, destinationClient.getCredentials().accessKeyId()) .property(S3BucketSchema.SECRET_ACCESS_KEY, destinationClient.getCredentials().secretAccessKey()) - .property(S3BucketSchema.ENDPOINT_OVERRIDE, DESTINATION_MINIO_ENDPOINT) + .property(S3BucketSchema.ENDPOINT_OVERRIDE, "http://localhost:" + destinationMinio.getFirstMappedPort()) .build(); var request = DataFlowStartMessage.Builder.newInstance() @@ -194,7 +215,7 @@ void should_copy_to_folder_case_property_is_present(String[] objectNames) { } } - private DataAddress createDataAddress(String[] assetNames, boolean isSingleObject) { + private DataAddress createDataAddress(List assetNames, boolean isSingleObject) { var dataAddressBuilder = DataAddress.Builder.newInstance() .type(S3BucketSchema.TYPE) .keyName(KEY_NAME) @@ -203,21 +224,25 @@ private DataAddress createDataAddress(String[] assetNames, boolean isSingleObjec .property(S3BucketSchema.ACCESS_KEY_ID, sourceClient.getCredentials().accessKeyId()) .property(S3BucketSchema.SECRET_ACCESS_KEY, sourceClient.getCredentials().secretAccessKey()); - return isSingleObject ? dataAddressBuilder.property(S3BucketSchema.OBJECT_NAME, assetNames[0]).build() : + return isSingleObject ? dataAddressBuilder.property(S3BucketSchema.OBJECT_NAME, assetNames.get(0)).build() : dataAddressBuilder.property(S3BucketSchema.OBJECT_PREFIX, OBJECT_PREFIX).build(); } private static class ObjectNamesToTransferProvider implements ArgumentsProvider { + + private static final String OBJECT_NAME = "text-document.txt"; + @Override public Stream provideArguments(ExtensionContext context) { return Stream.of( - Arguments.of((Object) new String[]{ + Arguments.of(List.of( OBJECT_PREFIX + "1-" + OBJECT_NAME, OBJECT_PREFIX + "2-" + OBJECT_NAME, - OBJECT_PREFIX + "3-" + OBJECT_NAME }), - Arguments.of((Object) new String[]{ - OBJECT_NAME })); + OBJECT_PREFIX + "3-" + OBJECT_NAME + )), + Arguments.of(List.of(OBJECT_NAME)) + ); } } } diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactoryTest.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactoryTest.java index 46ec3ed2..766da00a 100644 --- a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactoryTest.java +++ b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkFactoryTest.java @@ -18,32 +18,25 @@ import org.eclipse.edc.aws.s3.AwsClientProvider; import org.eclipse.edc.aws.s3.AwsSecretToken; import org.eclipse.edc.aws.s3.AwsTemporarySecretToken; -import org.eclipse.edc.aws.s3.S3BucketSchema; import org.eclipse.edc.aws.s3.S3ClientRequest; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; import org.eclipse.edc.json.JacksonTypeManager; import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.security.Vault; import org.eclipse.edc.spi.types.TypeManager; import org.eclipse.edc.spi.types.domain.DataAddress; import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage; +import org.eclipse.edc.validator.spi.DataAddressValidatorRegistry; +import org.eclipse.edc.validator.spi.ValidationResult; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; import org.mockito.ArgumentCaptor; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.stream.Stream; -import static java.lang.Integer.parseInt; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.eclipse.edc.aws.s3.S3BucketSchema.REGION; -import static org.eclipse.edc.connector.dataplane.aws.s3.DataPlaneS3Extension.DEFAULT_CHUNK_SIZE_IN_MB; +import static org.eclipse.edc.junit.assertions.AbstractResultAssert.assertThat; +import static org.eclipse.edc.validator.spi.Violation.violation; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -51,13 +44,13 @@ class S3DataSinkFactoryTest { - private final int defaultChunkSizeInBytes = 1024 * 1024 * parseInt(DEFAULT_CHUNK_SIZE_IN_MB); - private final AwsClientProvider clientProvider = mock(AwsClientProvider.class); - private final Vault vault = mock(Vault.class); + private final AwsClientProvider clientProvider = mock(); + private final Vault vault = mock(); private final TypeManager typeManager = new JacksonTypeManager(); - private final S3DataSinkFactory factory = new S3DataSinkFactory(clientProvider, mock(ExecutorService.class), mock(Monitor.class), vault, typeManager, defaultChunkSizeInBytes); - private final ArgumentCaptor s3ClientRequestArgumentCaptor = ArgumentCaptor.forClass(S3ClientRequest.class); + private final DataAddressValidatorRegistry validator = mock(); + private final S3DataSinkFactory factory = new S3DataSinkFactory(clientProvider, mock(), mock(), vault, typeManager, + 1024, validator); @Test void canHandle_returnsTrueWhenExpectedType() { @@ -78,31 +71,27 @@ void canHandle_returnsFalseWhenUnexpectedType() { } @Test - void validate_ShouldSucceedIfPropertiesAreValid() { + void validate_shouldSucceed_whenValidatorSucceeds() { + when(validator.validateDestination(any())).thenReturn(ValidationResult.success()); var destination = TestFunctions.s3DataAddressWithCredentials(); var request = createRequest(destination); var result = factory.validateRequest(request); - assertThat(result.succeeded()).isTrue(); + assertThat(result).isSucceeded(); + verify(validator).validateDestination(destination); } - @ParameterizedTest - @ArgumentsSource(InvalidInputs.class) - void validate_shouldFailIfMandatoryPropertiesAreMissing(String bucketName, String region, String accessKeyId, String secretAccessKey) { - var destination = DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .property(S3BucketSchema.BUCKET_NAME, bucketName) - .property(REGION, region) - .property(S3BucketSchema.ACCESS_KEY_ID, accessKeyId) - .property(S3BucketSchema.SECRET_ACCESS_KEY, secretAccessKey) - .build(); - + @Test + void validate_shouldFail_whenValidatorFails() { + when(validator.validateDestination(any())).thenReturn(ValidationResult.failure(violation("error", "path"))); + var destination = TestFunctions.s3DataAddressWithCredentials(); var request = createRequest(destination); var result = factory.validateRequest(request); - assertThat(result.failed()).isTrue(); + assertThat(result).isFailed(); + verify(validator).validateDestination(destination); } @Test @@ -110,16 +99,15 @@ void createSink_shouldGetTheTemporarySecretTokenFromTheVault() { var destination = TestFunctions.s3DataAddressWithCredentials(); var temporaryKey = new AwsTemporarySecretToken("temporaryId", "temporarySecret", "temporaryToken", 10); when(vault.resolveSecret(destination.getKeyName())).thenReturn(typeManager.writeValueAsString(temporaryKey)); + when(validator.validateDestination(any())).thenReturn(ValidationResult.success()); var request = createRequest(destination); var sink = factory.createSink(request); assertThat(sink).isNotNull().isInstanceOf(S3DataSink.class); - - verify(clientProvider).s3Client(s3ClientRequestArgumentCaptor.capture()); - - S3ClientRequest s3ClientRequest = s3ClientRequestArgumentCaptor.getValue(); - + var captor = ArgumentCaptor.forClass(S3ClientRequest.class); + verify(clientProvider).s3Client(captor.capture()); + var s3ClientRequest = captor.getValue(); assertThat(s3ClientRequest.region()).isEqualTo(TestFunctions.VALID_REGION); assertThat(s3ClientRequest.secretToken()).isInstanceOf(AwsTemporarySecretToken.class); assertThat(s3ClientRequest.endpointOverride()).isNull(); @@ -128,17 +116,16 @@ void createSink_shouldGetTheTemporarySecretTokenFromTheVault() { @Test void createSink_shouldCreateDataSinkWithCredentialsInDataAddressIfTheresNoSecret() { when(vault.resolveSecret(any())).thenReturn(null); + when(validator.validateDestination(any())).thenReturn(ValidationResult.success()); var destination = TestFunctions.s3DataAddressWithCredentials(); var request = createRequest(destination); var sink = factory.createSink(request); assertThat(sink).isNotNull().isInstanceOf(S3DataSink.class); - - verify(clientProvider).s3Client(s3ClientRequestArgumentCaptor.capture()); - - S3ClientRequest s3ClientRequest = s3ClientRequestArgumentCaptor.getValue(); - + var captor = ArgumentCaptor.forClass(S3ClientRequest.class); + verify(clientProvider).s3Client(captor.capture()); + var s3ClientRequest = captor.getValue(); assertThat(s3ClientRequest.region()).isEqualTo(TestFunctions.VALID_REGION); assertThat(s3ClientRequest.secretToken()).isEqualTo(new AwsSecretToken(TestFunctions.VALID_ACCESS_KEY_ID, TestFunctions.VALID_SECRET_ACCESS_KEY)); assertThat(s3ClientRequest.endpointOverride()).isNull(); @@ -147,17 +134,16 @@ void createSink_shouldCreateDataSinkWithCredentialsInDataAddressIfTheresNoSecret @Test void createSink_shouldLetTheProviderGetTheCredentialsAsFallback() { when(vault.resolveSecret(any())).thenReturn(null); + when(validator.validateDestination(any())).thenReturn(ValidationResult.success()); var destination = TestFunctions.s3DataAddressWithoutCredentials(); var request = createRequest(destination); var sink = factory.createSink(request); assertThat(sink).isNotNull().isInstanceOf(S3DataSink.class); - - verify(clientProvider).s3Client(s3ClientRequestArgumentCaptor.capture()); - - S3ClientRequest s3ClientRequest = s3ClientRequestArgumentCaptor.getValue(); - + var captor = ArgumentCaptor.forClass(S3ClientRequest.class); + verify(clientProvider).s3Client(captor.capture()); + var s3ClientRequest = captor.getValue(); assertThat(s3ClientRequest.region()).isEqualTo(TestFunctions.VALID_REGION); assertThat(s3ClientRequest.secretToken()).isNull(); assertThat(s3ClientRequest.endpointOverride()).isNull(); @@ -168,6 +154,7 @@ void createSink_shouldThrowExceptionIfValidationFails() { var destination = DataAddress.Builder.newInstance() .type(S3BucketSchema.TYPE) .build(); + when(validator.validateDestination(any())).thenReturn(ValidationResult.failure(violation("error", "path"))); var request = createRequest(destination); @@ -183,14 +170,4 @@ private DataFlowStartMessage createRequest(DataAddress destination) { .build(); } - private static class InvalidInputs implements ArgumentsProvider { - - @Override - public Stream provideArguments(ExtensionContext context) { - return Stream.of( - Arguments.of(TestFunctions.VALID_BUCKET_NAME, " ", TestFunctions.VALID_ACCESS_KEY_ID, TestFunctions.VALID_SECRET_ACCESS_KEY), - Arguments.of(" ", TestFunctions.VALID_REGION, TestFunctions.VALID_ACCESS_KEY_ID, TestFunctions.VALID_SECRET_ACCESS_KEY) - ); - } - } -} \ No newline at end of file +} diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkTest.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkTest.java index ac08eda0..fd818d20 100644 --- a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkTest.java +++ b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSinkTest.java @@ -14,7 +14,7 @@ package org.eclipse.edc.connector.dataplane.aws.s3; -import org.eclipse.edc.aws.s3.S3BucketSchema; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; import org.eclipse.edc.connector.dataplane.aws.s3.exceptions.S3DataSourceException; import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; import org.eclipse.edc.connector.dataplane.spi.pipeline.InputStreamDataSource; diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactoryTest.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactoryTest.java index 47ee3856..85c1ce19 100644 --- a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactoryTest.java +++ b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/S3DataSourceFactoryTest.java @@ -17,45 +17,39 @@ import org.eclipse.edc.aws.s3.AwsClientProvider; import org.eclipse.edc.aws.s3.AwsTemporarySecretToken; -import org.eclipse.edc.aws.s3.S3BucketSchema; import org.eclipse.edc.aws.s3.S3ClientRequest; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; import org.eclipse.edc.json.JacksonTypeManager; import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.security.Vault; import org.eclipse.edc.spi.types.TypeManager; import org.eclipse.edc.spi.types.domain.DataAddress; import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage; +import org.eclipse.edc.validator.spi.DataAddressValidatorRegistry; +import org.eclipse.edc.validator.spi.ValidationResult; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; import org.mockito.ArgumentCaptor; import java.util.UUID; -import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.VALID_ACCESS_KEY_ID; -import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.VALID_BUCKET_NAME; -import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.VALID_REGION; -import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.VALID_SECRET_ACCESS_KEY; import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.s3DataAddressWithCredentials; import static org.eclipse.edc.connector.dataplane.aws.s3.TestFunctions.s3DataAddressWithoutCredentials; +import static org.eclipse.edc.validator.spi.Violation.violation; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class S3DataSourceFactoryTest { - private final AwsClientProvider clientProvider = mock(AwsClientProvider.class); + private final AwsClientProvider clientProvider = mock(); private final TypeManager typeManager = new JacksonTypeManager(); - private final Vault vault = mock(Vault.class); - private final S3DataSourceFactory factory = new S3DataSourceFactory(clientProvider, mock(Monitor.class), vault, typeManager); - private final ArgumentCaptor s3ClientRequestArgumentCaptor = ArgumentCaptor.forClass(S3ClientRequest.class); + private final Vault vault = mock(); + private final DataAddressValidatorRegistry validator = mock(); + + private final S3DataSourceFactory factory = new S3DataSourceFactory(clientProvider, mock(), vault, typeManager, validator); @Test void canHandle_returnsTrueWhenExpectedType() { @@ -76,36 +70,33 @@ void canHandle_returnsFalseWhenUnexpectedType() { } @Test - void validate_shouldSucceedIfPropertiesAreValid() { + void validate_shouldSucceed_whenValidatorSucceeds() { + when(validator.validateSource(any())).thenReturn(ValidationResult.success()); var source = s3DataAddressWithCredentials(); var request = createRequest(source); var result = factory.validateRequest(request); assertThat(result.succeeded()).isTrue(); + verify(validator).validateSource(source); } - @ParameterizedTest - @ArgumentsSource(InvalidInputs.class) - void validate_shouldFailIfMandatoryPropertiesAreMissing(String bucketName, String region, String accessKeyId, String secretAccessKey) { - var source = DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .property(S3BucketSchema.BUCKET_NAME, bucketName) - .property(S3BucketSchema.REGION, region) - .property(S3BucketSchema.ACCESS_KEY_ID, accessKeyId) - .property(S3BucketSchema.SECRET_ACCESS_KEY, secretAccessKey) - .build(); - + @Test + void validate_shouldFail_whenValidatorFails() { + when(validator.validateSource(any())).thenReturn(ValidationResult.failure(violation("error", "path"))); + var source = s3DataAddressWithCredentials(); var request = createRequest(source); var result = factory.validateRequest(request); - assertThat(result.failed()).isTrue(); + assertThat(result.succeeded()).isFalse(); + verify(validator).validateSource(source); } @Test void createSource_shouldCreateDataSource() { - DataAddress source = s3DataAddressWithCredentials(); + when(validator.validateSource(any())).thenReturn(ValidationResult.success()); + var source = s3DataAddressWithCredentials(); var request = createRequest(source); var dataSource = factory.createSource(request); @@ -115,35 +106,24 @@ void createSource_shouldCreateDataSource() { @Test void createSink_shouldLetTheProviderGetTheCredentialsIfNotProvidedByTheAddress() { + when(validator.validateSource(any())).thenReturn(ValidationResult.success()); var destination = s3DataAddressWithoutCredentials(); var request = createRequest(destination); var sink = factory.createSource(request); assertThat(sink).isNotNull().isInstanceOf(S3DataSource.class); - - verify(clientProvider).s3Client(s3ClientRequestArgumentCaptor.capture()); - - S3ClientRequest s3ClientRequest = s3ClientRequestArgumentCaptor.getValue(); - + var captor = ArgumentCaptor.forClass(S3ClientRequest.class); + verify(clientProvider).s3Client(captor.capture()); + var s3ClientRequest = captor.getValue(); assertThat(s3ClientRequest.region()).isEqualTo(TestFunctions.VALID_REGION); assertThat(s3ClientRequest.secretToken()).isNull(); assertThat(s3ClientRequest.endpointOverride()).isNull(); } - @Test - void createSource_shouldThrowExceptionIfValidationFails() { - var source = DataAddress.Builder.newInstance() - .type(S3BucketSchema.TYPE) - .build(); - - var request = createRequest(source); - - assertThatThrownBy(() -> factory.createSource(request)).isInstanceOf(EdcException.class); - } - @Test void createSource_shouldGetTheSecretTokenFromTheVault() { + when(validator.validateSource(any())).thenReturn(ValidationResult.success()); var source = TestFunctions.s3DataAddressWithCredentials(); var temporaryKey = new AwsTemporarySecretToken("temporaryId", "temporarySecret", null, 0); when(vault.resolveSecret(source.getKeyName())).thenReturn(typeManager.writeValueAsString(temporaryKey)); @@ -152,16 +132,26 @@ void createSource_shouldGetTheSecretTokenFromTheVault() { var s3Source = factory.createSource(request); assertThat(s3Source).isNotNull().isInstanceOf(S3DataSource.class); - - verify(clientProvider).s3Client(s3ClientRequestArgumentCaptor.capture()); - - S3ClientRequest s3ClientRequest = s3ClientRequestArgumentCaptor.getValue(); - + var captor = ArgumentCaptor.forClass(S3ClientRequest.class); + verify(clientProvider).s3Client(captor.capture()); + var s3ClientRequest = captor.getValue(); assertThat(s3ClientRequest.region()).isEqualTo(TestFunctions.VALID_REGION); assertThat(s3ClientRequest.secretToken()).isInstanceOf(AwsTemporarySecretToken.class); assertThat(s3ClientRequest.endpointOverride()).isNull(); } + @Test + void createSource_shouldThrowExceptionIfValidationFails() { + when(validator.validateSource(any())).thenReturn(ValidationResult.failure(violation("error", "path"))); + var source = DataAddress.Builder.newInstance() + .type(S3BucketSchema.TYPE) + .build(); + + var request = createRequest(source); + + assertThatThrownBy(() -> factory.createSource(request)).isInstanceOf(EdcException.class); + } + private DataFlowStartMessage createRequest(DataAddress source) { return DataFlowStartMessage.Builder.newInstance() .id(UUID.randomUUID().toString()) @@ -171,14 +161,4 @@ private DataFlowStartMessage createRequest(DataAddress source) { .build(); } - private static class InvalidInputs implements ArgumentsProvider { - - @Override - public Stream provideArguments(ExtensionContext context) { - return Stream.of( - Arguments.of(VALID_BUCKET_NAME, " ", VALID_ACCESS_KEY_ID, VALID_SECRET_ACCESS_KEY), - Arguments.of(" ", VALID_REGION, VALID_ACCESS_KEY_ID, VALID_SECRET_ACCESS_KEY) - ); - } - } } diff --git a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/TestFunctions.java b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/TestFunctions.java index f8d58f2e..4766c6c5 100644 --- a/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/TestFunctions.java +++ b/extensions/data-plane/data-plane-aws-s3/src/test/java/org/eclipse/edc/connector/dataplane/aws/s3/TestFunctions.java @@ -14,7 +14,7 @@ package org.eclipse.edc.connector.dataplane.aws.s3; -import org.eclipse.edc.aws.s3.S3BucketSchema; +import org.eclipse.edc.aws.s3.spi.S3BucketSchema; import org.eclipse.edc.spi.types.domain.DataAddress; import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage; diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 91367e9e..2a97154d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -9,6 +9,7 @@ edc = "0.7.1-SNAPSHOT" failsafe = "3.3.1" junit = "5.10.2" restAssured = "5.4.0" +testcontainers = "1.19.8" [libraries] @@ -70,6 +71,7 @@ failsafe-core = { module = "dev.failsafe:failsafe", version.ref = "failsafe" } restAssured = { module = "io.rest-assured:rest-assured", version.ref = "restAssured" } junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit" } junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" } +testcontainers-junit-jupiter = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } # AWS dependencies aws-sts = { module = "software.amazon.awssdk:sts", version.ref = "aws" } diff --git a/resources/ci/discord_webhook.sh b/resources/ci/discord_webhook.sh deleted file mode 100755 index ef77df80..00000000 --- a/resources/ci/discord_webhook.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -# Copyright (c) 2022 Microsoft Corporation -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Contributors: -# Microsoft Corporation - initial implementation - -# This file is intended to be used as Post-Build action on CI pipelines to post a message to the EDC's #jenkins-ci Discord -# channel. -# It is loosely based on https://github.com/symboxtra/universal-ci-discord-webhook/blob/master/send.sh, but with many simplifications. - -# todo: make configurable -BRANCH_NAME="main" - -WEBHOOK_URL="$1" -STATUS="$2" -JENKINS_JOB="$3" -BUILD_NUMBER="$4" -REPO_URL="$5" -CONTENT="$6" - -# do not run script if required parameters are not supplied -if [ "$#" -lt 5 ]; then - echo "usage: discord_webhook.sh WEBHOOK_URL STATUS JOB_NAME BUILD_NUMBER REPO_URL CONTENT" - echo " WEBHOOK_URL = URL of the webhook to invoke, e.g. for discord" - echo " STATUS = \"success\" or \"failure\". Will use \"Unknown\" when anything else is passed" - echo " JOB_NAME = name of the job EXACTLY as configured in Jenkins. Use quotes if the job name contains blanks" - echo " BUILD_NUMBER = jenkins build number, must be an integer" - echo " REPO_URL = URL to the (Github) repository" - echo " CONTENT = [OPTIONAL] a string containing message content to be posted to. Defaults to \"I finished a job\"" - exit 1 -fi - -if [ -z "$6" ]; then - echo "No content supplied, using default." - CONTENT="I finished a job" -fi - -echo "'"WEBHOOK_URL: "${WEBHOOK_URL}""'" -echo "'"STATUS: "${STATUS}""'" -echo "'"JENKINS_JOB: "${JENKINS_JOB}""'" -echo "'"BUILD_NUMBER: "${BUILD_NUMBER}""'" -echo "'"REPO_URL: "${REPO_URL}""'" -echo "'"CONTENT: "${CONTENT}""'" - - -CI_PROVIDER="Jenkins" -DISCORD_AVATAR="https://wiki.jenkins.io/download/attachments/2916393/headshot.png?version=1&modificationDate=1302753947000&api=v2" -SUCCESS_AVATAR="https://jenkins.io/images/logos/cute/cute.png" -FAILURE_AVATAR="https://jenkins.io/images/logos/fire/fire.png" -UNKNOWN_AVATAR="https://www.jenkins.io/images/logos/mono/mono.png" - -JOB_URL="https://ci.eclipse.org/edc/job/${JENKINS_JOB}" -BUILD_URL="${JOB_URL}/${BUILD_NUMBER}" -BUILD_URL="${BUILD_URL}/console" - -echo -echo -e "[Webhook]: ${CI_PROVIDER} CI detected." -echo -e "[Webhook]: Sending webhook to Discord..." -echo - -case ${STATUS} in -"success") - EMBED_COLOR=3066993 - STATUS_MESSAGE="Passed" - AVATAR="${SUCCESS_AVATAR}" - ;; -"failure") - EMBED_COLOR=15158332 - STATUS_MESSAGE="Failed" - AVATAR="${FAILURE_AVATAR}" - ;; -*) - EMBED_COLOR=8421504 - STATUS_MESSAGE="Status Unknown" - echo "status \"${STATUS}\" --> ${STATUS_MESSAGE}" - AVATAR="${UNKNOWN_AVATAR}" - ;; -esac - -TIMESTAMP=$(date -u +%FT%TZ) -WEBHOOK_DATA='{ - "username": "Jenkins CI", - "content": "'"${CONTENT}"'", - "avatar_url": "'"${DISCORD_AVATAR}"'", - "embeds": [ { - "color": '${EMBED_COLOR}', - "author": { - "name": "'"${CI_PROVIDER}"' '"${JENKINS_JOB} #${BUILD_NUMBER}"' - '"${STATUS_MESSAGE}"'", - "url": "'"${BUILD_URL}"'", - "icon_url": "'"${AVATAR}"'" - }, - "title": "'"${JENKINS_JOB} - ${STATUS_MESSAGE}"'", - "url": "'"${JOB_URL}"'", - "fields": [ - { - "name": "Job Name", - "value": "'"[${JENKINS_JOB%}](${JOB_URL})"'", - "inline": true - }, - { - "name": "Build Number", - "value": "'"[${BUILD_NUMBER%.*}](${BUILD_URL})"'", - "inline": true - }, - { - "name": "Branch/Tag", - "value": "'"[\`${BRANCH_NAME}\`](${REPO_URL}/tree/${BRANCH_NAME})"'", - "inline": true - } - ], - "timestamp": "'"${TIMESTAMP}"'" - } ] -}' - -curl --fail --progress-bar -A "${CI_PROVIDER}-Webhook" -H "Content-Type:application/json" -d "${WEBHOOK_DATA}" "${WEBHOOK_URL}" - -if [ $? -ne 0 ]; then - echo -e "Webhook data:\\n${WEBHOOK_DATA}" - echo -e "\\n[Webhook]: Unable to send webhook." - - # Exit with an error signal - exit 1 -else - echo -e "\\n[Webhook]: Successfully sent the webhook." -fi diff --git a/settings.gradle.kts b/settings.gradle.kts index 1d042048..f85212a5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -36,14 +36,15 @@ dependencyResolutionManagement { } } -// common modules include(":extensions:common:aws:aws-s3-test") include(":extensions:common:aws:aws-s3-core") include(":extensions:common:vault:vault-aws") +include(":extensions:common:validator:validator-data-address-s3") -// control plane modules include(":extensions:control-plane:provision:provision-aws-s3") -// data plane include(":extensions:data-plane:data-plane-aws-s3") + +include(":spi:common:aws-spi") + include(":version-catalog") diff --git a/spi/common/aws-spi/build.gradle.kts b/spi/common/aws-spi/build.gradle.kts new file mode 100644 index 00000000..38e4b357 --- /dev/null +++ b/spi/common/aws-spi/build.gradle.kts @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + + +plugins { + `java-library` +} + +dependencies { + +} + + diff --git a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3BucketSchema.java b/spi/common/aws-spi/src/main/java/org/eclipse/edc/aws/s3/spi/S3BucketSchema.java similarity index 96% rename from extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3BucketSchema.java rename to spi/common/aws-spi/src/main/java/org/eclipse/edc/aws/s3/spi/S3BucketSchema.java index c57bb86b..6d6e5b6c 100644 --- a/extensions/common/aws/aws-s3-core/src/main/java/org/eclipse/edc/aws/s3/S3BucketSchema.java +++ b/spi/common/aws-spi/src/main/java/org/eclipse/edc/aws/s3/spi/S3BucketSchema.java @@ -13,7 +13,7 @@ * */ -package org.eclipse.edc.aws.s3; +package org.eclipse.edc.aws.s3.spi; public interface S3BucketSchema { String TYPE = "AmazonS3";