diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cce37297e788..91be80e76013d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `1password/load-secrets-action` from 2 to 3 ([#19100](https://github.com/opensearch-project/OpenSearch/pull/19100)) - Bump `com.nimbusds:nimbus-jose-jwt` from 10.3 to 10.4.2 ([#19099](https://github.com/opensearch-project/OpenSearch/pull/19099), [#19101](https://github.com/opensearch-project/OpenSearch/pull/19101)) - Bump netty from 4.1.121.Final to 4.1.124.Final ([#19103](https://github.com/opensearch-project/OpenSearch/pull/19103)) +- Bump google cloud storage from 1.113.1 to 2.55.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547)) ### Deprecated diff --git a/buildSrc/src/main/groovy/org/opensearch/gradle/test/TestWithSslPlugin.java b/buildSrc/src/main/groovy/org/opensearch/gradle/test/TestWithSslPlugin.java index 33e8966bd32c1..efba21419ebfc 100644 --- a/buildSrc/src/main/groovy/org/opensearch/gradle/test/TestWithSslPlugin.java +++ b/buildSrc/src/main/groovy/org/opensearch/gradle/test/TestWithSslPlugin.java @@ -77,6 +77,7 @@ public void apply(Project project) { File keystoreDir = new File(project.getBuildDir(), "keystore/test/ssl"); File nodeKeystore = new File(keystoreDir, "test-node.jks"); File clientKeyStore = new File(keystoreDir, "test-client.jks"); + @SuppressWarnings("unchecked") NamedDomainObjectContainer clusters = (NamedDomainObjectContainer) project.getExtensions() .getByName(TestClustersPlugin.EXTENSION_NAME); clusters.all(c -> { diff --git a/client/rest/licenses/commons-codec-1.16.1.jar.sha1 b/client/rest/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/client/rest/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/client/rest/licenses/commons-codec-1.18.0.jar.sha1 b/client/rest/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/client/rest/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/client/sniffer/licenses/commons-codec-1.16.1.jar.sha1 b/client/sniffer/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/client/sniffer/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/client/sniffer/licenses/commons-codec-1.18.0.jar.sha1 b/client/sniffer/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/client/sniffer/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 38833b530a6fa..3cd058d83ba4b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -21,7 +21,7 @@ woodstox = "6.4.0" kotlin = "1.7.10" antlr4 = "4.13.1" guava = "33.2.1-jre" -protobuf = "3.25.5" +protobuf = "3.25.8" jakarta_annotation = "1.3.5" google_http_client = "1.44.1" google_auth = "1.29.0" @@ -48,7 +48,7 @@ httpclient = "4.5.14" httpcore = "4.4.16" httpasyncclient = "4.1.5" commonslogging = "1.2" -commonscodec = "1.16.1" +commonscodec = "1.18.0" commonslang = "3.18.0" commonscompress = "1.26.1" commonsio = "2.16.0" diff --git a/plugins/analysis-phonetic/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/analysis-phonetic/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/plugins/analysis-phonetic/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/commons-codec-1.18.0.jar.sha1 b/plugins/analysis-phonetic/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/arrow-flight-rpc/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/commons-codec-1.18.0.jar.sha1 b/plugins/arrow-flight-rpc/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/crypto-kms/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/plugins/crypto-kms/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/commons-codec-1.18.0.jar.sha1 b/plugins/crypto-kms/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/plugins/crypto-kms/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/plugins/discovery-azure-classic/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/commons-codec-1.18.0.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/plugins/discovery-ec2/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.18.0.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/plugins/discovery-ec2/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/plugins/discovery-gce/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/commons-codec-1.18.0.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/plugins/discovery-gce/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/ingest-attachment/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-codec-1.18.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/plugins/ingest-attachment/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/ingestion-kinesis/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/plugins/ingestion-kinesis/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/commons-codec-1.18.0.jar.sha1 b/plugins/ingestion-kinesis/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 038d46a53cc67..e8338976fae5d 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -21,6 +21,7 @@ import java.security.KeyPair import java.security.KeyPairGenerator import static org.opensearch.gradle.PropertyNormalization.IGNORE_VALUE + /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with @@ -42,56 +43,88 @@ import static org.opensearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'opensearch.yaml-rest-test' apply plugin: 'opensearch.internal-cluster-test' +ext { + guava_version = "33.4.0-jre" +} + opensearchplugin { description = 'The GCS repository plugin adds Google Cloud Storage support for repositories.' classname = 'org.opensearch.repositories.gcs.GoogleCloudStoragePlugin' } dependencies { - api 'com.google.api:api-common:2.46.1' - api 'com.google.api:gax:2.63.1' - api 'com.google.api:gax-httpjson:2.42.0' - - api 'com.google.apis:google-api-services-storage:v1-rev20230617-2.0.0' - - api 'com.google.api-client:google-api-client:2.7.0' - api 'com.google.api.grpc:proto-google-common-protos:2.54.1' - api 'com.google.api.grpc:proto-google-iam-v1:1.49.1' + // dependencies consistent with 'com.google.cloud:google-cloud-storage-bom:2.55.0' + implementation "com.google.cloud:google-cloud-storage:2.55.0" + implementation "com.google.cloud:google-cloud-core:2.59.0" + implementation "com.google.cloud:google-cloud-core-http:2.59.0" + + runtimeOnly "com.google.guava:guava:${guava_version}" + runtimeOnly "com.google.guava:failureaccess:1.0.2" + compileOnly "com.google.errorprone:error_prone_annotations:2.38.0" + + runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}" // 2.0.16 in bom + runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}" // 1.18.0 in bom + implementation "com.google.api:api-common:2.52.0" + implementation "com.google.api:gax:2.69.0" + runtimeOnly "com.google.api:gax-httpjson:2.69.0" + implementation "org.threeten:threetenbp:1.7.0" + runtimeOnly "com.google.protobuf:protobuf-java-util:${versions.protobuf}" + runtimeOnly "com.google.protobuf:protobuf-java:${versions.protobuf}" + runtimeOnly "com.google.code.gson:gson:2.12.1" + runtimeOnly "com.google.api.grpc:proto-google-common-protos:2.60.0" + runtimeOnly "com.google.api.grpc:proto-google-iam-v1:1.55.0" + implementation "com.google.auth:google-auth-library-credentials:1.37.1" + implementation "com.google.auth:google-auth-library-oauth2-http:1.37.1" + runtimeOnly "com.google.oauth-client:google-oauth-client:1.39.0" // 1.39.0 in bom + implementation "com.google.api-client:google-api-client:2.7.2" + implementation "com.google.http-client:google-http-client:1.47.1" + runtimeOnly "com.google.http-client:google-http-client-gson:1.47.1" + runtimeOnly "com.google.http-client:google-http-client-appengine:1.47.1" + runtimeOnly "com.google.http-client:google-http-client-jackson2:1.47.1" + runtimeOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" // 2.18.2 in bom + runtimeOnly "io.opencensus:opencensus-api:0.31.1" + runtimeOnly "io.opencensus:opencensus-contrib-http-util:0.31.1" + implementation "com.google.apis:google-api-services-storage:v1-rev20250718-2.0.0" + + implementation "org.checkerframework:checker-qual:3.49.0" + + runtimeOnly "io.opentelemetry:opentelemetry-api:1.47.0" + runtimeOnly "io.opentelemetry:opentelemetry-context:1.47.0" + runtimeOnly "com.google.api.grpc:proto-google-cloud-storage-v2:2.55.0" + runtimeOnly "io.grpc:grpc-api:1.71.0" - api "com.google.auth:google-auth-library-credentials:${versions.google_auth}" - api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}" - - api 'com.google.cloud:google-cloud-core:2.30.0' - api 'com.google.cloud:google-cloud-core-http:2.47.0' - api 'com.google.cloud:google-cloud-storage:1.113.1' - - api 'com.google.code.gson:gson:2.13.0' - - runtimeOnly "com.google.guava:guava:${versions.guava}" - api 'com.google.guava:failureaccess:1.0.1' - - api "com.google.http-client:google-http-client:${versions.google_http_client}" - api "com.google.http-client:google-http-client-appengine:${versions.google_http_client}" - api "com.google.http-client:google-http-client-gson:${versions.google_http_client}" - api "com.google.http-client:google-http-client-jackson2:${versions.google_http_client}" + testImplementation project(':test:fixtures:gcs-fixture') +} - api 'com.google.oauth-client:google-oauth-client:1.34.1' +compileJava { + configurations.runtimeClasspath.files.forEach { + if (it.name.contains(guava_version)) { + classpath += files(it.toString()) + } + } +} - api "commons-logging:commons-logging:${versions.commonslogging}" - api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" - api "commons-codec:commons-codec:${versions.commonscodec}" - api 'org.threeten:threetenbp:1.4.4' - api "io.grpc:grpc-api:${versions.grpc}" - api 'io.opencensus:opencensus-api:0.31.1' - api 'io.opencensus:opencensus-contrib-http-util:0.31.1' +compileTestJava { + configurations.runtimeClasspath.files.forEach { + if (it.name.contains(guava_version)) { + classpath += files(it.toString()) + } + } +} - testImplementation project(':test:fixtures:gcs-fixture') +javadoc { + configurations.runtimeClasspath.files.forEach { + if (it.name.contains(guava_version)) { + classpath += files(it.toString()) + } + } } + restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count' + includeCore '_common', 'cluster', 'nodes', 'snapshot', 'indices', 'index', 'bulk', 'count' } } @@ -121,10 +154,28 @@ thirdPartyAudit { 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', + 'com.google.protobuf.MessageSchema', + 'com.google.protobuf.UnsafeUtil', + 'com.google.protobuf.UnsafeUtil$1', + 'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor', + 'com.google.protobuf.UnsafeUtil$MemoryAccessor', ) ignoreMissingClasses( + // GCS api 'com.google.api.client.http.apache.v2.ApacheHttpTransport', + 'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider$HardBoundTokenTypes', + 'com.google.api.gax.grpc.GrpcCallContext', + 'com.google.api.gax.grpc.GrpcCallSettings', + 'com.google.api.gax.grpc.GrpcCallSettings$Builder', + 'com.google.api.gax.grpc.GrpcInterceptorProvider', + 'com.google.api.gax.grpc.GrpcStatusCode', + 'com.google.api.gax.grpc.GrpcStubCallableFactory', + 'com.google.api.gax.grpc.GrpcTransportChannel', + 'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider', + 'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider$Builder', 'com.google.appengine.api.datastore.Blob', 'com.google.appengine.api.datastore.DatastoreService', 'com.google.appengine.api.datastore.DatastoreServiceFactory', @@ -144,14 +195,42 @@ thirdPartyAudit { 'com.google.appengine.api.urlfetch.HTTPResponse', 'com.google.appengine.api.urlfetch.URLFetchService', 'com.google.appengine.api.urlfetch.URLFetchServiceFactory', - 'com.google.protobuf.util.JsonFormat', - 'com.google.protobuf.util.JsonFormat$Parser', - 'com.google.protobuf.util.JsonFormat$Printer', - 'com.google.protobuf.util.Timestamps', - // commons-logging optional dependencies - 'org.apache.avalon.framework.logger.Logger', - 'org.apache.log.Hierarchy', - 'org.apache.log.Logger', + 'com.google.cloud.grpc.GrpcTransportOptions', + 'com.google.cloud.grpc.GrpcTransportOptions$Builder', + 'com.google.cloud.opentelemetry.metric.GoogleCloudMetricExporter', + 'com.google.cloud.opentelemetry.metric.MetricConfiguration', + 'com.google.cloud.opentelemetry.metric.MetricConfiguration$Builder', + 'com.google.storage.v2.StorageClient', + 'com.google.storage.v2.StorageClient$ListBucketsPagedResponse', + 'com.google.storage.v2.StorageSettings', + 'com.google.storage.v2.StorageSettings$Builder', + 'com.google.storage.v2.stub.GrpcStorageStub', + 'com.google.storage.v2.stub.StorageStub', + 'com.google.storage.v2.stub.StorageStubSettings', + 'com.google.storage.v2.stub.StorageStubSettings$Builder', + // IO grpc + 'io.grpc.opentelemetry.GrpcOpenTelemetry', + 'io.grpc.opentelemetry.GrpcOpenTelemetry$Builder', + 'io.grpc.protobuf.ProtoUtils', + 'io.opentelemetry.contrib.gcp.resource.GCPResourceProvider', + 'io.opentelemetry.sdk.OpenTelemetrySdk', + 'io.opentelemetry.sdk.OpenTelemetrySdkBuilder', + 'io.opentelemetry.sdk.common.CompletableResultCode', + 'io.opentelemetry.sdk.common.export.MemoryMode', + 'io.opentelemetry.sdk.metrics.Aggregation', + 'io.opentelemetry.sdk.metrics.InstrumentSelector', + 'io.opentelemetry.sdk.metrics.InstrumentSelectorBuilder', + 'io.opentelemetry.sdk.metrics.InstrumentType', + 'io.opentelemetry.sdk.metrics.SdkMeterProvider', + 'io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder', + 'io.opentelemetry.sdk.metrics.View', + 'io.opentelemetry.sdk.metrics.ViewBuilder', + 'io.opentelemetry.sdk.metrics.data.AggregationTemporality', + 'io.opentelemetry.sdk.metrics.export.DefaultAggregationSelector', + 'io.opentelemetry.sdk.metrics.export.MetricExporter', + 'io.opentelemetry.sdk.metrics.export.PeriodicMetricReader', + 'io.opentelemetry.sdk.metrics.export.PeriodicMetricReaderBuilder', + 'io.opentelemetry.sdk.resources.Resource', // optional apache http client dependencies 'org.apache.http.ConnectionReuseStrategy', 'org.apache.http.Header', @@ -198,30 +277,11 @@ thirdPartyAudit { 'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess', 'org.graalvm.nativeimage.hosted.Feature$FeatureAccess', 'org.graalvm.nativeimage.hosted.RuntimeReflection', - // commons-logging provided dependencies - 'javax.jms.Message', - 'javax.servlet.ServletContextEvent', - 'javax.servlet.ServletContextListener', - - // opentelemetry-api is an optional dependency of com.google.api:gax - 'io.opentelemetry.api.OpenTelemetry', - 'io.opentelemetry.api.common.Attributes', - 'io.opentelemetry.api.common.AttributesBuilder', - 'io.opentelemetry.api.metrics.DoubleHistogram', - 'io.opentelemetry.api.metrics.DoubleHistogramBuilder', - 'io.opentelemetry.api.metrics.LongCounter', - 'io.opentelemetry.api.metrics.LongCounterBuilder', - 'io.opentelemetry.api.metrics.Meter', - 'io.opentelemetry.api.metrics.MeterBuilder', - - // slf4j is an optional dependency of com.google.api:gax - 'org.slf4j.ILoggerFactory', - 'org.slf4j.Logger', - 'org.slf4j.LoggerFactory', - 'org.slf4j.MDC', - 'org.slf4j.event.Level', - 'org.slf4j.helpers.NOPLogger', - 'org.slf4j.spi.LoggingEventBuilder' + //slf4j dependencies + 'org.slf4j.impl.StaticLoggerBinder', + 'org.slf4j.impl.StaticMDCBinder', + 'org.slf4j.impl.StaticMarkerBinder', + 'org.slf4j.spi.LoggingEventBuilder', ) } @@ -332,10 +392,10 @@ task largeBlobYamlRestTest(type: RestIntegTestTask) { if (useFixture) { dependsOn createServiceAccountFile } - SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); - SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) - setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) - setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) + setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) } check.dependsOn largeBlobYamlRestTest diff --git a/plugins/repository-gcs/licenses/api-common-2.46.1.jar.sha1 b/plugins/repository-gcs/licenses/api-common-2.46.1.jar.sha1 deleted file mode 100644 index 19b87717499be..0000000000000 --- a/plugins/repository-gcs/licenses/api-common-2.46.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b38a684c734963a72c204aa208dd31018d79bf3a \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/api-common-2.52.0.jar.sha1 b/plugins/repository-gcs/licenses/api-common-2.52.0.jar.sha1 new file mode 100644 index 0000000000000..9b6d61a078f3f --- /dev/null +++ b/plugins/repository-gcs/licenses/api-common-2.52.0.jar.sha1 @@ -0,0 +1 @@ +504d2e98835a8e3f4d06433a53cfc2a03e0dd648 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/checker-qual-3.49.0.jar.sha1 b/plugins/repository-gcs/licenses/checker-qual-3.49.0.jar.sha1 new file mode 100644 index 0000000000000..6d96be486ce3b --- /dev/null +++ b/plugins/repository-gcs/licenses/checker-qual-3.49.0.jar.sha1 @@ -0,0 +1 @@ +54be36cb42c9b991c109e467e2bfa82af4cda44e \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/checker-qual-LICENSE.txt b/plugins/repository-gcs/licenses/checker-qual-LICENSE.txt new file mode 100644 index 0000000000000..9837c6b69fdab --- /dev/null +++ b/plugins/repository-gcs/licenses/checker-qual-LICENSE.txt @@ -0,0 +1,22 @@ +Checker Framework qualifiers +Copyright 2004-present by the Checker Framework developers + +MIT License: + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/plugins/repository-gcs/licenses/checker-qual-NOTICE.txt b/plugins/repository-gcs/licenses/checker-qual-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-gcs/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/repository-gcs/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/plugins/repository-gcs/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/commons-codec-1.18.0.jar.sha1 b/plugins/repository-gcs/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/plugins/repository-gcs/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/commons-logging-1.2.jar.sha1 b/plugins/repository-gcs/licenses/commons-logging-1.2.jar.sha1 deleted file mode 100644 index f40f0242448e8..0000000000000 --- a/plugins/repository-gcs/licenses/commons-logging-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4bfc12adfe4842bf07b657f0369c4cb522955686 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/commons-logging-NOTICE.txt b/plugins/repository-gcs/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index d3d6e140ce4f3..0000000000000 --- a/plugins/repository-gcs/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Commons Logging -Copyright 2003-2014 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/repository-gcs/licenses/failureaccess-1.0.1.jar.sha1 b/plugins/repository-gcs/licenses/failureaccess-1.0.1.jar.sha1 deleted file mode 100644 index 4798b37e20691..0000000000000 --- a/plugins/repository-gcs/licenses/failureaccess-1.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1dcf1de382a0bf95a3d8b0849546c88bac1292c9 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/failureaccess-1.0.2.jar.sha1 b/plugins/repository-gcs/licenses/failureaccess-1.0.2.jar.sha1 new file mode 100644 index 0000000000000..e1dbdc6bf7320 --- /dev/null +++ b/plugins/repository-gcs/licenses/failureaccess-1.0.2.jar.sha1 @@ -0,0 +1 @@ +c4a06a64e650562f30b7bf9aaec1bfed43aca12b \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/gax-2.63.1.jar.sha1 b/plugins/repository-gcs/licenses/gax-2.63.1.jar.sha1 deleted file mode 100644 index d438c0b04fcb9..0000000000000 --- a/plugins/repository-gcs/licenses/gax-2.63.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c9a340608a63e24dc8acd8da84afd8ffecca4b7 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/gax-2.69.0.jar.sha1 b/plugins/repository-gcs/licenses/gax-2.69.0.jar.sha1 new file mode 100644 index 0000000000000..eb476ea5ef3a6 --- /dev/null +++ b/plugins/repository-gcs/licenses/gax-2.69.0.jar.sha1 @@ -0,0 +1 @@ +3bfa5b525b9c7f1101ea481786fdda4c87cacd9f \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/gax-httpjson-2.42.0.jar.sha1 b/plugins/repository-gcs/licenses/gax-httpjson-2.42.0.jar.sha1 deleted file mode 100644 index 672506572ed4d..0000000000000 --- a/plugins/repository-gcs/licenses/gax-httpjson-2.42.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4db06bc31c2fb34b0490362e8666c20fdc1fb3f2 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/gax-httpjson-2.69.0.jar.sha1 b/plugins/repository-gcs/licenses/gax-httpjson-2.69.0.jar.sha1 new file mode 100644 index 0000000000000..8f0e2fb693586 --- /dev/null +++ b/plugins/repository-gcs/licenses/gax-httpjson-2.69.0.jar.sha1 @@ -0,0 +1 @@ +d58a1f8803ac63df943e4155b260aa2df33034aa \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-api-client-2.7.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-client-2.7.0.jar.sha1 deleted file mode 100644 index dcbd27a0009bf..0000000000000 --- a/plugins/repository-gcs/licenses/google-api-client-2.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -59c8e5e3c03f146561a83051af3ca945d40e02c6 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-api-client-2.7.2.jar.sha1 b/plugins/repository-gcs/licenses/google-api-client-2.7.2.jar.sha1 new file mode 100644 index 0000000000000..bc3dc91d301c5 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-api-client-2.7.2.jar.sha1 @@ -0,0 +1 @@ +495d58d6e31c2c5e24a707a50d6355ba92dd3d0c \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20230617-2.0.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20230617-2.0.0.jar.sha1 deleted file mode 100644 index 1a1452f773b96..0000000000000 --- a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20230617-2.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fc3f225b405303fe7cb760d578348b6b07e7ea8b \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20250718-2.0.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20250718-2.0.0.jar.sha1 new file mode 100644 index 0000000000000..5c89ad09bff22 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20250718-2.0.0.jar.sha1 @@ -0,0 +1 @@ +2a52ad55f9d1f78e6aeba2a54358ea3e6e92c0c9 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.29.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.29.0.jar.sha1 deleted file mode 100644 index e2f931a1e876f..0000000000000 --- a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.29.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19af4907301816d9328c1eb1fcc6dd05c8a0b544 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.37.1.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.37.1.jar.sha1 new file mode 100644 index 0000000000000..fd3bcfebb878e --- /dev/null +++ b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.37.1.jar.sha1 @@ -0,0 +1 @@ +894c1cd371380e254290ac7c7df04372bf547a8f \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.29.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.29.0.jar.sha1 deleted file mode 100644 index 98d0d1beda43d..0000000000000 --- a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.29.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2a42aead6cdc5d2cd22cdda1b9d7922e6135240f \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.37.1.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.37.1.jar.sha1 new file mode 100644 index 0000000000000..a0e34c8071d43 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.37.1.jar.sha1 @@ -0,0 +1 @@ +86a3c90a6b80128fccac09dead6158fe7cc5e7bd \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-core-2.30.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-2.30.0.jar.sha1 deleted file mode 100644 index 10f8f90df108f..0000000000000 --- a/plugins/repository-gcs/licenses/google-cloud-core-2.30.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b48ea27cbdccd5f225d8a35ea28e2cd01c25918b \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-core-2.59.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-2.59.0.jar.sha1 new file mode 100644 index 0000000000000..27c91368e8487 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-cloud-core-2.59.0.jar.sha1 @@ -0,0 +1 @@ +ad74095c899ca73b1a5a449eeef1a00742360382 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.47.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.47.0.jar.sha1 deleted file mode 100644 index 224893caeaafb..0000000000000 --- a/plugins/repository-gcs/licenses/google-cloud-core-http-2.47.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bfc8c587e8f2f1f1158cf36b0e515ef84f9e0a95 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.59.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.59.0.jar.sha1 new file mode 100644 index 0000000000000..b069b3a1ff669 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-cloud-core-http-2.59.0.jar.sha1 @@ -0,0 +1 @@ +9e4ea9e18a2ecd5487afe717648fdfd090f1b276 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-storage-1.113.1.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-storage-1.113.1.jar.sha1 deleted file mode 100644 index 22fc078b36aa1..0000000000000 --- a/plugins/repository-gcs/licenses/google-cloud-storage-1.113.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fd291ed57c1223bbb31363c4aa88c55faf0000c7 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-storage-2.55.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-storage-2.55.0.jar.sha1 new file mode 100644 index 0000000000000..8f4aa26c5cf1e --- /dev/null +++ b/plugins/repository-gcs/licenses/google-cloud-storage-2.55.0.jar.sha1 @@ -0,0 +1 @@ +faacc755d115d83aac04c4eecf45d65f6e8ce258 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-1.44.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-1.44.1.jar.sha1 deleted file mode 100644 index 501f268254fbc..0000000000000 --- a/plugins/repository-gcs/licenses/google-http-client-1.44.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d8956bacb8a4011365fa15a690482c49a70c78c5 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-1.47.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-1.47.1.jar.sha1 new file mode 100644 index 0000000000000..a6e55ecc00c5a --- /dev/null +++ b/plugins/repository-gcs/licenses/google-http-client-1.47.1.jar.sha1 @@ -0,0 +1 @@ +eabad78d440226732a453d6a300663a9770f5b7e \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-appengine-1.44.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-appengine-1.44.1.jar.sha1 deleted file mode 100644 index 7b27b165453cd..0000000000000 --- a/plugins/repository-gcs/licenses/google-http-client-appengine-1.44.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -da4f9f691edb7a9f00cd806157a4990cb7e07711 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-appengine-1.47.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-appengine-1.47.1.jar.sha1 new file mode 100644 index 0000000000000..6a202a1f01ca7 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-http-client-appengine-1.47.1.jar.sha1 @@ -0,0 +1 @@ +487e80e93247912e7f9e33dcd1f5cb6aa2fce107 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.44.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.44.1.jar.sha1 deleted file mode 100644 index 90ddf3ddc5ee6..0000000000000 --- a/plugins/repository-gcs/licenses/google-http-client-gson-1.44.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f3b8967c6f7078da6380687859d0873105f84d39 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.47.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.47.1.jar.sha1 new file mode 100644 index 0000000000000..d27c4b919285a --- /dev/null +++ b/plugins/repository-gcs/licenses/google-http-client-gson-1.47.1.jar.sha1 @@ -0,0 +1 @@ +04331c43544544d60df28055c295949e22ba60a4 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.44.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.44.1.jar.sha1 deleted file mode 100644 index 4472ffbbebe1c..0000000000000 --- a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.44.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3f1947de0fd9eb250af16abe6103c11e68d11635 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.47.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.47.1.jar.sha1 new file mode 100644 index 0000000000000..e9f5b4b75e272 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.47.1.jar.sha1 @@ -0,0 +1 @@ +47dc687cfe1a06bf1c9501e0d1b67d7dd6935c2d \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-oauth-client-1.34.1.jar.sha1 b/plugins/repository-gcs/licenses/google-oauth-client-1.34.1.jar.sha1 deleted file mode 100644 index a8434bd380761..0000000000000 --- a/plugins/repository-gcs/licenses/google-oauth-client-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4a4f88c5e13143f882268c98239fb85c3b2c6cb2 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-oauth-client-1.39.0.jar.sha1 b/plugins/repository-gcs/licenses/google-oauth-client-1.39.0.jar.sha1 new file mode 100644 index 0000000000000..dfd0b018aa63f --- /dev/null +++ b/plugins/repository-gcs/licenses/google-oauth-client-1.39.0.jar.sha1 @@ -0,0 +1 @@ +99f02f3c46c68c01dfc95878456b009a13229c88 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/grpc-api-1.68.2.jar.sha1 b/plugins/repository-gcs/licenses/grpc-api-1.68.2.jar.sha1 deleted file mode 100644 index 1844172dec982..0000000000000 --- a/plugins/repository-gcs/licenses/grpc-api-1.68.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a257a5dd25dda1c97a99b56d5b9c1e56c12ae554 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/grpc-api-1.71.0.jar.sha1 b/plugins/repository-gcs/licenses/grpc-api-1.71.0.jar.sha1 new file mode 100644 index 0000000000000..64e535b459e84 --- /dev/null +++ b/plugins/repository-gcs/licenses/grpc-api-1.71.0.jar.sha1 @@ -0,0 +1 @@ +239e7363a238943ff5ac5ba0d243d8b1d4f02ab7 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/gson-2.12.1.jar.sha1 b/plugins/repository-gcs/licenses/gson-2.12.1.jar.sha1 new file mode 100644 index 0000000000000..7d57e885daa08 --- /dev/null +++ b/plugins/repository-gcs/licenses/gson-2.12.1.jar.sha1 @@ -0,0 +1 @@ +4e773a317740b83b43cfc3d652962856041697cb \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/gson-2.13.0.jar.sha1 b/plugins/repository-gcs/licenses/gson-2.13.0.jar.sha1 deleted file mode 100644 index 7cf8ab0bbe08e..0000000000000 --- a/plugins/repository-gcs/licenses/gson-2.13.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -111ac98ad3d2d099d81d53b0549748144a8d2659 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/guava-33.2.1-jre.jar.sha1 b/plugins/repository-gcs/licenses/guava-33.2.1-jre.jar.sha1 deleted file mode 100644 index 27d5304e326df..0000000000000 --- a/plugins/repository-gcs/licenses/guava-33.2.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -818e780da2c66c63bbb6480fef1f3855eeafa3e4 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/guava-33.4.0-jre.jar.sha1 b/plugins/repository-gcs/licenses/guava-33.4.0-jre.jar.sha1 new file mode 100644 index 0000000000000..42b66665a578a --- /dev/null +++ b/plugins/repository-gcs/licenses/guava-33.4.0-jre.jar.sha1 @@ -0,0 +1 @@ +03fcc0a259f724c7de54a6a55ea7e26d3d5c0cac \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/log4j-1.2-api-2.21.0.jar.sha1 b/plugins/repository-gcs/licenses/log4j-1.2-api-2.21.0.jar.sha1 deleted file mode 100644 index 39d9177cb2fac..0000000000000 --- a/plugins/repository-gcs/licenses/log4j-1.2-api-2.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -12bad3819a9570807f3c97315930699584c12152 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/log4j-NOTICE.txt b/plugins/repository-gcs/licenses/log4j-NOTICE.txt deleted file mode 100644 index 0375732360047..0000000000000 --- a/plugins/repository-gcs/licenses/log4j-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache log4j -Copyright 2007 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/opentelemetry-api-1.47.0.jar.sha1 b/plugins/repository-gcs/licenses/opentelemetry-api-1.47.0.jar.sha1 new file mode 100644 index 0000000000000..1806d8e42714a --- /dev/null +++ b/plugins/repository-gcs/licenses/opentelemetry-api-1.47.0.jar.sha1 @@ -0,0 +1 @@ +9de168f2c648c33b86136f51a4584bde9a705ff1 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/commons-logging-LICENSE.txt b/plugins/repository-gcs/licenses/opentelemetry-api-LICENSE.txt similarity index 100% rename from plugins/repository-gcs/licenses/commons-logging-LICENSE.txt rename to plugins/repository-gcs/licenses/opentelemetry-api-LICENSE.txt index d645695673349..57bc88a15a0ee 100644 --- a/plugins/repository-gcs/licenses/commons-logging-LICENSE.txt +++ b/plugins/repository-gcs/licenses/opentelemetry-api-LICENSE.txt @@ -1,4 +1,3 @@ - Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -200,3 +199,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + diff --git a/plugins/repository-gcs/licenses/opentelemetry-api-NOTICE.txt b/plugins/repository-gcs/licenses/opentelemetry-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-gcs/licenses/opentelemetry-context-1.47.0.jar.sha1 b/plugins/repository-gcs/licenses/opentelemetry-context-1.47.0.jar.sha1 new file mode 100644 index 0000000000000..af4d69f26d333 --- /dev/null +++ b/plugins/repository-gcs/licenses/opentelemetry-context-1.47.0.jar.sha1 @@ -0,0 +1 @@ +86e49fe98ce06c279f7b9f028af8658cb7bc972a \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/log4j-LICENSE.txt b/plugins/repository-gcs/licenses/opentelemetry-context-LICENSE.txt similarity index 99% rename from plugins/repository-gcs/licenses/log4j-LICENSE.txt rename to plugins/repository-gcs/licenses/opentelemetry-context-LICENSE.txt index 6279e5206de13..57bc88a15a0ee 100644 --- a/plugins/repository-gcs/licenses/log4j-LICENSE.txt +++ b/plugins/repository-gcs/licenses/opentelemetry-context-LICENSE.txt @@ -1,4 +1,3 @@ - Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -187,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 1999-2005 The Apache Software Foundation + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -200,3 +199,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + diff --git a/plugins/repository-gcs/licenses/opentelemetry-context-NOTICE.txt b/plugins/repository-gcs/licenses/opentelemetry-context-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-gcs/licenses/proto-google-cloud-storage-v2-2.55.0.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-cloud-storage-v2-2.55.0.jar.sha1 new file mode 100644 index 0000000000000..c915f2bedc6ae --- /dev/null +++ b/plugins/repository-gcs/licenses/proto-google-cloud-storage-v2-2.55.0.jar.sha1 @@ -0,0 +1 @@ +3ef0b31ee17ae022ac6c20d4638ee44d47a61780 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/proto-google-common-protos-2.54.1.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-common-protos-2.54.1.jar.sha1 deleted file mode 100644 index a2cb686dc7bf6..0000000000000 --- a/plugins/repository-gcs/licenses/proto-google-common-protos-2.54.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aa342c093e2b75ecc341f28d2ee6c2b4480169c2 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/proto-google-common-protos-2.60.0.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-common-protos-2.60.0.jar.sha1 new file mode 100644 index 0000000000000..9363c672abef7 --- /dev/null +++ b/plugins/repository-gcs/licenses/proto-google-common-protos-2.60.0.jar.sha1 @@ -0,0 +1 @@ +8486130fd25ef43c3bef5ff1a6dcd82c17278c06 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/proto-google-iam-v1-1.49.1.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-iam-v1-1.49.1.jar.sha1 deleted file mode 100644 index 242da16cddf42..0000000000000 --- a/plugins/repository-gcs/licenses/proto-google-iam-v1-1.49.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3340df39c56ae913b068f17818bf016a4b4c4177 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/proto-google-iam-v1-1.55.0.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-iam-v1-1.55.0.jar.sha1 new file mode 100644 index 0000000000000..fc31be2e80131 --- /dev/null +++ b/plugins/repository-gcs/licenses/proto-google-iam-v1-1.55.0.jar.sha1 @@ -0,0 +1 @@ +71793e3db64f906c54ff0db45ef1332795aaa167 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/protobuf-LICENSE.txt b/plugins/repository-gcs/licenses/protobuf-LICENSE.txt new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/plugins/repository-gcs/licenses/protobuf-LICENSE.txt @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/plugins/repository-gcs/licenses/protobuf-NOTICE.txt b/plugins/repository-gcs/licenses/protobuf-NOTICE.txt new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/plugins/repository-gcs/licenses/protobuf-NOTICE.txt @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/plugins/repository-gcs/licenses/protobuf-java-util-3.25.8.jar.sha1 b/plugins/repository-gcs/licenses/protobuf-java-util-3.25.8.jar.sha1 new file mode 100644 index 0000000000000..f5ae8d3f32b92 --- /dev/null +++ b/plugins/repository-gcs/licenses/protobuf-java-util-3.25.8.jar.sha1 @@ -0,0 +1 @@ +0be3cb8bef1415d3b87cf5bf4de0b9149f6a0990 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-gcs/licenses/slf4j-api-1.7.36.jar.sha1 new file mode 100644 index 0000000000000..77b9917528382 --- /dev/null +++ b/plugins/repository-gcs/licenses/slf4j-api-1.7.36.jar.sha1 @@ -0,0 +1 @@ +6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/slf4j-api-LICENSE.txt b/plugins/repository-gcs/licenses/slf4j-api-LICENSE.txt new file mode 100644 index 0000000000000..8fda22f4d72f6 --- /dev/null +++ b/plugins/repository-gcs/licenses/slf4j-api-LICENSE.txt @@ -0,0 +1,21 @@ +Copyright (c) 2004-2014 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/repository-gcs/licenses/slf4j-api-NOTICE.txt b/plugins/repository-gcs/licenses/slf4j-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-gcs/licenses/threetenbp-1.4.4.jar.sha1 b/plugins/repository-gcs/licenses/threetenbp-1.4.4.jar.sha1 deleted file mode 100644 index 0f7ee08a6d2fc..0000000000000 --- a/plugins/repository-gcs/licenses/threetenbp-1.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bbe3cc15e8ea16863435009af8ca40dd97770240 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/threetenbp-1.7.0.jar.sha1 b/plugins/repository-gcs/licenses/threetenbp-1.7.0.jar.sha1 new file mode 100644 index 0000000000000..6c36a0e68fd97 --- /dev/null +++ b/plugins/repository-gcs/licenses/threetenbp-1.7.0.jar.sha1 @@ -0,0 +1 @@ +8703e893440e550295aa358281db468625bc9a05 \ No newline at end of file diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java index f5c20003ea7b6..5a49abf00a806 100644 --- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java +++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java @@ -300,7 +300,18 @@ private void writeBlobResumable(BlobInfo blobInfo, InputStream inputStream, long @SuppressForbidden(reason = "channel is based on a socket") @Override public int write(final ByteBuffer src) throws IOException { - return SocketAccess.doPrivilegedIOException(() -> writeChannel.write(src)); + try { + return SocketAccess.doPrivilegedIOException(() -> writeChannel.write(src)); + } catch (final IOException ioe) { + final StorageException storageException = (StorageException) ExceptionsHelper.unwrap( + ioe, + StorageException.class + ); + if (storageException != null) { + throw storageException; + } + throw ioe; + } } @Override diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java index 83a4146c99b99..bbcb30f640944 100644 --- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java +++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java @@ -216,7 +216,8 @@ StorageOptions createStorageOptions( mapBuilder.put("user-agent", clientSettings.getApplicationName()); } return mapBuilder.immutableMap(); - }); + }) + .setStorageRetryStrategy(new GoogleShouldRetryStorageStrategy()); if (Strings.hasLength(clientSettings.getHost())) { storageOptionsBuilder.setHost(clientSettings.getHost()); } diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleShouldRetryStorageStrategy.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleShouldRetryStorageStrategy.java new file mode 100644 index 0000000000000..6697a74b4bf77 --- /dev/null +++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleShouldRetryStorageStrategy.java @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.gcs; + +import com.google.api.gax.retrying.ResultRetryAlgorithm; +import com.google.api.gax.retrying.TimedAttemptSettings; +import com.google.cloud.BaseService; +import com.google.cloud.storage.StorageRetryStrategy; +import org.opensearch.ExceptionsHelper; + +import java.net.SocketException; +import java.net.UnknownHostException; +import java.util.concurrent.CancellationException; + +import static java.util.Objects.nonNull; + +public class GoogleShouldRetryStorageStrategy implements StorageRetryStrategy { + + private final DelagateResultRetryAlgorithm idempotentHandler = new DelagateResultRetryAlgorithm<>(BaseService.EXCEPTION_HANDLER); + + private final DelagateResultRetryAlgorithm nonIdempotentHandler = new DelagateResultRetryAlgorithm<>(BaseService.EXCEPTION_HANDLER); + + private static final class DelagateResultRetryAlgorithm implements ResultRetryAlgorithm { + + private final ResultRetryAlgorithm resultRetryAlgorithm; + + private DelagateResultRetryAlgorithm(ResultRetryAlgorithm resultRetryAlgorithm) { + this.resultRetryAlgorithm = resultRetryAlgorithm; + } + + @Override + public TimedAttemptSettings createNextAttempt(Throwable prevThrowable, T prevResponse, TimedAttemptSettings prevSettings) { + return resultRetryAlgorithm.createNextAttempt(prevThrowable, prevResponse, prevSettings); + } + + @Override + public boolean shouldRetry(Throwable prevThrowable, T prevResponse) throws CancellationException { + if (nonNull(ExceptionsHelper.unwrap(prevThrowable, UnknownHostException.class))) { + return true; + } + if (nonNull(ExceptionsHelper.unwrap(prevThrowable, SocketException.class))) { + return true; + } + return resultRetryAlgorithm.shouldRetry(prevThrowable, prevResponse); + } + }; + + @Override + public ResultRetryAlgorithm getIdempotentHandler() { + return idempotentHandler; + } + + @Override + public ResultRetryAlgorithm getNonidempotentHandler() { + return nonIdempotentHandler; + } +} diff --git a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java index 23c006c9d2ce6..f23edd7160785 100644 --- a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java +++ b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java @@ -52,7 +52,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.CountDown; -import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.unit.ByteSizeValue; @@ -75,6 +74,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import fixture.gcs.ContentHttpHeadersParser; import fixture.gcs.FakeOAuth2HttpHandler; import org.threeten.bp.Duration; @@ -92,9 +92,6 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeEnd; -import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeLimit; -import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeStart; import static fixture.gcs.GoogleCloudStorageHttpHandler.parseMultipartRequestBody; @SuppressForbidden(reason = "use a http server") @@ -152,7 +149,6 @@ StorageOptions createStorageOptions( .setInitialRetryDelay(Duration.ofMillis(10L)) .setRetryDelayMultiplier(1.0d) .setMaxRetryDelay(Duration.ofSeconds(1L)) - .setJittered(false) .setInitialRpcTimeout(Duration.ofSeconds(1)) .setRpcTimeoutMultiplier(options.getRetrySettings().getRpcTimeoutMultiplier()) .setMaxRpcTimeout(Duration.ofSeconds(1)); @@ -163,6 +159,7 @@ StorageOptions createStorageOptions( .setHost(options.getHost()) .setCredentials(options.getCredentials()) .setRetrySettings(retrySettingsBuilder.build()) + .setStorageRetryStrategy(new GoogleShouldRetryStorageStrategy()) .build(); } }; @@ -220,7 +217,8 @@ public void testWriteBlobWithRetries() throws Exception { assertThat(content.isPresent(), is(true)); assertThat(content.get().v1(), equalTo("write_blob_max_retries")); if (Objects.deepEquals(bytes, BytesReference.toBytes(content.get().v2()))) { - byte[] response = ("{\"bucket\":\"bucket\",\"name\":\"" + content.get().v1() + "\"}").getBytes(UTF_8); + byte[] response = String.format(""" + {"bucket":"bucket","name":"%s"}""", content.get().v1()).getBytes(UTF_8); exchange.getResponseHeaders().add("Content-Type", "application/json"); exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); exchange.getResponseBody().write(response); @@ -274,8 +272,7 @@ public void testWriteBlobWithReadTimeouts() { } public void testWriteLargeBlob() throws IOException { - // See {@link BaseWriteChannel#DEFAULT_CHUNK_SIZE} - final int defaultChunkSize = 60 * 256 * 1024; + final int defaultChunkSize = Math.toIntExact(ByteSizeValue.parseBytesSizeValue("16mb", "aaa").getBytes()); final int nbChunks = randomIntBetween(3, 5); final int lastChunkSize = randomIntBetween(1, defaultChunkSize - 1); final int totalChunks = nbChunks + 1; @@ -295,6 +292,7 @@ public void testWriteLargeBlob() throws IOException { final AtomicInteger countUploads = new AtomicInteger(nbErrors * totalChunks); final AtomicBoolean allow410Gone = new AtomicBoolean(randomBoolean()); final AtomicBoolean allowReadTimeout = new AtomicBoolean(rarely()); + final AtomicInteger bytesReceived = new AtomicInteger(); final int wrongChunk = randomIntBetween(1, totalChunks); final AtomicReference sessionUploadId = new AtomicReference<>(UUIDs.randomBase64UUID()); @@ -325,7 +323,6 @@ public void testWriteLargeBlob() throws IOException { assertThat(wrongChunk, greaterThan(0)); return; } - } else if ("PUT".equals(exchange.getRequestMethod())) { final String uploadId = params.get("upload_id"); if (uploadId.equals(sessionUploadId.get()) == false) { @@ -348,29 +345,43 @@ public void testWriteLargeBlob() throws IOException { // we must reset the counters because the whole object upload will be retried countInits.set(nbErrors); countUploads.set(nbErrors * totalChunks); + bytesReceived.set(0); exchange.sendResponseHeaders(HttpStatus.SC_GONE, -1); return; } } - final String range = exchange.getRequestHeaders().getFirst("Content-Range"); - assertTrue(Strings.hasLength(range)); + final String contentRangeHeaderValue = exchange.getRequestHeaders().getFirst("Content-Range"); + final var contentRange = ContentHttpHeadersParser.parseContentRangeHeader(contentRangeHeaderValue); + assertNotNull("Invalid content range header: " + contentRangeHeaderValue, contentRange); + + if (!contentRange.hasRange()) { + // Content-Range: */... is a status check + // https://cloud.google.com/storage/docs/performing-resumable-uploads#status-check + final int receivedSoFar = bytesReceived.get(); + if (receivedSoFar > 0) { + exchange.getResponseHeaders().add("Range", String.format("bytes=0-%s", receivedSoFar)); + } + exchange.getResponseHeaders().add("Content-Length", "0"); + exchange.sendResponseHeaders(308 /* Resume Incomplete */, -1); + return; + } if (countUploads.decrementAndGet() % 2 == 0) { assertThat(Math.toIntExact(requestBody.length()), anyOf(equalTo(defaultChunkSize), equalTo(lastChunkSize))); - - final int rangeStart = getContentRangeStart(range); - final int rangeEnd = getContentRangeEnd(range); + final int rangeStart = Math.toIntExact(contentRange.start()); + final int rangeEnd = Math.toIntExact(contentRange.end()); assertThat(rangeEnd + 1 - rangeStart, equalTo(Math.toIntExact(requestBody.length()))); assertThat(new BytesArray(data, rangeStart, rangeEnd - rangeStart + 1), is(requestBody)); + bytesReceived.updateAndGet(existing -> Math.max(existing, rangeEnd)); - final Integer limit = getContentRangeLimit(range); - if (limit != null) { + if (contentRange.size() != null) { + exchange.getResponseHeaders().add("x-goog-stored-content-length", String.valueOf(bytesReceived.get() + 1)); exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1); return; } else { - exchange.getResponseHeaders().add("Range", String.format(Locale.ROOT, "bytes=%d/%d", rangeStart, rangeEnd)); + exchange.getResponseHeaders().add("Range", String.format("bytes=%s-%s", rangeStart, rangeEnd)); exchange.getResponseHeaders().add("Content-Length", "0"); exchange.sendResponseHeaders(308 /* Resume Incomplete */, -1); return; diff --git a/plugins/repository-hdfs/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/repository-hdfs/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-codec-1.18.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/plugins/repository-s3/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/plugins/repository-s3/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/commons-codec-1.18.0.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/plugins/repository-s3/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file diff --git a/server/licenses/protobuf-java-3.25.5.jar.sha1 b/server/licenses/protobuf-java-3.25.5.jar.sha1 deleted file mode 100644 index 72b42c9efc85a..0000000000000 --- a/server/licenses/protobuf-java-3.25.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5ae5c9ec39930ae9b5a61b32b93288818ec05ec1 \ No newline at end of file diff --git a/server/licenses/protobuf-java-3.25.8.jar.sha1 b/server/licenses/protobuf-java-3.25.8.jar.sha1 new file mode 100644 index 0000000000000..7c966c247a5e9 --- /dev/null +++ b/server/licenses/protobuf-java-3.25.8.jar.sha1 @@ -0,0 +1 @@ +2ba593767658038775b2ea9724c3686609874470 \ No newline at end of file diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/ContentHttpHeadersParser.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/ContentHttpHeadersParser.java new file mode 100644 index 0000000000000..92debe801b95d --- /dev/null +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/ContentHttpHeadersParser.java @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package fixture.gcs; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ContentHttpHeadersParser { + + private static final Pattern PATTERN_CONTENT_RANGE = Pattern.compile("bytes=([0-9]+)-([0-9]+)"); + private static final Pattern PATTERN_CONTENT_RANGE_BYTES = Pattern.compile("bytes (?:(\\d+)-(\\d+)|\\*)/(?:(\\d+)|\\*)"); + + + public static Range parseRangeHeader(String rangeHeaderValue) { + final Matcher matcher = PATTERN_CONTENT_RANGE.matcher(rangeHeaderValue); + if (matcher.matches()) { + try { + return new Range(Integer.parseInt(matcher.group(1)), Integer.parseInt(matcher.group(2))); + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + + public record Range(int start, int end) { + + public String header() { + return String.format("bytes=%s-%s", start, end); + } + + } + + public static ContentRange parseContentRangeHeader(String contentRangeHeaderValue) { + final Matcher matcher = PATTERN_CONTENT_RANGE_BYTES.matcher(contentRangeHeaderValue); + if (matcher.matches()) { + try { + if (matcher.groupCount() == 3) { + final Integer start = parseIntegerValue(matcher.group(1)); + final Integer end = parseIntegerValue(matcher.group(2)); + final Integer size = parseIntegerValue(matcher.group(3)); + return new ContentRange(start, end, size); + } + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + + private static Integer parseIntegerValue(String value) { + return value == null ? null : Integer.parseInt(value); + } + + public record ContentRange(Integer start, Integer end, Integer size) { + + public ContentRange { + assert (start == null) == (end == null) : "Must have either start and end or neither"; + } + + public boolean hasRange() { + return start != null && end != null; + } + + public boolean hasSize() { + return size != null; + } + + public String headerString() { + final String rangeString = hasRange() ? start + "-" + end : "*"; + final String sizeString = hasSize() ? String.valueOf(size) : "*"; + return "bytes " + rangeString + "/" + sizeString; + } + } + +} diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java index f4ef1be4a9a3d..e8182e8f17701 100644 --- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java @@ -48,9 +48,12 @@ public class FakeOAuth2HttpHandler implements HttpHandler { @Override public void handle(final HttpExchange exchange) throws IOException { try { - while (exchange.getRequestBody().read(BUFFER) >= 0) ; - byte[] response = ("{\"access_token\":\"foo\",\"token_type\":\"Bearer\",\"expires_in\":3600}").getBytes(UTF_8); + while (exchange.getRequestBody().read(BUFFER) >= 0) { + } + byte[] response = """ + {"access_token":"foo","token_type":"Bearer","expires_in":3600}""".getBytes(UTF_8); exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.getResponseHeaders().add("Metadata-Flavor", "Google"); exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); exchange.getResponseBody().write(response); } finally { diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java index 358d573629e0a..90f8ce196a3c3 100644 --- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java @@ -38,13 +38,13 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.UUIDs; -import org.opensearch.core.common.bytes.BytesArray; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Streams; import org.opensearch.common.regex.Regex; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestUtils; @@ -64,7 +64,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.function.BiFunction; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -80,14 +79,58 @@ public class GoogleCloudStorageHttpHandler implements HttpHandler { private static final Logger logger = LogManager.getLogger(GoogleCloudStorageHttpHandler.class); - private static final Pattern RANGE_MATCHER = Pattern.compile("bytes=([0-9]*)-([0-9]*)"); + record BlobResumableUpload( + String uploadId, + BytesReference contents, + Integer finalLength, + boolean completed + ) { + + BlobResumableUpload(String uploadId) { + this(uploadId, BytesArray.EMPTY, null, false); + } + + public BlobResumableUpload update(BytesReference contents) { + if (completed) { + throw new IllegalStateException("Blob already completed"); + } + return new BlobResumableUpload(uploadId, contents, null, false); + } + public BlobResumableUpload complete() { + if (completed) { + throw new IllegalStateException("Blob already completed"); + } + return new BlobResumableUpload(uploadId, null, contents.length(), true); + } + + public ContentHttpHeadersParser.Range getRange() { + int length = length(); + if (length > 0) { + return new ContentHttpHeadersParser.Range(0, length - 1); + } else { + return null; + } + } + + public int length() { + if (finalLength != null) { + return finalLength; + } + if (contents != null) { + return contents.length(); + } + return 0; + } + } private final ConcurrentMap blobs; + private final ConcurrentMap resumableUploads; private final String bucket; public GoogleCloudStorageHttpHandler(final String bucket) { this.bucket = Objects.requireNonNull(bucket); this.blobs = new ConcurrentHashMap<>(); + this.resumableUploads = new ConcurrentHashMap<>(); } @Override @@ -100,7 +143,11 @@ public void handle(final HttpExchange exchange) throws IOException { try { // Request body is closed in the finally block final BytesReference requestBody = Streams.readFully(Streams.noCloseStream(exchange.getRequestBody())); - if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "/o/*", request)) { + if (request.equals("GET /") && "Google".equals(exchange.getRequestHeaders().getFirst("Metadata-Flavor"))) { + // the SDK checks this endpoint to determine if it's running within Google Compute Engine + exchange.getResponseHeaders().add("Metadata-Flavor", "Google"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), 0); + } else if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "/o/*", request)) { final String key = exchange.getRequestURI().getPath().replace("/storage/v1/b/" + bucket + "/o/", ""); final BytesReference blob = blobs.get(key); if (blob == null) { @@ -151,19 +198,19 @@ public void handle(final HttpExchange exchange) throws IOException { // Download Object https://cloud.google.com/storage/docs/request-body BytesReference blob = blobs.get(exchange.getRequestURI().getPath().replace("/download/storage/v1/b/" + bucket + "/o/", "")); if (blob != null) { - final String range = exchange.getRequestHeaders().getFirst("Range"); + final String rangeHeader = exchange.getRequestHeaders().getFirst("Range"); final int offset; final int end; - if (range == null) { + if (rangeHeader == null) { offset = 0; end = blob.length() - 1; } else { - Matcher matcher = RANGE_MATCHER.matcher(range); - if (matcher.find() == false) { - throw new AssertionError("Range bytes header does not match expected format: " + range); + final var rande = ContentHttpHeadersParser.parseRangeHeader(rangeHeader); + if (rande == null) { + throw new AssertionError("Range bytes header does not match expected format: " + rangeHeader); } - offset = Integer.parseInt(matcher.group(1)); - end = Integer.parseInt(matcher.group(2)); + offset = rande.start(); + end = rande.end(); } BytesReference response = blob; exchange.getResponseHeaders().add("Content-Type", "application/octet-stream"); @@ -203,9 +250,13 @@ public void handle(final HttpExchange exchange) throws IOException { Optional> content = parseMultipartRequestBody(requestBody.streamInput()); if (content.isPresent()) { blobs.put(content.get().v1(), content.get().v2()); - - byte[] response = ("{\"bucket\":\"" + bucket + "\",\"name\":\"" + content.get().v1() + "\"}").getBytes(UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json"); + byte[] response = String.format(""" + { + "bucket":"%s", + "name":"%s" + } + """, bucket, content.get().v1()).getBytes(UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8"); exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); exchange.getResponseBody().write(response); } else { @@ -220,15 +271,17 @@ public void handle(final HttpExchange exchange) throws IOException { final String blobName = params.get("name"); blobs.put(blobName, BytesArray.EMPTY); + final String uploadId = UUIDs.randomBase64UUID(); + resumableUploads.put(uploadId, new BlobResumableUpload(uploadId)); + byte[] response = requestBody.utf8ToString().getBytes(UTF_8); exchange.getResponseHeaders().add("Content-Type", "application/json"); exchange.getResponseHeaders().add("Location", httpServerUrl(exchange) + "/upload/storage/v1/b/" + bucket + "/o?" + "uploadType=resumable" - + "&upload_id=" + UUIDs.randomBase64UUID() + + "&upload_id=" + uploadId + "&test_blob_name=" + blobName); // not a Google Storage parameter, but it allows to pass the blob name exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); exchange.getResponseBody().write(response); - } else if (Regex.simpleMatch("PUT /upload/storage/v1/b/" + bucket + "/o?*uploadType=resumable*", request)) { // Resumable upload https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload final Map params = new HashMap<>(); @@ -239,25 +292,36 @@ public void handle(final HttpExchange exchange) throws IOException { exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1); return; } - BytesReference blob = blobs.get(blobName); - final String range = exchange.getRequestHeaders().getFirst("Content-Range"); - final Integer limit = getContentRangeLimit(range); - final int start = getContentRangeStart(range); - final int end = getContentRangeEnd(range); - - blob = CompositeBytesReference.of(blob, requestBody); - blobs.put(blobName, blob); - - if (limit == null) { - exchange.getResponseHeaders().add("Range", String.format(Locale.ROOT, "bytes=%d/%d", start, end)); - exchange.getResponseHeaders().add("Content-Length", "0"); - exchange.sendResponseHeaders(308 /* Resume Incomplete */, -1); - } else { - if (limit > blob.length()) { - throw new AssertionError("Requesting more bytes than available for blob"); + final var resumableUpload = resumableUploads.compute(params.get("upload_id"), (uuid, existing) -> { + BlobResumableUpload valueToReturn = existing; + final String contentRangeHeader = exchange.getRequestHeaders().getFirst("Content-Range"); + final var contentRange = ContentHttpHeadersParser.parseContentRangeHeader(contentRangeHeader); + + // Handle the request, a range indicates a chunk of data was submitted + if (contentRange.hasRange()) { + final int offset = Math.toIntExact(existing.contents.length() - contentRange.start()); + final BytesReference updatedContent = CompositeBytesReference.of( + existing.contents, + requestBody.slice(offset, requestBody.length()) + ); + valueToReturn = existing.update(updatedContent); } - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1); + + // Next we determine the response + if (contentRange.hasSize() && contentRange.size() == valueToReturn.contents.length()) { + blobs.put(blobName, valueToReturn.contents); + valueToReturn = valueToReturn.complete(); + } + + return valueToReturn; + }); + + final int httpStatus = resumableUpload.completed ? RestStatus.OK.getStatus() : 308; + if (resumableUpload.getRange() != null) { + exchange.getResponseHeaders().add("Range", resumableUpload.getRange().header()); } + exchange.getResponseHeaders().add("x-goog-stored-content-length", String.valueOf(resumableUpload.length())); + exchange.sendResponseHeaders(httpStatus, -1); } else { exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1); } @@ -269,12 +333,15 @@ public void handle(final HttpExchange exchange) throws IOException { } private String buildBlobInfoJson(String blobName, int size) { - return "{\"kind\":\"storage#object\"," - + "\"bucket\":\"" + bucket + "\"," - + "\"name\":\"" + blobName + "\"," - + "\"id\":\"" + blobName + "\"," - + "\"size\":\"" + size + "\"" - + "}"; + return String.format(""" + { + "kind": "storage#object", + "bucket": "%s", + "name": "%s", + "id": "%s", + "size": "%s" + } + """, bucket, blobName, blobName, size); } public Map blobs() { @@ -353,31 +420,4 @@ private static boolean isEndOfPart(BytesReference fullRequestBody, int endPos) { } return true; } - - private static final Pattern PATTERN_CONTENT_RANGE = Pattern.compile("bytes ([^/]*)/([0-9\\*]*)"); - private static final Pattern PATTERN_CONTENT_RANGE_BYTES = Pattern.compile("([0-9]*)-([0-9]*)"); - - private static Integer parse(final Pattern pattern, final String contentRange, final BiFunction fn) { - final Matcher matcher = pattern.matcher(contentRange); - if (matcher.matches() == false || matcher.groupCount() != 2) { - throw new IllegalArgumentException("Unable to parse content range header"); - } - return fn.apply(matcher.group(1), matcher.group(2)); - } - - public static Integer getContentRangeLimit(final String contentRange) { - return parse(PATTERN_CONTENT_RANGE, contentRange, (bytes, limit) -> "*".equals(limit) ? null : Integer.parseInt(limit)); - } - - public static int getContentRangeStart(final String contentRange) { - return parse(PATTERN_CONTENT_RANGE, contentRange, - (bytes, limit) -> parse(PATTERN_CONTENT_RANGE_BYTES, bytes, - (start, end) -> Integer.parseInt(start))); - } - - public static int getContentRangeEnd(final String contentRange) { - return parse(PATTERN_CONTENT_RANGE, contentRange, - (bytes, limit) -> parse(PATTERN_CONTENT_RANGE_BYTES, bytes, - (start, end) -> Integer.parseInt(end))); - } } diff --git a/test/framework/licenses/commons-codec-1.16.1.jar.sha1 b/test/framework/licenses/commons-codec-1.16.1.jar.sha1 deleted file mode 100644 index 6b8803089c6d7..0000000000000 --- a/test/framework/licenses/commons-codec-1.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/test/framework/licenses/commons-codec-1.18.0.jar.sha1 b/test/framework/licenses/commons-codec-1.18.0.jar.sha1 new file mode 100644 index 0000000000000..01a6a8f446302 --- /dev/null +++ b/test/framework/licenses/commons-codec-1.18.0.jar.sha1 @@ -0,0 +1 @@ +ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f \ No newline at end of file