From 2105529a2c632b89dd4796fba78ede9cc4b3f34c Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Fri, 31 Jan 2025 15:01:29 -0800 Subject: [PATCH] Deprecate support for IBM COS via Hive --- docs/src/main/sphinx/object-storage.md | 2 - .../main/sphinx/object-storage/legacy-cos.md | 105 ------------------ docs/src/main/sphinx/redirects.txt | 1 + docs/src/main/sphinx/release/release-401.md | 2 +- .../trino/hdfs/cos/HiveCosServiceConfig.java | 2 + 5 files changed, 4 insertions(+), 108 deletions(-) delete mode 100644 docs/src/main/sphinx/object-storage/legacy-cos.md diff --git a/docs/src/main/sphinx/object-storage.md b/docs/src/main/sphinx/object-storage.md index bc1f07b2fc1b..7d883c064e9b 100644 --- a/docs/src/main/sphinx/object-storage.md +++ b/docs/src/main/sphinx/object-storage.md @@ -80,7 +80,6 @@ with the Hive connector and relevant dedicated pages: - [](/connector/hive) - [](/object-storage/legacy-azure) - [](/object-storage/legacy-gcs) -- [](/object-storage/legacy-cos) - [](/object-storage/legacy-s3) (object-storage-other)= @@ -103,7 +102,6 @@ storage: /object-storage/file-system-s3 /object-storage/file-system-hdfs /object-storage/legacy-azure -/object-storage/legacy-cos /object-storage/legacy-gcs /object-storage/legacy-s3 /object-storage/file-system-cache diff --git a/docs/src/main/sphinx/object-storage/legacy-cos.md b/docs/src/main/sphinx/object-storage/legacy-cos.md deleted file mode 100644 index 51c1f75d6b4b..000000000000 --- a/docs/src/main/sphinx/object-storage/legacy-cos.md +++ /dev/null @@ -1,105 +0,0 @@ -# Legacy IBM COS support - -Configure the {doc}`/connector/hive` to support [IBM Cloud Object Storage COS](https://www.ibm.com/cloud/object-storage) access. - -:::{warning} -Legacy support is not recommended and will be removed. Use [](file-system-s3). -::: - -## Configuration - -To use legacy support, the `fs.hadoop.enabled` property must be set to `true` in -your catalog configuration file. - -To use COS, you need to configure a catalog file to use the Hive -connector. For example, create a file `etc/ibmcos.properties` and -specify the path to the COS service config file with the -`hive.cos.service-config` property. - -```properties -connector.name=hive -hive.cos.service-config=etc/cos-service.properties -``` - -The service configuration file contains the access and secret keys, as well as -the endpoints for one or multiple COS services: - -```properties -service1.access-key= -service1.secret-key= -service1.endpoint= -service2.access-key= -service2.secret-key= -service2.endpoint= -``` - -The endpoints property is optional. `service1` and `service2` are -placeholders for unique COS service names. The granularity for providing access -credentials is at the COS service level. - -To use IBM COS service, specify the service name, for example: `service1` in -the COS path. The general URI path pattern is -`cos://./object(s)`. - -``` -cos://example-bucket.service1/orders_tiny -``` - -Trino translates the COS path, and uses the `service1` endpoint and -credentials from `cos-service.properties` to access -`cos://example-bucket.service1/object`. - -The Hive Metastore (HMS) does not support the IBM COS filesystem, by default. -The [Stocator library](https://github.com/CODAIT/stocator) is a possible -solution to this problem. Download the [Stocator JAR](https://repo1.maven.org/maven2/com/ibm/stocator/stocator/1.1.4/stocator-1.1.4.jar), -and place it in Hadoop PATH. The [Stocator IBM COS configuration](https://github.com/CODAIT/stocator#reference-stocator-in-the-core-sitexml) -should be placed in `core-site.xml`. For example: - -``` - - fs.stocator.scheme.list - cos - - - fs.cos.impl - com.ibm.stocator.fs.ObjectStoreFileSystem - - - fs.stocator.cos.impl - com.ibm.stocator.fs.cos.COSAPIClient - - - fs.stocator.cos.scheme - cos - - - fs.cos.service1.endpoint - http://s3.eu-de.cloud-object-storage.appdomain.cloud - - - fs.cos.service1.access.key - access-key - - - fs.cos.service1.secret.key - secret-key - -``` - -## Alternative configuration using S3 compatibility - -Use the S3 properties for the Hive connector in the catalog file. If only one -IBM Cloud Object Storage endpoint is used, then the configuration can be -simplified: - -``` -hive.s3.endpoint=http://s3.eu-de.cloud-object-storage.appdomain.cloud -hive.s3.aws-access-key=access-key -hive.s3.aws-secret-key=secret-key -``` - -Use `s3` protocol instead of `cos` for the table location: - -``` -s3://example-bucket/object/ -``` diff --git a/docs/src/main/sphinx/redirects.txt b/docs/src/main/sphinx/redirects.txt index 533f8b40d81d..e22fee37f421 100644 --- a/docs/src/main/sphinx/redirects.txt +++ b/docs/src/main/sphinx/redirects.txt @@ -10,6 +10,7 @@ connector/atop.md connector/removed.md connector/localfile.md connector/removed.md connector/accumulo.md connector/removed.md connector/kinesis.md connector/removed.md +object-storage/legacy-cos.md object-storage/file-system-s3.md security/apache-ranger-access-control.md security/ranger-access-control.md routines.md udf.md routines/function.md udf/function.md diff --git a/docs/src/main/sphinx/release/release-401.md b/docs/src/main/sphinx/release/release-401.md index 4cb8f0711eac..96df05ceb84b 100644 --- a/docs/src/main/sphinx/release/release-401.md +++ b/docs/src/main/sphinx/release/release-401.md @@ -31,7 +31,7 @@ ## Hive connector -* Add support for [IBM Cloud Object Storage](/object-storage/legacy-cos). ({issue}`14625`) +* Add support for IBM Cloud Object Storage. ({issue}`14625`) * Allow creating tables with an Avro schema literal using the new table property `avro_schema_literal`. ({issue}`14426`) * Fix potential query failure or incorrect results when reading from a table diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/cos/HiveCosServiceConfig.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/cos/HiveCosServiceConfig.java index 8e5c106daff8..393724b15817 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/cos/HiveCosServiceConfig.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/cos/HiveCosServiceConfig.java @@ -22,12 +22,14 @@ public class HiveCosServiceConfig { private File serviceConfig; + @Deprecated @FileExists public File getServiceConfig() { return serviceConfig; } + @Deprecated @Config("hive.cos.service-config") public HiveCosServiceConfig setServiceConfig(File serviceConfig) {