diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/RESTCatalogConfigResponse.java b/core/src/main/java/org/apache/iceberg/rest/responses/RESTCatalogConfigResponse.java new file mode 100644 index 000000000000..6f6933c54862 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/responses/RESTCatalogConfigResponse.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.rest.responses; + +import java.util.Map; +import java.util.Objects; +import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; + +/** + * Represents a response to requesting server-side provided configuration for the REST catalog. + * This allows client provided values to be overridden by the server or defaulted if not provided by the client. + *

+ * The catalog properties, with overrides and defaults applied, should be used to configure the catalog and for all + * subsequent requests after this initial config request. + *

+ * Configuration from the server consists of two sets of key/value pairs. + *

+ */ +public class RESTCatalogConfigResponse { + + private Map defaults; + private Map overrides; + + public RESTCatalogConfigResponse() { + // Required for Jackson deserialization + } + + private RESTCatalogConfigResponse(Map defaults, Map overrides) { + this.defaults = defaults; + this.overrides = overrides; + validate(); + } + + RESTCatalogConfigResponse validate() { + return this; + } + + /** + * Properties that should be used as default configuration. {@code defaults} have the lowest priority + * and should be applied before the client provided configuration. + * + * @return properties that should be used as default configuration + */ + public Map defaults() { + return defaults != null ? defaults : ImmutableMap.of(); + } + + /** + * Properties that should be used to override client configuration. {@code overrides} have the highest priority + * and should be applied after defaults and any client-provided configuration properties. + * + * @return properties that should be given higher precedence than any client provided input + */ + public Map overrides() { + return overrides != null ? overrides : ImmutableMap.of(); + } + + /** + * Merge client-provided config with server side provided configuration to return a single + * properties map which will be used for instantiating and configuring the REST catalog. + * + * @param clientProperties - Client provided configuration + * @return Merged configuration, with precedence in the order overrides, then client properties, and then defaults. + */ + public Map merge(Map clientProperties) { + Preconditions.checkNotNull(clientProperties, + "Cannot merge client properties with server-provided properties. Invalid client configuration: null"); + Map merged = Maps.newHashMap(defaults); + merged.putAll(clientProperties); + merged.putAll(overrides); + return ImmutableMap.copyOf(Maps.filterValues(merged, Objects::nonNull)); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("defaults", defaults) + .add("overrides", overrides) + .toString(); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private final Map defaults; + private final Map overrides; + + private Builder() { + this.defaults = Maps.newHashMap(); + this.overrides = Maps.newHashMap(); + } + + public Builder withDefault(String key, String value) { + Preconditions.checkNotNull(key, "Invalid default property: null"); + defaults.put(key, value); + return this; + } + + public Builder withOverride(String key, String value) { + Preconditions.checkNotNull(key, "Invalid override property: null"); + overrides.put(key, value); + return this; + } + + /** + * Adds the passed in map entries to the existing `defaults` of this Builder. + */ + public Builder withDefaults(Map defaultsToAdd) { + Preconditions.checkNotNull(defaultsToAdd, "Invalid default properties map: null"); + Preconditions.checkArgument(!defaultsToAdd.containsKey(null), "Invalid default property: null"); + defaults.putAll(defaultsToAdd); + return this; + } + + /** + * Adds the passed in map entries to the existing `overrides` of this Builder. + */ + public Builder withOverrides(Map overridesToAdd) { + Preconditions.checkNotNull(overridesToAdd, "Invalid override properties map: null"); + Preconditions.checkArgument(!overridesToAdd.containsKey(null), "Invalid override property: null"); + overrides.putAll(overridesToAdd); + return this; + } + + public RESTCatalogConfigResponse build() { + return new RESTCatalogConfigResponse(defaults, overrides); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestRESTCatalogConfigResponse.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestRESTCatalogConfigResponse.java new file mode 100644 index 000000000000..4b3cd79de3f7 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestRESTCatalogConfigResponse.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.rest.responses; + +import com.fasterxml.jackson.core.JsonProcessingException; +import java.util.Map; +import org.apache.iceberg.AssertHelpers; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.rest.RequestResponseTestBase; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestRESTCatalogConfigResponse extends RequestResponseTestBase { + + private static final Map DEFAULTS = ImmutableMap.of("warehouse", "s3://bucket/warehouse"); + private static final Map OVERRIDES = ImmutableMap.of("clients", "5"); + + private static final Map DEFAULTS_WITH_NULL_VALUE = Maps.newHashMap(); + private static final Map OVERRIDES_WITH_NULL_VALUE = Maps.newHashMap(); + + @BeforeClass + public static void beforeAllForRestCatalogConfig() { + DEFAULTS_WITH_NULL_VALUE.put("warehouse", null); + OVERRIDES_WITH_NULL_VALUE.put("clients", null); + } + + @Test + // Test cases that are JSON that can be created via the Builder + public void testRoundTripSerDe() throws JsonProcessingException { + // Both fields have values without nulls + String fullJson = "{\"defaults\":{\"warehouse\":\"s3://bucket/warehouse\"},\"overrides\":{\"clients\":\"5\"}}"; + assertRoundTripSerializesEquallyFrom( + fullJson, + RESTCatalogConfigResponse.builder() + .withOverrides(OVERRIDES).withDefaults(DEFAULTS).build()); + assertRoundTripSerializesEquallyFrom( + fullJson, + RESTCatalogConfigResponse.builder() + .withOverride("clients", "5").withDefault("warehouse", "s3://bucket/warehouse").build()); + + // `defaults` is empty + String jsonEmptyDefaults = "{\"defaults\":{},\"overrides\":{\"clients\":\"5\"}}"; + assertRoundTripSerializesEquallyFrom( + jsonEmptyDefaults, + RESTCatalogConfigResponse.builder().withOverrides(OVERRIDES).build()); + assertRoundTripSerializesEquallyFrom( + jsonEmptyDefaults, + RESTCatalogConfigResponse.builder().withOverrides(OVERRIDES).withDefaults(ImmutableMap.of()).build()); + assertRoundTripSerializesEquallyFrom( + jsonEmptyDefaults, + RESTCatalogConfigResponse.builder().withOverride("clients", "5").build()); + + // `overrides` is empty + String jsonEmptyOverrides = "{\"defaults\":{\"warehouse\":\"s3://bucket/warehouse\"},\"overrides\":{}}"; + assertRoundTripSerializesEquallyFrom( + jsonEmptyOverrides, + RESTCatalogConfigResponse.builder().withDefaults(DEFAULTS).build()); + assertRoundTripSerializesEquallyFrom( + jsonEmptyOverrides, + RESTCatalogConfigResponse.builder().withDefault("warehouse", "s3://bucket/warehouse").build()); + assertRoundTripSerializesEquallyFrom( + jsonEmptyOverrides, + RESTCatalogConfigResponse.builder().withDefaults(DEFAULTS).withOverrides(ImmutableMap.of()).build()); + + // Both are empty + String emptyJson = "{\"defaults\":{},\"overrides\":{}}"; + assertRoundTripSerializesEquallyFrom( + emptyJson, + RESTCatalogConfigResponse.builder().build()); + assertRoundTripSerializesEquallyFrom( + emptyJson, + RESTCatalogConfigResponse.builder().withOverrides(ImmutableMap.of()).withDefaults(ImmutableMap.of()).build()); + } + + @Test + // Test cases that cannot be built with our builder, but that are accepted when parsed + public void testCanDeserializeWithoutDefaultValues() throws JsonProcessingException { + RESTCatalogConfigResponse noOverrides = RESTCatalogConfigResponse.builder().withDefaults(DEFAULTS).build(); + String jsonMissingOverrides = "{\"defaults\":{\"warehouse\":\"s3://bucket/warehouse\"}}"; + assertEquals(deserialize(jsonMissingOverrides), noOverrides); + String jsonNullOverrides = "{\"defaults\":{\"warehouse\":\"s3://bucket/warehouse\"},\"overrides\":null}"; + assertEquals(deserialize(jsonNullOverrides), noOverrides); + + RESTCatalogConfigResponse noDefaults = RESTCatalogConfigResponse.builder().withOverrides(OVERRIDES).build(); + String jsonMissingDefaults = "{\"overrides\":{\"clients\":\"5\"}}"; + assertEquals(deserialize(jsonMissingDefaults), noDefaults); + String jsonNullDefaults = "{\"defaults\":null,\"overrides\":{\"clients\":\"5\"}}"; + assertEquals(deserialize(jsonNullDefaults), noDefaults); + + RESTCatalogConfigResponse noValues = RESTCatalogConfigResponse.builder().build(); + String jsonEmptyObject = "{}"; + assertEquals(deserialize(jsonEmptyObject), noValues); + String jsonNullForAllFields = "{\"defaults\":null,\"overrides\":null}"; + assertEquals(deserialize(jsonNullForAllFields), noValues); + } + + @Test + public void testCanUseNullAsPropertyValue() throws JsonProcessingException { + String jsonNullValueInDefaults = + "{\"defaults\":{\"warehouse\":null},\"overrides\":{\"clients\":\"5\"}}"; + assertRoundTripSerializesEquallyFrom( + jsonNullValueInDefaults, + RESTCatalogConfigResponse.builder() + .withDefaults(DEFAULTS_WITH_NULL_VALUE).withOverrides(OVERRIDES).build()); + assertRoundTripSerializesEquallyFrom( + jsonNullValueInDefaults, + RESTCatalogConfigResponse.builder() + .withDefault("warehouse", null).withOverrides(OVERRIDES).build()); + + String jsonNullValueInOverrides = + "{\"defaults\":{\"warehouse\":\"s3://bucket/warehouse\"},\"overrides\":{\"clients\":null}}"; + assertRoundTripSerializesEquallyFrom( + jsonNullValueInOverrides, + RESTCatalogConfigResponse.builder() + .withDefaults(DEFAULTS).withOverrides(OVERRIDES_WITH_NULL_VALUE).build()); + assertRoundTripSerializesEquallyFrom( + jsonNullValueInOverrides, + RESTCatalogConfigResponse.builder() + .withDefaults(DEFAULTS).withOverride("clients", null).build()); + } + + @Test + public void testDeserializeInvalidResponse() { + String jsonDefaultsHasWrongType = + "{\"defaults\":[\"warehouse\",\"s3://bucket/warehouse\"],\"overrides\":{\"clients\":\"5\"}}"; + AssertHelpers.assertThrows( + "A JSON response with the wrong type for the defaults field should fail to deserialize", + JsonProcessingException.class, + () -> deserialize(jsonDefaultsHasWrongType) + ); + + String jsonOverridesHasWrongType = + "{\"defaults\":{\"warehouse\":\"s3://bucket/warehouse\"},\"overrides\":\"clients\"}"; + AssertHelpers.assertThrows( + "A JSON response with the wrong type for the overrides field should fail to deserialize", + JsonProcessingException.class, + () -> deserialize(jsonOverridesHasWrongType) + ); + + String jsonMisspelledKeys = + "{\"defaultzzzzzzz\":{\"warehouse\":\"s3://bucket/warehouse\"},\"overrrrrrrrides\":{\"clients\":\"5\"}}"; + AssertHelpers.assertThrows( + "A JSON response with the keys spelled incorrectly should fail to deserialize and validate", + JsonProcessingException.class, + () -> deserialize(jsonMisspelledKeys) + ); + + AssertHelpers.assertThrows( + "A null JSON response body should fail to deserialize", + IllegalArgumentException.class, + () -> deserialize(null) + ); + } + + @Test + public void testBuilderDoesNotCreateInvalidObjects() { + AssertHelpers.assertThrows( + "The builder should not allow using null as a key in the properties to override", + NullPointerException.class, + "Invalid override property: null", + () -> RESTCatalogConfigResponse.builder().withOverride(null, "100").build() + ); + + AssertHelpers.assertThrows( + "The builder should not allow using null as a key in the default properties", + NullPointerException.class, + "Invalid default property: null", + () -> RESTCatalogConfigResponse.builder().withDefault(null, "100").build() + ); + + AssertHelpers.assertThrows( + "The builder should not allow passing a null map of config properties to override", + NullPointerException.class, + "Invalid override properties map: null", + () -> RESTCatalogConfigResponse.builder().withOverrides(null).build() + ); + + AssertHelpers.assertThrows( + "The builder should not allow passing a null map of default config properties", + NullPointerException.class, + "Invalid default properties map: null", + () -> RESTCatalogConfigResponse.builder().withDefaults(null).build() + ); + + Map mapWithNullKey = Maps.newHashMap(); + mapWithNullKey.put(null, "a"); + mapWithNullKey.put("b", "b"); + AssertHelpers.assertThrows( + "The builder should not allow passing a map of default config properties with a null key", + IllegalArgumentException.class, + "Invalid default property: null", + () -> RESTCatalogConfigResponse.builder().withDefaults(mapWithNullKey).build() + ); + + AssertHelpers.assertThrows( + "The builder should not allow passing a map of properties to override with a null key", + IllegalArgumentException.class, + "Invalid override property: null", + () -> RESTCatalogConfigResponse.builder().withOverrides(mapWithNullKey).build() + ); + } + + @Test + public void testMergeStripsNullValuedEntries() { + Map mapWithNullValue = Maps.newHashMap(); + mapWithNullValue.put("a", null); + mapWithNullValue.put("b", "from_overrides"); + + Map overrides = mapWithNullValue; + Map defaults = ImmutableMap.of("a", "from_defaults"); + Map clientConfig = ImmutableMap.of("a", "from_client", "c", "from_client"); + + RESTCatalogConfigResponse resp = RESTCatalogConfigResponse.builder() + .withOverrides(overrides).withDefaults(defaults).build(); + + // "a" isn't present as it was marked as `null` in the overrides, so the provided client configuration is discarded + Map merged = resp.merge(clientConfig); + Map expected = ImmutableMap.of( + "b", "from_overrides", + "c", "from_client" + ); + + Assert.assertEquals( + "The merged properties map should use values from defaults, then client config, and finally overrides", + expected, merged); + Assert.assertFalse("The merged properties map should omit keys with null values", merged.containsValue(null)); + } + + @Override + public String[] allFieldsFromSpec() { + return new String[] {"defaults", "overrides"}; + } + + @Override + public RESTCatalogConfigResponse createExampleInstance() { + return RESTCatalogConfigResponse.builder() + .withDefaults(DEFAULTS) + .withOverrides(OVERRIDES) + .build(); + } + + @Override + public void assertEquals(RESTCatalogConfigResponse actual, RESTCatalogConfigResponse expected) { + Assert.assertEquals("Config properties to use as defaults should be equal", + actual.defaults(), expected.defaults()); + Assert.assertEquals("Config properties to use as overrides should be equal", + actual.overrides(), expected.overrides()); + } + + @Override + public RESTCatalogConfigResponse deserialize(String json) throws JsonProcessingException { + return mapper().readValue(json, RESTCatalogConfigResponse.class).validate(); + } +}