diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 4bfde0b516c1..5ac91ec0a96f 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -743,6 +743,37 @@ acceptedBreaks: new: "method java.util.List org.apache.iceberg.rest.requests.UpdateTableRequest::requirements()" justification: "Signature changed to an interface, but this is safe because\ \ of type erasure and the original type is always returned" + "1.3.0": + org.apache.iceberg:iceberg-api: + - code: "java.class.removed" + old: "class org.apache.iceberg.view.ImmutableSQLViewRepresentation" + justification: "Moving from iceberg-api to iceberg-core" + - code: "java.class.removed" + old: "class org.apache.iceberg.view.ImmutableViewHistoryEntry" + justification: "Moving from iceberg-api to iceberg-core" + - code: "java.class.removed" + old: "class org.apache.iceberg.view.ImmutableViewVersion" + justification: "Moving from iceberg-api to iceberg-core" + - code: "java.class.removed" + old: "interface org.apache.iceberg.view.SQLViewRepresentation" + justification: "Moving from iceberg-api to iceberg-core" + - code: "java.method.numberOfParametersChanged" + old: "method org.apache.iceberg.view.ViewBuilder org.apache.iceberg.view.ViewBuilder::withQuery(java.lang.String)" + new: "method T org.apache.iceberg.view.VersionBuilder::withQuery(java.lang.String,\ + \ java.lang.String) @ org.apache.iceberg.view.ViewBuilder" + justification: "Acceptable break due to updating View APIs and the View Spec" + - code: "java.method.removed" + old: "method org.apache.iceberg.view.ViewBuilder org.apache.iceberg.view.ViewBuilder::withDialect(java.lang.String)" + justification: "Acceptable break due to updating View APIs and the View Spec" + - code: "java.method.removed" + old: "method org.apache.iceberg.view.ViewBuilder org.apache.iceberg.view.ViewBuilder::withFieldAliases(java.util.List)" + justification: "Acceptable break due to updating View APIs and the View Spec" + - code: "java.method.removed" + old: "method org.apache.iceberg.view.ViewBuilder org.apache.iceberg.view.ViewBuilder::withFieldComments(java.util.List)" + justification: "Acceptable break due to updating View APIs and the View Spec" + - code: "java.method.removed" + old: "method org.apache.iceberg.view.ViewBuilder org.apache.iceberg.view.ViewBuilder::withQueryColumnNames(java.util.List)" + justification: "Acceptable break due to updating View APIs and the View Spec" apache-iceberg-0.14.0: org.apache.iceberg:iceberg-api: - code: "java.class.defaultSerializationChanged" diff --git a/api/src/main/java/org/apache/iceberg/view/ReplaceViewVersion.java b/api/src/main/java/org/apache/iceberg/view/ReplaceViewVersion.java new file mode 100644 index 000000000000..b876933345ed --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/view/ReplaceViewVersion.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.view; + +import org.apache.iceberg.PendingUpdate; + +/** + * API for replacing a view's version. + * + *

Apply returns the updated view version for validation. + * + *

When committing, these changes will be applied to the current view metadata. Commit conflicts + * will be resolved by applying the pending changes to the new view metadata. + */ +public interface ReplaceViewVersion + extends PendingUpdate, VersionBuilder {} diff --git a/api/src/main/java/org/apache/iceberg/view/VersionBuilder.java b/api/src/main/java/org/apache/iceberg/view/VersionBuilder.java new file mode 100644 index 000000000000..0c69c8fc9e76 --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/view/VersionBuilder.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.view; + +import org.apache.iceberg.Schema; +import org.apache.iceberg.catalog.Namespace; + +public interface VersionBuilder { + /** + * Set the view schema. + * + * @param schema The schema to use for this view version + * @return this for method chaining + */ + T withSchema(Schema schema); + + /** + * Add a view representation for the given dialect and the SQL to the view. + * + * @param dialect The dialect of the view representation + * @param sql The SQL of the view representation + * @return this for method chaining + */ + T withQuery(String dialect, String sql); + + /** + * Set the default catalog to use for the view. + * + * @param catalog The default catalog to use when the SQL does not contain a catalog + * @return this for method chaining + */ + T withDefaultCatalog(String catalog); + + /** + * Set the default namespace to use for the view. + * + * @param namespace The default namespace to use when the SQL does not contain a namespace + * @return this for method chaining + */ + T withDefaultNamespace(Namespace namespace); +} diff --git a/api/src/main/java/org/apache/iceberg/view/View.java b/api/src/main/java/org/apache/iceberg/view/View.java index 0f4d1cc9c0e8..284c561b7848 100644 --- a/api/src/main/java/org/apache/iceberg/view/View.java +++ b/api/src/main/java/org/apache/iceberg/view/View.java @@ -83,4 +83,13 @@ public interface View { * @return a new {@link UpdateViewProperties} */ UpdateViewProperties updateProperties(); + + /** + * Create a new {@link ReplaceViewVersion} to replace the view's current version. + * + * @return a new {@link ReplaceViewVersion} + */ + default ReplaceViewVersion replaceVersion() { + throw new UnsupportedOperationException("Replacing a view's version is not supported"); + } } diff --git a/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java b/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java index 57f8a970867b..02620de72289 100644 --- a/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java +++ b/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java @@ -18,10 +18,7 @@ */ package org.apache.iceberg.view; -import java.util.List; import java.util.Map; -import org.apache.iceberg.Schema; -import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.ViewCatalog; /** @@ -29,70 +26,7 @@ * *

Call {@link ViewCatalog#buildView} to create a new builder. */ -public interface ViewBuilder { - /** - * Set the view schema. - * - * @param schema view schema - * @return this for method chaining - */ - ViewBuilder withSchema(Schema schema); - - /** - * Set the view query. - * - * @param query view query - * @return this for method chaining - */ - ViewBuilder withQuery(String query); - - /** - * Set the view SQL dialect. - * - * @param dialect view SQL dialect - * @return this for method chaining - */ - ViewBuilder withDialect(String dialect); - - /** - * Set the view default catalog. - * - * @param defaultCatalog view default catalog - * @return this for method chaining - */ - ViewBuilder withDefaultCatalog(String defaultCatalog); - - /** - * Set the view default namespace. - * - * @param defaultNamespace view default namespace - * @return this for method chaining - */ - ViewBuilder withDefaultNamespace(Namespace defaultNamespace); - - /** - * Set the view query column names. - * - * @param queryColumnNames view query column names - * @return this for method chaining - */ - ViewBuilder withQueryColumnNames(List queryColumnNames); - - /** - * Set the view field aliases. - * - * @param fieldAliases view field aliases - * @return this for method chaining - */ - ViewBuilder withFieldAliases(List fieldAliases); - - /** - * Set the view field comments. - * - * @param fieldComments view field comments - * @return this for method chaining - */ - ViewBuilder withFieldComments(List fieldComments); +public interface ViewBuilder extends VersionBuilder { /** * Add key/value properties to the view. diff --git a/api/src/main/java/org/apache/iceberg/view/ViewHistoryEntry.java b/api/src/main/java/org/apache/iceberg/view/ViewHistoryEntry.java index 351a27ce81c5..2840560655ed 100644 --- a/api/src/main/java/org/apache/iceberg/view/ViewHistoryEntry.java +++ b/api/src/main/java/org/apache/iceberg/view/ViewHistoryEntry.java @@ -18,15 +18,12 @@ */ package org.apache.iceberg.view; -import org.immutables.value.Value; - /** * View history entry. * *

An entry contains a change to the view state. At the given timestamp, the current version was * set to the given version ID. */ -@Value.Immutable public interface ViewHistoryEntry { /** Return the timestamp in milliseconds of the change */ long timestampMillis(); diff --git a/api/src/main/java/org/apache/iceberg/view/ViewVersion.java b/api/src/main/java/org/apache/iceberg/view/ViewVersion.java index 11dec1ff16c9..bf292f1073f4 100644 --- a/api/src/main/java/org/apache/iceberg/view/ViewVersion.java +++ b/api/src/main/java/org/apache/iceberg/view/ViewVersion.java @@ -20,8 +20,8 @@ import java.util.List; import java.util.Map; +import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.immutables.value.Value; /** * A version of the view at a point in time. @@ -30,7 +30,6 @@ * *

Versions are created by view operations, like Create and Replace. */ -@Value.Immutable public interface ViewVersion { /** Return this version's id. Version ids are monotonically increasing */ @@ -66,7 +65,6 @@ public interface ViewVersion { * * @return the string operation which produced the view version */ - @Value.Lazy default String operation() { return summary().get("operation"); } @@ -74,7 +72,16 @@ default String operation() { /** The query output schema at version create time, without aliases */ int schemaId(); - @Value.Check + /** The default catalog when the view is created. */ + default String defaultCatalog() { + return null; + } + + /** The default namespace to use when the SQL does not contain a namespace. */ + default Namespace defaultNamespace() { + return null; + } + default void check() { Preconditions.checkArgument( summary().containsKey("operation"), "Invalid view version summary, missing operation"); diff --git a/core/src/main/java/org/apache/iceberg/view/BaseViewHistoryEntry.java b/core/src/main/java/org/apache/iceberg/view/BaseViewHistoryEntry.java new file mode 100644 index 000000000000..0562af5f7fad --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/view/BaseViewHistoryEntry.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.view; + +import org.immutables.value.Value; + +/** + * View history entry. + * + *

An entry contains a change to the view state. At the given timestamp, the current version was + * set to the given version ID. + */ +@Value.Immutable +// https://github.com/immutables/immutables/issues/291 does not apply here because we're not adding +// any Immutable-specific class to the classpath +@SuppressWarnings("ImmutablesStyle") +@Value.Style(typeImmutable = "ImmutableViewHistoryEntry") +public interface BaseViewHistoryEntry extends ViewHistoryEntry {} diff --git a/core/src/main/java/org/apache/iceberg/view/BaseViewVersion.java b/core/src/main/java/org/apache/iceberg/view/BaseViewVersion.java new file mode 100644 index 000000000000..ca6c2dfc286a --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/view/BaseViewVersion.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.view; + +import javax.annotation.Nullable; +import org.apache.iceberg.catalog.Namespace; +import org.immutables.value.Value; + +/** + * A version of the view at a point in time. + * + *

A version consists of a view metadata file. + * + *

Versions are created by view operations, like Create and Replace. + */ +@Value.Immutable +// https://github.com/immutables/immutables/issues/291 does not apply here because we're not adding +// any Immutable-specific class to the classpath +@SuppressWarnings("ImmutablesStyle") +@Value.Style(typeImmutable = "ImmutableViewVersion") +public interface BaseViewVersion extends ViewVersion { + + @Override + @Value.Lazy + default String operation() { + return summary().get("operation"); + } + + @Override + @Nullable + String defaultCatalog(); + + @Override + @Nullable + Namespace defaultNamespace(); + + @Override + @Value.Check + default void check() { + ViewVersion.super.check(); + } +} diff --git a/api/src/main/java/org/apache/iceberg/view/SQLViewRepresentation.java b/core/src/main/java/org/apache/iceberg/view/SQLViewRepresentation.java similarity index 72% rename from api/src/main/java/org/apache/iceberg/view/SQLViewRepresentation.java rename to core/src/main/java/org/apache/iceberg/view/SQLViewRepresentation.java index c308a6289b94..41124d9dde03 100644 --- a/api/src/main/java/org/apache/iceberg/view/SQLViewRepresentation.java +++ b/core/src/main/java/org/apache/iceberg/view/SQLViewRepresentation.java @@ -18,9 +18,6 @@ */ package org.apache.iceberg.view; -import java.util.List; -import javax.annotation.Nullable; -import org.apache.iceberg.catalog.Namespace; import org.immutables.value.Value; @Value.Immutable @@ -36,18 +33,4 @@ default String type() { /** The view query SQL dialect. */ String dialect(); - - /** The default catalog when the view is created. */ - @Nullable - String defaultCatalog(); - - /** The default namespace when the view is created. */ - @Nullable - Namespace defaultNamespace(); - - /** The view field comments. */ - List fieldComments(); - - /** The view field aliases. */ - List fieldAliases(); } diff --git a/core/src/main/java/org/apache/iceberg/view/SQLViewRepresentationParser.java b/core/src/main/java/org/apache/iceberg/view/SQLViewRepresentationParser.java index ae3676e8bd18..bd1d84a6031f 100644 --- a/core/src/main/java/org/apache/iceberg/view/SQLViewRepresentationParser.java +++ b/core/src/main/java/org/apache/iceberg/view/SQLViewRepresentationParser.java @@ -21,21 +21,12 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.util.JsonUtil; class SQLViewRepresentationParser { private static final String SQL = "sql"; private static final String DIALECT = "dialect"; - private static final String SCHEMA_ID = "schema-id"; - private static final String DEFAULT_CATALOG = "default-catalog"; - private static final String DEFAULT_NAMESPACE = "default-namespace"; - private static final String FIELD_ALIASES = "field-aliases"; - private static final String FIELD_COMMENTS = "field-comments"; private SQLViewRepresentationParser() {} @@ -50,23 +41,6 @@ static void toJson(SQLViewRepresentation view, JsonGenerator generator) throws I generator.writeStringField(SQL, view.sql()); generator.writeStringField(DIALECT, view.dialect()); - if (view.defaultCatalog() != null) { - generator.writeStringField(DEFAULT_CATALOG, view.defaultCatalog()); - } - - if (view.defaultNamespace() != null) { - JsonUtil.writeStringArray( - DEFAULT_NAMESPACE, Arrays.asList(view.defaultNamespace().levels()), generator); - } - - if (!view.fieldAliases().isEmpty()) { - JsonUtil.writeStringArray(FIELD_ALIASES, view.fieldAliases(), generator); - } - - if (!view.fieldComments().isEmpty()) { - JsonUtil.writeStringArray(FIELD_COMMENTS, view.fieldComments(), generator); - } - generator.writeEndObject(); } @@ -83,27 +57,6 @@ static SQLViewRepresentation fromJson(JsonNode node) { ImmutableSQLViewRepresentation.builder() .sql(JsonUtil.getString(SQL, node)) .dialect(JsonUtil.getString(DIALECT, node)); - String defaultCatalog = JsonUtil.getStringOrNull(DEFAULT_CATALOG, node); - if (defaultCatalog != null) { - builder.defaultCatalog(defaultCatalog); - } - - Integer schemaId = JsonUtil.getIntOrNull(SCHEMA_ID, node); - - List namespace = JsonUtil.getStringListOrNull(DEFAULT_NAMESPACE, node); - if (namespace != null && !namespace.isEmpty()) { - builder.defaultNamespace(Namespace.of(Iterables.toArray(namespace, String.class))); - } - - List fieldAliases = JsonUtil.getStringListOrNull(FIELD_ALIASES, node); - if (fieldAliases != null) { - builder.fieldAliases(fieldAliases); - } - - List fieldComments = JsonUtil.getStringListOrNull(FIELD_COMMENTS, node); - if (fieldComments != null) { - builder.fieldComments(fieldComments); - } return builder.build(); } diff --git a/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java b/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java index a0df4564985d..39783557e4b0 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java @@ -21,9 +21,13 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import java.util.Map; +import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.util.JsonUtil; class ViewVersionParser { @@ -33,6 +37,8 @@ class ViewVersionParser { private static final String SUMMARY = "summary"; private static final String REPRESENTATIONS = "representations"; private static final String SCHEMA_ID = "schema-id"; + private static final String DEFAULT_CATALOG = "default-catalog"; + private static final String DEFAULT_NAMESPACE = "default-namespace"; private ViewVersionParser() {} @@ -45,6 +51,15 @@ static void toJson(ViewVersion version, JsonGenerator generator) throws IOExcept generator.writeNumberField(SCHEMA_ID, version.schemaId()); JsonUtil.writeStringMap(SUMMARY, version.summary(), generator); + if (version.defaultCatalog() != null) { + generator.writeStringField(DEFAULT_CATALOG, version.defaultCatalog()); + } + + if (version.defaultNamespace() != null) { + JsonUtil.writeStringArray( + DEFAULT_NAMESPACE, Arrays.asList(version.defaultNamespace().levels()), generator); + } + generator.writeArrayFieldStart(REPRESENTATIONS); for (ViewRepresentation representation : version.representations()) { ViewRepresentationParser.toJson(representation, generator); @@ -81,11 +96,21 @@ static ViewVersion fromJson(JsonNode node) { representations.add(representation); } + String defaultCatalog = JsonUtil.getStringOrNull(DEFAULT_CATALOG, node); + + List namespace = JsonUtil.getStringListOrNull(DEFAULT_NAMESPACE, node); + Namespace defaultNamespace = null; + if (namespace != null && !namespace.isEmpty()) { + defaultNamespace = Namespace.of(Iterables.toArray(namespace, String.class)); + } + return ImmutableViewVersion.builder() .versionId(versionId) .timestampMillis(timestamp) .schemaId(schemaId) .summary(summary) + .defaultNamespace(defaultNamespace) + .defaultCatalog(defaultCatalog) .representations(representations.build()) .build(); } diff --git a/core/src/test/java/org/apache/iceberg/view/TestSQLViewRepresentationParser.java b/core/src/test/java/org/apache/iceberg/view/TestSQLViewRepresentationParser.java index 245bbbb33e1f..8eae11a4e67d 100644 --- a/core/src/test/java/org/apache/iceberg/view/TestSQLViewRepresentationParser.java +++ b/core/src/test/java/org/apache/iceberg/view/TestSQLViewRepresentationParser.java @@ -18,8 +18,7 @@ */ package org.apache.iceberg.view; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import com.fasterxml.jackson.databind.JsonNode; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -39,20 +38,12 @@ public void testParseSqlViewRepresentation() { .isEqualTo(viewRepresentation); String requiredAndOptionalFields = - "{\"type\":\"sql\", \"sql\": \"select * from foo\", \"dialect\": \"spark-sql\", " - + "\"default-catalog\":\"cat\", " - + "\"default-namespace\":[\"part1\",\"part2\"], " - + "\"field-aliases\":[\"col1\", \"col2\"], " - + "\"field-comments\":[\"Comment col1\", \"Comment col2\"]}"; + "{\"type\":\"sql\", \"sql\": \"select * from foo\", \"dialect\": \"spark-sql\"}"; SQLViewRepresentation viewWithOptionalFields = ImmutableSQLViewRepresentation.builder() .sql("select * from foo") .dialect("spark-sql") - .defaultCatalog("cat") - .fieldAliases(ImmutableList.of("col1", "col2")) - .fieldComments(ImmutableList.of("Comment col1", "Comment col2")) - .defaultNamespace(Namespace.of("part1", "part2")) .build(); Assertions.assertThat(SQLViewRepresentationParser.fromJson(requiredAndOptionalFields)) .as("Should be able to parse valid SQL view representation") @@ -74,8 +65,7 @@ public void testParseSqlViewRepresentationMissingRequiredFields() { @Test public void testViewRepresentationSerialization() { - String requiredFields = - "{\"type\":\"sql\",\"sql\":\"select * from foo\",\"dialect\":\"spark-sql\"}"; + String json = "{\"type\":\"sql\",\"sql\":\"select * from foo\",\"dialect\":\"spark-sql\"}"; SQLViewRepresentation viewRepresentation = ImmutableSQLViewRepresentation.builder() .sql("select * from foo") @@ -83,28 +73,10 @@ public void testViewRepresentationSerialization() { .build(); Assertions.assertThat(ViewRepresentationParser.toJson(viewRepresentation)) .as("Should be able to serialize valid SQL view representation") - .isEqualTo(requiredFields); - - String requiredAndOptionalFields = - "{\"type\":\"sql\",\"sql\":\"select * from foo\",\"dialect\":\"spark-sql\"," - + "\"default-catalog\":\"cat\"," - + "\"default-namespace\":[\"part1\",\"part2\"]," - + "\"field-aliases\":[\"col1\",\"col2\"]," - + "\"field-comments\":[\"Comment col1\",\"Comment col2\"]}"; - - SQLViewRepresentation viewWithOptionalFields = - ImmutableSQLViewRepresentation.builder() - .sql("select * from foo") - .dialect("spark-sql") - .defaultCatalog("cat") - .fieldAliases(ImmutableList.of("col1", "col2")) - .fieldComments(ImmutableList.of("Comment col1", "Comment col2")) - .defaultNamespace(Namespace.of("part1", "part2")) - .build(); - - Assertions.assertThat(ViewRepresentationParser.toJson(viewWithOptionalFields)) - .as("Should be able to serialize valid SQL view representation") - .isEqualTo(requiredAndOptionalFields); + .isEqualTo(json); + Assertions.assertThat( + ViewRepresentationParser.fromJson(ViewRepresentationParser.toJson(viewRepresentation))) + .isEqualTo(viewRepresentation); } @Test @@ -112,5 +84,9 @@ public void testNullSqlViewRepresentation() { Assertions.assertThatThrownBy(() -> SQLViewRepresentationParser.toJson(null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid SQL view representation: null"); + + Assertions.assertThatThrownBy(() -> SQLViewRepresentationParser.fromJson((JsonNode) null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse SQL view representation from null object"); } } diff --git a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java index d704c7226d5a..f4bf2118f140 100644 --- a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java +++ b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java @@ -62,11 +62,11 @@ public void readAndWriteValidViewMetadata() throws Exception { .timestampMillis(4353L) .summary(ImmutableMap.of("operation", "create")) .schemaId(1) + .defaultCatalog("some-catalog") .addRepresentations( ImmutableSQLViewRepresentation.builder() .sql("select 'foo' foo") .dialect("spark-sql") - .defaultCatalog("some-catalog") .build()) .build(); @@ -79,10 +79,10 @@ public void readAndWriteValidViewMetadata() throws Exception { .schemaId(1) .timestampMillis(5555L) .summary(ImmutableMap.of("operation", "replace")) + .defaultCatalog("some-catalog") .addRepresentations( ImmutableSQLViewRepresentation.builder() .sql("select 1 id, 'abc' data") - .defaultCatalog("some-catalog") .dialect("spark-sql") .build()) .build(); diff --git a/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java b/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java index a2cbf25968d4..3272a012b548 100644 --- a/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java +++ b/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java @@ -19,6 +19,7 @@ package org.apache.iceberg.view; import com.fasterxml.jackson.databind.JsonNode; +import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -80,6 +81,8 @@ public void testSerializeViewVersion() { .timestampMillis(12345) .addRepresentations(firstRepresentation, secondRepresentation) .summary(ImmutableMap.of("operation", "create", "user", "some-user")) + .defaultNamespace(Namespace.of("one", "two")) + .defaultCatalog("catalog") .schemaId(1) .build(); @@ -89,7 +92,8 @@ public void testSerializeViewVersion() { String expectedViewVersion = String.format( - "{\"version-id\":1,\"timestamp-ms\":12345,\"schema-id\":1,\"summary\":{\"operation\":\"create\",\"user\":\"some-user\"},\"representations\":%s}", + "{\"version-id\":1,\"timestamp-ms\":12345,\"schema-id\":1,\"summary\":{\"operation\":\"create\",\"user\":\"some-user\"}," + + "\"default-catalog\":\"catalog\",\"default-namespace\":[\"one\",\"two\"],\"representations\":%s}", expectedRepresentations); Assertions.assertThat(ViewVersionParser.toJson(viewVersion)) diff --git a/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json b/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json index a5be230aaca2..9c0ae0ecbef6 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json +++ b/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json @@ -37,15 +37,13 @@ "timestamp-ms": 4353, "summary": {"operation":"create"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 'foo' foo", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] }, @@ -54,15 +52,13 @@ "timestamp-ms": 5555, "summary": {"operation": "replace"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 1 id, 'abc' data", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] } diff --git a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json index 16e048881b86..b90befe515c2 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json +++ b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json @@ -37,15 +37,13 @@ "timestamp-ms": 4353, "summary": {"operation":"create"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 'foo' foo", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] }, @@ -54,15 +52,13 @@ "timestamp-ms": 5555, "summary": {"operation": "replace"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 1 id, 'abc' data", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] } diff --git a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json index 8b160f404d39..fbcb2c9a4176 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json +++ b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json @@ -37,15 +37,13 @@ "timestamp-ms": 4353, "summary": {"operation":"create"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 'foo' foo", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] }, @@ -54,15 +52,13 @@ "timestamp-ms": 5555, "summary": {"operation": "replace"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 1 id, 'abc' data", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] } diff --git a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataLimitedVersions.json b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataLimitedVersions.json index 5b4a99b55337..db80a3baefb7 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataLimitedVersions.json +++ b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataLimitedVersions.json @@ -37,15 +37,13 @@ "timestamp-ms": 4353, "summary": {"operation":"create"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 'foo' foo", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] }, @@ -54,15 +52,13 @@ "timestamp-ms": 5555, "summary": {"operation": "replace"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 1 id, 'abc' data", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] } diff --git a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentSchema.json b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentSchema.json index ede054648ae8..f2dc76723ddb 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentSchema.json +++ b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentSchema.json @@ -36,15 +36,13 @@ "timestamp-ms": 4353, "summary": {"operation":"create"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 'foo' foo", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] }, @@ -53,15 +51,13 @@ "timestamp-ms": 5555, "summary": {"operation": "replace"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 1 id, 'abc' data", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] } diff --git a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json index 4ca969d5d310..f09a7a4aa6b5 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json +++ b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json @@ -36,15 +36,13 @@ "timestamp-ms": 4353, "summary": {"operation":"create"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 'foo' foo", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] }, @@ -53,15 +51,13 @@ "timestamp-ms": 5555, "summary": {"operation": "replace"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 1 id, 'abc' data", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] } diff --git a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json index 54de512b98ca..d0fa7d9392a9 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json +++ b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json @@ -36,15 +36,13 @@ "timestamp-ms": 4353, "summary": {"operation":"create"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 'foo' foo", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] }, @@ -53,15 +51,13 @@ "timestamp-ms": 5555, "summary": {"operation": "replace"}, "schema-id": 1, + "default-catalog": "some-catalog", + "default-namespace": [], "representations": [ { "type": "sql", "sql": "select 1 id, 'abc' data", - "dialect": "spark-sql", - "default-catalog": "some-catalog", - "default-namespace": [], - "field-aliases": [], - "field-comments": [] + "dialect": "spark-sql" } ] } diff --git a/format/view-spec.md b/format/view-spec.md index a9826a32cf3e..fe23b4efb893 100644 --- a/format/view-spec.md +++ b/format/view-spec.md @@ -75,13 +75,17 @@ Notes: Each version in `versions` is a struct with the following fields: -| Requirement | Field name | Description | -|-------------|-------------------|--------------------------------------------------------------------------| -| _required_ | `version-id` | ID for the version | -| _required_ | `schema-id` | ID of the schema for the view version | -| _required_ | `timestamp-ms` | Timestamp when the version was created (ms from epoch) | -| _required_ | `summary` | A string to string map of [summary metadata](#summary) about the version | -| _required_ | `representations` | A list of [representations](#representations) for the view definition | +| Requirement | Field name | Description | +|-------------|---------------------|-------------------------------------------------------------------------------| +| _required_ | `version-id` | ID for the version | +| _required_ | `schema-id` | ID of the schema for the view version | +| _required_ | `timestamp-ms` | Timestamp when the version was created (ms from epoch) | +| _required_ | `summary` | A string to string map of [summary metadata](#summary) about the version | +| _required_ | `representations` | A list of [representations](#representations) for the view definition | +| _optional_ | `default-catalog` | Catalog name to use when a reference in the SELECT does not contain a catalog | +| _required_ | `default-namespace` | Namespace to use when a reference in the SELECT is a single identifier | + +When `default-catalog` is `null` or not set, the catalog in which the view is stored must be used as the default catalog. #### Summary @@ -117,10 +121,6 @@ A view version can have multiple SQL representations of different dialects, but | _required_ | `type` | `string` | Must be `sql` | | _required_ | `sql` | `string` | A SQL SELECT statement | | _required_ | `dialect` | `string` | The dialect of the `sql` SELECT statement (e.g., "trino" or "spark") | -| _optional_ | `default-catalog` | `string` | Catalog name to use when a reference in the SELECT does not contain a catalog | -| _optional_ | `default-namespace` | `list` | Namespace to use when a reference in the SELECT is a single identifier | -| _optional_ | `field-aliases` | `list` | Column names optionally specified in the create statement | -| _optional_ | `field-comments` | `list` | Column descriptions (COMMENT) optionally specified in the create statement | For example: @@ -144,13 +144,11 @@ This create statement would produce the following `sql` representation metadata: | `type` | `"sql"` | | `sql` | `"SELECT\n COUNT(1), CAST(event_ts AS DATE)\nFROM events\nGROUP BY 2"` | | `dialect` | `"spark"` | -| `default-catalog` | `"prod"` | -| `default-namespace` | `["default"]` | -| `field-aliases` | `["event_count", "event_date"]` | -| `field-comments` | `["Count of events", null]` | If a create statement does not include column names or comments before `AS`, the fields should be omitted. +The `event_count` (with the `Count of events` comment) and `event_date` field aliases must be part of the view version's `schema`. + #### Version log The version log tracks changes to the view's current version. This is the view's history and allows reconstructing what version of the view would have been used at some point in time. @@ -205,6 +203,8 @@ s3://bucket/warehouse/default.db/event_agg/metadata/00001-(uuid).metadata.json "version-id" : 1, "timestamp-ms" : 1573518431292, "schema-id" : 1, + "default-catalog" : "prod", + "default-namespace" : [ "default" ], "summary" : { "operation" : "create", "engine-name" : "Spark", @@ -213,11 +213,7 @@ s3://bucket/warehouse/default.db/event_agg/metadata/00001-(uuid).metadata.json "representations" : [ { "type" : "sql", "sql" : "SELECT\n COUNT(1), CAST(event_ts AS DATE)\nFROM events\nGROUP BY 2", - "dialect" : "spark", - "default-catalog" : "prod", - "default-namespace" : [ "default" ], - "field-aliases" : ["event_count", "event_date"], - "field-comments" : ["Count of events", null] + "dialect" : "spark" } ] } ], "current-schema-id": 1, @@ -226,12 +222,13 @@ s3://bucket/warehouse/default.db/event_agg/metadata/00001-(uuid).metadata.json "type" : "struct", "fields" : [ { "id" : 1, - "name" : "col1", + "name" : "event_count", "required" : false, - "type" : "int" + "type" : "int", + "doc" : "Count of events" }, { "id" : 2, - "name" : "col2", + "name" : "event_date", "required" : false, "type" : "date" } ] @@ -274,6 +271,8 @@ s3://bucket/warehouse/default.db/event_agg/metadata/00002-(uuid).metadata.json "version-id" : 1, "timestamp-ms" : 1573518431292, "schema-id" : 1, + "default-catalog" : "prod", + "default-namespace" : [ "default" ], "summary" : { "operation" : "create", "engine-name" : "Spark", @@ -282,15 +281,14 @@ s3://bucket/warehouse/default.db/event_agg/metadata/00002-(uuid).metadata.json "representations" : [ { "type" : "sql", "sql" : "SELECT\n COUNT(1), CAST(event_ts AS DATE)\nFROM events\nGROUP BY 2", - "dialect" : "spark", - "default-catalog" : "prod", - "default-namespace" : [ "default" ], - "field-aliases" : ["event_count", "event_date"], - "field-comments" : ["Count of events", null] + "dialect" : "spark" } ] }, { "version-id" : 2, "timestamp-ms" : 1573518981593, + "schema-id" : 1, + "default-catalog" : "prod", + "default-namespace" : [ "default" ], "summary" : { "operation" : "create", "engine-name" : "Spark", @@ -299,10 +297,7 @@ s3://bucket/warehouse/default.db/event_agg/metadata/00002-(uuid).metadata.json "representations" : [ { "type" : "sql", "sql" : "SELECT\n COUNT(1), CAST(event_ts AS DATE)\nFROM prod.default.events\nGROUP BY 2", - "dialect" : "spark", - "default-catalog" : "prod", - "default-namespace" : [ "default" ], - "field-aliases" : ["event_count", "event_date"] + "dialect" : "spark" } ] } ], "current-schema-id": 1, @@ -311,12 +306,13 @@ s3://bucket/warehouse/default.db/event_agg/metadata/00002-(uuid).metadata.json "type" : "struct", "fields" : [ { "id" : 1, - "name" : "col1", + "name" : "event_count", "required" : false, - "type" : "int" + "type" : "int", + "doc" : "Count of events" }, { "id" : 2, - "name" : "col2", + "name" : "event_date", "required" : false, "type" : "date" } ]