From db8de9b2ce440f93fd18880c6dfa71070fb8f7fe Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 2 Feb 2022 03:35:26 +0800 Subject: [PATCH 1/2] Spark: Allow create table in hadoop catalog root namespace --- .../java/org/apache/iceberg/spark/SparkCatalog.java | 5 +++++ .../org/apache/iceberg/spark/sql/TestCreateTable.java | 11 +++++++++++ .../java/org/apache/iceberg/spark/SparkCatalog.java | 5 +++++ .../org/apache/iceberg/spark/sql/TestCreateTable.java | 11 +++++++++++ .../java/org/apache/iceberg/spark/SparkCatalog.java | 5 +++++ .../org/apache/iceberg/spark/sql/TestCreateTable.java | 11 +++++++++++ 6 files changed, 48 insertions(+) diff --git a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index ae921c5200a8..f750680d69ce 100644 --- a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -488,6 +488,10 @@ private Pair load(Identifier ident) { return Pair.of(icebergCatalog.loadTable(buildIdentifier(ident)), null); } catch (org.apache.iceberg.exceptions.NoSuchTableException e) { + if (ident.namespace().length == 0) { + throw e; + } + // if the original load didn't work, the identifier may be extended and include a snapshot selector TableIdentifier namespaceAsIdent = buildIdentifier(namespaceToIdentifier(ident.namespace())); Table table; @@ -567,6 +571,7 @@ private Pair loadFromPathIdentifier(PathIdentifier ident) { } private Identifier namespaceToIdentifier(String[] namespace) { + Preconditions.checkArgument(namespace.length > 0); String[] ns = Arrays.copyOf(namespace, namespace.length - 1); String name = namespace[ns.length]; return Identifier.of(ns, name); diff --git a/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 303cbb5f932b..0a4c9368cb96 100644 --- a/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -67,6 +67,17 @@ public void testCreateTable() { table.properties().get(TableProperties.DEFAULT_FILE_FORMAT)); } + @Test + public void testCreateTableInRootNamespace() { + Assume.assumeTrue("Hadoop has no default namespace configured", "testhadoop".equals(catalogName)); + + try { + sql("CREATE TABLE %s.table (id bigint) USING iceberg", catalogName); + } finally { + sql("DROP TABLE IF EXISTS %s.table", catalogName); + } + } + @Test public void testCreateTableUsingParquet() { Assume.assumeTrue( diff --git a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index ae921c5200a8..f750680d69ce 100644 --- a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -488,6 +488,10 @@ private Pair load(Identifier ident) { return Pair.of(icebergCatalog.loadTable(buildIdentifier(ident)), null); } catch (org.apache.iceberg.exceptions.NoSuchTableException e) { + if (ident.namespace().length == 0) { + throw e; + } + // if the original load didn't work, the identifier may be extended and include a snapshot selector TableIdentifier namespaceAsIdent = buildIdentifier(namespaceToIdentifier(ident.namespace())); Table table; @@ -567,6 +571,7 @@ private Pair loadFromPathIdentifier(PathIdentifier ident) { } private Identifier namespaceToIdentifier(String[] namespace) { + Preconditions.checkArgument(namespace.length > 0); String[] ns = Arrays.copyOf(namespace, namespace.length - 1); String name = namespace[ns.length]; return Identifier.of(ns, name); diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 303cbb5f932b..0a4c9368cb96 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -67,6 +67,17 @@ public void testCreateTable() { table.properties().get(TableProperties.DEFAULT_FILE_FORMAT)); } + @Test + public void testCreateTableInRootNamespace() { + Assume.assumeTrue("Hadoop has no default namespace configured", "testhadoop".equals(catalogName)); + + try { + sql("CREATE TABLE %s.table (id bigint) USING iceberg", catalogName); + } finally { + sql("DROP TABLE IF EXISTS %s.table", catalogName); + } + } + @Test public void testCreateTableUsingParquet() { Assume.assumeTrue( diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index ae921c5200a8..f750680d69ce 100644 --- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -488,6 +488,10 @@ private Pair load(Identifier ident) { return Pair.of(icebergCatalog.loadTable(buildIdentifier(ident)), null); } catch (org.apache.iceberg.exceptions.NoSuchTableException e) { + if (ident.namespace().length == 0) { + throw e; + } + // if the original load didn't work, the identifier may be extended and include a snapshot selector TableIdentifier namespaceAsIdent = buildIdentifier(namespaceToIdentifier(ident.namespace())); Table table; @@ -567,6 +571,7 @@ private Pair loadFromPathIdentifier(PathIdentifier ident) { } private Identifier namespaceToIdentifier(String[] namespace) { + Preconditions.checkArgument(namespace.length > 0); String[] ns = Arrays.copyOf(namespace, namespace.length - 1); String name = namespace[ns.length]; return Identifier.of(ns, name); diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 303cbb5f932b..0a4c9368cb96 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -67,6 +67,17 @@ public void testCreateTable() { table.properties().get(TableProperties.DEFAULT_FILE_FORMAT)); } + @Test + public void testCreateTableInRootNamespace() { + Assume.assumeTrue("Hadoop has no default namespace configured", "testhadoop".equals(catalogName)); + + try { + sql("CREATE TABLE %s.table (id bigint) USING iceberg", catalogName); + } finally { + sql("DROP TABLE IF EXISTS %s.table", catalogName); + } + } + @Test public void testCreateTableUsingParquet() { Assume.assumeTrue( From 7d8723984825b21127ebe8dad2c84a07f8d266e8 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Thu, 3 Feb 2022 02:15:43 +0800 Subject: [PATCH 2/2] nit --- .../src/main/java/org/apache/iceberg/spark/SparkCatalog.java | 3 ++- .../src/main/java/org/apache/iceberg/spark/SparkCatalog.java | 3 ++- .../src/main/java/org/apache/iceberg/spark/SparkCatalog.java | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index f750680d69ce..d80b9bf5476d 100644 --- a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -571,7 +571,8 @@ private Pair loadFromPathIdentifier(PathIdentifier ident) { } private Identifier namespaceToIdentifier(String[] namespace) { - Preconditions.checkArgument(namespace.length > 0); + Preconditions.checkArgument(namespace.length > 0, + "Cannot convert empty namespace to identifier"); String[] ns = Arrays.copyOf(namespace, namespace.length - 1); String name = namespace[ns.length]; return Identifier.of(ns, name); diff --git a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index f750680d69ce..d80b9bf5476d 100644 --- a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -571,7 +571,8 @@ private Pair loadFromPathIdentifier(PathIdentifier ident) { } private Identifier namespaceToIdentifier(String[] namespace) { - Preconditions.checkArgument(namespace.length > 0); + Preconditions.checkArgument(namespace.length > 0, + "Cannot convert empty namespace to identifier"); String[] ns = Arrays.copyOf(namespace, namespace.length - 1); String name = namespace[ns.length]; return Identifier.of(ns, name); diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index f750680d69ce..d80b9bf5476d 100644 --- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -571,7 +571,8 @@ private Pair loadFromPathIdentifier(PathIdentifier ident) { } private Identifier namespaceToIdentifier(String[] namespace) { - Preconditions.checkArgument(namespace.length > 0); + Preconditions.checkArgument(namespace.length > 0, + "Cannot convert empty namespace to identifier"); String[] ns = Arrays.copyOf(namespace, namespace.length - 1); String name = namespace[ns.length]; return Identifier.of(ns, name);