From bcfebb5b4783ba1c6ff0d44d20d1a390bf1792a8 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 16 Nov 2023 22:50:43 -0800 Subject: [PATCH 1/2] [SPARK-45963][SQL][DOCS] Restore documentation for DSv2 API This PR restores the DSv2 documentation. https://github.com/apache/spark/pull/38392 mistakenly added `org/apache/spark/sql/connect` as a private that includes `org/apache/spark/sql/connector`. For end users to read DSv2 documentation. Yes, it restores the DSv2 API documentation that used to be there https://spark.apache.org/docs/3.3.0/api/scala/org/apache/spark/sql/connector/catalog/index.html Manually tested via: ``` SKIP_PYTHONDOC=1 SKIP_RDOC=1 SKIP_SQLDOC=1 bundle exec jekyll build ``` No. Closes #43855 from HyukjinKwon/connector-docs. Authored-by: Hyukjin Kwon Signed-off-by: Dongjoon Hyun (cherry picked from commit a7147c8e04711a552009d513d900d29fcb258315) Signed-off-by: Hyukjin Kwon --- project/SparkBuild.scala | 2 +- .../spark/sql/connector/catalog/SupportsMetadataColumns.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 718f2bb28cec4..60d52368de458 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1401,7 +1401,7 @@ object Unidoc { .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/util/io"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/util/kvstore"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/catalyst"))) - .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/connect"))) + .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/connect/"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/execution"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/internal"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/hive"))) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsMetadataColumns.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsMetadataColumns.java index 894184dbcc82d..e42424268b44d 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsMetadataColumns.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsMetadataColumns.java @@ -58,8 +58,8 @@ public interface SupportsMetadataColumns extends Table { * Determines how this data source handles name conflicts between metadata and data columns. *

* If true, spark will automatically rename the metadata column to resolve the conflict. End users - * can reliably select metadata columns (renamed or not) with {@link Dataset.metadataColumn}, and - * internal code can use {@link MetadataAttributeWithLogicalName} to extract the logical name from + * can reliably select metadata columns (renamed or not) with {@code Dataset.metadataColumn}, and + * internal code can use {@code MetadataAttributeWithLogicalName} to extract the logical name from * a metadata attribute. *

* If false, the data column will hide the metadata column. It is recommended that Table From b785f547d0d46bf4723bb9c5dbe4a2d29c4896ba Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 17 Nov 2023 20:16:40 +0900 Subject: [PATCH 2/2] fix javadoc errors --- .../spark/sql/connector/catalog/InMemoryBaseTable.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala index a0a4d8bdee9f5..a309db341d8e6 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala @@ -619,9 +619,9 @@ class BufferedRows(val key: Seq[Any] = Seq.empty) extends WriterCommitMessage } /** - * Theoretically, [[InternalRow]] returned by [[HasPartitionKey#partitionKey()]] + * Theoretically, `InternalRow` returned by `HasPartitionKey#partitionKey()` * does not need to implement equal and hashcode methods. - * But [[GenericInternalRow]] implements equals and hashcode methods already. Here we override it + * But `GenericInternalRow` implements equals and hashcode methods already. Here we override it * to simulate that it has not been implemented to verify codes correctness. */ case class PartitionInternalRow(keys: Array[Any])