From 66d947dc7f0cdd087744b6cb13aca1a2b9374bbf Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 14 Jan 2025 11:58:53 -0700 Subject: [PATCH 1/4] start 0.6.0 development --- common/pom.xml | 2 +- docs/source/user-guide/compatibility.md | 1 - fuzz-testing/pom.xml | 2 +- native/Cargo.lock | 6 +++--- native/Cargo.toml | 6 +++--- pom.xml | 2 +- spark-integration/pom.xml | 2 +- spark/pom.xml | 2 +- 8 files changed, 11 insertions(+), 12 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 91109edf5d..b6cd75a32d 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.datafusion comet-parent-spark${spark.version.short}_${scala.binary.version} - 0.5.0-SNAPSHOT + 0.6.0-SNAPSHOT ../pom.xml diff --git a/docs/source/user-guide/compatibility.md b/docs/source/user-guide/compatibility.md index 10459d4d28..94fd9ea7ed 100644 --- a/docs/source/user-guide/compatibility.md +++ b/docs/source/user-guide/compatibility.md @@ -133,7 +133,6 @@ The following cast operations are generally compatible with Spark except for the | string | date | Only supports years between 262143 BC and 262142 AD | | date | string | | | timestamp | long | | -| timestamp | decimal | | | timestamp | string | | | timestamp | date | | diff --git a/fuzz-testing/pom.xml b/fuzz-testing/pom.xml index 2184e54eea..0b45025c67 100644 --- a/fuzz-testing/pom.xml +++ b/fuzz-testing/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.datafusion comet-parent-spark${spark.version.short}_${scala.binary.version} - 0.5.0-SNAPSHOT + 0.6.0-SNAPSHOT ../pom.xml diff --git a/native/Cargo.lock b/native/Cargo.lock index 5a98d74afc..70c8c4295f 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -878,7 +878,7 @@ dependencies = [ [[package]] name = "datafusion-comet" -version = "0.5.0" +version = "0.6.0" dependencies = [ "arrow", "arrow-array", @@ -926,7 +926,7 @@ dependencies = [ [[package]] name = "datafusion-comet-proto" -version = "0.5.0" +version = "0.6.0" dependencies = [ "prost 0.12.6", "prost-build", @@ -934,7 +934,7 @@ dependencies = [ [[package]] name = "datafusion-comet-spark-expr" -version = "0.5.0" +version = "0.6.0" dependencies = [ "arrow", "arrow-array", diff --git a/native/Cargo.toml b/native/Cargo.toml index 0b39334bd4..ad5e1a141c 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -20,7 +20,7 @@ members = ["core", "spark-expr", "proto"] resolver = "2" [workspace.package] -version = "0.5.0" +version = "0.6.0" homepage = "https://datafusion.apache.org/comet" repository = "https://github.com/apache/datafusion-comet" authors = ["Apache DataFusion "] @@ -48,8 +48,8 @@ datafusion-expr-common = { version = "44.0.0", default-features = false } datafusion-execution = { version = "44.0.0", default-features = false } datafusion-physical-plan = { version = "44.0.0", default-features = false } datafusion-physical-expr = { version = "44.0.0", default-features = false } -datafusion-comet-spark-expr = { path = "spark-expr", version = "0.5.0" } -datafusion-comet-proto = { path = "proto", version = "0.5.0" } +datafusion-comet-spark-expr = { path = "spark-expr", version = "0.6.0" } +datafusion-comet-proto = { path = "proto", version = "0.6.0" } chrono = { version = "0.4", default-features = false, features = ["clock"] } chrono-tz = { version = "0.8" } futures = "0.3.28" diff --git a/pom.xml b/pom.xml index 76e2288ccc..4559d67412 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ under the License. org.apache.datafusion comet-parent-spark${spark.version.short}_${scala.binary.version} - 0.5.0-SNAPSHOT + 0.6.0-SNAPSHOT pom Comet Project Parent POM diff --git a/spark-integration/pom.xml b/spark-integration/pom.xml index 84c09c1c97..24b1f7a002 100644 --- a/spark-integration/pom.xml +++ b/spark-integration/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.datafusion comet-parent-spark${spark.version.short}_${scala.binary.version} - 0.5.0-SNAPSHOT + 0.6.0-SNAPSHOT ../pom.xml diff --git a/spark/pom.xml b/spark/pom.xml index ad7590dbc4..f15b0b2e8f 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.datafusion comet-parent-spark${spark.version.short}_${scala.binary.version} - 0.5.0-SNAPSHOT + 0.6.0-SNAPSHOT ../pom.xml From cc718702491a3189812d7677939ac5510ebb0003 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 14 Jan 2025 12:01:19 -0700 Subject: [PATCH 2/4] update some docs --- docs/source/contributor-guide/debugging.md | 2 +- docs/source/user-guide/installation.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/contributor-guide/debugging.md b/docs/source/contributor-guide/debugging.md index 47d1f04c87..8a368cca26 100644 --- a/docs/source/contributor-guide/debugging.md +++ b/docs/source/contributor-guide/debugging.md @@ -130,7 +130,7 @@ Then build the Comet as [described](https://github.com/apache/arrow-datafusion-c Start Comet with `RUST_BACKTRACE=1` ```console -RUST_BACKTRACE=1 $SPARK_HOME/spark-shell --jars spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar --conf spark.plugins=org.apache.spark.CometPlugin --conf spark.comet.enabled=true --conf spark.comet.exec.enabled=true +RUST_BACKTRACE=1 $SPARK_HOME/spark-shell --jars spark/target/comet-spark-spark3.4_2.12-0.6.0-SNAPSHOT.jar --conf spark.plugins=org.apache.spark.CometPlugin --conf spark.comet.enabled=true --conf spark.comet.exec.enabled=true ``` Get the expanded exception details diff --git a/docs/source/user-guide/installation.md b/docs/source/user-guide/installation.md index 22d482e475..390c926387 100644 --- a/docs/source/user-guide/installation.md +++ b/docs/source/user-guide/installation.md @@ -74,7 +74,7 @@ See the [Comet Kubernetes Guide](kubernetes.md) guide. Make sure `SPARK_HOME` points to the same Spark version as Comet was built for. ```console -export COMET_JAR=spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar +export COMET_JAR=spark/target/comet-spark-spark3.4_2.12-0.6.0-SNAPSHOT.jar $SPARK_HOME/bin/spark-shell \ --jars $COMET_JAR \ @@ -130,7 +130,7 @@ explicitly contain Comet otherwise Spark may use a different class-loader for th components which will then fail at runtime. For example: ``` ---driver-class-path spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar +--driver-class-path spark/target/comet-spark-spark3.4_2.12-0.6.0-SNAPSHOT.jar ``` Some cluster managers may require additional configuration, see From e6e9914539d5c1b691d7a916e07fc9867b3d0cea Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 14 Jan 2025 12:32:40 -0700 Subject: [PATCH 3/4] Revert a change --- docs/source/user-guide/compatibility.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/user-guide/compatibility.md b/docs/source/user-guide/compatibility.md index 94fd9ea7ed..10459d4d28 100644 --- a/docs/source/user-guide/compatibility.md +++ b/docs/source/user-guide/compatibility.md @@ -133,6 +133,7 @@ The following cast operations are generally compatible with Spark except for the | string | date | Only supports years between 262143 BC and 262142 AD | | date | string | | | timestamp | long | | +| timestamp | decimal | | | timestamp | string | | | timestamp | date | | From 3972b2c39f74e03fd74282ad45f159a31ead3022 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 14 Jan 2025 12:32:58 -0700 Subject: [PATCH 4/4] update CI --- .github/workflows/spark_sql_test.yml | 2 +- .github/workflows/spark_sql_test_ansi.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml index 477e3a1ab9..238fbb2715 100644 --- a/.github/workflows/spark_sql_test.yml +++ b/.github/workflows/spark_sql_test.yml @@ -71,7 +71,7 @@ jobs: with: spark-version: ${{ matrix.spark-version.full }} spark-short-version: ${{ matrix.spark-version.short }} - comet-version: '0.5.0-SNAPSHOT' # TODO: get this from pom.xml + comet-version: '0.6.0-SNAPSHOT' # TODO: get this from pom.xml - name: Run Spark tests run: | cd apache-spark diff --git a/.github/workflows/spark_sql_test_ansi.yml b/.github/workflows/spark_sql_test_ansi.yml index e1d8388fb1..14ec6366f4 100644 --- a/.github/workflows/spark_sql_test_ansi.yml +++ b/.github/workflows/spark_sql_test_ansi.yml @@ -69,7 +69,7 @@ jobs: with: spark-version: ${{ matrix.spark-version.full }} spark-short-version: ${{ matrix.spark-version.short }} - comet-version: '0.5.0-SNAPSHOT' # TODO: get this from pom.xml + comet-version: '0.6.0-SNAPSHOT' # TODO: get this from pom.xml - name: Run Spark tests run: | cd apache-spark