From 3d3620773eda9503b16f25f0f8bf68e139a123bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Arroyo?= Date: Wed, 21 Sep 2022 11:29:15 +0200 Subject: [PATCH 1/5] add version core function --- clouds/databricks/CHANGELOG.md | 6 +++++- clouds/databricks/libraries/scala/build.sbt | 4 ++++ .../scala/core/src/main/resources/sql/createUDFs.sql | 6 +++++- clouds/databricks/libraries/scala/project/plugins.sbt | 2 +- clouds/databricks/modules/sql/product/VERSION_CORE.sql | 3 +++ 5 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 clouds/databricks/modules/sql/product/VERSION_CORE.sql diff --git a/clouds/databricks/CHANGELOG.md b/clouds/databricks/CHANGELOG.md index 2e7835eae..202a36b51 100644 --- a/clouds/databricks/CHANGELOG.md +++ b/clouds/databricks/CHANGELOG.md @@ -6,7 +6,11 @@ All notable changes to this project will be documented in this file. The format is inspired by [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [2022.09.20] - 2022-09-21 +## [2022.09.21] - 2022-09-21 + +### All modules +#### Added +- Add VERSION_CORE function ### Accessors #### Changed diff --git a/clouds/databricks/libraries/scala/build.sbt b/clouds/databricks/libraries/scala/build.sbt index 6d1501dcf..608388090 100644 --- a/clouds/databricks/libraries/scala/build.sbt +++ b/clouds/databricks/libraries/scala/build.sbt @@ -113,6 +113,10 @@ lazy val core = project .dependsOn(jts) .settings(commonSettings) .settings(name := "core") + .enablePlugins(BuildInfoPlugin) + .settings( + buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion) + ) .settings( libraryDependencies ++= Seq( "com.azavea" %% "hiveless-core" % hivelessVersion, diff --git a/clouds/databricks/libraries/scala/core/src/main/resources/sql/createUDFs.sql b/clouds/databricks/libraries/scala/core/src/main/resources/sql/createUDFs.sql index a441590ca..159fc0639 100644 --- a/clouds/databricks/libraries/scala/core/src/main/resources/sql/createUDFs.sql +++ b/clouds/databricks/libraries/scala/core/src/main/resources/sql/createUDFs.sql @@ -91,4 +91,8 @@ CREATE OR REPLACE FUNCTION ST_EXTENTTOGEOM as 'com.carto.analyticstoolbox.module CREATE OR REPLACE FUNCTION ST_GEOMREPROJECT as 'com.carto.analyticstoolbox.modules.index.ST_GeomReproject'; CREATE OR REPLACE FUNCTION ST_MAKEEXTENT as 'com.carto.analyticstoolbox.modules.index.ST_MakeExtent'; CREATE OR REPLACE FUNCTION ST_PARTITIONCENTROID as 'com.carto.analyticstoolbox.modules.index.ST_PartitionCentroid'; -CREATE OR REPLACE FUNCTION ST_Z2LATLON as 'com.carto.analyticstoolbox.modules.index.ST_Z2LatLon'; \ No newline at end of file +CREATE OR REPLACE FUNCTION ST_Z2LATLON as 'com.carto.analyticstoolbox.modules.index.ST_Z2LatLon'; +-- Product +CREATE OR REPLACE FUNCTION VERSION_CORE() + RETURNS STRING + RETURN '2022.09.21'; \ No newline at end of file diff --git a/clouds/databricks/libraries/scala/project/plugins.sbt b/clouds/databricks/libraries/scala/project/plugins.sbt index d2ee480b9..3d403d023 100644 --- a/clouds/databricks/libraries/scala/project/plugins.sbt +++ b/clouds/databricks/libraries/scala/project/plugins.sbt @@ -4,4 +4,4 @@ addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.3") addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.10") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.10.1") - +addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0") diff --git a/clouds/databricks/modules/sql/product/VERSION_CORE.sql b/clouds/databricks/modules/sql/product/VERSION_CORE.sql new file mode 100644 index 000000000..08412d533 --- /dev/null +++ b/clouds/databricks/modules/sql/product/VERSION_CORE.sql @@ -0,0 +1,3 @@ +CREATE OR REPLACE FUNCTION VERSION_CORE() + RETURNS STRING + RETURN '2022.09.21'; \ No newline at end of file From 39c7053f1809eccd851eb4e6976a1e7b6e05b563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Arroyo?= Date: Wed, 21 Sep 2022 13:09:45 +0200 Subject: [PATCH 2/5] ignore linter for get_version function --- clouds/databricks/modules/sql/.sqlfluffignore | 1 + clouds/databricks/modules/sql/product/VERSION_CORE.sql | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 clouds/databricks/modules/sql/.sqlfluffignore diff --git a/clouds/databricks/modules/sql/.sqlfluffignore b/clouds/databricks/modules/sql/.sqlfluffignore new file mode 100644 index 000000000..451dacab1 --- /dev/null +++ b/clouds/databricks/modules/sql/.sqlfluffignore @@ -0,0 +1 @@ +VERSION_CORE.sql diff --git a/clouds/databricks/modules/sql/product/VERSION_CORE.sql b/clouds/databricks/modules/sql/product/VERSION_CORE.sql index 08412d533..4beaa4a3b 100644 --- a/clouds/databricks/modules/sql/product/VERSION_CORE.sql +++ b/clouds/databricks/modules/sql/product/VERSION_CORE.sql @@ -1,3 +1,3 @@ CREATE OR REPLACE FUNCTION VERSION_CORE() RETURNS STRING - RETURN '2022.09.21'; \ No newline at end of file + RETURN '2022.09.21'; From f1bb4d05549761ad724531df11773b5c17018fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Arroyo?= Date: Wed, 21 Sep 2022 18:02:39 +0200 Subject: [PATCH 3/5] ignore get version lint and edit changelog --- clouds/databricks/CHANGELOG.md | 10 ++++++++++ clouds/databricks/modules/sql/.sqlfluffignore | 1 + 2 files changed, 11 insertions(+) create mode 100644 clouds/databricks/modules/sql/.sqlfluffignore diff --git a/clouds/databricks/CHANGELOG.md b/clouds/databricks/CHANGELOG.md index 2e7835eae..7c5187ba0 100644 --- a/clouds/databricks/CHANGELOG.md +++ b/clouds/databricks/CHANGELOG.md @@ -8,14 +8,24 @@ The format is inspired by [Keep a Changelog](https://keepachangelog.com/en/1.0.0 ## [2022.09.20] - 2022-09-21 +### All modules + +#### Added + +- Add GET_VERSION function + ### Accessors + #### Changed + - Change type returned by ST_X and ST_Y to Double ## [2022.09.20] - 2022-09-20 ### All modules + #### Changed + - Set default compression codec to snappy ## [2022.09.16] - 2022-09-16 diff --git a/clouds/databricks/modules/sql/.sqlfluffignore b/clouds/databricks/modules/sql/.sqlfluffignore new file mode 100644 index 000000000..451dacab1 --- /dev/null +++ b/clouds/databricks/modules/sql/.sqlfluffignore @@ -0,0 +1 @@ +VERSION_CORE.sql From c814bcd9b6ea096d781594950ddb71dbe68e6bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Arroyo?= Date: Wed, 21 Sep 2022 18:08:26 +0200 Subject: [PATCH 4/5] lint changelog --- clouds/databricks/CHANGELOG.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/clouds/databricks/CHANGELOG.md b/clouds/databricks/CHANGELOG.md index 0a700f5fd..0e314c943 100644 --- a/clouds/databricks/CHANGELOG.md +++ b/clouds/databricks/CHANGELOG.md @@ -8,15 +8,11 @@ The format is inspired by [Keep a Changelog](https://keepachangelog.com/en/1.0.0 ## [2022.09.21] - 2022-09-21 -### All modules -#### Added -- Add VERSION_CORE function - ### All modules #### Added -- Add GET_VERSION function +- Add VERSION_CORE function ### Accessors From 3cd8ee19b43fd78bea488d5eaa4f747aa0bdf2fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Arroyo?= Date: Wed, 21 Sep 2022 18:40:28 +0200 Subject: [PATCH 5/5] fix remove modules in databricks --- clouds/databricks/common/python_utils/create_drop_functions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clouds/databricks/common/python_utils/create_drop_functions.py b/clouds/databricks/common/python_utils/create_drop_functions.py index 1b5a5d201..182c4ff15 100644 --- a/clouds/databricks/common/python_utils/create_drop_functions.py +++ b/clouds/databricks/common/python_utils/create_drop_functions.py @@ -24,6 +24,8 @@ def write_queries(final_query): if __name__ == '__main__': sql_path = os.path.join(python_util_path, '..', '..', 'modules', 'sql') modules = os.listdir(sql_path) + # We filter paths that aren't a directory + modules = list(filter(lambda x: os.path.isdir(os.path.join(sql_path, x)), modules)) queries_list = [] for module in modules: module_path = os.path.join(sql_path, module)