Bug/sc 253458/error casting string to utf8 (#311)

- Fix - Fixed the error casting string to UTD8String in GeomReproject function

Author: aarroyosal

clouds/databricks/CHANGELOG.md

@@ -3,6 +3,12 @@ Changelog for databricks cloud. All notable changes to this project will be docu
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [1.0.3] - 2022-09-06
+
+### Fixed
+- [accessors] Fix the bug in ST_COORDDIM that was adding z coordinate to geometries.
+- [indexing] Fix the cast ClassCastException in ST_GEOMREPROJECT.
+
 ## [1.0.2] - 2022-09-01
 
 ### Changed

clouds/databricks/Makefile

@@ -39,6 +39,7 @@ deploy-modules:
 
 test:
 	$(MAKE) test-modules
+	$(MAKE) test-libraries
 
 test-libraries:
 	$(MAKE) -C libraries/scala test

clouds/databricks/libraries/scala/core/src/main/scala/com/carto/analyticstoolbox/modules/index/package.scala

@@ -13,7 +13,7 @@ import org.apache.spark.sql.types.{DataType, StringType}
 
 package object index extends StandardEncoders {
   implicit def crsConverter: HConverter[CRS] = new HConverter[CRS] {
-    def convert(argument: Any): CRS = CRS.fromString(argument.convert[String])
+    def convert(argument: Any): CRS = CRS.fromString(argument.toString)
   }
 
   implicit def extentConverter: HConverter[Extent] = new HConverter[Extent] {

clouds/databricks/libraries/scala/core/src/test/scala/com/carto/analyticstoolbox/modules/accessors/ST_CoordDimTest.scala

@@ -1,8 +1,6 @@
 package com.carto.analyticstoolbox.modules.accessors
 
-import com.carto.hiveless.spatial.util.TWKBUtils
-import org.locationtech.geomesa.spark.jts.udf.{GeometricConstructorFunctions, GeometricOutputFunctions}
-import org.locationtech.jts.geom.{Coordinate, CoordinateXY, Geometry, GeometryFactory, Point}
+import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions
 import org.scalatest.funspec.AnyFunSpec
 import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
 

clouds/databricks/libraries/scala/core/src/test/scala/com/carto/analyticstoolbox/modules/index/STIndexSpec.scala

@@ -22,12 +22,12 @@ import org.scalatest.funspec.AnyFunSpec
 class STIndexSpec extends AnyFunSpec with HiveTestEnvironment with TestTables {
   describe("ST index functions spec") {
     it("ST_GEOMREPROJECT test") {
-      val df2 = ssc.sql(
-        """SELECT ST_GEOMREPROJECT(ST_POINT(3, 5),
+      val df = ssc.sql(
+        """SELECT ST_ASTEXT(ST_GEOMREPROJECT(ST_POINT(3, 5),
           |ST_CRSFROMTEXT('+proj=merc +lat_ts=56.5 +ellps=GRS80'),
-          |ST_CRSFROMTEXT('+proj=longlat +ellps=GRS80 +datum=NAD83 +no_defs'))""".stripMargin
+          |ST_CRSFROMTEXT('+proj=longlat +ellps=GRS80 +datum=NAD83 +no_defs')))""".stripMargin
       )
-      print(df2.take(1).head.get(0))
+      df.head.get(0) shouldEqual "POINT (0.0000269 0.0000452)"
     }
   }
 }

clouds/databricks/libraries/scala/core/src/test/scala/com/carto/analyticstoolbox/modules/index/ST_CrsFromTextTest.scala

@@ -1,16 +1,13 @@
 package com.carto.analyticstoolbox.modules.index
 
-import org.apache.spark.unsafe.types.UTF8String
 import org.scalatest.funspec.AnyFunSpec
+import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
 
 class ST_CrsFromTextTest extends AnyFunSpec {
   describe("ST_CrsFromText") {
     it ("Should return a CRS") {
-      val cls = new ST_CrsFromText().function("+proj=merc +lat_ts=56.5 +ellps=GRS80")
-    }
-    it ("CRS string to utf8 string") {
-      "+proj=merc +lat_ts=56.5 +ellps=GRS80".asInstanceOf[UTF8String]     //esto falla
-      UTF8String.fromString("+proj=merc +lat_ts=56.5 +ellps=GRS80")  //esto no
+      val crs = new ST_CrsFromText().function("+proj=merc +lat_ts=56.5 +ellps=GRS80")
+      crs.proj4jCrs.getName shouldEqual "merc-CS"
     }
   }
 }

clouds/databricks/modules/Makefile

@@ -34,5 +34,7 @@ remove: check
 
 drop-udfs:
 	sed -e 's/@@DB_CLUSTER_ID@@/${DB_CLUSTER_ID}/g' -e 's!@@SQLPath@@!$(SQL_DEPLOY_PATH)/dropUDF!g' $(COMMON_DIR)/submit-run-template.json > $(COMMON_DIR)/submit-run.json
-	databricks runs submit --json-file $(COMMON_DIR)/submit-run.json
+	databricks runs submit --json-file $(COMMON_DIR)/submit-run.json --wait
+	echo "- Removing drop script from the workspace"
+	databricks workspace rm -r $(SQL_DEPLOY_PATH)
 	echo "- Modules removed"

clouds/databricks/modules/doc/indexing/ST_GEOMREPROJECT.md_x renamed to clouds/databricks/modules/doc/indexing/ST_GEOMREPROJECT.md

@@ -19,6 +19,10 @@ Transform a `Geometry` from The Common Reference System _crsA_ to _crsB_.
 **Example**
 
 ```sql
-
--- 
+WITH t AS (
+  SELECT carto.ST_POINT(3, 5) AS point, 
+  carto.ST_CRSFROMTEXT('+proj=merc +lat_ts=56.5 +ellps=GRS80') AS crsa,
+  carto.ST_CRSFROMTEXT('+proj=longlat +ellps=GRS80 +datum=NAD83 +no_defs') AS crsb
+) SELECT carto.ST_ASTEXT(carto.ST_GEOMREPROJECT(point, crsa, crsb)) FROM t;
+-- POINT (0.0000269 0.0000452)
 ```

clouds/databricks/modules/test/indexing/test_ST_GEOMREPROJECT.py

@@ -0,0 +1,11 @@
+from python_utils.test_utils import run_query
+
+
+def test_st_geomreproject_success():
+    query = """WITH t AS (
+  SELECT @@DB_SCHEMA@@.ST_POINT(3, 5) AS point, 
+  @@DB_SCHEMA@@.ST_CRSFROMTEXT('+proj=merc +lat_ts=56.5 +ellps=GRS80') AS crsa,
+  @@DB_SCHEMA@@.ST_CRSFROMTEXT('+proj=longlat +ellps=GRS80 +datum=NAD83 +no_defs') AS crsb
+) SELECT @@DB_SCHEMA@@.ST_ASTEXT(@@DB_SCHEMA@@.ST_GEOMREPROJECT(point, crsa, crsb)) FROM t;"""
+    result = run_query(query)
+    assert result[0][0] == 'POINT (0.0000269 0.0000452)'