release: 2024-03-18

.github/workflows/redshift-ded.yml

@@ -33,6 +33,19 @@ jobs:
         uses: actions/checkout@v2
       - name: Check diff
         uses: technote-space/get-diff-action@v4
+      - name: Set RS_PREFIX for releases
+        if: startsWith(github.event.pull_request.head.ref, 'release/')
+        run: |
+          echo "RS_PREFIX=dedicated_release_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
+      - name: Set RS_PREFIX for hotfixes
+        if: startsWith(github.event.pull_request.head.ref, 'hotfix/')
+        run: |
+          echo "RS_PREFIX=dedicated_hotfix_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
+      - name: Set RS_PREFIX for the rest
+        if: |
+          !(startsWith(github.event.pull_request.head.ref, 'hotfix/')) &&
+          !(startsWith(github.event.pull_request.head.ref, 'release/'))
+        run: echo "RS_PREFIX=dedicated_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Setup node
         uses: actions/setup-node@v1
         with:
@@ -63,7 +76,7 @@ jobs:
         if: github.event.action == 'synchronize' || github.event.action == 'labeled'
         run: |
           cd clouds/redshift
-          make deploy
+          make deploy dropfirst=1
       - name: Run remove
         id: remove
         if: github.event.action == 'unlabeled' || github.event.action == 'closed'

CHANGELOG.md

@@ -4,6 +4,14 @@ CARTO Analytics Toolbox Core.
 
 All notable commits to this project will be documented in this file.
 
+## 2024-03-18
+
+- fix(sf): CI and CD not working because of snowflake driver breaking changes (#484)
+- fix(bq,sf,rs,pg): drop schemas when dedicateds gets released (#485)
+- fix(bq|random): ST_GENERATEPOINTS returning exact name of points (#486)
+- chore(rs,pg): remove installer tool (#483)
+- chore(deps): bump jinja2 from 3.1.2 to 3.1.3 in /clouds/databricks/common (#468)
+
 ## 2024-02-15
 
 - chore(sf): add additional tables to native apps (#473)

clouds/bigquery/CHANGELOG.md

@@ -4,6 +4,10 @@ CARTO Analytics Toolbox Core for BigQuery.
 
 All notable commits to this project will be documented in this file.
 
+## [1.2.1] - 2024-03-18
+
+- fix(random): ST_GENERATEPOINTS returning exact name of points (#486)
+
 ## [1.2.0] - 2024-01-17
 
 - chore: increase tests timeout to 200000 (#455)

clouds/bigquery/libraries/javascript/src/random.js

@@ -1,5 +1,24 @@
-import { bbox } from '@turf/turf';
+import { bbox, booleanPointInPolygon, randomPoint } from '@turf/turf';
+
+function generateRandomPointsInPolygon (polygon, numPoints) {
+    const randomPoints = [];
+    while (randomPoints.length < numPoints) {
+        const point = randomPoint(1, { bbox: bbox(polygon) }).features[0];
+        if (booleanPointInPolygon(point, polygon)) {
+            randomPoints.push(JSON.stringify(point.geometry));
+        }
+    }
+    return randomPoints;
+}
+
+function generateRandomPointInPolygon (polygon) {
+    let point
+    do  {
+        point = randomPoint(1, { bbox: bbox(polygon) }).features[0];
+    } while (!booleanPointInPolygon(point, polygon))
+    return JSON.stringify(point.geometry);
+}
 
 export default {
-    bbox
+    generateRandomPointsInPolygon
 };

clouds/bigquery/libraries/javascript/test/random.test.js

@@ -1,5 +1,5 @@
 const lib = require('../build/index');
 
 test('random library defined', () => {
-    expect(lib.random.bbox).toBeDefined();
+    expect(lib.random.generateRandomPointsInPolygon).toBeDefined();
 });

clouds/bigquery/modules/Makefile

@@ -34,10 +34,10 @@ include $(COMMON_DIR)/Makefile
 
 .SILENT:
 
-.PHONY: help lint build build-share deploy deploy-share test remove remove-share clean
+.PHONY: help lint build build-share deploy deploy-share test remove remove-functions remove-share clean
 
 help:
-	echo "Available targets: lint build deploy test remove clean"
+	echo "Available targets: lint build deploy test remove remove-functions clean"
 
 lint: venv3 $(NODE_MODULES_DEV)
 	echo "Linting modules..."
@@ -85,7 +85,7 @@ test: check $(NODE_MODULES_DEV)
 		if [ ! -z "$$TESTS" ]; then \
 			GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \
 			PATH="$(NODE_MODULES_DEV)/.bin/:$(PATH)" \
-			jest --testTimeout=200000 $(BAIL) --verbose --slowTestThreshold=20 --maxConcurrency=10 $$TESTS \
+			jest --testTimeout=250000 $(BAIL) --verbose --slowTestThreshold=20 --maxConcurrency=10 $$TESTS \
 			     --setupFilesAfterEnv "$(COMMON_DIR)/test-extend.js" || exit 1; \
 			OLD_TEST=$(TEST_DIR)/$$m/old-test; \
 			if [ -d $$OLD_TEST ]; then \
@@ -101,6 +101,12 @@ remove: check
 	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \
 	$(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql
 
+remove-functions: check
+	echo "Removing functions..."
+	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \
+	$(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql
+
 clean:
 	echo "Cleaning modules..."
 	rm -rf $(BUILD_DIR) $(NODE_MODULES_DEV)

clouds/bigquery/modules/doc/random/ST_GENERATEPOINTS.md

@@ -10,13 +10,6 @@ Generates randomly placed points inside a polygon and returns them in an array o
 
 The distribution of the generated points is spherically uniform (i.e. if the coordinates are interpreted as longitude and latitude on a sphere); this means that WGS84 coordinates will be only approximately uniformly distributed, since WGS84 is based on an ellipsoidal model.
 
-````hint:warning
-**warning**
-
-It never generates more than the requested number of points, but there is a small chance of generating less points.
-
-````
-
 * `geog`: `GEOGRAPHY` a polygon; the random points generated will be inside this polygon.
 * `npoints`: `INT64` number of points to generate.
 

clouds/bigquery/modules/doc/transformations/ST_CONCAVEHULL.md

@@ -9,7 +9,7 @@ ST_CONCAVEHULL(geog, maxEdge, units)
 Takes a set of points and returns a concave hull Polygon or MultiPolygon. In case that a single or a couple of points are passed as input, the function will return that point or a segment respectively.
 
 * `geog`: `ARRAY<GEOGRAPHY>` input points.
-* `maxEdge`: `FLOAT64`|`NULL` the length (in 'units') of an edge necessary for part of the hull to become concave. If `NULL`the default value `infinity` is used.
+* `maxEdge`: `FLOAT64`|`NULL` the maximum length allowed for an edge of the concave hull. Higher `maxEdge` values will produce more convex-like hulls. If `NULL`, the default value `infinity` is used and it would be equivalent to a Convex Hull.
 * `units`: `STRING`|`NULL` units of length, the supported options are: miles, kilometers, degrees or radians. If `NULL`the default value `kilometers` is used.
 
 **Return type**

clouds/bigquery/modules/sql/random/ST_GENERATEPOINTS.sql

@@ -1,74 +1,23 @@
-----------------------------
--- Copyright (C) 2021 CARTO
-----------------------------
+--------------------------------
+-- Copyright (C) 2021-2024 CARTO
+--------------------------------
+
+CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__ST_GENERATEPOINTS`
+(geojson STRING, npoints INT64)
+RETURNS ARRAY<STRING>
+DETERMINISTIC
+LANGUAGE js
+OPTIONS (library = ["@@BQ_LIBRARY_BUCKET@@"])
+AS """
+    return lib.random.generateRandomPointsInPolygon(JSON.parse(geojson), npoints);
+""";
 
 CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.ST_GENERATEPOINTS`
 (geog GEOGRAPHY, npoints INT64)
 RETURNS ARRAY<GEOGRAPHY>
 AS (
     (
-        WITH bbox AS (
-            -- compute the bounding box of the polygon
-            SELECT `@@BQ_DATASET@@.__BBOX_FROM_GEOJSON`(ST_ASGEOJSON(geog)) AS box
-        ),
-
-        bbox_coords AS (
-            -- break down the bbox array into minx, miny, maxx, maxy
-            SELECT
-                box[ORDINAL(1)] AS minx,
-                box[ORDINAL(2)] AS miny,
-                box[ORDINAL(3)] AS maxx,
-                box[ORDINAL(4)] AS maxy
-            FROM bbox
-        ),
-
-        bbox_data AS (
-            -- compute area of bbox and put some handy values here too
-            SELECT
-                minx,
-                miny,
-                maxx,
-                maxy,
-                1.2 AS k, -- security factor to make it more likely that at least npoints fall within the polygon
-                ST_AREA(
-                    ST_MAKEPOLYGON(ST_MAKELINE([
-                        ST_GEOGPOINT(minx, miny),
-                        ST_GEOGPOINT(minx, maxy),
-                        ST_GEOGPOINT(maxx, maxy),
-                        ST_GEOGPOINT(maxx, miny),
-                        ST_GEOGPOINT(minx, miny)
-                        ]))
-                ) AS bbox_area
-            FROM bbox_coords
-        ),
-
-        point_seeds AS (
-            -- generate enough values so that we will hopefully have at least npoints of them randomly placed inside geog
-            SELECT
-                GENERATE_ARRAY(1, CEIL(k * npoints * bbox_area / ST_AREA(geog))) AS i,
-                SIN(miny * ACOS(-1) / 180.0) AS minsin,
-                SIN(maxy * ACOS(-1) / 180.0) AS maxsin,
-                180.0 / ACOS(-1) AS radtodeg
-            FROM bbox_data
-        ),
-
-        bbox_points AS (
-            -- compute the random points uniformly in the bbox;
-            SELECT ST_GEOGPOINT(minx + RAND() * (maxx - minx), radtodeg * ASIN(minsin + RAND() * (maxsin - minsin))) AS point
-            FROM bbox_coords, point_seeds, UNNEST(i)
-        ),
-
-        poly_points AS (
-            -- now we need to select the points inside the polygon and number them, so we can limit
-            -- the end result to npoints (note that we can't have a dynamic LIMIT)
-            SELECT
-                point,
-                ROW_NUMBER() OVER () AS rn
-            FROM bbox_points
-            WHERE ST_WITHIN(point, geog)
-        )
-
-        -- finally select at most npoints  and return them in an array
-        SELECT ARRAY(SELECT point FROM poly_points WHERE rn <= npoints)
+        SELECT ARRAY_AGG(ST_GEOGFROMGEOJSON(point))
+        FROM UNNEST(`@@BQ_DATASET@@.__ST_GENERATEPOINTS`(ST_ASGEOJSON(geog), npoints)) AS point
     )
 );

clouds/bigquery/version

@@ -1 +1 @@
-1.2.0
+1.2.1

clouds/databricks/common/requirements_create_it.txt

@@ -1 +1 @@
-Jinja2==3.1.2
+Jinja2==3.1.3

clouds/postgres/modules/Makefile

@@ -21,10 +21,10 @@ include $(COMMON_DIR)/Makefile
 
 .SILENT:
 
-.PHONY: help lint build deploy test remove clean
+.PHONY: help lint build deploy test remove remove-functions clean
 
 help:
-	echo "Available targets: lint build deploy test remove clean"
+	echo "Available targets: lint build deploy test remove remove-functions clean"
 
 lint: venv3 $(NODE_MODULES_DEV)
 	echo "Linting modules..."
@@ -64,6 +64,11 @@ remove: venv3
 	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
 	$(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql
 
+remove-functions: venv3
+	echo "Removing functions..."
+	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
+	$(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql
+
 clean:
 	echo "Cleaning modules..."
 	rm -rf $(BUILD_DIR) $(VENV3_DIR) $(NODE_MODULES_DEV)

clouds/redshift/modules/Makefile

@@ -21,10 +21,10 @@ include $(COMMON_DIR)/Makefile
 
 .SILENT:
 
-.PHONY: help lint build deploy test remove clean
+.PHONY: help lint build deploy test remove remove-functions clean
 
 help:
-	echo "Available targets: lint build deploy test remove clean"
+	echo "Available targets: lint build deploy test remove remove-functions clean"
 
 lint: venv3 $(NODE_MODULES_DEV)
 	echo "Linting modules..."
@@ -64,6 +64,11 @@ remove: venv3
 	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
 	$(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql
 
+remove-functions: venv3
+	echo "Removing functions..."
+	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
+	$(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql
+
 clean:
 	echo "Cleaning modules..."
 	rm -rf $(BUILD_DIR) $(VENV3_DIR) $(NODE_MODULES_DEV)

clouds/redshift/modules/doc/random/ST_GENERATEPOINTS.md

@@ -10,13 +10,6 @@ Generates randomly placed points inside a polygon and returns them in an array o
 
 The distribution of the generated points is spherically uniform (i.e. if the coordinates are interpreted as longitude and latitude on a sphere); this means that WGS84 coordinates will be only approximately uniformly distributed, since WGS84 is based on an ellipsoidal model.
 
-````hint:warning
-**warning**
-
-It never generates more than the requested number of points, but there is a small chance of generating less points.
-
-````
-
 * `geog`: `GEOMETRY` a polygon; the random points generated will be inside this polygon.
 * `npoints`: `INT` number of points to generate.
 

clouds/snowflake/common/package.json

@@ -11,6 +11,6 @@
     "rollup": "^2.47.0",
     "rollup-plugin-bundle-size": "^1.0.3",
     "rollup-plugin-terser": "^7.0.2",
-    "snowflake-sdk": "^1.6.6"
+    "snowflake-sdk": "1.9.3"
   }
 }

clouds/snowflake/modules/Makefile

@@ -32,10 +32,10 @@ include $(COMMON_DIR)/Makefile
 
 .SILENT:
 
-.PHONY: help lint build build-share build-native-app-setup-script deploy deploy-share test remove remove-share clean
+.PHONY: help lint build build-share build-native-app-setup-script deploy deploy-share test remove remove-functions remove-share clean
 
 help:
-	echo "Available targets: lint build deploy test remove clean"
+	echo "Available targets: lint build deploy test remove remove-functions clean"
 
 lint: $(NODE_MODULES_DEV) venv3
 	echo "Linting modules..."
@@ -110,6 +110,11 @@ remove: $(NODE_MODULES_DEV)
 	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
 	$(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql
 
+remove-functions: $(NODE_MODULES_DEV)
+	echo "Removing functions..."
+	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
+	$(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql
+
 remove-share: $(NODE_MODULES_DEV)
 	echo "Removing share..."
 	$(COMMON_DIR)/run-query.js "DROP SHARE $(SF_SHARE);"

clouds/snowflake/modules/doc/random/ST_GENERATEPOINTS.md

@@ -10,13 +10,6 @@ Generates randomly placed points inside a polygon and returns them in an array o
 
 The distribution of the generated points is spherically uniform (i.e. if the coordinates are interpreted as longitude and latitude on a sphere); this means that WGS84 coordinates will be only approximately uniformly distributed, since WGS84 is based on an ellipsoidal model.
 
-````hint:warning
-**warning**
-
-It never generates more than the requested number of points, but there is a small chance of generating less points.
-
-````
-
 * `geog`: `GEOGRAPHY` a polygon; the random points generated will be inside this polygon.
 * `npoints`: `INT` number of points to generate.
 

clouds/snowflake/modules/doc/transformations/ST_CONCAVEHULL.md

@@ -9,7 +9,7 @@ ST_CONCAVEHULL(geojsons [, maxEdge] [, units])
 Takes a set of points and returns a concave hull Polygon or MultiPolygon. In case that a single or a couple of points are passed as input, the function will return that point or a segment respectively.
 
 * `geojsons`: `ARRAY` array of features in GeoJSON format casted to STRING.
-* `maxEdge` (optional): `DOUBLE` the length (in 'units') of an edge necessary for part of the hull to become concave. By default `maxEdge` is `infinity`.
+* `maxEdge` (optional): `DOUBLE` the maximum length allowed for an edge of the concave hull. Higher `maxEdge` values will produce more convex-like hulls. If not provided, the default value `infinity` is used and it would be equivalent to a Convex Hull.
 * `units` (optional): `STRING` units of length, the supported options are: miles, kilometers, degrees or radians. By default `units` is `kilometers`.
 
 **Return type**