From 532fc226ede0d8fc5aa80d191515f86a64af99f4 Mon Sep 17 00:00:00 2001 From: Valentin de la Cruz Barquero <6054336+vdelacruzb@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:37:52 +0100 Subject: [PATCH] release: 2024-03-18 (#487) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: dependabot[bot] Co-authored-by: Javier Goizueta Co-authored-by: Alberto Hernández Co-authored-by: Jesús Arroyo Torrens Co-authored-by: Pedro-Juan Ferrer Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/redshift-ded.yml | 15 +- CHANGELOG.md | 8 + clouds/bigquery/CHANGELOG.md | 4 + .../libraries/javascript/src/random.js | 23 ++- .../libraries/javascript/test/random.test.js | 2 +- clouds/bigquery/modules/Makefile | 12 +- .../modules/doc/random/ST_GENERATEPOINTS.md | 7 - .../doc/transformations/ST_CONCAVEHULL.md | 2 +- .../modules/sql/random/ST_GENERATEPOINTS.sql | 81 ++------- .../sql/random/__BBOX_FROM_GEOJSON.sql | 13 -- clouds/bigquery/version | 2 +- .../common/requirements_create_it.txt | 2 +- clouds/postgres/modules/Makefile | 9 +- clouds/redshift/modules/Makefile | 9 +- .../modules/doc/random/ST_GENERATEPOINTS.md | 7 - clouds/snowflake/common/package.json | 2 +- clouds/snowflake/modules/Makefile | 9 +- .../modules/doc/random/ST_GENERATEPOINTS.md | 7 - .../doc/transformations/ST_CONCAVEHULL.md | 2 +- tools/installer/.gitignore | 170 ------------------ tools/installer/README.md | 71 -------- tools/installer/requirements.txt | 1 - tools/installer/setup.cfg | 44 ----- tools/installer/setup.py | 3 - tools/installer/src/__init__.py | 170 ------------------ 25 files changed, 98 insertions(+), 577 deletions(-) delete mode 100644 clouds/bigquery/modules/sql/random/__BBOX_FROM_GEOJSON.sql delete mode 100644 tools/installer/.gitignore delete mode 100644 tools/installer/README.md delete mode 100644 tools/installer/requirements.txt delete mode 100644 tools/installer/setup.cfg delete mode 100644 tools/installer/setup.py delete mode 100644 tools/installer/src/__init__.py diff --git a/.github/workflows/redshift-ded.yml b/.github/workflows/redshift-ded.yml index 0c5e71c2b..333159823 100644 --- a/.github/workflows/redshift-ded.yml +++ b/.github/workflows/redshift-ded.yml @@ -33,6 +33,19 @@ jobs: uses: actions/checkout@v2 - name: Check diff uses: technote-space/get-diff-action@v4 + - name: Set RS_PREFIX for releases + if: startsWith(github.event.pull_request.head.ref, 'release/') + run: | + echo "RS_PREFIX=dedicated_release_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV + - name: Set RS_PREFIX for hotfixes + if: startsWith(github.event.pull_request.head.ref, 'hotfix/') + run: | + echo "RS_PREFIX=dedicated_hotfix_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV + - name: Set RS_PREFIX for the rest + if: | + !(startsWith(github.event.pull_request.head.ref, 'hotfix/')) && + !(startsWith(github.event.pull_request.head.ref, 'release/')) + run: echo "RS_PREFIX=dedicated_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV - name: Setup node uses: actions/setup-node@v1 with: @@ -63,7 +76,7 @@ jobs: if: github.event.action == 'synchronize' || github.event.action == 'labeled' run: | cd clouds/redshift - make deploy + make deploy dropfirst=1 - name: Run remove id: remove if: github.event.action == 'unlabeled' || github.event.action == 'closed' diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d97a718d..f939a6f08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ CARTO Analytics Toolbox Core. All notable commits to this project will be documented in this file. +## 2024-03-18 + +- fix(sf): CI and CD not working because of snowflake driver breaking changes (#484) +- fix(bq,sf,rs,pg): drop schemas when dedicateds gets released (#485) +- fix(bq|random): ST_GENERATEPOINTS returning exact name of points (#486) +- chore(rs,pg): remove installer tool (#483) +- chore(deps): bump jinja2 from 3.1.2 to 3.1.3 in /clouds/databricks/common (#468) + ## 2024-02-15 - chore(sf): add additional tables to native apps (#473) diff --git a/clouds/bigquery/CHANGELOG.md b/clouds/bigquery/CHANGELOG.md index 3a1801276..f510577fd 100644 --- a/clouds/bigquery/CHANGELOG.md +++ b/clouds/bigquery/CHANGELOG.md @@ -4,6 +4,10 @@ CARTO Analytics Toolbox Core for BigQuery. All notable commits to this project will be documented in this file. +## [1.2.1] - 2024-03-18 + +- fix(random): ST_GENERATEPOINTS returning exact name of points (#486) + ## [1.2.0] - 2024-01-17 - chore: increase tests timeout to 200000 (#455) diff --git a/clouds/bigquery/libraries/javascript/src/random.js b/clouds/bigquery/libraries/javascript/src/random.js index 939a44b0c..fde5772e1 100644 --- a/clouds/bigquery/libraries/javascript/src/random.js +++ b/clouds/bigquery/libraries/javascript/src/random.js @@ -1,5 +1,24 @@ -import { bbox } from '@turf/turf'; +import { bbox, booleanPointInPolygon, randomPoint } from '@turf/turf'; + +function generateRandomPointsInPolygon (polygon, numPoints) { + const randomPoints = []; + while (randomPoints.length < numPoints) { + const point = randomPoint(1, { bbox: bbox(polygon) }).features[0]; + if (booleanPointInPolygon(point, polygon)) { + randomPoints.push(JSON.stringify(point.geometry)); + } + } + return randomPoints; +} + +function generateRandomPointInPolygon (polygon) { + let point + do { + point = randomPoint(1, { bbox: bbox(polygon) }).features[0]; + } while (!booleanPointInPolygon(point, polygon)) + return JSON.stringify(point.geometry); +} export default { - bbox + generateRandomPointsInPolygon }; \ No newline at end of file diff --git a/clouds/bigquery/libraries/javascript/test/random.test.js b/clouds/bigquery/libraries/javascript/test/random.test.js index e8855ee2c..f21b4dd35 100644 --- a/clouds/bigquery/libraries/javascript/test/random.test.js +++ b/clouds/bigquery/libraries/javascript/test/random.test.js @@ -1,5 +1,5 @@ const lib = require('../build/index'); test('random library defined', () => { - expect(lib.random.bbox).toBeDefined(); + expect(lib.random.generateRandomPointsInPolygon).toBeDefined(); }); \ No newline at end of file diff --git a/clouds/bigquery/modules/Makefile b/clouds/bigquery/modules/Makefile index ca2f9abfe..521fe1ce2 100644 --- a/clouds/bigquery/modules/Makefile +++ b/clouds/bigquery/modules/Makefile @@ -34,10 +34,10 @@ include $(COMMON_DIR)/Makefile .SILENT: -.PHONY: help lint build build-share deploy deploy-share test remove remove-share clean +.PHONY: help lint build build-share deploy deploy-share test remove remove-functions remove-share clean help: - echo "Available targets: lint build deploy test remove clean" + echo "Available targets: lint build deploy test remove remove-functions clean" lint: venv3 $(NODE_MODULES_DEV) echo "Linting modules..." @@ -85,7 +85,7 @@ test: check $(NODE_MODULES_DEV) if [ ! -z "$$TESTS" ]; then \ GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \ PATH="$(NODE_MODULES_DEV)/.bin/:$(PATH)" \ - jest --testTimeout=200000 $(BAIL) --verbose --slowTestThreshold=20 --maxConcurrency=10 $$TESTS \ + jest --testTimeout=250000 $(BAIL) --verbose --slowTestThreshold=20 --maxConcurrency=10 $$TESTS \ --setupFilesAfterEnv "$(COMMON_DIR)/test-extend.js" || exit 1; \ OLD_TEST=$(TEST_DIR)/$$m/old-test; \ if [ -d $$OLD_TEST ]; then \ @@ -101,6 +101,12 @@ remove: check GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \ $(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql +remove-functions: check + echo "Removing functions..." + REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ + GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \ + $(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql + clean: echo "Cleaning modules..." rm -rf $(BUILD_DIR) $(NODE_MODULES_DEV) diff --git a/clouds/bigquery/modules/doc/random/ST_GENERATEPOINTS.md b/clouds/bigquery/modules/doc/random/ST_GENERATEPOINTS.md index cc119f749..91a082e72 100644 --- a/clouds/bigquery/modules/doc/random/ST_GENERATEPOINTS.md +++ b/clouds/bigquery/modules/doc/random/ST_GENERATEPOINTS.md @@ -10,13 +10,6 @@ Generates randomly placed points inside a polygon and returns them in an array o The distribution of the generated points is spherically uniform (i.e. if the coordinates are interpreted as longitude and latitude on a sphere); this means that WGS84 coordinates will be only approximately uniformly distributed, since WGS84 is based on an ellipsoidal model. -````hint:warning -**warning** - -It never generates more than the requested number of points, but there is a small chance of generating less points. - -```` - * `geog`: `GEOGRAPHY` a polygon; the random points generated will be inside this polygon. * `npoints`: `INT64` number of points to generate. diff --git a/clouds/bigquery/modules/doc/transformations/ST_CONCAVEHULL.md b/clouds/bigquery/modules/doc/transformations/ST_CONCAVEHULL.md index 3c1b94410..10070fa05 100644 --- a/clouds/bigquery/modules/doc/transformations/ST_CONCAVEHULL.md +++ b/clouds/bigquery/modules/doc/transformations/ST_CONCAVEHULL.md @@ -9,7 +9,7 @@ ST_CONCAVEHULL(geog, maxEdge, units) Takes a set of points and returns a concave hull Polygon or MultiPolygon. In case that a single or a couple of points are passed as input, the function will return that point or a segment respectively. * `geog`: `ARRAY` input points. -* `maxEdge`: `FLOAT64`|`NULL` the length (in 'units') of an edge necessary for part of the hull to become concave. If `NULL`the default value `infinity` is used. +* `maxEdge`: `FLOAT64`|`NULL` the maximum length allowed for an edge of the concave hull. Higher `maxEdge` values will produce more convex-like hulls. If `NULL`, the default value `infinity` is used and it would be equivalent to a Convex Hull. * `units`: `STRING`|`NULL` units of length, the supported options are: miles, kilometers, degrees or radians. If `NULL`the default value `kilometers` is used. **Return type** diff --git a/clouds/bigquery/modules/sql/random/ST_GENERATEPOINTS.sql b/clouds/bigquery/modules/sql/random/ST_GENERATEPOINTS.sql index 7c67550f8..e63946a21 100644 --- a/clouds/bigquery/modules/sql/random/ST_GENERATEPOINTS.sql +++ b/clouds/bigquery/modules/sql/random/ST_GENERATEPOINTS.sql @@ -1,74 +1,23 @@ ----------------------------- --- Copyright (C) 2021 CARTO ----------------------------- +-------------------------------- +-- Copyright (C) 2021-2024 CARTO +-------------------------------- + +CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__ST_GENERATEPOINTS` +(geojson STRING, npoints INT64) +RETURNS ARRAY +DETERMINISTIC +LANGUAGE js +OPTIONS (library = ["@@BQ_LIBRARY_BUCKET@@"]) +AS """ + return lib.random.generateRandomPointsInPolygon(JSON.parse(geojson), npoints); +"""; CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.ST_GENERATEPOINTS` (geog GEOGRAPHY, npoints INT64) RETURNS ARRAY AS ( ( - WITH bbox AS ( - -- compute the bounding box of the polygon - SELECT `@@BQ_DATASET@@.__BBOX_FROM_GEOJSON`(ST_ASGEOJSON(geog)) AS box - ), - - bbox_coords AS ( - -- break down the bbox array into minx, miny, maxx, maxy - SELECT - box[ORDINAL(1)] AS minx, - box[ORDINAL(2)] AS miny, - box[ORDINAL(3)] AS maxx, - box[ORDINAL(4)] AS maxy - FROM bbox - ), - - bbox_data AS ( - -- compute area of bbox and put some handy values here too - SELECT - minx, - miny, - maxx, - maxy, - 1.2 AS k, -- security factor to make it more likely that at least npoints fall within the polygon - ST_AREA( - ST_MAKEPOLYGON(ST_MAKELINE([ - ST_GEOGPOINT(minx, miny), - ST_GEOGPOINT(minx, maxy), - ST_GEOGPOINT(maxx, maxy), - ST_GEOGPOINT(maxx, miny), - ST_GEOGPOINT(minx, miny) - ])) - ) AS bbox_area - FROM bbox_coords - ), - - point_seeds AS ( - -- generate enough values so that we will hopefully have at least npoints of them randomly placed inside geog - SELECT - GENERATE_ARRAY(1, CEIL(k * npoints * bbox_area / ST_AREA(geog))) AS i, - SIN(miny * ACOS(-1) / 180.0) AS minsin, - SIN(maxy * ACOS(-1) / 180.0) AS maxsin, - 180.0 / ACOS(-1) AS radtodeg - FROM bbox_data - ), - - bbox_points AS ( - -- compute the random points uniformly in the bbox; - SELECT ST_GEOGPOINT(minx + RAND() * (maxx - minx), radtodeg * ASIN(minsin + RAND() * (maxsin - minsin))) AS point - FROM bbox_coords, point_seeds, UNNEST(i) - ), - - poly_points AS ( - -- now we need to select the points inside the polygon and number them, so we can limit - -- the end result to npoints (note that we can't have a dynamic LIMIT) - SELECT - point, - ROW_NUMBER() OVER () AS rn - FROM bbox_points - WHERE ST_WITHIN(point, geog) - ) - - -- finally select at most npoints and return them in an array - SELECT ARRAY(SELECT point FROM poly_points WHERE rn <= npoints) + SELECT ARRAY_AGG(ST_GEOGFROMGEOJSON(point)) + FROM UNNEST(`@@BQ_DATASET@@.__ST_GENERATEPOINTS`(ST_ASGEOJSON(geog), npoints)) AS point ) ); diff --git a/clouds/bigquery/modules/sql/random/__BBOX_FROM_GEOJSON.sql b/clouds/bigquery/modules/sql/random/__BBOX_FROM_GEOJSON.sql deleted file mode 100644 index 37994f268..000000000 --- a/clouds/bigquery/modules/sql/random/__BBOX_FROM_GEOJSON.sql +++ /dev/null @@ -1,13 +0,0 @@ ----------------------------- --- Copyright (C) 2021 CARTO ----------------------------- - -CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__BBOX_FROM_GEOJSON` -(geojson STRING) -RETURNS ARRAY -DETERMINISTIC -LANGUAGE js -OPTIONS (library = ["@@BQ_LIBRARY_BUCKET@@"]) -AS """ - return lib.random.bbox(JSON.parse(geojson)); -"""; diff --git a/clouds/bigquery/version b/clouds/bigquery/version index 26aaba0e8..6085e9465 100644 --- a/clouds/bigquery/version +++ b/clouds/bigquery/version @@ -1 +1 @@ -1.2.0 +1.2.1 diff --git a/clouds/databricks/common/requirements_create_it.txt b/clouds/databricks/common/requirements_create_it.txt index 769ef83ec..cd1737d67 100644 --- a/clouds/databricks/common/requirements_create_it.txt +++ b/clouds/databricks/common/requirements_create_it.txt @@ -1 +1 @@ -Jinja2==3.1.2 +Jinja2==3.1.3 diff --git a/clouds/postgres/modules/Makefile b/clouds/postgres/modules/Makefile index ba3603b36..4e62a6d7d 100644 --- a/clouds/postgres/modules/Makefile +++ b/clouds/postgres/modules/Makefile @@ -21,10 +21,10 @@ include $(COMMON_DIR)/Makefile .SILENT: -.PHONY: help lint build deploy test remove clean +.PHONY: help lint build deploy test remove remove-functions clean help: - echo "Available targets: lint build deploy test remove clean" + echo "Available targets: lint build deploy test remove remove-functions clean" lint: venv3 $(NODE_MODULES_DEV) echo "Linting modules..." @@ -64,6 +64,11 @@ remove: venv3 REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ $(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql +remove-functions: venv3 + echo "Removing functions..." + REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ + $(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql + clean: echo "Cleaning modules..." rm -rf $(BUILD_DIR) $(VENV3_DIR) $(NODE_MODULES_DEV) diff --git a/clouds/redshift/modules/Makefile b/clouds/redshift/modules/Makefile index d56d2c3c4..75a1d8554 100644 --- a/clouds/redshift/modules/Makefile +++ b/clouds/redshift/modules/Makefile @@ -21,10 +21,10 @@ include $(COMMON_DIR)/Makefile .SILENT: -.PHONY: help lint build deploy test remove clean +.PHONY: help lint build deploy test remove remove-functions clean help: - echo "Available targets: lint build deploy test remove clean" + echo "Available targets: lint build deploy test remove remove-functions clean" lint: venv3 $(NODE_MODULES_DEV) echo "Linting modules..." @@ -64,6 +64,11 @@ remove: venv3 REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ $(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql +remove-functions: venv3 + echo "Removing functions..." + REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ + $(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql + clean: echo "Cleaning modules..." rm -rf $(BUILD_DIR) $(VENV3_DIR) $(NODE_MODULES_DEV) diff --git a/clouds/redshift/modules/doc/random/ST_GENERATEPOINTS.md b/clouds/redshift/modules/doc/random/ST_GENERATEPOINTS.md index 341eb15b6..c1add4884 100644 --- a/clouds/redshift/modules/doc/random/ST_GENERATEPOINTS.md +++ b/clouds/redshift/modules/doc/random/ST_GENERATEPOINTS.md @@ -10,13 +10,6 @@ Generates randomly placed points inside a polygon and returns them in an array o The distribution of the generated points is spherically uniform (i.e. if the coordinates are interpreted as longitude and latitude on a sphere); this means that WGS84 coordinates will be only approximately uniformly distributed, since WGS84 is based on an ellipsoidal model. -````hint:warning -**warning** - -It never generates more than the requested number of points, but there is a small chance of generating less points. - -```` - * `geog`: `GEOMETRY` a polygon; the random points generated will be inside this polygon. * `npoints`: `INT` number of points to generate. diff --git a/clouds/snowflake/common/package.json b/clouds/snowflake/common/package.json index f852c7570..c227179cd 100644 --- a/clouds/snowflake/common/package.json +++ b/clouds/snowflake/common/package.json @@ -11,6 +11,6 @@ "rollup": "^2.47.0", "rollup-plugin-bundle-size": "^1.0.3", "rollup-plugin-terser": "^7.0.2", - "snowflake-sdk": "^1.6.6" + "snowflake-sdk": "1.9.3" } } diff --git a/clouds/snowflake/modules/Makefile b/clouds/snowflake/modules/Makefile index c2f7ac8e2..d557139f4 100644 --- a/clouds/snowflake/modules/Makefile +++ b/clouds/snowflake/modules/Makefile @@ -32,10 +32,10 @@ include $(COMMON_DIR)/Makefile .SILENT: -.PHONY: help lint build build-share build-native-app-setup-script deploy deploy-share test remove remove-share clean +.PHONY: help lint build build-share build-native-app-setup-script deploy deploy-share test remove remove-functions remove-share clean help: - echo "Available targets: lint build deploy test remove clean" + echo "Available targets: lint build deploy test remove remove-functions clean" lint: $(NODE_MODULES_DEV) venv3 echo "Linting modules..." @@ -110,6 +110,11 @@ remove: $(NODE_MODULES_DEV) REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ $(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql +remove-functions: $(NODE_MODULES_DEV) + echo "Removing functions..." + REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ + $(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql + remove-share: $(NODE_MODULES_DEV) echo "Removing share..." $(COMMON_DIR)/run-query.js "DROP SHARE $(SF_SHARE);" diff --git a/clouds/snowflake/modules/doc/random/ST_GENERATEPOINTS.md b/clouds/snowflake/modules/doc/random/ST_GENERATEPOINTS.md index c94c76136..93e8f6cf9 100644 --- a/clouds/snowflake/modules/doc/random/ST_GENERATEPOINTS.md +++ b/clouds/snowflake/modules/doc/random/ST_GENERATEPOINTS.md @@ -10,13 +10,6 @@ Generates randomly placed points inside a polygon and returns them in an array o The distribution of the generated points is spherically uniform (i.e. if the coordinates are interpreted as longitude and latitude on a sphere); this means that WGS84 coordinates will be only approximately uniformly distributed, since WGS84 is based on an ellipsoidal model. -````hint:warning -**warning** - -It never generates more than the requested number of points, but there is a small chance of generating less points. - -```` - * `geog`: `GEOGRAPHY` a polygon; the random points generated will be inside this polygon. * `npoints`: `INT` number of points to generate. diff --git a/clouds/snowflake/modules/doc/transformations/ST_CONCAVEHULL.md b/clouds/snowflake/modules/doc/transformations/ST_CONCAVEHULL.md index 6f5ec8169..0a0173894 100644 --- a/clouds/snowflake/modules/doc/transformations/ST_CONCAVEHULL.md +++ b/clouds/snowflake/modules/doc/transformations/ST_CONCAVEHULL.md @@ -9,7 +9,7 @@ ST_CONCAVEHULL(geojsons [, maxEdge] [, units]) Takes a set of points and returns a concave hull Polygon or MultiPolygon. In case that a single or a couple of points are passed as input, the function will return that point or a segment respectively. * `geojsons`: `ARRAY` array of features in GeoJSON format casted to STRING. -* `maxEdge` (optional): `DOUBLE` the length (in 'units') of an edge necessary for part of the hull to become concave. By default `maxEdge` is `infinity`. +* `maxEdge` (optional): `DOUBLE` the maximum length allowed for an edge of the concave hull. Higher `maxEdge` values will produce more convex-like hulls. If not provided, the default value `infinity` is used and it would be equivalent to a Convex Hull. * `units` (optional): `STRING` units of length, the supported options are: miles, kilometers, degrees or radians. By default `units` is `kilometers`. **Return type** diff --git a/tools/installer/.gitignore b/tools/installer/.gitignore deleted file mode 100644 index 3bedfacaf..000000000 --- a/tools/installer/.gitignore +++ /dev/null @@ -1,170 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env*/ -venv*/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# Images -*.svg - -# Visual Studio Code -.vscode - -# Custom -config.yml -carto-analytics-toolbox-redshift-* diff --git a/tools/installer/README.md b/tools/installer/README.md deleted file mode 100644 index 4c5907bc1..000000000 --- a/tools/installer/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# CARTO Analytics Toolbox installer - -A Python script to install the CARTO Analytics Toolbox in Redshift and Postgres. - -## Install - -1. Install Python >= 3.7: https://www.python.org/downloads/ - - Create a virtual environment (optional for Linux, macOS): - - ``` - python -m venv env - source env/bin/activate - ``` - -2. Install the tool: - - ``` - pip install -U pip - pip install git+https://github.com/cartodb/analytics-toolbox-core.git@main#subdirectory=tools/installer - ``` - -> Note: if `python` does not point to Python 3, use `python3` instead. - -## Usage - -1. Create a `config.yml` file: - - - Redshift packages `config.yml` file. This file must contain the information of the Redshift connection and LDS. - ```yml - connection: - cloud: redshift - host: CLUSTER.ACCOUNT.REGION.redshift.amazonaws.com - database: DATABASE - user: USER - password: PASSWORD - lds: - lambda: lds-function-europe-west1 - roles: arn:aws:iam::XXXXXXXXXXXX:role/CartoFunctionsRedshiftRole,arn:aws:iam::000955892807:role/CartoFunctionsRole - api_base_url: https://gcp-europe-west1.api.carto.com - token: eyJhbGciOiJ... - ``` - - > Note: Redshift core does not require setting up a `lds` configuration. - - - Postgres packages `config.yml` file. This file must contain the information of the Postgres connection. - ```yml - connection: - cloud: postgres - host: HOST - database: DATABASE - user: USER - password: PASSWORD - ``` - -2. Download the installation package (zip file). - -3. Run the script: - - ``` - cat-installer carto-analytics-toolbox-redshift-latest.zip - ``` - - ``` - Reading config file: config.yml - Reading package file: carto-analytics-toolbox-redshift-latest.zip - Installing libraries... - 100%|█████████████████████████████████████████████████████████████████| 8/8 [00:07<00:00, 1.05it/s] - Installing modules... - 100%|█████████████████████████████████████████████████████████████| 246/246 [02:09<00:00, 1.90it/s] - ``` diff --git a/tools/installer/requirements.txt b/tools/installer/requirements.txt deleted file mode 100644 index ecf975e2f..000000000 --- a/tools/installer/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . \ No newline at end of file diff --git a/tools/installer/setup.cfg b/tools/installer/setup.cfg deleted file mode 100644 index 6c889c9dd..000000000 --- a/tools/installer/setup.cfg +++ /dev/null @@ -1,44 +0,0 @@ -[metadata] -name = cat-installer -version = 0.2.1 -description = Python script to install the CARTO Analytics Toolbox in Redshift and Postgres -long_description = file: README.md -long_description_content_type = text/markdown -keywords = - carto - analytics - toolbox - redshift -author = CARTO -url = https://github.com/cartodb/analytics-toolbox-core -license = BSD 3-Clause -classifiers = - Development Status :: 4 - Beta - Intended Audience :: Developers - License :: OSI Approved :: BSD License - Natural Language :: English - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 -[options] -packages = find: -python_requires = >=3.7 -install_requires = - pyyaml>=6.0 - click>8.1 - tqdm>4.64 - sqlparse>0.4 - redshift-connector>2.0 - psycopg2-binary>=2.9.1 - pyjwt==2.7.0 - validator-collection==1.5.0 -zip_safe = False -[options.entry_points] -console_scripts = - cat-installer = src:main -[flake8] -max-line-length = 88 -ignore = E203 diff --git a/tools/installer/setup.py b/tools/installer/setup.py deleted file mode 100644 index 606849326..000000000 --- a/tools/installer/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import setup - -setup() diff --git a/tools/installer/src/__init__.py b/tools/installer/src/__init__.py deleted file mode 100644 index 4e776f6cd..000000000 --- a/tools/installer/src/__init__.py +++ /dev/null @@ -1,170 +0,0 @@ -import os -import re -import sys -import yaml -import click -import zipfile -import redshift_connector -import jwt -from validator_collection import checkers - -from tqdm import trange -from sqlparse import split -from psycopg2 import connect - - -def read_config(filename): - try: - with open(filename, 'r') as yamlfile: - config = yaml.load(yamlfile, Loader=yaml.FullLoader) - validate_config(config) - return config - except FileNotFoundError: - exit(f'ERROR: configuration file not found: {filename}') - - -def read_package(filename, config): - with zipfile.ZipFile(filename, 'r') as zip_ref: - folder = zip_ref.infolist()[0].filename - zip_ref.extractall('.') - - package = {} - - if config['connection']['cloud'] == 'redshift': - with open(os.path.join(folder, 'libraries.sql'), 'r') as lib_file: - package['libraries'] = lib_file.read() - - with open(os.path.join(folder, 'modules.sql'), 'r') as mod_file: - package['modules'] = mod_file.read() - - return package - - -def run_sql(sql, config): - if config['connection']['cloud'] == 'redshift': - with redshift_connector.connect( - host=config['connection']['host'], - database=config['connection']['database'], - user=config['connection']['user'], - password=config['connection']['password'], - ) as conn: - conn.autocommit = True - with conn.cursor() as cursor: - lds = config.get('lds') - if lds is not None: - sql = ( - sql.replace('@@API_BASE_URL@@', lds['api_base_url']) - .replace('@@LDS_LAMBDA@@', lds['lambda']) - .replace('@@LDS_ROLES@@', lds['roles']) - .replace('@@LDS_TOKEN@@', lds['token']) - ) - queries = split(sql) - for i in trange(len(queries), ncols=100): - query = queries[i] - cursor.execute(query) - elif config['connection']['cloud'] == 'postgres': - with connect( - host=config['connection']['host'], - database=config['connection']['database'], - user=config['connection']['user'], - password=config['connection']['password'], - ) as conn: - conn.autocommit = True - with conn.cursor() as cursor: - queries = split(sql) - for i in trange(len(queries), ncols=97): - query = queries[i] - cursor.execute(query) - for notice in list(set(conn.notices)): - print(notice.strip()) - - -def validate_lds_config(lds_config): - pattern = r'^(lds-function-asia-northeast1|lds-function-australia-southeast1|lds-function-europe-west1|lds-function-us-east1)$' # noqa: E501 - if not validate_str(lds_config.get('lambda'), pattern): - exit('incorrect configuration: missing or invalid lds.lambda') - - pattern = r'^arn:aws:iam::[0-9]+:role/CartoFunctionsRedshiftRole,arn:aws:iam::000955892807:role/CartoFunctionsRole$' # noqa: E501 - if not validate_str(lds_config.get('roles'), pattern): - exit('incorrect configuration: missing or invalid lds.roles') - - if not validate_str(lds_config.get('api_base_url')): - exit('incorrect configuration: missing lds.api_base_url') - - if not checkers.is_url(lds_config.get('api_base_url')): - exit('incorrect configuration: invalid lds.api_base_url') - - token = lds_config.get('token') - if not validate_str(token): - exit('incorrect configuration: missing lds.token') - algorithm = jwt.get_unverified_header(token).get('alg') - if not algorithm: - exit('incorrect configuration: invalid lds.token') - jwt_payload = jwt.decode( - token, algorithms=[algorithm], options={'verify_signature': False} - ) - if not jwt_payload.get('a') or not jwt_payload.get('jti'): - exit('incorrect configuration: invalid lds.token') - - -def validate_config(config): - connection = config.get('connection') - - cloud = connection.get('cloud') - if not validate_str(cloud): - exit('incorrect configuration: missing connection.cloud') - - if cloud not in ['redshift', 'postgres']: - exit('incorrect configuration: invalid connection.cloud') - - if connection is None: - exit('incorrect configuration: missing connection') - - if not validate_str(connection.get('host')): - exit('incorrect configuration: missing connection.host') - - if cloud == 'redshift': - pattern = r'^([^.]+)\.([^.]+)\.([^.]+)\.redshift(-serverless)?\.amazonaws\.com$' - if not validate_str(connection.get('host'), pattern): - exit('incorrect configuration: invalid connection.host') - - if not validate_str(connection.get('database')): - exit('incorrect configuration: missing connection.database') - - if not validate_str(connection.get('user')): - exit('incorrect configuration: missing connection.user') - - if not validate_str(connection.get('password')): - exit('incorrect configuration: missing connection.password') - - lds_config = config.get('lds') - if cloud == 'redshift' and lds_config is not None: - validate_lds_config(lds_config) - - -def validate_str(string, pattern=None): - return ( - isinstance(string, str) - and len(string) > 0 - and (not pattern or re.compile(pattern).match(string)) - ) - - -def exit(message): - print(f'ERROR: {message}') - sys.exit(1) - - -@click.command(help='Python installer for the CARTO Analytics Toolbox in Redshift.') -@click.argument('package_file', type=click.Path(exists=True), required=True) -def main(package_file): - config_file = 'config.yml' - print(f'Reading config file: {config_file}') - config = read_config(config_file) - print(f'Reading package file: {package_file}') - package = read_package(package_file, config) - if 'libraries' in package: - print('Installing libraries...') - run_sql(package['libraries'], config) - print('Installing modules...') - run_sql(package['modules'], config)