Skip to content

Commit b9cb656

Browse files
vdelacruzbvdelacruzb
authored
release: 2023-07-11 (#435)
1 parent 3265bba commit b9cb656

30 files changed

+700
-238
lines changed

‎.github/workflows/bigquery.yml

+11-5
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,16 @@ jobs:
6969
needs: test
7070
runs-on: ubuntu-20.04
7171
timeout-minutes: 20
72+
strategy:
73+
matrix:
74+
include:
75+
- project: BQ_PROJECT_CD
76+
bucket: BQ_BUCKET_CD
77+
region: BQ_REGION_CD
7278
env:
73-
BQ_PROJECT: ${{ secrets.BQ_PROJECT_CD }}
74-
BQ_BUCKET: ${{ secrets.BQ_BUCKET_CD }}
75-
BQ_REGION: ${{ secrets.BQ_REGION_CD }}
79+
BQ_PROJECT: ${{ secrets[matrix.project] }}
80+
BQ_BUCKET: ${{ secrets[matrix.bucket] }}
81+
BQ_REGION: ${{ secrets[matrix.region] }}
7682
steps:
7783
- name: Checkout repo
7884
uses: actions/checkout@v2
@@ -82,13 +88,13 @@ jobs:
8288
uses: google-github-actions/auth@v0
8389
with:
8490
credentials_json: ${{ secrets.BQCARTOCD_DEPLOY_CLOUD_EXTENSIONS_SA_BASE64 }}
85-
project_id: ${{ env.BQ_PROJECT_CD }}
91+
project_id: ${{ env.BQ_PROJECT }}
8692
create_credentials_file: true
8793
- name: Setup gcloud
8894
uses: google-github-actions/setup-gcloud@v0
8995
with:
9096
version: ${{ env.GCLOUD_VERSION }}
91-
project_id: ${{ env.BQ_PROJECT_CD }}
97+
project_id: ${{ env.BQ_PROJECT }}
9298
- name: Run deploy
9399
run: |
94100
cd clouds/bigquery

‎.github/workflows/databricks.yml

+9-3
Original file line numberDiff line numberDiff line change
@@ -73,10 +73,16 @@ jobs:
7373
needs: test
7474
runs-on: ubuntu-20.04
7575
timeout-minutes: 20
76+
strategy:
77+
matrix:
78+
include:
79+
- cluster_id: DB_CLUSTER_ID_CD
80+
http_path: DB_HTTP_PATH_CD
81+
host: DB_HOST_CD
7682
env:
77-
DB_CLUSTER_ID: ${{ secrets.DB_CLUSTER_ID_CD }}
78-
DB_HTTP_PATH: ${{ secrets.DB_HTTP_PATH_CD }}
79-
DB_HOST: ${{ secrets.DB_HOST_CD }}
83+
DB_CLUSTER_ID: ${{ secrets[matrix.cluster_id] }}
84+
DB_HTTP_PATH: ${{ secrets[matrix.http_path] }}
85+
DB_HOST: ${{ secrets[matrix.host] }}
8086
steps:
8187
- name: Checkout repo
8288
uses: actions/checkout@v3

‎.github/workflows/postgres.yml

+15-4
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,22 @@ jobs:
5959
needs: test
6060
runs-on: ubuntu-20.04
6161
timeout-minutes: 20
62+
strategy:
63+
matrix:
64+
include:
65+
- host: PG_HOST_CD
66+
database: PG_DATABASE_CD
67+
user: PG_USER_CD
68+
password: PG_PASSWORD_CD
69+
- host: PG_HOST_CI
70+
database: PG_DATABASE_CI
71+
user: PG_USER_CI
72+
password: PG_PASSWORD_CI
6273
env:
63-
PG_HOST: ${{ secrets.PG_HOST_CD }}
64-
PG_DATABASE: ${{ secrets.PG_DATABASE_CD }}
65-
PG_USER: ${{ secrets.PG_USER_CD }}
66-
PG_PASSWORD: ${{ secrets.PG_PASSWORD_CD }}
74+
PG_HOST: ${{ secrets[matrix.host] }}
75+
PG_DATABASE: ${{ secrets[matrix.database] }}
76+
PG_USER: ${{ secrets[matrix.user] }}
77+
PG_PASSWORD: ${{ secrets[matrix.password] }}
6778
steps:
6879
- name: Checkout repo
6980
uses: actions/checkout@v2

‎.github/workflows/redshift-ded.yml

+12-9
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ env:
88
NODE_VERSION: 14
99
PYTHON2_VERSION: 2.7.18
1010
PYTHON3_VERSION: 3.8.10
11-
VIRTUALENV_VERSION: 20.21.1
11+
VIRTUALENV_VERSION: 20.15.1
1212

1313
jobs:
1414

@@ -51,14 +51,17 @@ jobs:
5151
uses: actions/setup-node@v1
5252
with:
5353
node-version: ${{ env.NODE_VERSION }}
54-
- name: Setup python 2
55-
uses: actions/setup-python@v2
56-
with:
57-
python-version: ${{ env.PYTHON2_VERSION }}
58-
- name: Setup python 3
59-
uses: actions/setup-python@v2
60-
with:
61-
python-version: ${{ env.PYTHON3_VERSION }}
54+
- name: Setup yarn
55+
run: npm install -g yarn
56+
- name: Install dependencies
57+
run: |
58+
apt-get update
59+
apt-get -y install rsync zip unzip curl glibc-source groff less
60+
- name: Install AWS CLI
61+
run: |
62+
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
63+
unzip awscliv2.zip
64+
./aws/install
6265
- name: Configure AWS Credentials
6366
uses: aws-actions/configure-aws-credentials@v1
6467
with:

‎.github/workflows/redshift.yml

+67-31
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,14 @@ env:
1818
NODE_VERSION: 14
1919
PYTHON2_VERSION: 2.7.18
2020
PYTHON3_VERSION: 3.8.10
21-
VIRTUALENV_VERSION: 20.21.1
21+
VIRTUALENV_VERSION: 20.15.1
2222
GCLOUD_VERSION: 290.0.1
2323

2424
jobs:
2525

2626
test:
2727
runs-on: ubuntu-20.04
28+
container: python:2.7.18-buster
2829
timeout-minutes: 30
2930
env:
3031
RS_PREFIX: ci_${{ github.sha }}_${{ github.run_id }}_${{ github.run_attempt }}_
@@ -44,14 +45,17 @@ jobs:
4445
uses: actions/setup-node@v1
4546
with:
4647
node-version: ${{ env.NODE_VERSION }}
47-
- name: Setup python 2
48-
uses: actions/setup-python@v2
49-
with:
50-
python-version: ${{ env.PYTHON2_VERSION }}
51-
- name: Setup python 3
52-
uses: actions/setup-python@v2
53-
with:
54-
python-version: ${{ env.PYTHON3_VERSION }}
48+
- name: Setup yarn
49+
run: npm install -g yarn
50+
- name: Install dependencies
51+
run: |
52+
apt-get update
53+
apt-get -y install rsync zip unzip curl glibc-source groff less
54+
- name: Install AWS CLI
55+
run: |
56+
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
57+
unzip awscliv2.zip
58+
./aws/install
5559
- name: Setup virtualenv
5660
run: pip install virtualenv==${{ env.VIRTUALENV_VERSION }}
5761
- name: Configure AWS Credentials
@@ -72,15 +76,36 @@ jobs:
7276
if: github.ref_name == 'main'
7377
needs: test
7478
runs-on: ubuntu-20.04
79+
container: python:2.7.18-buster
7580
timeout-minutes: 20
81+
strategy:
82+
matrix:
83+
include:
84+
- host: RS_HOST_CD
85+
database: RS_DATABASE_CD
86+
user: RS_USER_CD
87+
password: RS_PASSWORD_CD
88+
bucket: RS_BUCKET_CD
89+
region: RS_REGION_CD
90+
aws_access_key_id: RS_AWS_ACCESS_KEY_ID_CD
91+
aws_secret_access_key: RS_AWS_SECRET_ACCESS_KEY_CD
92+
- host: RS_HOST_CI
93+
database: RS_DATABASE_CI
94+
user: RS_USER_CI
95+
password: RS_PASSWORD_CI
96+
bucket: RS_BUCKET_CI
97+
region: RS_REGION_CI
98+
aws_access_key_id: RS_AWS_ACCESS_KEY_ID_CI
99+
aws_secret_access_key: RS_AWS_SECRET_ACCESS_KEY_CI
76100
env:
77-
RS_HOST: ${{ secrets.RS_HOST_CD }}
78-
RS_DATABASE: ${{ secrets.RS_DATABASE_CD }}
79-
RS_USER: ${{ secrets.RS_USER_CD }}
80-
RS_PASSWORD: ${{ secrets.RS_PASSWORD_CD }}
81-
RS_BUCKET: ${{ secrets.RS_BUCKET_CD }}
82-
AWS_ACCESS_KEY_ID: ${{ secrets.RS_AWS_ACCESS_KEY_ID_CD }}
83-
AWS_SECRET_ACCESS_KEY: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY_CD }}
101+
RS_HOST: ${{ secrets[matrix.host] }}
102+
RS_DATABASE: ${{ secrets[matrix.database] }}
103+
RS_USER: ${{ secrets[matrix.user] }}
104+
RS_PASSWORD: ${{ secrets[matrix.password] }}
105+
RS_BUCKET: ${{ secrets[matrix.bucket] }}
106+
RS_REGION: ${{ secrets[matrix.region] }}
107+
AWS_ACCESS_KEY_ID: ${{ secrets[matrix.aws_access_key_id] }}
108+
AWS_SECRET_ACCESS_KEY: ${{ secrets[matrix.aws_secret_access_key] }}
84109
steps:
85110
- name: Checkout repo
86111
uses: actions/checkout@v2
@@ -90,22 +115,25 @@ jobs:
90115
uses: actions/setup-node@v1
91116
with:
92117
node-version: ${{ env.NODE_VERSION }}
93-
- name: Setup python 2
94-
uses: actions/setup-python@v2
95-
with:
96-
python-version: ${{ env.PYTHON2_VERSION }}
97-
- name: Setup python 3
98-
uses: actions/setup-python@v2
99-
with:
100-
python-version: ${{ env.PYTHON3_VERSION }}
118+
- name: Setup yarn
119+
run: npm install -g yarn
120+
- name: Install dependencies
121+
run: |
122+
apt-get update
123+
apt-get -y install rsync zip unzip curl glibc-source groff less
124+
- name: Install AWS CLI
125+
run: |
126+
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
127+
unzip awscliv2.zip
128+
./aws/install
101129
- name: Setup virtualenv
102130
run: pip install virtualenv==${{ env.VIRTUALENV_VERSION }}
103131
- name: Configure AWS Credentials
104132
uses: aws-actions/configure-aws-credentials@v1
105133
with:
106-
aws-access-key-id: ${{ secrets.RS_AWS_ACCESS_KEY_ID_CD }}
107-
aws-secret-access-key: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY_CD }}
108-
aws-region: ${{ secrets.RS_REGION_CD }}
134+
aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
135+
aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
136+
aws-region: ${{ env.RS_REGION }}
109137
- name: Run deploy
110138
run: |
111139
cd clouds/redshift
@@ -115,6 +143,7 @@ jobs:
115143
if: github.ref_name == 'stable'
116144
needs: test
117145
runs-on: ubuntu-20.04
146+
container: python:2.7.18-buster
118147
timeout-minutes: 10
119148
env:
120149
PACKAGE_BUCKET: gs://carto-analytics-toolbox-core/redshift
@@ -126,12 +155,19 @@ jobs:
126155
uses: actions/setup-node@v1
127156
with:
128157
node-version: ${{ env.NODE_VERSION }}
129-
- name: Setup python 2
130-
uses: actions/setup-python@v2
131-
with:
132-
python-version: ${{ env.PYTHON2_VERSION }}
158+
- name: Setup yarn
159+
run: npm install -g yarn
133160
- name: Setup virtualenv
134161
run: pip install virtualenv==${{ env.VIRTUALENV_VERSION }}
162+
- name: Install dependencies
163+
run: |
164+
apt-get update
165+
apt-get -y install rsync zip unzip curl glibc-source groff less
166+
- name: Install AWS CLI
167+
run: |
168+
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
169+
unzip awscliv2.zip
170+
./aws/install
135171
- name: Auth google
136172
uses: google-github-actions/auth@v0
137173
with:

‎.github/workflows/snowflake.yml

+13-5
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,20 @@ jobs:
6060
needs: test
6161
runs-on: ubuntu-20.04
6262
timeout-minutes: 20
63+
strategy:
64+
matrix:
65+
include:
66+
- account: SF_ACCOUNT_CD
67+
database: SF_DATABASE_CD
68+
user: SF_USER_CD
69+
password: SF_PASSWORD_CD
70+
role: SF_ROLE_CD
6371
env:
64-
SF_ACCOUNT: ${{ secrets.SF_ACCOUNT_CD }}
65-
SF_DATABASE: ${{ secrets.SF_DATABASE_CD }}
66-
SF_USER: ${{ secrets.SF_USER_CD }}
67-
SF_PASSWORD: ${{ secrets.SF_PASSWORD_CD }}
68-
SF_ROLE: ${{ secrets.SF_ROLE_CD }}
72+
SF_ACCOUNT: ${{ secrets[matrix.account] }}
73+
SF_DATABASE: ${{ secrets[matrix.database] }}
74+
SF_USER: ${{ secrets[matrix.user] }}
75+
SF_PASSWORD: ${{ secrets[matrix.password] }}
76+
SF_ROLE: ${{ secrets[matrix.role] }}
6977
steps:
7078
- name: Checkout repo
7179
uses: actions/checkout@v2

‎CHANGELOG.md

+9-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,15 @@ CARTO Analytics Toolbox Core.
44

55
All notable commits to this project will be documented in this file.
66

7-
## 2023-06-01
7+
## 2023-07-11
8+
9+
- chore(bq,rs,pg|quadbin): update QUADBIN_FROMLONGLAT formula (#409, #411)
10+
- chore(bq,rs,pg|quadbin): optimize QUADBIN_TOCHILDREN performance (#412, #413)
11+
- chore(bq,sf,pg): introduce skip_progress_bar option to allow deploying faster (#416, #419)
12+
- chore(all): use matrix in deploy internal and deploy in ci (#430, #433)
13+
- fix(rs): update github actions to prevents python2 incompatibility (#431, #434)
14+
15+
## 2023-06-01
816

917
- fix(tools|installer): verify lds config when using cat-installer (#406)
1018

‎clouds/bigquery/CHANGELOG.md

+5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@ CARTO Analytics Toolbox Core for BigQuery.
44

55
All notable commits to this project will be documented in this file.
66

7+
## [1.0.2] - 2023-07-11
8+
9+
- chore(quadbin): update QUADBIN_FROMLONGLAT formula (#409, #411)
10+
- chore(quadbin): optimize QUADBIN_TOCHILDREN performance (#412, #413)
11+
712
## [1.0.1] - 2023-03-06
813

914
- fix(transformations): great circle crashing for equal origin and end (#390)

‎clouds/bigquery/common/run-script.js

+15-6
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,21 @@ async function runQueries (queries) {
3535
}
3636

3737
const script = process.argv[2];
38-
const content = fs.readFileSync(script).toString();
38+
let content = fs.readFileSync(script).toString();
3939
const separator = '\n-->\n';
40-
const queries = content.split(separator).filter(q => !!q);
4140
let sqlFunction = '';
4241

43-
runQueries(queries).catch(error => {
44-
console.log(`\n[${sqlFunction}] ERROR: ${error}`);
45-
process.exit(1);
46-
});
42+
if (process.env.SKIP_PROGRESS_BAR) {
43+
content = content.replaceAll(separator, '')
44+
runQueries([content]).catch(error => {
45+
console.log(`\nERROR: ${error}`);
46+
process.exit(1);
47+
});
48+
} else {
49+
const queries = content.split(separator).filter(q => !!q);
50+
51+
runQueries(queries).catch(error => {
52+
console.log(`\n[${sqlFunction}] ERROR: ${error}`);
53+
process.exit(1);
54+
});
55+
}

0 commit comments

Comments
 (0)