chores(bq): split JS libraries generation

clouds/bigquery/Makefile

@@ -4,6 +4,7 @@ ROOT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
 
 DIST_DIR ?= $(ROOT_DIR)/dist
 BUILD_DIR ?= $(ROOT_DIR)/build
+MODULES_DIRS ?= $(ROOT_DIR)/modules
 ESLINTRC_DIR ?= $(ROOT_DIR)/../..
 COMMON_DIR = $(ROOT_DIR)/common
 PACKAGE_VERSION ?= $(shell cat $(ROOT_DIR)/version)
@@ -49,7 +50,10 @@ build:
 build-libraries:
 	mkdir -p $(BUILD_DIR)/libs
 	$(MAKE) -C libraries/javascript build
-	cp libraries/javascript/build/index.js $(BUILD_DIR)/libs/$(BQ_LIBRARY_DEFAULT).js
+	$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB) 1>/dev/null # Check errors
+	for f in `$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB)`; do \
+		cp libraries/javascript/build/$${f}.js $(BUILD_DIR)/libs/$(BQ_LIBRARY_DEFAULT)_$${f}.js; \
+	done
 ifdef MAKE_LIB
 	cp libraries/javascript/build/index_$(MAKE_LIB).js $(BUILD_DIR)/libs/$(BQ_LIBRARY_DEFAULT)_$(MAKE_LIB).js
 endif

clouds/bigquery/common/Makefile

@@ -10,11 +10,11 @@ BQ_DATASET_DEFAULT = carto
 BQ_LIBRARY_DEFAULT ?= carto_analytics_toolbox_core
 
 ifeq ($(production),1)
-export BQ_LIBRARY_BUCKET = $(BQ_BUCKET)/$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT).js
-export BQ_LIBRARY_TILER_BUCKET = $(BQ_BUCKET)/$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT)_tiler.js
+export BQ_LIBRARY_BUCKET = $(BQ_BUCKET)/$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT)
+export BQ_LIBRARY_TILER_BUCKET = $(BQ_LIBRARY_BUCKET)_tiler.js
 else
-export BQ_LIBRARY_BUCKET = $(BQ_BUCKET)/$(BQ_PREFIX)$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT).js
-export BQ_LIBRARY_TILER_BUCKET = $(BQ_BUCKET)/$(BQ_PREFIX)$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT)_tiler.js
+export BQ_LIBRARY_BUCKET = $(BQ_BUCKET)/$(BQ_PREFIX)$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT)
+export BQ_LIBRARY_TILER_BUCKET = $(BQ_LIBRARY_BUCKET)_tiler.js
 
 endif
 

clouds/bigquery/common/build_modules.js

@@ -14,8 +14,11 @@ const argv = require('minimist')(process.argv.slice(2));
 
 const inputDirs = argv._[0] && argv._[0].split(',');
 const outputDir = argv.output || 'build';
+const libsBuildDir = argv.libs_build_dir || '../libraries/javascript/build';
 const diff = argv.diff || [];
 const nodeps = argv.nodeps;
+const libraryBucket = argv.librarybucket;
+const makelib = argv.makelib;
 let modulesFilter = (argv.modules && argv.modules.split(',')) || [];
 let functionsFilter = (argv.functions && argv.functions.split(',')) || [];
 let all = !(diff.length || modulesFilter.length || functionsFilter.length);
@@ -156,6 +159,23 @@ if (argv.production) {
 let content = output.map(f => f.content).join(separator);
 
 function apply_replacements (text) {
+    const libraries = [... new Set(text.match(new RegExp('@@BQ_LIBRARY_.*_BUCKET@@', 'g')))];
+    for (let library of libraries) {
+        let libraryName = library.replace('@@BQ_LIBRARY_', '').replace('_BUCKET@@', '').toLowerCase();
+        if (makelib == libraryName) {
+            continue;
+        }
+        libraryName += '.js';
+        const libraryPath = path.join(libsBuildDir, libraryName);
+        if (fs.existsSync(libraryPath)) {
+            const libraryBucketPath = libraryBucket + '_' + libraryName;
+            text = text.replace(new RegExp(library, 'g'), libraryBucketPath);
+        }
+        else {
+            console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`);
+            process.exit(1);
+        }
+    }
     const replacements = process.env.REPLACEMENTS.split(' ');
     for (let replacement of replacements) {
         if (replacement) {

clouds/bigquery/common/list_libraries.js

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+
+// List the JavaScript libraries based on the input filters to the SQL functions
+
+
+// ./build_modules.js modules --output=build --diff="clouds/bigquery/modules/sql/quadbin/QUADBIN_TOZXY.sql"
+// ./build_modules.js modules --output=build --functions=ST_TILEENVELOPE
+// ./build_modules.js modules --output=build --modules=quadbin
+// ./build_modules.js modules --output=build --production --dropfirst
+
+const fs = require('fs');
+const path = require('path');
+const argv = require('minimist')(process.argv.slice(2));
+
+const inputDirs = argv._[0] && argv._[0].split(',');
+const diff = argv.diff || [];
+const nodeps = argv.nodeps;
+const makelib = argv.makelib;
+let modulesFilter = (argv.modules && argv.modules.split(',')) || [];
+let functionsFilter = (argv.functions && argv.functions.split(',')) || [];
+let all = !(diff.length || modulesFilter.length || functionsFilter.length);
+
+// Convert diff to modules/functions
+if (diff.length) {
+    const patternsAll = [
+        /\.github\/workflows\/bigquery\.yml/,
+        /clouds\/bigquery\/common\/.+/,
+        /clouds\/bigquery\/libraries\/.+/,
+        /clouds\/bigquery\/.*Makefile/,
+        /clouds\/bigquery\/version/
+    ];
+    const patternModulesSql = /clouds\/bigquery\/modules\/sql\/([^\s]*?)\//g;
+    const patternModulesTest = /clouds\/bigquery\/modules\/test\/([^\s]*?)\//g;
+    const diffAll = patternsAll.some(p => diff.match(p));
+    if (diffAll) {
+        all = diffAll;
+    } else {
+        const modulesSql = [...diff.matchAll(patternModulesSql)].map(m => m[1]);
+        const modulesTest = [...diff.matchAll(patternModulesTest)].map(m => m[1]);
+        const diffModulesFilter = [...new Set(modulesSql.concat(modulesTest))];
+        if (diffModulesFilter) {
+            modulesFilter = diffModulesFilter;
+        }
+    }
+}
+
+// Extract functions
+const functions = [];
+for (let inputDir of inputDirs) {
+    const sqldir = path.join(inputDir, 'sql');
+    const modules = fs.readdirSync(sqldir);
+    modules.forEach(module => {
+        const moduledir = path.join(sqldir, module);
+        if (fs.statSync(moduledir).isDirectory()) {
+            const files = fs.readdirSync(moduledir);
+            files.forEach(file => {
+                if (file.endsWith('.sql')) {
+                    const name = path.parse(file).name;
+                    const content = fs.readFileSync(path.join(moduledir, file)).toString().replace(/--.*\n/g, '');
+                    functions.push({
+                        name,
+                        module,
+                        content,
+                        dependencies: []
+                    });
+                }
+            });
+        }
+    });
+}
+
+// Check filters
+modulesFilter.forEach(m => {
+    if (!functions.map(fn => fn.module).includes(m)) {
+        process.stderr.write(`ERROR: Module not found ${m}\n`);
+        process.exit(1);
+    }
+});
+functionsFilter.forEach(f => {
+    if (!functions.map(fn => fn.name).includes(f)) {
+        process.stderr.write(`ERROR: Function not found ${f}`);
+        process.exit(1);
+    }
+});
+
+// Extract function dependencies
+if (!nodeps) {
+    functions.forEach(mainFunction => {
+        functions.forEach(depFunction => {
+            if (mainFunction.name != depFunction.name) {
+                const depFunctionMatches = [];
+                depFunctionMatches.push(...depFunction.content.replace(/(\r\n|\n|\r)/gm,' ').matchAll(new RegExp('(?<=(?<!TEMP )FUNCTION)(.*?)(?=AS |RETURNS)','g')));
+                depFunctionMatches.push(...depFunction.content.replace(/(\r\n|\n|\r)/gm,' ').matchAll(new RegExp('(?<=PROCEDURE)(.*?)(?=BEGIN)','g')));
+                const depFunctionNames = [];
+                depFunctionMatches.forEach((depFunctionMatch) => {
+                    let qualifiedDepFunctName = depFunctionMatch[0].replace(/[ \p{Diacritic}]/gu, '').split('(')[0];
+                    qualifiedDepFunctName = qualifiedDepFunctName.split('.');
+                    depFunctionNames.push(qualifiedDepFunctName[qualifiedDepFunctName.length - 1]);
+                })
+                if (depFunctionNames.some((depFunctionName) => mainFunction.content.includes(`DATASET@@.${depFunctionName}\`(`))) {
+                    mainFunction.dependencies.push(depFunction.name);
+                }
+            }
+        });
+    });
+}
+
+// Check circular dependencies
+functions.forEach(mainFunction => {
+    functions.forEach(depFunction => {
+        if (mainFunction.dependencies.includes(depFunction.name) &&
+            depFunction.dependencies.includes(mainFunction.name)) {
+            process.stderr.write(`ERROR: Circular dependency between ${mainFunction.name} and ${depFunction.name}`);
+            process.exit(1);
+        }
+    });
+});
+
+
+// Filter functions
+const output = [];
+function add (f, include) {
+    include = include || all || functionsFilter.includes(f.name) || modulesFilter.includes(f.module);
+    for (const dependency of f.dependencies) {
+        add(functions.find(f => f.name === dependency), include);
+    }
+    if (!output.map(f => f.name).includes(f.name) && include) {
+        output.push({
+            name: f.name,
+            content: f.content
+        });
+    }
+}
+functions.forEach(f => add(f));
+
+const content = output.map(f => f.content).join('\n');
+let libraries = [... new Set(content.match(new RegExp('@@BQ_LIBRARY_.*_BUCKET@@', 'g')))]
+    .map(l => l.replace('@@BQ_LIBRARY_', '').replace('_BUCKET@@', '').toLowerCase());
+
+// Exclude libraries pointed by makelib as they are deployed separately
+if (makelib) {
+    libraries = libraries.filter(l => l !== makelib);
+}
+
+process.stdout.write(libraries.join(' '));

clouds/bigquery/common/rollup.config.js

@@ -18,6 +18,11 @@ for (let dir of dirs) {
     }
 }
 
+if (!input && filename) {
+    console.log(`Error: library "${filename}" does not exist. Add it or revisit the replacement "@@BQ_LIBRARY_${path.parse(filename).name.toUpperCase()}@@" in one of your sql files.`);
+    process.exit(1);
+}
+
 // Format library name to camel case
 const name = process.env.NAME.replace(/(_\w)/g, k => k[1].toUpperCase());
 

clouds/bigquery/libraries/javascript/Makefile

@@ -45,12 +45,27 @@ ifdef MAKE_LIB
 endif
 
 build-libs: build-install $(NODE_MODULES_DEV)
-	NAME=lib \
-	PATH="$(NODE_MODULES_DEV)/.bin/:$(PATH)" \
-	DIRS=$(LIBS_DIR) \
-	FILENAME=index.js \
-	OUTPUT=$(BUILD_DIR)/index.js \
-	rollup --config $(COMMON_DIR)/rollup.config.js $(BUILD_PARAMS); \
+ifdef UNIT_TEST
+	$(COMMON_DIR)/list_libraries.js $(MODULES_DIR) --all --makelib=$(MAKE_LIB) 1>/dev/null # Check errors
+	for f in `$(COMMON_DIR)/list_libraries.js $(MODULES_DIR) --all --makelib=$(MAKE_LIB)`; do \
+		NAME=$${f}Lib \
+		PATH="$(NODE_MODULES_DEV)/.bin/:$(PATH)" \
+		DIRS=$(LIBS_DIR) \
+		FILENAME=$${f}.js \
+		OUTPUT=$(BUILD_DIR)/$${f}.js \
+		rollup --config $(COMMON_DIR)/rollup.config.js $(BUILD_PARAMS); \
+	done
+else
+	$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB) 1>/dev/null # Check errors
+	for f in `$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB)`; do \
+		NAME=$${f}Lib \
+		PATH="$(NODE_MODULES_DEV)/.bin/:$(PATH)" \
+		DIRS=$(LIBS_DIRS) \
+		FILENAME=$${f}.js \
+		OUTPUT=$(BUILD_DIR)/$${f}.js \
+		rollup --config $(COMMON_DIR)/rollup.config.js $(BUILD_PARAMS); \
+	done
+endif
 
 build-install:
 	for d in $(shell echo $(LIBS_DIRS) | tr "," "\n"); do \
@@ -59,7 +74,10 @@ build-install:
 
 deploy: check build
 	echo "Deploying libraries..."
-	$(GSUTIL) cp -r $(BUILD_DIR)/index.js $(BQ_LIBRARY_BUCKET)
+	$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB) 1>/dev/null # Check errors
+	for f in `$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB)`; do \
+		$(GSUTIL) cp -r $(BUILD_DIR)/$${f}.js $(BQ_LIBRARY_BUCKET)_$${f}.js; \
+	done
 	# Provisional for WASM version of tiler
 ifdef MAKE_LIB
 	echo "Deploying tiler library..."

clouds/bigquery/libraries/javascript/libs/quadkey.js

@@ -0,0 +1,25 @@
+import {
+    bbox,
+    toParent,
+    toChildren,
+    quadkeyFromQuadint,
+    quadintFromQuadkey,
+    quadintFromLocation,
+    quadintToGeoJSON,
+    quadintFromZXY,
+    geojsonToQuadints,
+    ZXYFromQuadint
+} from '../src/quadkey';
+
+export default {
+    bbox,
+    toParent,
+    toChildren,
+    quadkeyFromQuadint,
+    quadintFromQuadkey,
+    quadintFromLocation,
+    quadintToGeoJSON,
+    quadintFromZXY,
+    geojsonToQuadints,
+    ZXYFromQuadint
+};

clouds/bigquery/libraries/javascript/libs/random.js

@@ -0,0 +1,8 @@
+import {
+    generateRandomPointsInPolygon
+
+} from '../src/random';
+
+export default {
+    generateRandomPointsInPolygon
+};

clouds/bigquery/libraries/javascript/libs/s2.js

@@ -0,0 +1,9 @@
+import { S2 } from '../src/s2';
+
+export default {
+    keyToId: S2.keyToId,
+    idToKey: S2.idToKey,
+    latLngToKey: S2.latLngToKey,
+    FromHilbertQuadKey: S2.S2Cell.FromHilbertQuadKey,
+    idToLatLng: S2.idToLatLng
+};