From b2c2b5eb28b62f8d08e0ac7f24c045093800343b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Mon, 10 Jun 2024 06:49:19 +0200 Subject: [PATCH 1/8] optimize polyfill --- .../modules/sql/quadbin/QUADBIN_POLYFILL.sql | 105 ++++++++++++++---- 1 file changed, 84 insertions(+), 21 deletions(-) diff --git a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql index 09c835317..db79b90b2 100644 --- a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql +++ b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql @@ -85,11 +85,11 @@ AS (( ) -- return the min value between the target and intermediate -- resolutions to return between 1 and 256 cells - SELECT LEAST( + SELECT LEAST(26, LEAST( resolution, -- compute the resolution of cells that match the geog area - -- by comparing with the area of the quadbin 0, plus 3 levels - IF(geog_area > 0, CAST(-LOG(geog_area / 508164597540055.75, 4) AS INT64) + 3, resolution)) + -- by comparing with the area of the quadbin 0, plus 6 levels + IF(geog_area > 0, CAST(-LOG(geog_area / 508164597540055.75, 4) AS INT64) + 6, resolution))) FROM __params )); @@ -97,45 +97,108 @@ CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CHILDREN_INTERSECT (geog GEOGRAPHY, resolution INT64) RETURNS ARRAY AS (( - WITH __cells AS ( - SELECT quadbin + WITH __parents AS ( + SELECT parent, ST_CONTAINS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)) AS inside FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`(geog, - `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z`(geog, resolution))) AS parent, - UNNEST(`@@BQ_DATASET@@.QUADBIN_TOCHILDREN`(parent, resolution)) AS quadbin + `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z`(geog, resolution))) AS parent + ), + __children AS ( + SELECT child, inside + FROM __parents, + UNNEST(`@@BQ_DATASET@@.QUADBIN_TOCHILDREN`(parent, resolution)) AS child + ), + __children_inside AS ( + SELECT child + FROM __children + WHERE inside + ), + __children_border AS ( + SELECT child + FROM __children + WHERE NOT inside + ), + __cells AS ( + SELECT child + FROM __children_inside + UNION ALL + SELECT child + FROM __children_border + WHERE ST_INTERSECTS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(child)) ) - SELECT ARRAY_AGG(quadbin) + SELECT ARRAY_AGG(child) FROM __cells - WHERE ST_INTERSECTS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(quadbin)) )); CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CHILDREN_CONTAINS` (geog GEOGRAPHY, resolution INT64) RETURNS ARRAY AS (( - WITH __cells AS ( - SELECT quadbin + WITH __parents AS ( + SELECT parent, ST_CONTAINS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)) AS inside FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`(geog, - `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z`(geog, resolution))) AS parent, - UNNEST(`@@BQ_DATASET@@.QUADBIN_TOCHILDREN`(parent, resolution)) AS quadbin + `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z`(geog, resolution))) AS parent + ), + __children AS ( + SELECT child, inside + FROM __parents, + UNNEST(`@@BQ_DATASET@@.QUADBIN_TOCHILDREN`(parent, resolution)) AS child + ), + __children_inside AS ( + SELECT child + FROM __children + WHERE inside + ), + __children_border AS ( + SELECT child + FROM __children + WHERE not inside + ), + __cells AS ( + SELECT child + FROM __children_inside + UNION ALL + SELECT child + FROM __children_border + WHERE ST_CONTAINS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(child)) ) - SELECT ARRAY_AGG(quadbin) + SELECT ARRAY_AGG(child) FROM __cells - WHERE ST_CONTAINS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(quadbin)) )); CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CHILDREN_CENTER` (geog GEOGRAPHY, resolution INT64) RETURNS ARRAY AS (( - WITH __cells AS ( - SELECT quadbin + WITH __parents AS ( + SELECT parent, ST_CONTAINS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)) AS inside FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`(geog, - `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z`(geog, resolution))) AS parent, - UNNEST(`@@BQ_DATASET@@.QUADBIN_TOCHILDREN`(parent, resolution)) AS quadbin + `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z`(geog, resolution))) AS parent + ), + __children AS ( + SELECT child, inside + FROM __parents, + UNNEST(`@@BQ_DATASET@@.QUADBIN_TOCHILDREN`(parent, resolution)) AS child + ), + __children_inside AS ( + SELECT child + FROM __children + WHERE inside + ), + __children_border AS ( + SELECT child + FROM __children + WHERE not inside + ), + __cells AS ( + SELECT child + FROM __children_inside + UNION ALL + SELECT child + FROM __children_border + WHERE ST_INTERSECTS(geog, `@@BQ_DATASET@@.QUADBIN_CENTER`(child)) ) - SELECT ARRAY_AGG(quadbin) + SELECT ARRAY_AGG(child) FROM __cells - WHERE ST_INTERSECTS(geog, `@@BQ_DATASET@@.QUADBIN_CENTER`(quadbin)) )); CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.QUADBIN_POLYFILL_MODE` From e4b6f7de4a252dd42c29549cd7714ef93b784674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Mon, 24 Jun 2024 08:28:05 +0200 Subject: [PATCH 2/8] add internal quadbin functions without check to gain efficiency in the compilation --- .../sql/quadbin/QUADBIN_FROMGEOGPOINT.sql | 8 +++++++ .../sql/quadbin/QUADBIN_FROMLONGLAT.sql | 24 +++++++++++++++++++ .../modules/sql/quadbin/QUADBIN_TOPARENT.sql | 22 +++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/clouds/snowflake/modules/sql/quadbin/QUADBIN_FROMGEOGPOINT.sql b/clouds/snowflake/modules/sql/quadbin/QUADBIN_FROMGEOGPOINT.sql index 2917c5395..b130ee2a6 100644 --- a/clouds/snowflake/modules/sql/quadbin/QUADBIN_FROMGEOGPOINT.sql +++ b/clouds/snowflake/modules/sql/quadbin/QUADBIN_FROMGEOGPOINT.sql @@ -9,3 +9,11 @@ IMMUTABLE AS $$ @@SF_SCHEMA@@.QUADBIN_FROMLONGLAT(ST_X(point), ST_Y(point), resolution) $$; + +CREATE OR REPLACE SECURE FUNCTION @@SF_SCHEMA@@._QUADBIN_FROMGEOGPOINT +(point GEOGRAPHY, resolution INT) +RETURNS BIGINT +IMMUTABLE +AS $$ + @@SF_SCHEMA@@._QUADBIN_FROMLONGLAT(ST_X(point), ST_Y(point), resolution) +$$; diff --git a/clouds/snowflake/modules/sql/quadbin/QUADBIN_FROMLONGLAT.sql b/clouds/snowflake/modules/sql/quadbin/QUADBIN_FROMLONGLAT.sql index 35b7bc677..e56c62d90 100644 --- a/clouds/snowflake/modules/sql/quadbin/QUADBIN_FROMLONGLAT.sql +++ b/clouds/snowflake/modules/sql/quadbin/QUADBIN_FROMLONGLAT.sql @@ -30,3 +30,27 @@ AS $$ ) ) $$; + +CREATE OR REPLACE SECURE FUNCTION @@SF_SCHEMA@@._QUADBIN_FROMLONGLAT +(longitude FLOAT, latitude FLOAT, resolution INT) +RETURNS BIGINT +IMMUTABLE +AS $$ + WITH + __params AS ( + SELECT + resolution AS z, + ACOS(-1) AS PI, + GREATEST(-89, LEAST(89, latitude)) AS params_latitude + ), + __zxy AS ( + SELECT + z, + BITAND( CAST(FLOOR(BITSHIFTLEFT(1, z) * ((longitude / 360.0) + 0.5)) AS INT), (BITSHIFTLEFT(1, z) - 1)) AS x, + BITAND( CAST(FLOOR(GREATEST(0, LEAST(BITSHIFTLEFT(1, z) - 1, BITSHIFTLEFT(1, z) * (0.5 - (LN(TAN(PI/4.0 + params_latitude/2.0 * PI/180.0)) / (2*PI)))))) AS INT),(BITSHIFTLEFT(1, z) - 1)) AS y + FROM __params + ) + SELECT @@SF_SCHEMA@@._QUADBIN_STRING_TOINT(@@SF_SCHEMA@@._QUADBIN_FROMZXY(z, x, y)) + FROM __zxy +$$; + diff --git a/clouds/snowflake/modules/sql/quadbin/QUADBIN_TOPARENT.sql b/clouds/snowflake/modules/sql/quadbin/QUADBIN_TOPARENT.sql index 6854ce95c..6e0dcf188 100644 --- a/clouds/snowflake/modules/sql/quadbin/QUADBIN_TOPARENT.sql +++ b/clouds/snowflake/modules/sql/quadbin/QUADBIN_TOPARENT.sql @@ -26,3 +26,25 @@ AS $$ ) ) $$; + +CREATE OR REPLACE SECURE FUNCTION @@SF_SCHEMA@@._QUADBIN_TOPARENT +(quadbin BIGINT, resolution INT) +RETURNS BIGINT +IMMUTABLE +AS $$ + bitor( + bitor( + bitand( + quadbin, + bitnot( + bitshiftleft(31, 52) + ) + ), + bitshiftleft(resolution, 52) + ), + bitshiftright( + 4503599627370495, + resolution * 2 + ) + ) +$$; \ No newline at end of file From e1d0e5dd487c95b2fa1400f05596cad0797b70b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Fri, 28 Jun 2024 16:50:50 +0200 Subject: [PATCH 3/8] optimize quadbin_polyfill in BigQuery for big geographies --- .../modules/sql/quadbin/QUADBIN_POLYFILL.sql | 222 ++++++++++-------- 1 file changed, 120 insertions(+), 102 deletions(-) diff --git a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql index db79b90b2..25ea7bf40 100644 --- a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql +++ b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql @@ -1,106 +1,116 @@ --------------------------------- --- Copyright (C) 2022-2023 CARTO +-- Copyright (C) 2022-2024 CARTO --------------------------------- CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT` (geog GEOGRAPHY, resolution INT64) RETURNS ARRAY AS (( - IF(geog IS NULL OR resolution IS NULL, - NULL, - IF(resolution < 0 OR resolution > 26, - ERROR('Invalid resolution, should be between 0 and 26'), ( - WITH __bbox AS ( - SELECT ST_BOUNDINGBOX(geog) AS box - ), - __params AS ( - SELECT - box.xmin AS minlon, - box.xmax AS maxlon, - GREATEST(-89, LEAST(89, box.ymin)) AS minlat, - GREATEST(-89, LEAST(89, box.ymax)) AS maxlat, - (1 << resolution) AS z2, - ACOS(-1) AS pi - FROM __bbox - ), - __sinlat AS ( - SELECT - SIN(minlat * pi / 180.0) AS sinlat_min, - SIN(maxlat * pi / 180.0) AS sinlat_max - FROM __params - ), - __tile_coords_range AS ( - SELECT - resolution AS z, - CAST( - FLOOR(z2 * ((minlon / 360.0) + 0.5)) AS INT64 - ) AS xmin, - CAST( - FLOOR( - GREATEST(0, LEAST(z2 - 1, - z2 * ( - 0.5 - 0.25 * LN( - (1 + sinlat_max) / (1 - sinlat_max) - ) / pi - ) - )) - ) AS INT64 - ) AS ymin, - CAST( - FLOOR(z2 * ((maxlon / 360.0) + 0.5)) AS INT64 - ) AS xmax, - CAST( - FLOOR( - GREATEST(0, LEAST(z2 - 1, - z2 * ( - 0.5 - 0.25 * LN( - (1 + sinlat_min) / (1 - sinlat_min) - ) / pi - ) - )) - ) AS INT64 - ) AS ymax - FROM __params, __sinlat - ), - -- compute all the quadbin cells contained in the bounding box - __cells AS ( - SELECT `@@BQ_DATASET@@.QUADBIN_FROMZXY`(z, x, y) AS quadbin - FROM __tile_coords_range, - UNNEST(GENERATE_ARRAY(xmin, xmax)) AS x, - UNNEST(GENERATE_ARRAY(ymin, ymax)) AS y - ) - SELECT ARRAY_AGG(quadbin) - FROM __cells - WHERE ST_INTERSECTS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(quadbin)) - )) + WITH __bbox AS ( + SELECT ST_BOUNDINGBOX(geog) AS box + ), + __params AS ( + SELECT + box.xmin AS minlon, + box.xmax AS maxlon, + GREATEST(-89, LEAST(89, box.ymin)) AS minlat, + GREATEST(-89, LEAST(89, box.ymax)) AS maxlat, + (1 << resolution) AS z2, + ACOS(-1) AS pi + FROM __bbox + ), + __sinlat AS ( + SELECT + SIN(minlat * pi / 180.0) AS sinlat_min, + SIN(maxlat * pi / 180.0) AS sinlat_max + FROM __params + ), + __tile_coords_range AS ( + SELECT + resolution AS z, + CAST( + FLOOR(z2 * ((minlon / 360.0) + 0.5)) AS INT64 + ) AS xmin, + CAST( + FLOOR( + GREATEST(0, LEAST(z2 - 1, + z2 * ( + 0.5 - 0.25 * LN( + (1 + sinlat_max) / (1 - sinlat_max) + ) / pi + ) + )) + ) AS INT64 + ) AS ymin, + CAST( + FLOOR(z2 * ((maxlon / 360.0) + 0.5)) AS INT64 + ) AS xmax, + CAST( + FLOOR( + GREATEST(0, LEAST(z2 - 1, + z2 * ( + 0.5 - 0.25 * LN( + (1 + sinlat_min) / (1 - sinlat_min) + ) / pi + ) + )) + ) AS INT64 + ) AS ymax + FROM __params, __sinlat + ), + -- compute all the quadbin cells contained in the bounding box + __cells AS ( + SELECT `@@BQ_DATASET@@.QUADBIN_FROMZXY`(z, x, y) AS quadbin + FROM __tile_coords_range, + UNNEST(GENERATE_ARRAY(xmin, xmax)) AS x, + UNNEST(GENERATE_ARRAY(ymin, ymax)) AS y ) + SELECT ARRAY_AGG(quadbin) + FROM __cells + WHERE ST_INTERSECTS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(quadbin)) )); -CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z` +CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__INIT_RESOLUTION` (geog GEOGRAPHY, resolution INT64) RETURNS INT64 AS (( - WITH __params AS ( + WITH __geog_area AS ( SELECT ST_AREA(geog) AS geog_area - ) - -- return the min value between the target and intermediate - -- resolutions to return between 1 and 256 cells - SELECT LEAST(26, LEAST( - resolution, + ), + __geog_resolution AS ( -- compute the resolution of cells that match the geog area - -- by comparing with the area of the quadbin 0, plus 6 levels - IF(geog_area > 0, CAST(-LOG(geog_area / 508164597540055.75, 4) AS INT64) + 6, resolution))) - FROM __params + SELECT IF(geog_area > 0, CAST(-LOG(geog_area / 508164597540055.75, 4) AS INT64), resolution) AS geog_resolution + FROM __geog_area + ) + -- compute the average resolution for the initial polyfill + SELECT CAST((resolution + geog_resolution) / 2 AS INT64) + FROM __geog_resolution )); -CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CHILDREN_INTERSECTS` -(geog GEOGRAPHY, resolution INT64) +CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__USE_PARENT_CONTAINS` +(geog GEOGRAPHY) +RETURNS BOOL +AS (( + -- the algorithm using the parent contains can be used if the + -- area is more than 20% of the max coverage area, otherwise, + -- the computation of the contains is not worth it + WITH __geog_area AS ( + SELECT ST_AREA(geog) AS geog_area, + POW(ST_MAXDISTANCE(geog, geog), 2) AS max_area + ) + SELECT IF(max_area > 0, geog_area / max_area > 0.2, FALSE) + FROM __geog_area +)); + +CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INTERSECTS` +(geog GEOGRAPHY, use_parent_contains BOOLEAN, resolution INT64) RETURNS ARRAY AS (( WITH __parents AS ( - SELECT parent, ST_CONTAINS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)) AS inside - FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`(geog, - `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z`(geog, resolution))) AS parent + SELECT parent, IF(use_parent_contains, ST_CONTAINS( + geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)), FALSE) AS inside + FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`( + geog, `@@BQ_DATASET@@.__INIT_RESOLUTION`(geog, resolution))) AS parent ), __children AS ( SELECT child, inside @@ -129,14 +139,15 @@ AS (( FROM __cells )); -CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CHILDREN_CONTAINS` -(geog GEOGRAPHY, resolution INT64) +CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CONTAINS` +(geog GEOGRAPHY, use_parent_contains BOOLEAN, resolution INT64) RETURNS ARRAY AS (( WITH __parents AS ( - SELECT parent, ST_CONTAINS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)) AS inside - FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`(geog, - `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z`(geog, resolution))) AS parent + SELECT parent, IF(use_parent_contains, ST_CONTAINS( + geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)), FALSE) AS inside + FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`( + geog, `@@BQ_DATASET@@.__INIT_RESOLUTION`(geog, resolution))) AS parent ), __children AS ( SELECT child, inside @@ -151,7 +162,7 @@ AS (( __children_border AS ( SELECT child FROM __children - WHERE not inside + WHERE NOT inside ), __cells AS ( SELECT child @@ -165,14 +176,15 @@ AS (( FROM __cells )); -CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CHILDREN_CENTER` -(geog GEOGRAPHY, resolution INT64) +CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CENTER` +(geog GEOGRAPHY, use_parent_contains BOOLEAN, resolution INT64) RETURNS ARRAY AS (( WITH __parents AS ( - SELECT parent, ST_CONTAINS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)) AS inside - FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`(geog, - `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z`(geog, resolution))) AS parent + SELECT parent, IF(use_parent_contains, ST_CONTAINS( + geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)), FALSE) AS inside + FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`( + geog, `@@BQ_DATASET@@.__INIT_RESOLUTION`(geog, resolution))) AS parent ), __children AS ( SELECT child, inside @@ -187,7 +199,7 @@ AS (( __children_border AS ( SELECT child FROM __children - WHERE not inside + WHERE NOT inside ), __cells AS ( SELECT child @@ -195,7 +207,7 @@ AS (( UNION ALL SELECT child FROM __children_border - WHERE ST_INTERSECTS(geog, `@@BQ_DATASET@@.QUADBIN_CENTER`(child)) + WHERE ST_CONTAINS(geog, `@@BQ_DATASET@@.QUADBIN_CENTER`(child)) ) SELECT ARRAY_AGG(child) FROM __cells @@ -205,16 +217,22 @@ CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.QUADBIN_POLYFILL_MODE` (geog GEOGRAPHY, resolution INT64, mode STRING) RETURNS ARRAY AS (( - CASE mode - WHEN 'intersects' THEN `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CHILDREN_INTERSECTS`(geog, resolution) - WHEN 'contains' THEN `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CHILDREN_CONTAINS`(geog, resolution) - WHEN 'center' THEN `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CHILDREN_CENTER`(geog, resolution) - END + IF(resolution IS NULL OR resolution < 0 OR resolution > 26, + ERROR('Invalid resolution, should be between 0 and 26'), + (SELECT CASE mode + WHEN 'intersects' THEN `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INTERSECTS`( + geog, `@@BQ_DATASET@@.__USE_PARENT_CONTAINS`(geog), resolution) + WHEN 'contains' THEN `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CONTAINS`( + geog, `@@BQ_DATASET@@.__USE_PARENT_CONTAINS`(geog), resolution) + WHEN 'center' THEN `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CENTER`( + geog, `@@BQ_DATASET@@.__USE_PARENT_CONTAINS`(geog), resolution) + END) + ) )); CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.QUADBIN_POLYFILL` (geog GEOGRAPHY, resolution INT64) RETURNS ARRAY AS (( - `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CHILDREN_CENTER`(geog, resolution) + `@@BQ_DATASET@@.QUADBIN_POLYFILL_MODE`(geog, resolution, 'center') )); From d7231493eef9a622e0c18b141af2824f1428608d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Fri, 28 Jun 2024 18:30:28 +0200 Subject: [PATCH 4/8] remove core badge from docs --- clouds/bigquery/modules/doc/accessors/_INTRO.md | 1 - clouds/bigquery/modules/doc/clustering/_INTRO.md | 1 - clouds/bigquery/modules/doc/constructors/_INTRO.md | 1 - clouds/bigquery/modules/doc/geohash/_INTRO.md | 1 - clouds/bigquery/modules/doc/h3/_INTRO.md | 1 - clouds/bigquery/modules/doc/measurements/_INTRO.md | 1 - clouds/bigquery/modules/doc/placekey/_INTRO.md | 1 - clouds/bigquery/modules/doc/processing/_INTRO.md | 1 - clouds/bigquery/modules/doc/quadbin/_INTRO.md | 1 - clouds/bigquery/modules/doc/random/_INTRO.md | 1 - clouds/bigquery/modules/doc/s2/_INTRO.md | 1 - clouds/bigquery/modules/doc/transformations/_INTRO.md | 1 - clouds/databricks/modules/doc/accessors/_INTRO.md | 1 - clouds/databricks/modules/doc/constructors/_INTRO.md | 1 - clouds/databricks/modules/doc/formatters/_INTRO.md | 1 - clouds/databricks/modules/doc/indexing/_INTRO.md | 1 - clouds/databricks/modules/doc/measurements/_INTRO.md | 1 - clouds/databricks/modules/doc/parsers/_INTRO.md | 1 - clouds/databricks/modules/doc/predicates/_INTRO.md | 1 - clouds/databricks/modules/doc/transformations/_INTRO.md | 1 - clouds/postgres/modules/doc/h3/_INTRO.md | 1 - clouds/postgres/modules/doc/quadbin/_INTRO.md | 1 - clouds/redshift/modules/doc/clustering/_INTRO.md | 1 - clouds/redshift/modules/doc/constructors/_INTRO.md | 1 - clouds/redshift/modules/doc/placekey/_INTRO.md | 1 - clouds/redshift/modules/doc/processing/_INTRO.md | 1 - clouds/redshift/modules/doc/quadbin/_INTRO.md | 1 - clouds/redshift/modules/doc/random/_INTRO.md | 1 - clouds/redshift/modules/doc/s2/_INTRO.md | 1 - clouds/redshift/modules/doc/transformations/_INTRO.md | 1 - clouds/snowflake/modules/doc/accessors/_INTRO.md | 1 - clouds/snowflake/modules/doc/clustering/_INTRO.md | 1 - clouds/snowflake/modules/doc/constructors/_INTRO.md | 1 - clouds/snowflake/modules/doc/h3/_INTRO.md | 1 - clouds/snowflake/modules/doc/measurements/_INTRO.md | 1 - clouds/snowflake/modules/doc/placekey/_INTRO.md | 1 - clouds/snowflake/modules/doc/processing/_INTRO.md | 1 - clouds/snowflake/modules/doc/quadbin/_INTRO.md | 1 - clouds/snowflake/modules/doc/random/_INTRO.md | 1 - clouds/snowflake/modules/doc/s2/_INTRO.md | 1 - clouds/snowflake/modules/doc/transformations/_INTRO.md | 1 - 41 files changed, 41 deletions(-) diff --git a/clouds/bigquery/modules/doc/accessors/_INTRO.md b/clouds/bigquery/modules/doc/accessors/_INTRO.md index d3e7a730e..45bee0962 100644 --- a/clouds/bigquery/modules/doc/accessors/_INTRO.md +++ b/clouds/bigquery/modules/doc/accessors/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # accessors diff --git a/clouds/bigquery/modules/doc/clustering/_INTRO.md b/clouds/bigquery/modules/doc/clustering/_INTRO.md index 66a4f1944..32d841535 100644 --- a/clouds/bigquery/modules/doc/clustering/_INTRO.md +++ b/clouds/bigquery/modules/doc/clustering/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # clustering diff --git a/clouds/bigquery/modules/doc/constructors/_INTRO.md b/clouds/bigquery/modules/doc/constructors/_INTRO.md index 7baa0bb31..176ceefce 100644 --- a/clouds/bigquery/modules/doc/constructors/_INTRO.md +++ b/clouds/bigquery/modules/doc/constructors/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # constructors diff --git a/clouds/bigquery/modules/doc/geohash/_INTRO.md b/clouds/bigquery/modules/doc/geohash/_INTRO.md index 1aa9f0d62..6bd559e0f 100644 --- a/clouds/bigquery/modules/doc/geohash/_INTRO.md +++ b/clouds/bigquery/modules/doc/geohash/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # geohash diff --git a/clouds/bigquery/modules/doc/h3/_INTRO.md b/clouds/bigquery/modules/doc/h3/_INTRO.md index 4e4883ff6..667c8ced7 100644 --- a/clouds/bigquery/modules/doc/h3/_INTRO.md +++ b/clouds/bigquery/modules/doc/h3/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # h3 diff --git a/clouds/bigquery/modules/doc/measurements/_INTRO.md b/clouds/bigquery/modules/doc/measurements/_INTRO.md index b80357a83..c5cea46a4 100644 --- a/clouds/bigquery/modules/doc/measurements/_INTRO.md +++ b/clouds/bigquery/modules/doc/measurements/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # measurements diff --git a/clouds/bigquery/modules/doc/placekey/_INTRO.md b/clouds/bigquery/modules/doc/placekey/_INTRO.md index e5fc4d56f..29eebc170 100644 --- a/clouds/bigquery/modules/doc/placekey/_INTRO.md +++ b/clouds/bigquery/modules/doc/placekey/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # placekey diff --git a/clouds/bigquery/modules/doc/processing/_INTRO.md b/clouds/bigquery/modules/doc/processing/_INTRO.md index 5736464d3..1f1048fe9 100644 --- a/clouds/bigquery/modules/doc/processing/_INTRO.md +++ b/clouds/bigquery/modules/doc/processing/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # processing diff --git a/clouds/bigquery/modules/doc/quadbin/_INTRO.md b/clouds/bigquery/modules/doc/quadbin/_INTRO.md index d92210fb1..6d28038f3 100644 --- a/clouds/bigquery/modules/doc/quadbin/_INTRO.md +++ b/clouds/bigquery/modules/doc/quadbin/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # quadbin diff --git a/clouds/bigquery/modules/doc/random/_INTRO.md b/clouds/bigquery/modules/doc/random/_INTRO.md index 357d0fdac..31ebf2175 100644 --- a/clouds/bigquery/modules/doc/random/_INTRO.md +++ b/clouds/bigquery/modules/doc/random/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # random diff --git a/clouds/bigquery/modules/doc/s2/_INTRO.md b/clouds/bigquery/modules/doc/s2/_INTRO.md index 1ef7f4da6..91a8966c8 100644 --- a/clouds/bigquery/modules/doc/s2/_INTRO.md +++ b/clouds/bigquery/modules/doc/s2/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # s2 diff --git a/clouds/bigquery/modules/doc/transformations/_INTRO.md b/clouds/bigquery/modules/doc/transformations/_INTRO.md index 96eeabdc9..c51442d91 100644 --- a/clouds/bigquery/modules/doc/transformations/_INTRO.md +++ b/clouds/bigquery/modules/doc/transformations/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # transformations diff --git a/clouds/databricks/modules/doc/accessors/_INTRO.md b/clouds/databricks/modules/doc/accessors/_INTRO.md index d3e7a730e..45bee0962 100644 --- a/clouds/databricks/modules/doc/accessors/_INTRO.md +++ b/clouds/databricks/modules/doc/accessors/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # accessors diff --git a/clouds/databricks/modules/doc/constructors/_INTRO.md b/clouds/databricks/modules/doc/constructors/_INTRO.md index 7baa0bb31..176ceefce 100644 --- a/clouds/databricks/modules/doc/constructors/_INTRO.md +++ b/clouds/databricks/modules/doc/constructors/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # constructors diff --git a/clouds/databricks/modules/doc/formatters/_INTRO.md b/clouds/databricks/modules/doc/formatters/_INTRO.md index cd5766b5f..70b2983ce 100644 --- a/clouds/databricks/modules/doc/formatters/_INTRO.md +++ b/clouds/databricks/modules/doc/formatters/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # formatters diff --git a/clouds/databricks/modules/doc/indexing/_INTRO.md b/clouds/databricks/modules/doc/indexing/_INTRO.md index 21bb990f6..7c7166add 100644 --- a/clouds/databricks/modules/doc/indexing/_INTRO.md +++ b/clouds/databricks/modules/doc/indexing/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # indexing diff --git a/clouds/databricks/modules/doc/measurements/_INTRO.md b/clouds/databricks/modules/doc/measurements/_INTRO.md index b80357a83..c5cea46a4 100644 --- a/clouds/databricks/modules/doc/measurements/_INTRO.md +++ b/clouds/databricks/modules/doc/measurements/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # measurements diff --git a/clouds/databricks/modules/doc/parsers/_INTRO.md b/clouds/databricks/modules/doc/parsers/_INTRO.md index 8a01f83fd..79df96623 100644 --- a/clouds/databricks/modules/doc/parsers/_INTRO.md +++ b/clouds/databricks/modules/doc/parsers/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # parsers diff --git a/clouds/databricks/modules/doc/predicates/_INTRO.md b/clouds/databricks/modules/doc/predicates/_INTRO.md index 01624ffe7..c228c18c7 100644 --- a/clouds/databricks/modules/doc/predicates/_INTRO.md +++ b/clouds/databricks/modules/doc/predicates/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # predicates diff --git a/clouds/databricks/modules/doc/transformations/_INTRO.md b/clouds/databricks/modules/doc/transformations/_INTRO.md index 96eeabdc9..c51442d91 100644 --- a/clouds/databricks/modules/doc/transformations/_INTRO.md +++ b/clouds/databricks/modules/doc/transformations/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # transformations diff --git a/clouds/postgres/modules/doc/h3/_INTRO.md b/clouds/postgres/modules/doc/h3/_INTRO.md index 035656bb1..ef9cfc866 100644 --- a/clouds/postgres/modules/doc/h3/_INTRO.md +++ b/clouds/postgres/modules/doc/h3/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # h3 diff --git a/clouds/postgres/modules/doc/quadbin/_INTRO.md b/clouds/postgres/modules/doc/quadbin/_INTRO.md index a43d6b7bd..503bcffdc 100644 --- a/clouds/postgres/modules/doc/quadbin/_INTRO.md +++ b/clouds/postgres/modules/doc/quadbin/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # quadbin diff --git a/clouds/redshift/modules/doc/clustering/_INTRO.md b/clouds/redshift/modules/doc/clustering/_INTRO.md index 66a4f1944..32d841535 100644 --- a/clouds/redshift/modules/doc/clustering/_INTRO.md +++ b/clouds/redshift/modules/doc/clustering/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # clustering diff --git a/clouds/redshift/modules/doc/constructors/_INTRO.md b/clouds/redshift/modules/doc/constructors/_INTRO.md index 7baa0bb31..176ceefce 100644 --- a/clouds/redshift/modules/doc/constructors/_INTRO.md +++ b/clouds/redshift/modules/doc/constructors/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # constructors diff --git a/clouds/redshift/modules/doc/placekey/_INTRO.md b/clouds/redshift/modules/doc/placekey/_INTRO.md index e487b1fbd..326060af0 100644 --- a/clouds/redshift/modules/doc/placekey/_INTRO.md +++ b/clouds/redshift/modules/doc/placekey/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # placekey diff --git a/clouds/redshift/modules/doc/processing/_INTRO.md b/clouds/redshift/modules/doc/processing/_INTRO.md index 5736464d3..1f1048fe9 100644 --- a/clouds/redshift/modules/doc/processing/_INTRO.md +++ b/clouds/redshift/modules/doc/processing/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # processing diff --git a/clouds/redshift/modules/doc/quadbin/_INTRO.md b/clouds/redshift/modules/doc/quadbin/_INTRO.md index 8936ecbfa..5e46821a0 100644 --- a/clouds/redshift/modules/doc/quadbin/_INTRO.md +++ b/clouds/redshift/modules/doc/quadbin/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # quadbin diff --git a/clouds/redshift/modules/doc/random/_INTRO.md b/clouds/redshift/modules/doc/random/_INTRO.md index 357d0fdac..31ebf2175 100644 --- a/clouds/redshift/modules/doc/random/_INTRO.md +++ b/clouds/redshift/modules/doc/random/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # random diff --git a/clouds/redshift/modules/doc/s2/_INTRO.md b/clouds/redshift/modules/doc/s2/_INTRO.md index ced3b362f..ca6f159ed 100644 --- a/clouds/redshift/modules/doc/s2/_INTRO.md +++ b/clouds/redshift/modules/doc/s2/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # s2 diff --git a/clouds/redshift/modules/doc/transformations/_INTRO.md b/clouds/redshift/modules/doc/transformations/_INTRO.md index 96eeabdc9..c51442d91 100644 --- a/clouds/redshift/modules/doc/transformations/_INTRO.md +++ b/clouds/redshift/modules/doc/transformations/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # transformations diff --git a/clouds/snowflake/modules/doc/accessors/_INTRO.md b/clouds/snowflake/modules/doc/accessors/_INTRO.md index d3e7a730e..45bee0962 100644 --- a/clouds/snowflake/modules/doc/accessors/_INTRO.md +++ b/clouds/snowflake/modules/doc/accessors/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # accessors diff --git a/clouds/snowflake/modules/doc/clustering/_INTRO.md b/clouds/snowflake/modules/doc/clustering/_INTRO.md index 66a4f1944..32d841535 100644 --- a/clouds/snowflake/modules/doc/clustering/_INTRO.md +++ b/clouds/snowflake/modules/doc/clustering/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # clustering diff --git a/clouds/snowflake/modules/doc/constructors/_INTRO.md b/clouds/snowflake/modules/doc/constructors/_INTRO.md index 7baa0bb31..176ceefce 100644 --- a/clouds/snowflake/modules/doc/constructors/_INTRO.md +++ b/clouds/snowflake/modules/doc/constructors/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # constructors diff --git a/clouds/snowflake/modules/doc/h3/_INTRO.md b/clouds/snowflake/modules/doc/h3/_INTRO.md index 65b2a3678..81d68eeb2 100644 --- a/clouds/snowflake/modules/doc/h3/_INTRO.md +++ b/clouds/snowflake/modules/doc/h3/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # h3 diff --git a/clouds/snowflake/modules/doc/measurements/_INTRO.md b/clouds/snowflake/modules/doc/measurements/_INTRO.md index b80357a83..c5cea46a4 100644 --- a/clouds/snowflake/modules/doc/measurements/_INTRO.md +++ b/clouds/snowflake/modules/doc/measurements/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # measurements diff --git a/clouds/snowflake/modules/doc/placekey/_INTRO.md b/clouds/snowflake/modules/doc/placekey/_INTRO.md index 06a5d85ff..6fb0561ae 100644 --- a/clouds/snowflake/modules/doc/placekey/_INTRO.md +++ b/clouds/snowflake/modules/doc/placekey/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # placekey diff --git a/clouds/snowflake/modules/doc/processing/_INTRO.md b/clouds/snowflake/modules/doc/processing/_INTRO.md index 5736464d3..1f1048fe9 100644 --- a/clouds/snowflake/modules/doc/processing/_INTRO.md +++ b/clouds/snowflake/modules/doc/processing/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # processing diff --git a/clouds/snowflake/modules/doc/quadbin/_INTRO.md b/clouds/snowflake/modules/doc/quadbin/_INTRO.md index eba16b566..8a8f5c709 100644 --- a/clouds/snowflake/modules/doc/quadbin/_INTRO.md +++ b/clouds/snowflake/modules/doc/quadbin/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # quadbin diff --git a/clouds/snowflake/modules/doc/random/_INTRO.md b/clouds/snowflake/modules/doc/random/_INTRO.md index 357d0fdac..31ebf2175 100644 --- a/clouds/snowflake/modules/doc/random/_INTRO.md +++ b/clouds/snowflake/modules/doc/random/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # random diff --git a/clouds/snowflake/modules/doc/s2/_INTRO.md b/clouds/snowflake/modules/doc/s2/_INTRO.md index 5f84cc858..97a029864 100644 --- a/clouds/snowflake/modules/doc/s2/_INTRO.md +++ b/clouds/snowflake/modules/doc/s2/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # s2 diff --git a/clouds/snowflake/modules/doc/transformations/_INTRO.md b/clouds/snowflake/modules/doc/transformations/_INTRO.md index 96eeabdc9..c51442d91 100644 --- a/clouds/snowflake/modules/doc/transformations/_INTRO.md +++ b/clouds/snowflake/modules/doc/transformations/_INTRO.md @@ -1,6 +1,5 @@ --- badges: -- core --- # transformations From 62ecd7a1b160d619a0932ee6856fce42c195d4be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Fri, 28 Jun 2024 19:30:15 +0200 Subject: [PATCH 5/8] upper limit init_resolution with resolution --- clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql index 25ea7bf40..bd8419100 100644 --- a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql +++ b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql @@ -83,7 +83,7 @@ AS (( FROM __geog_area ) -- compute the average resolution for the initial polyfill - SELECT CAST((resolution + geog_resolution) / 2 AS INT64) + SELECT LEAST(resolution, CAST((resolution + geog_resolution) / 2 AS INT64)) FROM __geog_resolution )); From 87d79ea093522717ae5d23133ad27fcd7a9e982f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Sun, 30 Jun 2024 13:41:08 +0200 Subject: [PATCH 6/8] optimize use_contains limit --- .../modules/sql/quadbin/QUADBIN_POLYFILL.sql | 85 +++++++++---------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql index bd8419100..024604a3e 100644 --- a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql +++ b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql @@ -70,47 +70,31 @@ AS (( WHERE ST_INTERSECTS(geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(quadbin)) )); -CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__INIT_RESOLUTION` +CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__GEOG_RESOLUTION` (geog GEOGRAPHY, resolution INT64) RETURNS INT64 AS (( WITH __geog_area AS ( - SELECT ST_AREA(geog) AS geog_area - ), - __geog_resolution AS ( - -- compute the resolution of cells that match the geog area - SELECT IF(geog_area > 0, CAST(-LOG(geog_area / 508164597540055.75, 4) AS INT64), resolution) AS geog_resolution - FROM __geog_area - ) - -- compute the average resolution for the initial polyfill - SELECT LEAST(resolution, CAST((resolution + geog_resolution) / 2 AS INT64)) - FROM __geog_resolution -)); - -CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__USE_PARENT_CONTAINS` -(geog GEOGRAPHY) -RETURNS BOOL -AS (( - -- the algorithm using the parent contains can be used if the - -- area is more than 20% of the max coverage area, otherwise, - -- the computation of the contains is not worth it - WITH __geog_area AS ( - SELECT ST_AREA(geog) AS geog_area, - POW(ST_MAXDISTANCE(geog, geog), 2) AS max_area + SELECT + ST_AREA(geog) AS geog_area, + COS(ST_Y(ST_CENTROID(geog)) * ACOS(-1) / 180) AS cos_geog_lat, + 508164597540055.75 AS q0_area ) - SELECT IF(max_area > 0, geog_area / max_area > 0.2, FALSE) + -- compute the resolution of cells that match the geog area + SELECT IF(geog_area > 0 AND cos_geog_lat > 0, + CAST(-LOG(geog_area / q0_area / cos_geog_lat, 4) AS INT64), resolution) FROM __geog_area )); CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INTERSECTS` -(geog GEOGRAPHY, use_parent_contains BOOLEAN, resolution INT64) +(geog GEOGRAPHY, use_contains BOOLEAN, init_resolution INT64, resolution INT64) RETURNS ARRAY AS (( WITH __parents AS ( - SELECT parent, IF(use_parent_contains, ST_CONTAINS( - geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)), FALSE) AS inside - FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`( - geog, `@@BQ_DATASET@@.__INIT_RESOLUTION`(geog, resolution))) AS parent + SELECT parent, COALESCE(ST_CONTAINS( + geog, IF(use_contains, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent), NULL) + ), FALSE) AS inside + FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`(geog, init_resolution)) AS parent ), __children AS ( SELECT child, inside @@ -140,14 +124,14 @@ AS (( )); CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CONTAINS` -(geog GEOGRAPHY, use_parent_contains BOOLEAN, resolution INT64) +(geog GEOGRAPHY, use_contains BOOLEAN, init_resolution INT64, resolution INT64) RETURNS ARRAY AS (( WITH __parents AS ( - SELECT parent, IF(use_parent_contains, ST_CONTAINS( - geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)), FALSE) AS inside - FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`( - geog, `@@BQ_DATASET@@.__INIT_RESOLUTION`(geog, resolution))) AS parent + SELECT parent, COALESCE(ST_CONTAINS( + geog, IF(use_contains, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent), NULL) + ), FALSE) AS inside + FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`(geog, init_resolution)) AS parent ), __children AS ( SELECT child, inside @@ -177,14 +161,14 @@ AS (( )); CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CENTER` -(geog GEOGRAPHY, use_parent_contains BOOLEAN, resolution INT64) +(geog GEOGRAPHY, use_contains BOOLEAN, init_resolution INT64, resolution INT64) RETURNS ARRAY AS (( WITH __parents AS ( - SELECT parent, IF(use_parent_contains, ST_CONTAINS( - geog, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent)), FALSE) AS inside - FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`( - geog, `@@BQ_DATASET@@.__INIT_RESOLUTION`(geog, resolution))) AS parent + SELECT parent, COALESCE(ST_CONTAINS( + geog, IF(use_contains, `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(parent), NULL) + ), FALSE) AS inside + FROM UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`(geog, init_resolution)) AS parent ), __children AS ( SELECT child, inside @@ -219,14 +203,27 @@ RETURNS ARRAY AS (( IF(resolution IS NULL OR resolution < 0 OR resolution > 26, ERROR('Invalid resolution, should be between 0 and 26'), - (SELECT CASE mode + ( + WITH __geog_params AS ( + SELECT + `@@BQ_DATASET@@.__GEOG_RESOLUTION`(geog, resolution) AS geog_resolution + ), + __params AS ( + SELECT + resolution - geog_resolution >= 10 AS use_contains, + LEAST(resolution, CAST((resolution + geog_resolution) / 2 AS INT64)) AS init_resolution + FROM __geog_params + ) + SELECT CASE mode WHEN 'intersects' THEN `@@BQ_DATASET@@.__QUADBIN_POLYFILL_INTERSECTS`( - geog, `@@BQ_DATASET@@.__USE_PARENT_CONTAINS`(geog), resolution) + geog, use_contains, init_resolution, resolution) WHEN 'contains' THEN `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CONTAINS`( - geog, `@@BQ_DATASET@@.__USE_PARENT_CONTAINS`(geog), resolution) + geog, use_contains, init_resolution, resolution) WHEN 'center' THEN `@@BQ_DATASET@@.__QUADBIN_POLYFILL_CENTER`( - geog, `@@BQ_DATASET@@.__USE_PARENT_CONTAINS`(geog), resolution) - END) + geog, use_contains, init_resolution, resolution) + END + FROM __params + ) ) )); From 396ac6bf4fe18128b4db6576f46b2a1335ba346a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Sun, 30 Jun 2024 16:02:23 +0200 Subject: [PATCH 7/8] update quadbin_polyfill_table --- .../modules/sql/quadbin/QUADBIN_POLYFILL_TABLE.sql | 12 +++--------- .../test/quadbin/QUADBIN_POLYFILL_TABLE.test.js | 9 +++------ 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL_TABLE.sql b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL_TABLE.sql index 65dd1f423..0c32ae2df 100644 --- a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL_TABLE.sql +++ b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL_TABLE.sql @@ -23,16 +23,10 @@ AS """ output_table = output_table.replace(/`/g, '') - const containmentFunction = (mode === 'contains') ? 'ST_CONTAINS' : 'ST_INTERSECTS' - const cellFunction = (mode === 'center') ? '@@BQ_DATASET@@.QUADBIN_CENTER' : '@@BQ_DATASET@@.QUADBIN_BOUNDARY' - return 'CREATE TABLE `' + output_table + '` CLUSTER BY (quadbin) AS\\n' + - 'WITH __input AS (' + input_query + '),\\n' + - '__cells AS (SELECT quadbin, i.* FROM __input AS i,\\n' + - 'UNNEST(`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT`(geom,`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z`(geom,' + resolution + '))) AS parent,\\n' + - 'UNNEST(`@@BQ_DATASET@@.QUADBIN_TOCHILDREN`(parent,' + resolution + ')) AS quadbin)\\n' + - 'SELECT * EXCEPT (geom) FROM __cells\\n' + - 'WHERE ' + containmentFunction + '(geom, `' + cellFunction + '`(quadbin));' + 'WITH __input AS (' + input_query + ')\\n' + + 'SELECT quadbin, i.* FROM __input AS i,\\n' + + 'UNNEST(`@@BQ_DATASET@@.QUADBIN_POLYFILL_MODE`(geom,' + resolution + ',\\'' + mode + '\\')) AS quadbin;' """; CREATE OR REPLACE PROCEDURE `@@BQ_DATASET@@.QUADBIN_POLYFILL_TABLE` diff --git a/clouds/bigquery/modules/test/quadbin/QUADBIN_POLYFILL_TABLE.test.js b/clouds/bigquery/modules/test/quadbin/QUADBIN_POLYFILL_TABLE.test.js index 977d996a9..19eb0ddfb 100644 --- a/clouds/bigquery/modules/test/quadbin/QUADBIN_POLYFILL_TABLE.test.js +++ b/clouds/bigquery/modules/test/quadbin/QUADBIN_POLYFILL_TABLE.test.js @@ -11,10 +11,7 @@ test('QUADBIN_POLYFILL_TABLE should generate the correct query', async () => { const rows = await runQuery(query); expect(rows.length).toEqual(1); expect(rows[0].output).toEqual(`CREATE TABLE \`..\` CLUSTER BY (quadbin) AS -WITH __input AS (SELECT geom, name, value FROM \`..\`), -__cells AS (SELECT quadbin, i.* FROM __input AS i, -UNNEST(\`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT\`(geom,\`@@BQ_DATASET@@.__QUADBIN_POLYFILL_INIT_Z\`(geom,12))) AS parent, -UNNEST(\`@@BQ_DATASET@@.QUADBIN_TOCHILDREN\`(parent,12)) AS quadbin) -SELECT * EXCEPT (geom) FROM __cells -WHERE ST_INTERSECTS(geom, \`@@BQ_DATASET@@.QUADBIN_CENTER\`(quadbin));`.replace(/@@BQ_DATASET@@/g, BQ_DATASET)); +WITH __input AS (SELECT geom, name, value FROM \`..
\`) +SELECT quadbin, i.* FROM __input AS i, +UNNEST(\`@@BQ_DATASET@@.QUADBIN_POLYFILL_MODE\`(geom,12,'center')) AS quadbin;`.replace(/@@BQ_DATASET@@/g, BQ_DATASET)); }); \ No newline at end of file From 7346173dd68d568575edf22461f2305c3f39e03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Tue, 2 Jul 2024 07:37:32 +0200 Subject: [PATCH 8/8] use intersects for the center --- clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql index 024604a3e..93b69f8b9 100644 --- a/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql +++ b/clouds/bigquery/modules/sql/quadbin/QUADBIN_POLYFILL.sql @@ -191,7 +191,7 @@ AS (( UNION ALL SELECT child FROM __children_border - WHERE ST_CONTAINS(geog, `@@BQ_DATASET@@.QUADBIN_CENTER`(child)) + WHERE ST_INTERSECTS(geog, `@@BQ_DATASET@@.QUADBIN_CENTER`(child)) ) SELECT ARRAY_AGG(child) FROM __cells