CARTO: Support gzip compression in RasterLayer (#9352)

modules/carto/src/layers/raster-tile-layer.ts

@@ -61,10 +61,10 @@ export default class RasterTileLayer<
   }
 
   renderLayers(): Layer | null | LayersList {
-    const tileJSON = this.props.data as TilejsonResult;
+    const tileJSON = this.props.data as TilejsonResult & {raster_metadata: any};
     if (!tileJSON) return null;
 
-    const {tiles: data, minzoom: minZoom, maxzoom: maxZoom} = tileJSON;
+    const {tiles: data, minzoom: minZoom, maxzoom: maxZoom, raster_metadata: metadata} = tileJSON;
     const SubLayerClass = this.getSubLayerClass('tile', PostProcessTileLayer);
     return new SubLayerClass(this.props, {
       id: `raster-tile-layer-${this.props.id}`,
@@ -74,7 +74,10 @@ export default class RasterTileLayer<
       renderSubLayers,
       minZoom,
       maxZoom,
-      loadOptions: this.getLoadOptions()
+      loadOptions: {
+        cartoRasterTile: {metadata},
+        ...this.getLoadOptions()
+      }
     });
   }
 }

modules/carto/src/layers/schema/carto-properties-tile-loader.ts

@@ -11,7 +11,7 @@ const CartoPropertiesTileLoader: LoaderWithParser = {
   extensions: ['pbf'],
   mimeTypes: ['application/vnd.carto-properties-tile'],
   category: 'geometry',
-  worker: false,
+  worker: true,
   parse: async (arrayBuffer, options) => parseCartoPropertiesTile(arrayBuffer, options),
   parseSync: parseCartoPropertiesTile,
   options: {}

modules/carto/src/layers/schema/carto-raster-tile-loader.ts

@@ -10,12 +10,14 @@ const id = 'cartoRasterTile';
 
 type CartoRasterTileLoaderOptions = LoaderOptions & {
   cartoRasterTile?: {
+    metadata: {compression: 'gzip' | null} | null;
     workerUrl: string;
   };
 };
 
 const DEFAULT_OPTIONS: CartoRasterTileLoaderOptions = {
   cartoRasterTile: {
+    metadata: null,
     workerUrl: getWorkerUrl(id, VERSION)
   }
 };
@@ -48,8 +50,11 @@ function parseCartoRasterTile(
   arrayBuffer: ArrayBuffer,
   options?: CartoRasterTileLoaderOptions
 ): Raster | null {
-  if (!arrayBuffer) return null;
-  const {bands, blockSize} = parsePbf(arrayBuffer, TileReader);
+  const metadata = options?.cartoRasterTile?.metadata;
+  if (!arrayBuffer || !metadata) return null;
+  TileReader.compression = metadata.compression;
+  const out = parsePbf(arrayBuffer, TileReader);
+  const {bands, blockSize} = out;
 
   const numericProps = {};
   for (let i = 0; i < bands.length; i++) {

modules/carto/src/layers/schema/carto-raster-tile.ts

@@ -29,12 +29,14 @@ export class BandReader {
         throw Error(`Invalid data type: ${obj.type}`);
       }
       obj.data = {};
-      readPackedTypedArray(TypedArray, pbf, obj.data);
+      const {compression} = TileReader;
+      readPackedTypedArray(TypedArray, pbf, obj.data, {compression});
     }
   }
 }
 
 export class TileReader {
+  public static compression: null | 'gzip';
   static read(pbf, end) {
     return pbf.readFields(TileReader._readField, {blockSize: 0, bands: []}, end);
   }

modules/carto/src/layers/schema/fast-pbf.ts

@@ -1,7 +1,22 @@
+import {GZipCompression} from '@loaders.gl/compression';
+
+type ReadPackedOptions = {
+  compression: null | 'gzip';
+};
+
 // Optimized (100X speed improvement) reading function for binary data
-export function readPackedTypedArray(TypedArray, pbf, obj) {
+export function readPackedTypedArray(TypedArray, pbf, obj, options?: ReadPackedOptions) {
   const end = pbf.type === 2 ? pbf.readVarint() + pbf.pos : pbf.pos + 1;
-  obj.value = new TypedArray(pbf.buf.buffer.slice(pbf.pos, end));
+  const data = pbf.buf.buffer.slice(pbf.pos, end);
+
+  if (options?.compression === 'gzip') {
+    const compression = new GZipCompression();
+    const decompressedData = compression.decompressSync(data);
+    obj.value = new TypedArray(decompressedData);
+  } else {
+    obj.value = new TypedArray(data);
+  }
+
   pbf.pos = end;
   return obj.value;
 }

test/modules/carto/index.ts

@@ -6,7 +6,6 @@ import './api/parse-map.spec';
 import './api/query.spec';
 import './api/request-with-parameters.spec';
 import './utils.spec';
-import './layers/carto-vector-tile.spec';
 import './layers/h3-tile-layer.spec';
 import './layers/h3-tileset-2d.spec';
 import './layers/raster.spec';
@@ -29,6 +28,12 @@ import './sources/raster-source.spec';
 import './sources/vector-query-source.spec';
 import './sources/vector-table-source.spec';
 import './sources/vector-tileset-source.spec';
+import './layers/schema/carto-properties-tile-loader.spec';
+import './layers/schema/carto-raster-tile-loader.spec';
+import './layers/schema/carto-raster-tile.spec';
+import './layers/schema/carto-spatial-tile-loader.spec';
+import './layers/schema/carto-vector-tile-loader.spec';
+import './layers/schema/carto-vector-tile.spec';
 import './style/carto-color-bins.spec';
 import './style/carto-color-categories.spec';
 import './style/carto-color-continuous.spec';

test/modules/carto/layers/raster.spec.ts

@@ -9,7 +9,9 @@ import binaryRasterTileData from '../data/binaryRasterTile.json';
 const BINARY_RASTER_TILE = new Uint8Array(binaryRasterTileData).buffer;
 
 test('Parse Carto Raster Tile', async t => {
-  const converted = CartoRasterTileLoader.parseSync(BINARY_RASTER_TILE, {});
+  const converted = CartoRasterTileLoader.parseSync(BINARY_RASTER_TILE, {
+    cartoRasterTile: {metadata: {}}
+  });
   const {numericProps} = converted.cells;
 
   const {band_1} = numericProps;

test/modules/carto/layers/schema/carto-properties-tile-loader.spec.ts

@@ -0,0 +1,18 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import test from 'tape-promise/tape';
+import CartoPropertiesTileLoader from '@deck.gl/carto/layers/schema/carto-properties-tile-loader';
+import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+
+test('CartoPropertiesTileLoader', t => {
+  const loader = CartoPropertiesTileLoader as LoaderWithParser;
+
+  t.ok(loader, 'CartoPropertiesTileLoader should be defined');
+  t.equals(loader.name, 'CARTO Properties Tile', 'Should have correct name');
+  t.equals(typeof loader.parse, 'function', 'Should have parse method');
+  t.equals(typeof loader.parseSync, 'function', 'Should have parseSync method');
+  t.equals(loader.worker, true, 'worker property should be true');
+  t.end();
+});

test/modules/carto/layers/schema/carto-raster-tile-loader.spec.ts

@@ -0,0 +1,39 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import test from 'tape-promise/tape';
+import CartoRasterTileLoader from '@deck.gl/carto/layers/schema/carto-raster-tile-loader';
+import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+import {BAND, COMPRESSED_BAND, TEST_DATA} from './carto-raster-tile.spec';
+
+test('CartoRasterTileLoader', t => {
+  const loader = CartoRasterTileLoader as LoaderWithParser;
+
+  t.ok(loader, 'CartoRasterTileLoader should be defined');
+  t.equals(loader.name, 'CARTO Raster Tile', 'Should have correct name');
+  t.equals(typeof loader.parse, 'function', 'Should have parse method');
+  t.equals(typeof loader.parseSync, 'function', 'Should have parseSync method');
+  t.equals(CartoRasterTileLoader.worker, true, 'worker property should be true');
+
+  const result = loader.parseSync!(TEST_DATA, {cartoRasterTile: {metadata: {compression: null}}});
+  t.equals(result.blockSize, 256, 'Should return correct blockSize');
+  t.ok(result.cells, 'Should return cells');
+  t.ok(result.cells.numericProps, 'Should return numericProps');
+  t.deepEqual(
+    result.cells.numericProps.band1.value,
+    COMPRESSED_BAND,
+    'Should return compressed band'
+  );
+
+  // Repeat with compressed data
+  const result2 = loader.parseSync!(TEST_DATA, {
+    cartoRasterTile: {metadata: {compression: 'gzip'}}
+  });
+  t.equals(result2.blockSize, 256, 'Should return correct blockSize');
+  t.ok(result2.cells, 'Should return cells');
+  t.ok(result2.cells.numericProps, 'Should return numericProps');
+  t.deepEqual(result2.cells.numericProps.band1.value, BAND, 'Should return uncompressed band');
+
+  t.end();
+});

test/modules/carto/layers/schema/carto-raster-tile.spec.ts

@@ -0,0 +1,55 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import test from 'tape-promise/tape';
+import {TileReader} from '@deck.gl/carto/layers/schema/carto-raster-tile';
+import Pbf from 'pbf';
+
+// GZIP compressed data for [1, 2, 3, 4]
+export const BAND = [1, 2, 3, 4];
+export const COMPRESSED_BAND = [
+  31, 139, 8, 0, 0, 0, 0, 0, 0, 3, 99, 100, 98, 102, 1, 0, 205, 251, 60, 182, 4, 0, 0, 0
+];
+const buffer = new Pbf();
+buffer.writeVarintField(1, 256); // blockSize
+buffer.writeMessage(2, (_, pbf) => {
+  // bands
+  pbf.writeStringField(1, 'band1');
+  pbf.writeStringField(2, 'uint8');
+  pbf.writeBytesField(3, new Uint8Array(COMPRESSED_BAND));
+});
+export const TEST_DATA = buffer.finish();
+
+/**
+ * syntax = "proto3";
+ * package carto;
+ *
+ * message Band {
+ *   string name = 1;
+ *   string type = 2;
+ *   bytes data = 3;
+ * }
+ *
+ * message Tile {
+ *   uint32 blockSize = 1;
+ *   repeated Band bands = 2;
+ * }
+ */
+test('TileReader', t => {
+  const tile = TileReader.read(new Pbf(TEST_DATA), TEST_DATA.byteLength);
+  t.equals(tile.blockSize, 256, 'Should read blockSize correctly');
+  t.equals(tile.bands.length, 1, 'Should have one band');
+  t.equals(tile.bands[0].name, 'band1', 'Band should have correct name');
+  t.deepEqual(tile.bands[0].data.value, COMPRESSED_BAND, 'Band should have compressed data');
+
+  // Repeat with compressed data
+  TileReader.compression = 'gzip';
+  const tile2 = TileReader.read(new Pbf(TEST_DATA), TEST_DATA.byteLength);
+  t.equals(tile.blockSize, 256, 'Should read blockSize correctly');
+  t.equals(tile.bands.length, 1, 'Should have one band');
+  t.equals(tile.bands[0].name, 'band1', 'Band should have correct name');
+  t.deepEqual(tile2.bands[0].data.value, BAND, 'Band should have decompressed data');
+
+  t.end();
+});

test/modules/carto/layers/schema/carto-spatial-tile-loader.spec.ts

@@ -0,0 +1,18 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import test from 'tape-promise/tape';
+import CartoSpatialTileLoader from '@deck.gl/carto/layers/schema/carto-spatial-tile-loader';
+import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+
+test('CartoSpatialTileLoader', t => {
+  const loader = CartoSpatialTileLoader as LoaderWithParser;
+
+  t.ok(loader, 'CartoSpatialTileLoader should be defined');
+  t.equals(loader.name, 'CARTO Spatial Tile', 'Should have correct name');
+  t.equals(typeof loader.parse, 'function', 'Should have parse method');
+  t.equals(typeof loader.parseSync, 'function', 'Should have parseSync method');
+  t.equals(loader.worker, true, 'worker property should be true');
+  t.end();
+});

test/modules/carto/layers/schema/carto-vector-tile-loader.spec.ts

@@ -0,0 +1,18 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import test from 'tape-promise/tape';
+import CartoVectorTileLoader from '@deck.gl/carto/layers/schema/carto-vector-tile-loader';
+import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+
+test('CartoVectorTileLoader', t => {
+  const loader = CartoVectorTileLoader as LoaderWithParser;
+
+  t.ok(loader, 'CartoVectorTileLoader should be defined');
+  t.equals(loader.name, 'CARTO Vector Tile', 'Should have correct name');
+  t.equals(typeof loader.parse, 'function', 'Should have parse method');
+  t.equals(typeof loader.parseSync, 'function', 'Should have parseSync method');
+  t.equals(loader.worker, true, 'worker property should be true');
+  t.end();
+});

test/modules/carto/layers/carto-vector-tile.spec.ts → test/modules/carto/layers/schema/carto-vector-tile.spec.ts

@@ -3,8 +3,8 @@ import test from 'tape-promise/tape';
 import CartoVectoTileLoader from '@deck.gl/carto/layers/schema/carto-vector-tile-loader';
 
 // See test/modules/carto/responseToJson for details for creating test data
-import binaryVectorTileData from '../data/binaryTilePolygon.json';
-import binaryNoTrianglesTileData from '../data/binaryTilePolygonNoTri.json';
+import binaryVectorTileData from '../../data/binaryTilePolygon.json';
+import binaryNoTrianglesTileData from '../../data/binaryTilePolygonNoTri.json';
 const BINARY_VECTOR_TILE = new Uint8Array(binaryVectorTileData).buffer;
 const BINARY_VECTOR_TILE_NOTRI = new Uint8Array(binaryNoTrianglesTileData).buffer;