diff --git a/modules/carto/src/layers/raster-tile-layer.ts b/modules/carto/src/layers/raster-tile-layer.ts index 7a180ca3bb7..e6cac30bcb6 100644 --- a/modules/carto/src/layers/raster-tile-layer.ts +++ b/modules/carto/src/layers/raster-tile-layer.ts @@ -61,10 +61,10 @@ export default class RasterTileLayer< } renderLayers(): Layer | null | LayersList { - const tileJSON = this.props.data as TilejsonResult; + const tileJSON = this.props.data as TilejsonResult & {raster_metadata: any}; if (!tileJSON) return null; - const {tiles: data, minzoom: minZoom, maxzoom: maxZoom} = tileJSON; + const {tiles: data, minzoom: minZoom, maxzoom: maxZoom, raster_metadata: metadata} = tileJSON; const SubLayerClass = this.getSubLayerClass('tile', PostProcessTileLayer); return new SubLayerClass(this.props, { id: `raster-tile-layer-${this.props.id}`, @@ -74,7 +74,10 @@ export default class RasterTileLayer< renderSubLayers, minZoom, maxZoom, - loadOptions: this.getLoadOptions() + loadOptions: { + cartoRasterTile: {metadata}, + ...this.getLoadOptions() + } }); } } diff --git a/modules/carto/src/layers/schema/carto-properties-tile-loader.ts b/modules/carto/src/layers/schema/carto-properties-tile-loader.ts index 59ce9f2b347..4608fb8e2af 100644 --- a/modules/carto/src/layers/schema/carto-properties-tile-loader.ts +++ b/modules/carto/src/layers/schema/carto-properties-tile-loader.ts @@ -11,7 +11,7 @@ const CartoPropertiesTileLoader: LoaderWithParser = { extensions: ['pbf'], mimeTypes: ['application/vnd.carto-properties-tile'], category: 'geometry', - worker: false, + worker: true, parse: async (arrayBuffer, options) => parseCartoPropertiesTile(arrayBuffer, options), parseSync: parseCartoPropertiesTile, options: {} diff --git a/modules/carto/src/layers/schema/carto-raster-tile-loader.ts b/modules/carto/src/layers/schema/carto-raster-tile-loader.ts index f8e647f413e..33dbf4d091a 100644 --- a/modules/carto/src/layers/schema/carto-raster-tile-loader.ts +++ b/modules/carto/src/layers/schema/carto-raster-tile-loader.ts @@ -10,12 +10,14 @@ const id = 'cartoRasterTile'; type CartoRasterTileLoaderOptions = LoaderOptions & { cartoRasterTile?: { + metadata: {compression: 'gzip' | null} | null; workerUrl: string; }; }; const DEFAULT_OPTIONS: CartoRasterTileLoaderOptions = { cartoRasterTile: { + metadata: null, workerUrl: getWorkerUrl(id, VERSION) } }; @@ -48,8 +50,11 @@ function parseCartoRasterTile( arrayBuffer: ArrayBuffer, options?: CartoRasterTileLoaderOptions ): Raster | null { - if (!arrayBuffer) return null; - const {bands, blockSize} = parsePbf(arrayBuffer, TileReader); + const metadata = options?.cartoRasterTile?.metadata; + if (!arrayBuffer || !metadata) return null; + TileReader.compression = metadata.compression; + const out = parsePbf(arrayBuffer, TileReader); + const {bands, blockSize} = out; const numericProps = {}; for (let i = 0; i < bands.length; i++) { diff --git a/modules/carto/src/layers/schema/carto-raster-tile.ts b/modules/carto/src/layers/schema/carto-raster-tile.ts index 1d3ba5f53a6..d5eb73b87c5 100644 --- a/modules/carto/src/layers/schema/carto-raster-tile.ts +++ b/modules/carto/src/layers/schema/carto-raster-tile.ts @@ -29,12 +29,14 @@ export class BandReader { throw Error(`Invalid data type: ${obj.type}`); } obj.data = {}; - readPackedTypedArray(TypedArray, pbf, obj.data); + const {compression} = TileReader; + readPackedTypedArray(TypedArray, pbf, obj.data, {compression}); } } } export class TileReader { + public static compression: null | 'gzip'; static read(pbf, end) { return pbf.readFields(TileReader._readField, {blockSize: 0, bands: []}, end); } diff --git a/modules/carto/src/layers/schema/fast-pbf.ts b/modules/carto/src/layers/schema/fast-pbf.ts index 564fd7bcd41..a275381969e 100644 --- a/modules/carto/src/layers/schema/fast-pbf.ts +++ b/modules/carto/src/layers/schema/fast-pbf.ts @@ -1,7 +1,22 @@ +import {GZipCompression} from '@loaders.gl/compression'; + +type ReadPackedOptions = { + compression: null | 'gzip'; +}; + // Optimized (100X speed improvement) reading function for binary data -export function readPackedTypedArray(TypedArray, pbf, obj) { +export function readPackedTypedArray(TypedArray, pbf, obj, options?: ReadPackedOptions) { const end = pbf.type === 2 ? pbf.readVarint() + pbf.pos : pbf.pos + 1; - obj.value = new TypedArray(pbf.buf.buffer.slice(pbf.pos, end)); + const data = pbf.buf.buffer.slice(pbf.pos, end); + + if (options?.compression === 'gzip') { + const compression = new GZipCompression(); + const decompressedData = compression.decompressSync(data); + obj.value = new TypedArray(decompressedData); + } else { + obj.value = new TypedArray(data); + } + pbf.pos = end; return obj.value; } diff --git a/test/modules/carto/index.ts b/test/modules/carto/index.ts index dbcd51b0cb1..b9b4d55434e 100644 --- a/test/modules/carto/index.ts +++ b/test/modules/carto/index.ts @@ -6,7 +6,6 @@ import './api/parse-map.spec'; import './api/query.spec'; import './api/request-with-parameters.spec'; import './utils.spec'; -import './layers/carto-vector-tile.spec'; import './layers/h3-tile-layer.spec'; import './layers/h3-tileset-2d.spec'; import './layers/raster.spec'; @@ -29,6 +28,12 @@ import './sources/raster-source.spec'; import './sources/vector-query-source.spec'; import './sources/vector-table-source.spec'; import './sources/vector-tileset-source.spec'; +import './layers/schema/carto-properties-tile-loader.spec'; +import './layers/schema/carto-raster-tile-loader.spec'; +import './layers/schema/carto-raster-tile.spec'; +import './layers/schema/carto-spatial-tile-loader.spec'; +import './layers/schema/carto-vector-tile-loader.spec'; +import './layers/schema/carto-vector-tile.spec'; import './style/carto-color-bins.spec'; import './style/carto-color-categories.spec'; import './style/carto-color-continuous.spec'; diff --git a/test/modules/carto/layers/raster.spec.ts b/test/modules/carto/layers/raster.spec.ts index 881c6898605..d4262f847bb 100644 --- a/test/modules/carto/layers/raster.spec.ts +++ b/test/modules/carto/layers/raster.spec.ts @@ -9,7 +9,9 @@ import binaryRasterTileData from '../data/binaryRasterTile.json'; const BINARY_RASTER_TILE = new Uint8Array(binaryRasterTileData).buffer; test('Parse Carto Raster Tile', async t => { - const converted = CartoRasterTileLoader.parseSync(BINARY_RASTER_TILE, {}); + const converted = CartoRasterTileLoader.parseSync(BINARY_RASTER_TILE, { + cartoRasterTile: {metadata: {}} + }); const {numericProps} = converted.cells; const {band_1} = numericProps; diff --git a/test/modules/carto/layers/schema/carto-properties-tile-loader.spec.ts b/test/modules/carto/layers/schema/carto-properties-tile-loader.spec.ts new file mode 100644 index 00000000000..2a458b98332 --- /dev/null +++ b/test/modules/carto/layers/schema/carto-properties-tile-loader.spec.ts @@ -0,0 +1,18 @@ +// deck.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import test from 'tape-promise/tape'; +import CartoPropertiesTileLoader from '@deck.gl/carto/layers/schema/carto-properties-tile-loader'; +import type {LoaderWithParser} from '@loaders.gl/loader-utils'; + +test('CartoPropertiesTileLoader', t => { + const loader = CartoPropertiesTileLoader as LoaderWithParser; + + t.ok(loader, 'CartoPropertiesTileLoader should be defined'); + t.equals(loader.name, 'CARTO Properties Tile', 'Should have correct name'); + t.equals(typeof loader.parse, 'function', 'Should have parse method'); + t.equals(typeof loader.parseSync, 'function', 'Should have parseSync method'); + t.equals(loader.worker, true, 'worker property should be true'); + t.end(); +}); diff --git a/test/modules/carto/layers/schema/carto-raster-tile-loader.spec.ts b/test/modules/carto/layers/schema/carto-raster-tile-loader.spec.ts new file mode 100644 index 00000000000..c352a115558 --- /dev/null +++ b/test/modules/carto/layers/schema/carto-raster-tile-loader.spec.ts @@ -0,0 +1,39 @@ +// deck.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import test from 'tape-promise/tape'; +import CartoRasterTileLoader from '@deck.gl/carto/layers/schema/carto-raster-tile-loader'; +import type {LoaderWithParser} from '@loaders.gl/loader-utils'; +import {BAND, COMPRESSED_BAND, TEST_DATA} from './carto-raster-tile.spec'; + +test('CartoRasterTileLoader', t => { + const loader = CartoRasterTileLoader as LoaderWithParser; + + t.ok(loader, 'CartoRasterTileLoader should be defined'); + t.equals(loader.name, 'CARTO Raster Tile', 'Should have correct name'); + t.equals(typeof loader.parse, 'function', 'Should have parse method'); + t.equals(typeof loader.parseSync, 'function', 'Should have parseSync method'); + t.equals(CartoRasterTileLoader.worker, true, 'worker property should be true'); + + const result = loader.parseSync!(TEST_DATA, {cartoRasterTile: {metadata: {compression: null}}}); + t.equals(result.blockSize, 256, 'Should return correct blockSize'); + t.ok(result.cells, 'Should return cells'); + t.ok(result.cells.numericProps, 'Should return numericProps'); + t.deepEqual( + result.cells.numericProps.band1.value, + COMPRESSED_BAND, + 'Should return compressed band' + ); + + // Repeat with compressed data + const result2 = loader.parseSync!(TEST_DATA, { + cartoRasterTile: {metadata: {compression: 'gzip'}} + }); + t.equals(result2.blockSize, 256, 'Should return correct blockSize'); + t.ok(result2.cells, 'Should return cells'); + t.ok(result2.cells.numericProps, 'Should return numericProps'); + t.deepEqual(result2.cells.numericProps.band1.value, BAND, 'Should return uncompressed band'); + + t.end(); +}); diff --git a/test/modules/carto/layers/schema/carto-raster-tile.spec.ts b/test/modules/carto/layers/schema/carto-raster-tile.spec.ts new file mode 100644 index 00000000000..bb2643e4ac8 --- /dev/null +++ b/test/modules/carto/layers/schema/carto-raster-tile.spec.ts @@ -0,0 +1,55 @@ +// deck.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import test from 'tape-promise/tape'; +import {TileReader} from '@deck.gl/carto/layers/schema/carto-raster-tile'; +import Pbf from 'pbf'; + +// GZIP compressed data for [1, 2, 3, 4] +export const BAND = [1, 2, 3, 4]; +export const COMPRESSED_BAND = [ + 31, 139, 8, 0, 0, 0, 0, 0, 0, 3, 99, 100, 98, 102, 1, 0, 205, 251, 60, 182, 4, 0, 0, 0 +]; +const buffer = new Pbf(); +buffer.writeVarintField(1, 256); // blockSize +buffer.writeMessage(2, (_, pbf) => { + // bands + pbf.writeStringField(1, 'band1'); + pbf.writeStringField(2, 'uint8'); + pbf.writeBytesField(3, new Uint8Array(COMPRESSED_BAND)); +}); +export const TEST_DATA = buffer.finish(); + +/** + * syntax = "proto3"; + * package carto; + * + * message Band { + * string name = 1; + * string type = 2; + * bytes data = 3; + * } + * + * message Tile { + * uint32 blockSize = 1; + * repeated Band bands = 2; + * } + */ +test('TileReader', t => { + const tile = TileReader.read(new Pbf(TEST_DATA), TEST_DATA.byteLength); + t.equals(tile.blockSize, 256, 'Should read blockSize correctly'); + t.equals(tile.bands.length, 1, 'Should have one band'); + t.equals(tile.bands[0].name, 'band1', 'Band should have correct name'); + t.deepEqual(tile.bands[0].data.value, COMPRESSED_BAND, 'Band should have compressed data'); + + // Repeat with compressed data + TileReader.compression = 'gzip'; + const tile2 = TileReader.read(new Pbf(TEST_DATA), TEST_DATA.byteLength); + t.equals(tile.blockSize, 256, 'Should read blockSize correctly'); + t.equals(tile.bands.length, 1, 'Should have one band'); + t.equals(tile.bands[0].name, 'band1', 'Band should have correct name'); + t.deepEqual(tile2.bands[0].data.value, BAND, 'Band should have decompressed data'); + + t.end(); +}); diff --git a/test/modules/carto/layers/schema/carto-spatial-tile-loader.spec.ts b/test/modules/carto/layers/schema/carto-spatial-tile-loader.spec.ts new file mode 100644 index 00000000000..e84183b6370 --- /dev/null +++ b/test/modules/carto/layers/schema/carto-spatial-tile-loader.spec.ts @@ -0,0 +1,18 @@ +// deck.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import test from 'tape-promise/tape'; +import CartoSpatialTileLoader from '@deck.gl/carto/layers/schema/carto-spatial-tile-loader'; +import type {LoaderWithParser} from '@loaders.gl/loader-utils'; + +test('CartoSpatialTileLoader', t => { + const loader = CartoSpatialTileLoader as LoaderWithParser; + + t.ok(loader, 'CartoSpatialTileLoader should be defined'); + t.equals(loader.name, 'CARTO Spatial Tile', 'Should have correct name'); + t.equals(typeof loader.parse, 'function', 'Should have parse method'); + t.equals(typeof loader.parseSync, 'function', 'Should have parseSync method'); + t.equals(loader.worker, true, 'worker property should be true'); + t.end(); +}); diff --git a/test/modules/carto/layers/schema/carto-vector-tile-loader.spec.ts b/test/modules/carto/layers/schema/carto-vector-tile-loader.spec.ts new file mode 100644 index 00000000000..69d5dacabb3 --- /dev/null +++ b/test/modules/carto/layers/schema/carto-vector-tile-loader.spec.ts @@ -0,0 +1,18 @@ +// deck.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import test from 'tape-promise/tape'; +import CartoVectorTileLoader from '@deck.gl/carto/layers/schema/carto-vector-tile-loader'; +import type {LoaderWithParser} from '@loaders.gl/loader-utils'; + +test('CartoVectorTileLoader', t => { + const loader = CartoVectorTileLoader as LoaderWithParser; + + t.ok(loader, 'CartoVectorTileLoader should be defined'); + t.equals(loader.name, 'CARTO Vector Tile', 'Should have correct name'); + t.equals(typeof loader.parse, 'function', 'Should have parse method'); + t.equals(typeof loader.parseSync, 'function', 'Should have parseSync method'); + t.equals(loader.worker, true, 'worker property should be true'); + t.end(); +}); diff --git a/test/modules/carto/layers/carto-vector-tile.spec.ts b/test/modules/carto/layers/schema/carto-vector-tile.spec.ts similarity index 90% rename from test/modules/carto/layers/carto-vector-tile.spec.ts rename to test/modules/carto/layers/schema/carto-vector-tile.spec.ts index d95899cac7d..d2597f0059f 100644 --- a/test/modules/carto/layers/carto-vector-tile.spec.ts +++ b/test/modules/carto/layers/schema/carto-vector-tile.spec.ts @@ -3,8 +3,8 @@ import test from 'tape-promise/tape'; import CartoVectoTileLoader from '@deck.gl/carto/layers/schema/carto-vector-tile-loader'; // See test/modules/carto/responseToJson for details for creating test data -import binaryVectorTileData from '../data/binaryTilePolygon.json'; -import binaryNoTrianglesTileData from '../data/binaryTilePolygonNoTri.json'; +import binaryVectorTileData from '../../data/binaryTilePolygon.json'; +import binaryNoTrianglesTileData from '../../data/binaryTilePolygonNoTri.json'; const BINARY_VECTOR_TILE = new Uint8Array(binaryVectorTileData).buffer; const BINARY_VECTOR_TILE_NOTRI = new Uint8Array(binaryNoTrianglesTileData).buffer;