From cc625558b54b394b6819add5860e54ea95e053ef Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Thu, 7 Jul 2016 18:47:21 -0700 Subject: [PATCH 1/3] Extract ArrayGroup --- js/data/array_group.js | 94 ++++++++++++++++++++++++++++++++ js/data/bucket.js | 84 +++++----------------------- test/js/data/fill_bucket.test.js | 12 ++-- 3 files changed, 113 insertions(+), 77 deletions(-) create mode 100644 js/data/array_group.js diff --git a/js/data/array_group.js b/js/data/array_group.js new file mode 100644 index 00000000000..ff10c1068b9 --- /dev/null +++ b/js/data/array_group.js @@ -0,0 +1,94 @@ +'use strict'; + +var util = require('../util/util'); + +module.exports = ArrayGroup; + +/** + * A class that manages vertex and element arrays for a range of features. It handles initialization, + * serialization for transfer to the main thread, and certain intervening mutations. + * + * Array elements are broken into array groups based on inherent limits of WebGL. Within a group is: + * + * * A "layout" vertex array, with fixed layout, containing values calculated from layout properties. + * * Zero, one, or two element arrays, with fixed layout, typically for eventual use in + * `gl.drawElements(gl.TRIANGLES, ...)`. + * * Zero or more "paint" vertex arrays keyed by layer ID, each with a dynamic layout which depends + * on which paint properties of that layer use data-driven-functions (property functions or + * property-and-zoom functions). Values are calculated by evaluating those functions. + * + * @private + */ +function ArrayGroup(arrayTypes) { + var LayoutVertexArrayType = arrayTypes.layoutVertexArrayType; + this.layoutVertexArray = new LayoutVertexArrayType(); + + var ElementArrayType = arrayTypes.elementArrayType; + if (ElementArrayType) this.elementArray = new ElementArrayType(); + + var ElementArrayType2 = arrayTypes.elementArrayType2; + if (ElementArrayType2) this.elementArray2 = new ElementArrayType2(); + + this.paintVertexArrays = util.mapObject(arrayTypes.paintVertexArrayTypes, function (PaintVertexArrayType) { + return new PaintVertexArrayType(); + }); +} + +/** + * The maximum size of a vertex array. This limit is imposed by WebGL's 16 bit + * addressing of vertex buffers. + * @private + * @readonly + */ +ArrayGroup.MAX_VERTEX_ARRAY_LENGTH = Math.pow(2, 16) - 1; + +ArrayGroup.prototype.hasCapacityFor = function(numVertices) { + return this.layoutVertexArray.length + numVertices <= ArrayGroup.MAX_VERTEX_ARRAY_LENGTH; +}; + +ArrayGroup.prototype.isEmpty = function() { + return this.layoutVertexArray.length === 0; +}; + +ArrayGroup.prototype.trim = function() { + this.layoutVertexArray.trim(); + + if (this.elementArray) { + this.elementArray.trim(); + } + + if (this.elementArray2) { + this.elementArray2.trim(); + } + + for (var layerName in this.paintVertexArrays) { + this.paintVertexArrays[layerName].trim(); + } +}; + +ArrayGroup.prototype.serialize = function() { + return { + layoutVertexArray: this.layoutVertexArray.serialize(), + elementArray: this.elementArray && this.elementArray.serialize(), + elementArray2: this.elementArray2 && this.elementArray2.serialize(), + paintVertexArrays: util.mapObject(this.paintVertexArrays, function(array) { + return array.serialize(); + }) + }; +}; + +ArrayGroup.prototype.getTransferables = function(transferables) { + transferables.push(this.layoutVertexArray.arrayBuffer); + + if (this.elementArray) { + transferables.push(this.elementArray.arrayBuffer); + } + + if (this.elementArray2) { + transferables.push(this.elementArray2.arrayBuffer); + } + + for (var layerName in this.paintVertexArrays) { + transferables.push(this.paintVertexArrays[layerName].arrayBuffer); + } +}; diff --git a/js/data/bucket.js b/js/data/bucket.js index 7a4dbb420dd..4542f70fc64 100644 --- a/js/data/bucket.js +++ b/js/data/bucket.js @@ -1,6 +1,7 @@ 'use strict'; var featureFilter = require('feature-filter'); +var ArrayGroup = require('./array_group'); var Buffer = require('./buffer'); var util = require('../util/util'); var StructArrayType = require('../util/struct_array'); @@ -41,14 +42,6 @@ Bucket.create = function(options) { */ Bucket.EXTENT = 8192; -/** - * The maximum size of a vertex array. This limit is imposed by WebGL's 16 bit - * addressing of vertex buffers. - * @private - * @readonly - */ -Bucket.MAX_VERTEX_ARRAY_LENGTH = Math.pow(2, 16) - 1; - /** * The `Bucket` class is the single point of knowledge about turning vector * tiles into WebGL buffers. @@ -144,15 +137,6 @@ Bucket.prototype.populateArrays = function() { * by `populateArrays` and its callees. * * Array groups are added to this.arrayGroups[programName]. - * An individual array group looks like: - * { - * index: number, - * layoutVertexArray: VertexArrayType, - * ?elementArray: ElementArrayType, - * ?elementArray2: ElementArrayType2, - * paintVertexArrays: { [layerName]: PaintArrayType, ... } - * } - * * * @private * @param {string} programName the name of the program associated with the buffer that will receive the vertices @@ -163,29 +147,15 @@ Bucket.prototype.prepareArrayGroup = function(programName, numVertices) { var groups = this.arrayGroups[programName]; var currentGroup = groups.length && groups[groups.length - 1]; - if (!currentGroup || currentGroup.layoutVertexArray.length + numVertices > Bucket.MAX_VERTEX_ARRAY_LENGTH) { - - var programInterface = this.programInterfaces[programName]; - var LayoutVertexArrayType = programInterface.layoutVertexArrayType; - - currentGroup = { - index: groups.length, - layoutVertexArray: new LayoutVertexArrayType(), - paintVertexArrays: {} - }; - - var ElementArrayType = programInterface.elementArrayType; - if (ElementArrayType) currentGroup.elementArray = new ElementArrayType(); - - var ElementArrayType2 = programInterface.elementArrayType2; - if (ElementArrayType2) currentGroup.elementArray2 = new ElementArrayType2(); + if (!currentGroup || !currentGroup.hasCapacityFor(numVertices)) { + currentGroup = new ArrayGroup({ + layoutVertexArrayType: this.programInterfaces[programName].layoutVertexArrayType, + elementArrayType: this.programInterfaces[programName].elementArrayType, + elementArrayType2: this.programInterfaces[programName].elementArrayType2, + paintVertexArrayTypes: this.paintVertexArrayTypes[programName] + }); - var paintVertexArrayTypes = this.paintVertexArrayTypes[programName]; - for (var i = 0; i < this.childLayers.length; i++) { - var layerName = this.childLayers[i].id; - var PaintVertexArrayType = paintVertexArrayTypes[layerName]; - currentGroup.paintVertexArrays[layerName] = new PaintVertexArrayType(); - } + currentGroup.index = groups.length; groups.push(currentGroup); } @@ -247,17 +217,7 @@ Bucket.prototype.trimArrays = function() { for (var programName in this.arrayGroups) { var arrayGroups = this.arrayGroups[programName]; for (var i = 0; i < arrayGroups.length; i++) { - var arrayGroup = arrayGroups[i]; - arrayGroup.layoutVertexArray.trim(); - if (arrayGroup.elementArray) { - arrayGroup.elementArray.trim(); - } - if (arrayGroup.elementArray2) { - arrayGroup.elementArray2.trim(); - } - for (var paintArray in arrayGroup.paintVertexArrays) { - arrayGroup.paintVertexArrays[paintArray].trim(); - } + arrayGroups[i].trim(); } } }; @@ -266,8 +226,7 @@ Bucket.prototype.isEmpty = function() { for (var programName in this.arrayGroups) { var arrayGroups = this.arrayGroups[programName]; for (var i = 0; i < arrayGroups.length; i++) { - var arrayGroup = arrayGroups[i]; - if (arrayGroup.layoutVertexArray.length > 0) { + if (!arrayGroups[i].isEmpty()) { return false; } } @@ -279,17 +238,7 @@ Bucket.prototype.getTransferables = function(transferables) { for (var programName in this.arrayGroups) { var arrayGroups = this.arrayGroups[programName]; for (var i = 0; i < arrayGroups.length; i++) { - var arrayGroup = arrayGroups[i]; - transferables.push(arrayGroup.layoutVertexArray.arrayBuffer); - if (arrayGroup.elementArray) { - transferables.push(arrayGroup.elementArray.arrayBuffer); - } - if (arrayGroup.elementArray2) { - transferables.push(arrayGroup.elementArray2.arrayBuffer); - } - for (var paintArray in arrayGroup.paintVertexArrays) { - transferables.push(arrayGroup.paintVertexArrays[paintArray].arrayBuffer); - } + arrayGroups[i].getTransferables(transferables); } } }; @@ -309,14 +258,7 @@ Bucket.prototype.serialize = function() { zoom: this.zoom, arrays: util.mapObject(this.arrayGroups, function(programArrayGroups) { return programArrayGroups.map(function(arrayGroup) { - return { - layoutVertexArray: arrayGroup.layoutVertexArray.serialize(), - elementArray: arrayGroup.elementArray && arrayGroup.elementArray.serialize(), - elementArray2: arrayGroup.elementArray2 && arrayGroup.elementArray2.serialize(), - paintVertexArrays: util.mapObject(arrayGroup.paintVertexArrays, function(array) { - return array.serialize(); - }) - }; + return arrayGroup.serialize(); }); }), paintVertexArrayTypes: util.mapObject(this.paintVertexArrayTypes, function(arrayTypes) { diff --git a/test/js/data/fill_bucket.test.js b/test/js/data/fill_bucket.test.js index a403a1b753a..d0faf097302 100644 --- a/test/js/data/fill_bucket.test.js +++ b/test/js/data/fill_bucket.test.js @@ -5,7 +5,7 @@ var fs = require('fs'); var Protobuf = require('pbf'); var VectorTile = require('vector-tile').VectorTile; var Point = require('point-geometry'); -var Bucket = require('../../../js/data/bucket'); +var ArrayGroup = require('../../../js/data/array_group'); var FillBucket = require('../../../js/data/bucket/fill_bucket'); var path = require('path'); var StyleLayer = require('../../../js/style/style_layer'); @@ -57,8 +57,8 @@ test('FillBucket', function(t) { test('FillBucket - feature split across array groups', function (t) { // temporarily reduce the max array length so we can test features // breaking across array groups without tests taking a _long_ time. - var prevMaxArrayLength = Bucket.MAX_VERTEX_ARRAY_LENGTH; - Bucket.MAX_VERTEX_ARRAY_LENGTH = 1023; + var prevMaxArrayLength = ArrayGroup.MAX_VERTEX_ARRAY_LENGTH; + ArrayGroup.MAX_VERTEX_ARRAY_LENGTH = 1023; var layer = new StyleLayer({ id: 'test', @@ -88,7 +88,7 @@ test('FillBucket - feature split across array groups', function (t) { // add a feature that will break across the group boundary (65536) bucket.addFeature(createFeature([ - Bucket.MAX_VERTEX_ARRAY_LENGTH - 20, // the first polygon fits within the bucket + ArrayGroup.MAX_VERTEX_ARRAY_LENGTH - 20, // the first polygon fits within the bucket 20 // but the second one breaks across the boundary. ].map(createPolygon))); @@ -100,7 +100,7 @@ test('FillBucket - feature split across array groups', function (t) { // feature and the first polygon of the second feature, and the second // group to include the _entire_ second polygon of the second feature. var expectedLengths = [ - 10 + (Bucket.MAX_VERTEX_ARRAY_LENGTH - 20), + 10 + (ArrayGroup.MAX_VERTEX_ARRAY_LENGTH - 20), 20 ]; t.equal(groups[0].paintVertexArrays.test.length, expectedLengths[0], 'group 0 length, paint'); @@ -126,7 +126,7 @@ test('FillBucket - feature split across array groups', function (t) { } // restore - Bucket.MAX_VERTEX_ARRAY_LENGTH = prevMaxArrayLength; + ArrayGroup.MAX_VERTEX_ARRAY_LENGTH = prevMaxArrayLength; t.end(); }); From 33891eba3d600d43bf5c70da5bcffea81ee6db2d Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Fri, 8 Jul 2016 10:19:21 -0700 Subject: [PATCH 2/3] Extract BufferGroup --- js/data/bucket.js | 55 ++++++----------------------------------- js/data/buffer_group.js | 53 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 47 deletions(-) create mode 100644 js/data/buffer_group.js diff --git a/js/data/bucket.js b/js/data/bucket.js index 4542f70fc64..26e2955ac4b 100644 --- a/js/data/bucket.js +++ b/js/data/bucket.js @@ -3,9 +3,9 @@ var featureFilter = require('feature-filter'); var ArrayGroup = require('./array_group'); var Buffer = require('./buffer'); +var BufferGroup = require('./buffer_group'); var util = require('../util/util'); var StructArrayType = require('../util/struct_array'); -var VertexArrayObject = require('../render/vertex_array_object'); var assert = require('assert'); module.exports = Bucket; @@ -78,39 +78,17 @@ function Bucket(options) { this.paintAttributes = createPaintAttributes(this); if (options.arrays) { - var childLayers = this.childLayers; var programInterfaces = this.programInterfaces; this.bufferGroups = util.mapObject(options.arrays, function(programArrayGroups, programName) { var programInterface = programInterfaces[programName]; var paintVertexArrayTypes = options.paintVertexArrayTypes[programName]; return programArrayGroups.map(function(arrayGroup) { - var group = { - layoutVertexBuffer: new Buffer(arrayGroup.layoutVertexArray, - programInterface.layoutVertexArrayType.serialize(), Buffer.BufferType.VERTEX), - paintVertexBuffers: util.mapObject(arrayGroup.paintVertexArrays, function(array, name) { - return new Buffer(array, paintVertexArrayTypes[name], Buffer.BufferType.VERTEX); - }), - vaos: {} - }; - - if (arrayGroup.elementArray) { - group.elementBuffer = new Buffer(arrayGroup.elementArray, - programInterface.elementArrayType.serialize(), Buffer.BufferType.ELEMENT); - } - - if (arrayGroup.elementArray2) { - group.elementBuffer2 = new Buffer(arrayGroup.elementArray2, - programInterface.elementArrayType2.serialize(), Buffer.BufferType.ELEMENT); - group.secondVaos = {}; - } - - for (var l = 0; l < childLayers.length; l++) { - var layerName = childLayers[l].id; - group.vaos[layerName] = new VertexArrayObject(); - if (arrayGroup.elementArray2) group.secondVaos[layerName] = new VertexArrayObject(); - } - - return group; + return new BufferGroup(arrayGroup, { + layoutVertexArrayType: programInterface.layoutVertexArrayType.serialize(), + elementArrayType: programInterface.elementArrayType && programInterface.elementArrayType.serialize(), + elementArrayType2: programInterface.elementArrayType2 && programInterface.elementArrayType2.serialize(), + paintVertexArrayTypes: paintVertexArrayTypes + }); }); }); } @@ -191,26 +169,9 @@ Bucket.prototype.destroy = function(gl) { for (var programName in this.bufferGroups) { var programBufferGroups = this.bufferGroups[programName]; for (var i = 0; i < programBufferGroups.length; i++) { - var bufferGroup = programBufferGroups[i]; - bufferGroup.layoutVertexBuffer.destroy(gl); - if (bufferGroup.elementBuffer) { - bufferGroup.elementBuffer.destroy(gl); - } - if (bufferGroup.elementBuffer2) { - bufferGroup.elementBuffer2.destroy(gl); - } - for (var n in bufferGroup.paintVertexBuffers) { - bufferGroup.paintVertexBuffers[n].destroy(gl); - } - for (var j in bufferGroup.vaos) { - bufferGroup.vaos[j].destroy(gl); - } - for (var k in bufferGroup.secondVaos) { - bufferGroup.secondVaos[k].destroy(gl); - } + programBufferGroups[i].destroy(gl); } } - }; Bucket.prototype.trimArrays = function() { diff --git a/js/data/buffer_group.js b/js/data/buffer_group.js new file mode 100644 index 00000000000..4a0f1b3bf95 --- /dev/null +++ b/js/data/buffer_group.js @@ -0,0 +1,53 @@ +'use strict'; + +var util = require('../util/util'); +var Buffer = require('./buffer'); +var VertexArrayObject = require('../render/vertex_array_object'); + +module.exports = BufferGroup; + +function BufferGroup(arrayGroup, arrayTypes) { + this.layoutVertexBuffer = new Buffer(arrayGroup.layoutVertexArray, + arrayTypes.layoutVertexArrayType, Buffer.BufferType.VERTEX); + + if (arrayGroup.elementArray) { + this.elementBuffer = new Buffer(arrayGroup.elementArray, + arrayTypes.elementArrayType, Buffer.BufferType.ELEMENT); + } + + var vaos = this.vaos = {}; + var secondVaos; + + if (arrayGroup.elementArray2) { + this.elementBuffer2 = new Buffer(arrayGroup.elementArray2, + arrayTypes.elementArrayType2, Buffer.BufferType.ELEMENT); + secondVaos = this.secondVaos = {}; + } + + this.paintVertexBuffers = util.mapObject(arrayGroup.paintVertexArrays, function(array, name) { + vaos[name] = new VertexArrayObject(); + if (arrayGroup.elementArray2) { + secondVaos[name] = new VertexArrayObject(); + } + return new Buffer(array, arrayTypes.paintVertexArrayTypes[name], Buffer.BufferType.VERTEX); + }); +} + +BufferGroup.prototype.destroy = function(gl) { + this.layoutVertexBuffer.destroy(gl); + if (this.elementBuffer) { + this.elementBuffer.destroy(gl); + } + if (this.elementBuffer2) { + this.elementBuffer2.destroy(gl); + } + for (var n in this.paintVertexBuffers) { + this.paintVertexBuffers[n].destroy(gl); + } + for (var j in this.vaos) { + this.vaos[j].destroy(gl); + } + for (var k in this.secondVaos) { + this.secondVaos[k].destroy(gl); + } +}; From 637d9a1986545f224f4c829d23ad4713bbae05fe Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Fri, 8 Jul 2016 11:04:42 -0700 Subject: [PATCH 3/3] Remove Buffer dependency from Bucket The constants are inlined into what are effective higher-abstracted constants. --- js/data/bucket.js | 15 ++++++++++++--- js/data/buffer.js | 16 ---------------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/js/data/bucket.js b/js/data/bucket.js index 26e2955ac4b..1272e3b8f86 100644 --- a/js/data/bucket.js +++ b/js/data/bucket.js @@ -2,7 +2,6 @@ var featureFilter = require('feature-filter'); var ArrayGroup = require('./array_group'); -var Buffer = require('./buffer'); var BufferGroup = require('./buffer_group'); var util = require('../util/util'); var StructArrayType = require('../util/struct_array'); @@ -278,17 +277,27 @@ Bucket.prototype.populatePaintArrays = function(interfaceName, globalProperties, } }; +/** + * A vertex array stores data for each vertex in a geometry. Elements are aligned to 4 byte + * boundaries for best performance in WebGL. + * @private + */ Bucket.VertexArrayType = function (members) { return new StructArrayType({ members: members, - alignment: Buffer.VERTEX_ATTRIBUTE_ALIGNMENT + alignment: 4 }); }; +/** + * An element array stores Uint16 indicies of vertexes in a corresponding vertex array. With no + * arguments, it defaults to three components per element, forming triangles. + * @private + */ Bucket.ElementArrayType = function (components) { return new StructArrayType({ members: [{ - type: Buffer.ELEMENT_ATTRIBUTE_TYPE, + type: 'Uint16', name: 'vertices', components: components || 3 }] diff --git a/js/data/buffer.js b/js/data/buffer.js index de59cd6881a..7593361f038 100644 --- a/js/data/buffer.js +++ b/js/data/buffer.js @@ -97,19 +97,3 @@ Buffer.BufferType = { VERTEX: 'ARRAY_BUFFER', ELEMENT: 'ELEMENT_ARRAY_BUFFER' }; - -/** - * An `BufferType.ELEMENT` buffer holds indicies of a corresponding `BufferType.VERTEX` buffer. - * These indicies are stored in the `BufferType.ELEMENT` buffer as `UNSIGNED_SHORT`s. - * - * @private - * @readonly - */ -Buffer.ELEMENT_ATTRIBUTE_TYPE = 'Uint16'; - -/** - * WebGL performs best if vertex attribute offsets are aligned to 4 byte boundaries. - * @private - * @readonly - */ -Buffer.VERTEX_ATTRIBUTE_ALIGNMENT = 4;