Skip to content

Commit

Permalink
feat: add custom protons options for limiting list/map sizes (#120)
Browse files Browse the repository at this point in the history
Adds the capability to define protons-specific custom options that control decoding behaviour initially around limiting the sizes of maps and lists.

```protobuf
// import the options definition - it will work without this but some
// code editor plugins may report an error if the def can't be found
import "protons.proto";

message Message {
  // define the size limit - here more than 10 repeated items will
  // cause decoding to fail
  repeated string repeatedField = 1 [(protons.options).limit = 10];
}
```

The definition is shipped with the `protons-runtime` module. There is a [pending PR](protocolbuffers/protobuf#14505) to reserve the `1186` field ID. This should be merged first and/or the field ID updated here if it changes due to that PR.

Fixes #113
  • Loading branch information
achingbrain authored Nov 1, 2023
1 parent cbfe768 commit a5ba36b
Show file tree
Hide file tree
Showing 8 changed files with 322 additions and 3 deletions.
1 change: 1 addition & 0 deletions packages/protons-runtime/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"type": "module",
"types": "./dist/src/index.d.ts",
"files": [
"protons.proto",
"src",
"dist",
"!dist/test",
Expand Down
15 changes: 15 additions & 0 deletions packages/protons-runtime/protons.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
syntax = "proto2";

import "google/protobuf/descriptor.proto";

package protons;

message ProtonsOptions {
// limit the number of repeated fields or map entries that will be decoded
optional int32 limit = 1;
}

// custom options available for use by protons
extend google.protobuf.FieldOptions {
optional ProtonsOptions options = 1186;
}
10 changes: 10 additions & 0 deletions packages/protons-runtime/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -326,3 +326,13 @@ export interface Reader {
*/
sfixed64String(): string
}

export class CodeError extends Error {
public code: string

constructor (message: string, code: string, options?: ErrorOptions) {
super(message, options)

this.code = code
}
}
6 changes: 6 additions & 0 deletions packages/protons/bin/protons.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ async function main (): Promise<void> {
Options
--output, -o Path to a directory to write transpiled typescript files into
--strict, -s Causes parsing warnings to become errors
--path, -p Adds a directory to the include path
Examples
$ protons ./path/to/file.proto ./path/to/other/file.proto
Expand All @@ -25,6 +26,11 @@ async function main (): Promise<void> {
strict: {
type: 'boolean',
shortFlag: 's'
},
path: {
type: 'string',
shortFlag: 'p',
isMultiple: true
}
}
})
Expand Down
41 changes: 38 additions & 3 deletions packages/protons/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,7 @@ interface FieldDef {
rule: string
optional: boolean
repeated: boolean
lengthLimit?: number
message: boolean
enum: boolean
map: boolean
Expand Down Expand Up @@ -685,13 +686,37 @@ export interface ${messageDef.name} {
const parseValue = `${decoderGenerators[type] == null ? `${codec}.decode(reader${type === 'message' ? ', reader.uint32()' : ''})` : decoderGenerators[type](jsTypeOverride)}`

if (fieldDef.map) {
return `case ${fieldDef.id}: {
let limit = ''

if (fieldDef.lengthLimit != null) {
moduleDef.imports.add('CodeError')

limit = `
if (obj.${fieldName}.size === ${fieldDef.lengthLimit}) {
throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_SIZE')
}
`
}

return `case ${fieldDef.id}: {${limit}
const entry = ${parseValue}
obj.${fieldName}.set(entry.key, entry.value)
break
}`
} else if (fieldDef.repeated) {
return `case ${fieldDef.id}: {
let limit = ''

if (fieldDef.lengthLimit != null) {
moduleDef.imports.add('CodeError')

limit = `
if (obj.${fieldName}.length === ${fieldDef.lengthLimit}) {
throw new CodeError('decode error - repeated field "${fieldName}" had too many elements', 'ERR_MAX_LENGTH')
}
`
}

return `case ${fieldDef.id}: {${limit}
obj.${fieldName}.push(${parseValue})
break
}`
Expand Down Expand Up @@ -801,6 +826,7 @@ function defineModule (def: ClassDef, flags: Flags): ModuleDef {
fieldDef.repeated = fieldDef.rule === 'repeated'
fieldDef.optional = !fieldDef.repeated && fieldDef.options?.proto3_optional === true
fieldDef.map = fieldDef.keyType != null
fieldDef.lengthLimit = fieldDef.options?.['(protons.options).limit']
fieldDef.proto2Required = false

if (fieldDef.rule === 'required') {
Expand Down Expand Up @@ -876,11 +902,20 @@ interface Flags {
* If true, warnings will be thrown as errors
*/
strict?: boolean

/**
* A list of directories to add to the include path
*/
path?: string[]
}

export async function generate (source: string, flags: Flags): Promise<void> {
// convert .protobuf to .json
const json = await promisify(pbjs)(['-t', 'json', source])
const json = await promisify(pbjs)([
'-t', 'json',
...(flags.path ?? []).map(p => ['--path', path.isAbsolute(p) ? p : path.resolve(process.cwd(), p)]).flat(),
source
])

if (json == null) {
throw new Error(`Could not convert ${source} to intermediate JSON format`)
Expand Down
11 changes: 11 additions & 0 deletions packages/protons/test/fixtures/protons-options.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
syntax = "proto3";

import "protons.proto";

message MessageWithSizeLimitedRepeatedField {
repeated string repeatedField = 1 [(protons.options).limit = 1];
}

message MessageWithSizeLimitedMap {
map<string, string> mapField = 1 [(protons.options).limit = 1];
}
213 changes: 213 additions & 0 deletions packages/protons/test/fixtures/protons-options.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
/* eslint-disable import/export */
/* eslint-disable complexity */
/* eslint-disable @typescript-eslint/no-namespace */
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
/* eslint-disable @typescript-eslint/no-empty-interface */

import { encodeMessage, decodeMessage, message, CodeError } from 'protons-runtime'
import type { Codec } from 'protons-runtime'
import type { Uint8ArrayList } from 'uint8arraylist'

export interface MessageWithSizeLimitedRepeatedField {
repeatedField: string[]
}

export namespace MessageWithSizeLimitedRepeatedField {
let _codec: Codec<MessageWithSizeLimitedRepeatedField>

export const codec = (): Codec<MessageWithSizeLimitedRepeatedField> => {
if (_codec == null) {
_codec = message<MessageWithSizeLimitedRepeatedField>((obj, w, opts = {}) => {
if (opts.lengthDelimited !== false) {
w.fork()
}

if (obj.repeatedField != null) {
for (const value of obj.repeatedField) {
w.uint32(10)
w.string(value)
}
}

if (opts.lengthDelimited !== false) {
w.ldelim()
}
}, (reader, length) => {
const obj: any = {
repeatedField: []
}

const end = length == null ? reader.len : reader.pos + length

while (reader.pos < end) {
const tag = reader.uint32()

switch (tag >>> 3) {
case 1: {
if (obj.repeatedField.length === 1) {
throw new CodeError('decode error - repeated field "repeatedField" had too many elements', 'ERR_MAX_LENGTH')
}

obj.repeatedField.push(reader.string())
break
}
default: {
reader.skipType(tag & 7)
break
}
}
}

return obj
})
}

return _codec
}

export const encode = (obj: Partial<MessageWithSizeLimitedRepeatedField>): Uint8Array => {
return encodeMessage(obj, MessageWithSizeLimitedRepeatedField.codec())
}

export const decode = (buf: Uint8Array | Uint8ArrayList): MessageWithSizeLimitedRepeatedField => {
return decodeMessage(buf, MessageWithSizeLimitedRepeatedField.codec())
}
}

export interface MessageWithSizeLimitedMap {
mapField: Map<string, string>
}

export namespace MessageWithSizeLimitedMap {
export interface MessageWithSizeLimitedMap$mapFieldEntry {
key: string
value: string
}

export namespace MessageWithSizeLimitedMap$mapFieldEntry {
let _codec: Codec<MessageWithSizeLimitedMap$mapFieldEntry>

export const codec = (): Codec<MessageWithSizeLimitedMap$mapFieldEntry> => {
if (_codec == null) {
_codec = message<MessageWithSizeLimitedMap$mapFieldEntry>((obj, w, opts = {}) => {
if (opts.lengthDelimited !== false) {
w.fork()
}

if ((obj.key != null && obj.key !== '')) {
w.uint32(10)
w.string(obj.key)
}

if ((obj.value != null && obj.value !== '')) {
w.uint32(18)
w.string(obj.value)
}

if (opts.lengthDelimited !== false) {
w.ldelim()
}
}, (reader, length) => {
const obj: any = {
key: '',
value: ''
}

const end = length == null ? reader.len : reader.pos + length

while (reader.pos < end) {
const tag = reader.uint32()

switch (tag >>> 3) {
case 1: {
obj.key = reader.string()
break
}
case 2: {
obj.value = reader.string()
break
}
default: {
reader.skipType(tag & 7)
break
}
}
}

return obj
})
}

return _codec
}

export const encode = (obj: Partial<MessageWithSizeLimitedMap$mapFieldEntry>): Uint8Array => {
return encodeMessage(obj, MessageWithSizeLimitedMap$mapFieldEntry.codec())
}

export const decode = (buf: Uint8Array | Uint8ArrayList): MessageWithSizeLimitedMap$mapFieldEntry => {
return decodeMessage(buf, MessageWithSizeLimitedMap$mapFieldEntry.codec())
}
}

let _codec: Codec<MessageWithSizeLimitedMap>

export const codec = (): Codec<MessageWithSizeLimitedMap> => {
if (_codec == null) {
_codec = message<MessageWithSizeLimitedMap>((obj, w, opts = {}) => {
if (opts.lengthDelimited !== false) {
w.fork()
}

if (obj.mapField != null && obj.mapField.size !== 0) {
for (const [key, value] of obj.mapField.entries()) {
w.uint32(10)
MessageWithSizeLimitedMap.MessageWithSizeLimitedMap$mapFieldEntry.codec().encode({ key, value }, w)
}
}

if (opts.lengthDelimited !== false) {
w.ldelim()
}
}, (reader, length) => {
const obj: any = {
mapField: new Map<string, string>()
}

const end = length == null ? reader.len : reader.pos + length

while (reader.pos < end) {
const tag = reader.uint32()

switch (tag >>> 3) {
case 1: {
if (obj.mapField.size === 1) {
throw new CodeError('decode error - map field "mapField" had too many elements', 'ERR_MAX_SIZE')
}

const entry = MessageWithSizeLimitedMap.MessageWithSizeLimitedMap$mapFieldEntry.codec().decode(reader, reader.uint32())
obj.mapField.set(entry.key, entry.value)
break
}
default: {
reader.skipType(tag & 7)
break
}
}
}

return obj
})
}

return _codec
}

export const encode = (obj: Partial<MessageWithSizeLimitedMap>): Uint8Array => {
return encodeMessage(obj, MessageWithSizeLimitedMap.codec())
}

export const decode = (buf: Uint8Array | Uint8ArrayList): MessageWithSizeLimitedMap => {
return decodeMessage(buf, MessageWithSizeLimitedMap.codec())
}
}
28 changes: 28 additions & 0 deletions packages/protons/test/protons-options.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/* eslint-env mocha */

import { expect } from 'aegir/chai'
import { MessageWithSizeLimitedMap, MessageWithSizeLimitedRepeatedField } from './fixtures/protons-options.js'

describe('protons options', () => {
it('should not decode message with map that is too big', () => {
const obj: MessageWithSizeLimitedMap = {
mapField: new Map<string, string>([['one', 'two'], ['three', 'four']])
}

const buf = MessageWithSizeLimitedMap.encode(obj)

expect(() => MessageWithSizeLimitedMap.decode(buf))
.to.throw().with.property('code', 'ERR_MAX_SIZE')
})

it('should not decode message with list that is too big', () => {
const obj: MessageWithSizeLimitedRepeatedField = {
repeatedField: ['0', '1']
}

const buf = MessageWithSizeLimitedRepeatedField.encode(obj)

expect(() => MessageWithSizeLimitedRepeatedField.decode(buf))
.to.throw().with.property('code', 'ERR_MAX_LENGTH')
})
})

0 comments on commit a5ba36b

Please sign in to comment.