Skip to content

Commit

Permalink
Merge pull request #17 from stoplightio/master
Browse files Browse the repository at this point in the history 
add onLineBreak visit function
  • Loading branch information
@aeschli
aeschli committed Mar 21, 2019
2 parents 77ae9d4 + 85b5249 commit b9ecc3b
Show file tree
Hide file tree
Showing 5 changed files with 156 additions and 46 deletions.
34 changes: 21 additions & 13 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ export interface JSONScanner {
* The length of the last read token.
*/
getTokenLength(): number;
/**
* The zero-based start line number of the last read token.
*/
getTokenStartLine(): number;
/**
* The zero-based start character (column) of the last read token.
*/
getTokenStartCharacter(): number;
/**
* An error code of the last scan.
*/
Expand Down Expand Up @@ -96,39 +104,39 @@ export interface JSONVisitor {
/**
* Invoked when an open brace is encountered and an object is started. The offset and length represent the location of the open brace.
*/
onObjectBegin?: (offset: number, length: number) => void;
onObjectBegin?: (offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when a property is encountered. The offset and length represent the location of the property name.
*/
onObjectProperty?: (property: string, offset: number, length: number) => void;
onObjectProperty?: (property: string, offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when a closing brace is encountered and an object is completed. The offset and length represent the location of the closing brace.
*/
onObjectEnd?: (offset: number, length: number) => void;
onObjectEnd?: (offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when an open bracket is encountered. The offset and length represent the location of the open bracket.
*/
onArrayBegin?: (offset: number, length: number) => void;
onArrayBegin?: (offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when a closing bracket is encountered. The offset and length represent the location of the closing bracket.
*/
onArrayEnd?: (offset: number, length: number) => void;
onArrayEnd?: (offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when a literal value is encountered. The offset and length represent the location of the literal value.
*/
onLiteralValue?: (value: any, offset: number, length: number) => void;
onLiteralValue?: (value: any, offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when a comma or colon separator is encountered. The offset and length represent the location of the separator.
*/
onSeparator?: (charcter: string, offset: number, length: number) => void;
/**
* When comments are allowed, invoked when a line or block comment is encountered. The offset and length represent the location of the comment.
*/
onComment?: (offset: number, length: number) => void;
onSeparator?: (character: string, offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* When comments are allowed, invoked when a line or block comment is encountered. The offset and length represent the location of the comment.
*/
onComment?: (offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked on an error.
*/
onError?: (error: ParseErrorCode, offset: number, length: number) => void;
onError?: (error: ParseErrorCode, offset: number, length: number, startLine: number, startCharacter: number) => void;
}

/**
Expand Down Expand Up @@ -298,4 +306,4 @@ License

(MIT License)

Copyright 2018, Microsoft
Copyright 2018, Microsoft
23 changes: 16 additions & 7 deletions src/impl/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,17 @@

import { createScanner } from './scanner';
import {
ScanError, SyntaxKind, Node, NodeType, Edit, JSONPath, FormattingOptions,
ModificationOptions, ParseError, ParseErrorCode, Location, Segment, ParseOptions, JSONVisitor
JSONPath,
JSONVisitor,
Location,
Node,
NodeType,
ParseError,
ParseErrorCode,
ParseOptions,
ScanError,
Segment,
SyntaxKind
} from '../main';

namespace ParseOptions {
Expand Down Expand Up @@ -376,11 +385,11 @@ export function visit(text: string, visitor: JSONVisitor, options: ParseOptions

let _scanner = createScanner(text, false);

function toNoArgVisit(visitFunction?: (offset: number, length: number) => void): () => void {
return visitFunction ? () => visitFunction(_scanner.getTokenOffset(), _scanner.getTokenLength()) : () => true;
function toNoArgVisit(visitFunction?: (offset: number, length: number, startLine: number, startCharacter: number) => void): () => void {
return visitFunction ? () => visitFunction(_scanner.getTokenOffset(), _scanner.getTokenLength(), _scanner.getTokenStartLine(), _scanner.getTokenStartCharacter()) : () => true;
}
function toOneArgVisit<T>(visitFunction?: (arg: T, offset: number, length: number) => void): (arg: T) => void {
return visitFunction ? (arg: T) => visitFunction(arg, _scanner.getTokenOffset(), _scanner.getTokenLength()) : () => true;
function toOneArgVisit<T>(visitFunction?: (arg: T, offset: number, length: number, startLine: number, startCharacter: number) => void): (arg: T) => void {
return visitFunction ? (arg: T) => visitFunction(arg, _scanner.getTokenOffset(), _scanner.getTokenLength(), _scanner.getTokenStartLine(), _scanner.getTokenStartCharacter()) : () => true;
}

let onObjectBegin = toNoArgVisit(visitor.onObjectBegin),
Expand Down Expand Up @@ -650,4 +659,4 @@ function getLiteralNodeType(value: any): NodeType {
case 'string': return 'string';
default: return 'null';
}
}
}
22 changes: 21 additions & 1 deletion src/impl/scanner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ export function createScanner(text: string, ignoreTrivia: boolean = false): JSON
value: string = '',
tokenOffset = 0,
token: SyntaxKind = SyntaxKind.Unknown,
lineNumber = 0,
lineStartOffset = 0,
tokenLineStartOffset = 0,
prevTokenLineStartOffset = 0,
scanError: ScanError = ScanError.None;

function scanHexDigits(count: number, exact?: boolean): number {
Expand Down Expand Up @@ -179,6 +183,8 @@ export function createScanner(text: string, ignoreTrivia: boolean = false): JSON
scanError = ScanError.None;

tokenOffset = pos;
lineStartOffset = lineNumber;
prevTokenLineStartOffset = tokenLineStartOffset;

if (pos >= len) {
// at the end
Expand Down Expand Up @@ -206,6 +212,8 @@ export function createScanner(text: string, ignoreTrivia: boolean = false): JSON
pos++;
value += '\n';
}
lineNumber++;
tokenLineStartOffset = pos;
return token = SyntaxKind.LineBreakTrivia;
}

Expand Down Expand Up @@ -268,7 +276,17 @@ export function createScanner(text: string, ignoreTrivia: boolean = false): JSON
commentClosed = true;
break;
}

pos++;

if (isLineBreak(ch)) {
if (ch === CharacterCodes.carriageReturn && text.charCodeAt(pos) === CharacterCodes.lineFeed) {
pos++;
}

lineNumber++;
tokenLineStartOffset = pos;
}
}

if (!commentClosed) {
Expand Down Expand Up @@ -365,7 +383,9 @@ export function createScanner(text: string, ignoreTrivia: boolean = false): JSON
getTokenValue: () => value,
getTokenOffset: () => tokenOffset,
getTokenLength: () => pos - tokenOffset,
getTokenError: () => scanError
getTokenStartLine: () => lineStartOffset,
getTokenStartCharacter: () => tokenOffset - prevTokenLineStartOffset,
getTokenError: () => scanError,
};
}

Expand Down
26 changes: 17 additions & 9 deletions src/main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,14 @@ export interface JSONScanner {
* The length of the last read token.
*/
getTokenLength(): number;
/**
* The zero-based start line number of the last read token.
*/
getTokenStartLine(): number;
/**
* The zero-based start character (column) of the last read token.
*/
getTokenStartCharacter(): number;
/**
* An error code of the last scan.
*/
Expand Down Expand Up @@ -225,47 +233,47 @@ export interface JSONVisitor {
/**
* Invoked when an open brace is encountered and an object is started. The offset and length represent the location of the open brace.
*/
onObjectBegin?: (offset: number, length: number) => void;
onObjectBegin?: (offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when a property is encountered. The offset and length represent the location of the property name.
*/
onObjectProperty?: (property: string, offset: number, length: number) => void;
onObjectProperty?: (property: string, offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when a closing brace is encountered and an object is completed. The offset and length represent the location of the closing brace.
*/
onObjectEnd?: (offset: number, length: number) => void;
onObjectEnd?: (offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when an open bracket is encountered. The offset and length represent the location of the open bracket.
*/
onArrayBegin?: (offset: number, length: number) => void;
onArrayBegin?: (offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when a closing bracket is encountered. The offset and length represent the location of the closing bracket.
*/
onArrayEnd?: (offset: number, length: number) => void;
onArrayEnd?: (offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when a literal value is encountered. The offset and length represent the location of the literal value.
*/
onLiteralValue?: (value: any, offset: number, length: number) => void;
onLiteralValue?: (value: any, offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when a comma or colon separator is encountered. The offset and length represent the location of the separator.
*/
onSeparator?: (character: string, offset: number, length: number) => void;
onSeparator?: (character: string, offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* When comments are allowed, invoked when a line or block comment is encountered. The offset and length represent the location of the comment.
*/
onComment?: (offset: number, length: number) => void;
onComment?: (offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked on an error.
*/
onError?: (error: ParseErrorCode, offset: number, length: number) => void;
onError?: (error: ParseErrorCode, offset: number, length: number, startLine: number, startCharacter: number) => void;
}

/**
Expand Down
97 changes: 81 additions & 16 deletions src/test/json.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,20 @@ function assertTree(input: string, expected: any, expectedErrors: ParseError[] =
interface VisitorCallback {
id: keyof JSONVisitor,
text: string;
startLine: number;
startCharacter: number;
arg?: any;
};
interface VisitorError extends ParseError {
startLine: number;
startCharacter: number;
}

function assertVisit(input: string, expected: VisitorCallback[], expectedErrors: ParseError[] = [], disallowComments = false): void {
let errors: ParseError[] = [];
function assertVisit(input: string, expected: VisitorCallback[], expectedErrors: VisitorError[] = [], disallowComments = false): void {
let errors: VisitorError[] = [];
let actuals: VisitorCallback[] = [];
let noArgHalder = (id: keyof JSONVisitor) => (offset: number, length: number) => actuals.push({ id, text: input.substr(offset, length) });
let oneArgHalder = (id: keyof JSONVisitor) => (arg: any, offset: number, length: number) => actuals.push({ id, text: input.substr(offset, length), arg });
let noArgHalder = (id: keyof JSONVisitor) => (offset: number, length: number, startLine: number, startCharacter: number) => actuals.push({ id, text: input.substr(offset, length), startLine, startCharacter });
let oneArgHalder = (id: keyof JSONVisitor) => (arg: any, offset: number, length: number, startLine: number, startCharacter: number) => actuals.push({ id, text: input.substr(offset, length), startLine, startCharacter, arg });
visit(input, {
onObjectBegin: noArgHalder('onObjectBegin'),
onObjectProperty: oneArgHalder('onObjectProperty'),
Expand All @@ -87,8 +93,8 @@ function assertVisit(input: string, expected: VisitorCallback[], expectedErrors:
onLiteralValue: oneArgHalder('onLiteralValue'),
onSeparator: oneArgHalder('onSeparator'),
onComment: noArgHalder('onComment'),
onError: (error: ParseErrorCode, offset: number, length: number) => {
errors.push({ error, offset, length })
onError: (error: ParseErrorCode, offset: number, length: number, startLine: number, startCharacter: number) => {
errors.push({ error, offset, length, startLine, startCharacter })
}
}, {
disallowComments
Expand Down Expand Up @@ -282,7 +288,7 @@ suite('JSON', () => {
assertInvalidParse('[ ,1, 2, 3, ]', [1, 2, 3]);
});

test('parse: disallow commments', () => {
test('parse: disallow comments', () => {
let options = { disallowComments: true };

assertValidParse('[ 1, 2, null, "foo" ]', [1, 2, null, 'foo'], options);
Expand Down Expand Up @@ -422,21 +428,80 @@ suite('JSON', () => {
});

test('visit: object', () => {
assertVisit('{ }', [{ id: 'onObjectBegin', text: '{' }, { id: 'onObjectEnd', text: '}' }]);
assertVisit('{ "foo": "bar" }', [{ id: 'onObjectBegin', text: '{' }, { id: 'onObjectProperty', text: '"foo"', arg: 'foo' }, { id: 'onSeparator', text: ':', arg: ':' }, { id: 'onLiteralValue', text: '"bar"', arg: 'bar' }, { id: 'onObjectEnd', text: '}' }]);
assertVisit('{ "foo": { "goo": 3 } }', [{ id: 'onObjectBegin', text: '{' }, { id: 'onObjectProperty', text: '"foo"', arg: 'foo' }, { id: 'onSeparator', text: ':', arg: ':' }, { id: 'onObjectBegin', text: '{' }, { id: 'onObjectProperty', text: '"goo"', arg: 'goo' }, { id: 'onSeparator', text: ':', arg: ':' }, { id: 'onLiteralValue', text: '3', arg: 3 }, { id: 'onObjectEnd', text: '}' }, { id: 'onObjectEnd', text: '}' }]);
assertVisit('{ }', [{ id: 'onObjectBegin', text: '{', startLine: 0, startCharacter: 0 }, { id: 'onObjectEnd', text: '}', startLine: 0, startCharacter: 2 }]);
assertVisit('{ "foo": "bar" }', [
{ id: 'onObjectBegin', text: '{', startLine: 0, startCharacter: 0 },
{ id: 'onObjectProperty', text: '"foo"', startLine: 0, startCharacter: 2, arg: 'foo' },
{ id: 'onSeparator', text: ':', startLine: 0, startCharacter: 7, arg: ':' },
{ id: 'onLiteralValue', text: '"bar"', startLine: 0, startCharacter: 9, arg: 'bar' },
{ id: 'onObjectEnd', text: '}', startLine: 0, startCharacter: 15 },
]);
assertVisit('{ "foo": { "goo": 3 } }', [
{ id: 'onObjectBegin', text: '{', startLine: 0, startCharacter: 0 },
{ id: 'onObjectProperty', text: '"foo"', startLine: 0, startCharacter: 2, arg: 'foo' },
{ id: 'onSeparator', text: ':', startLine: 0, startCharacter: 7, arg: ':' },
{ id: 'onObjectBegin', text: '{', startLine: 0, startCharacter: 9 },
{ id: 'onObjectProperty', text: '"goo"', startLine: 0, startCharacter: 11, arg: 'goo' },
{ id: 'onSeparator', text: ':', startLine: 0, startCharacter: 16, arg: ':' },
{ id: 'onLiteralValue', text: '3', startLine: 0, startCharacter: 18, arg: 3 },
{ id: 'onObjectEnd', text: '}', startLine: 0, startCharacter: 20 },
{ id: 'onObjectEnd', text: '}', startLine: 0, startCharacter: 22 },
]);
});

test('visit: array', () => {
assertVisit('[]', [{ id: 'onArrayBegin', text: '[' }, { id: 'onArrayEnd', text: ']' }]);
assertVisit('[ true, null, [] ]', [{ id: 'onArrayBegin', text: '[' }, { id: 'onLiteralValue', text: 'true', arg: true }, { id: 'onSeparator', text: ',', arg: ',' }, { id: 'onLiteralValue', text: 'null', arg: null }, { id: 'onSeparator', text: ',', arg: ',' }, { id: 'onArrayBegin', text: '[' }, { id: 'onArrayEnd', text: ']' }, { id: 'onArrayEnd', text: ']' }]);
assertVisit('[]', [{ id: 'onArrayBegin', text: '[', startLine: 0, startCharacter: 0 }, { id: 'onArrayEnd', text: ']', startLine: 0, startCharacter: 1 }]);
assertVisit('[ true, null, [] ]', [
{ id: 'onArrayBegin', text: '[', startLine: 0, startCharacter: 0 },
{ id: 'onLiteralValue', text: 'true', startLine: 0, startCharacter: 2, arg: true },
{ id: 'onSeparator', text: ',', startLine: 0, startCharacter: 6, arg: ',' },
{ id: 'onLiteralValue', text: 'null', startLine: 0, startCharacter: 8, arg: null },
{ id: 'onSeparator', text: ',', startLine: 0, startCharacter: 12, arg: ',' },
{ id: 'onArrayBegin', text: '[', startLine: 0, startCharacter: 14 },
{ id: 'onArrayEnd', text: ']', startLine: 0, startCharacter: 15 },
{ id: 'onArrayEnd', text: ']', startLine: 0, startCharacter: 17 },
]);
assertVisit('[\r\n0,\r\n1,\r\n2\r\n]', [
{ id: 'onArrayBegin', text: '[', startLine: 0, startCharacter: 0 },
{ id: 'onLiteralValue', text: '0', startLine: 1, startCharacter: 0, arg: 0 },
{ id: 'onSeparator', text: ',', startLine: 1, startCharacter: 1, arg: ',' },
{ id: 'onLiteralValue', text: '1', startLine: 2, startCharacter: 0, arg: 1 },
{ id: 'onSeparator', text: ',', startLine: 2, startCharacter: 1, arg: ',' },
{ id: 'onLiteralValue', text: '2', startLine: 3, startCharacter: 0, arg: 2 },
{ id: 'onArrayEnd', text: ']', startLine: 4, startCharacter: 0 }]);
});

test('visit: comment', () => {
assertVisit('/* g */ { "foo": //f\n"bar" }', [{ id: 'onComment', text: '/* g */' }, { id: 'onObjectBegin', text: '{' }, { id: 'onObjectProperty', text: '"foo"', arg: 'foo' }, { id: 'onSeparator', text: ':', arg: ':' }, { id: 'onComment', text: '//f' }, { id: 'onLiteralValue', text: '"bar"', arg: 'bar' }, { id: 'onObjectEnd', text: '}' }]);
assertVisit('/* g */ { "foo": //f\n"bar" }',
[{ id: 'onObjectBegin', text: '{' }, { id: 'onObjectProperty', text: '"foo"', arg: 'foo' }, { id: 'onSeparator', text: ':', arg: ':' }, { id: 'onLiteralValue', text: '"bar"', arg: 'bar' }, { id: 'onObjectEnd', text: '}' }],
[{ error: ParseErrorCode.InvalidCommentToken, offset: 0, length: 7 }, { error: ParseErrorCode.InvalidCommentToken, offset: 17, length: 3 }],
assertVisit('/* g */ { "foo": //f\n"bar" }', [
{ id: 'onComment', text: '/* g */', startLine: 0, startCharacter: 0 },
{ id: 'onObjectBegin', text: '{', startLine: 0, startCharacter: 8 },
{ id: 'onObjectProperty', text: '"foo"', startLine: 0, startCharacter: 10, arg: 'foo' },
{ id: 'onSeparator', text: ':', startLine: 0, startCharacter: 15, arg: ':' },
{ id: 'onComment', text: '//f', startLine: 0, startCharacter: 17 },
{ id: 'onLiteralValue', text: '"bar"', startLine: 1, startCharacter: 0, arg: 'bar' },
{ id: 'onObjectEnd', text: '}', startLine: 1, startCharacter: 6 },
]);
assertVisit('/* g\r\n */ { "foo": //f\n"bar" }', [
{ id: 'onComment', text: '/* g\r\n */', startLine: 0, startCharacter: 0 },
{ id: 'onObjectBegin', text: '{', startLine: 1, startCharacter: 4 },
{ id: 'onObjectProperty', text: '"foo"', startLine: 1, startCharacter: 6, arg: 'foo' },
{ id: 'onSeparator', text: ':', startLine: 1, startCharacter: 11, arg: ':' },
{ id: 'onComment', text: '//f', startLine: 1, startCharacter: 13 },
{ id: 'onLiteralValue', text: '"bar"', startLine: 2, startCharacter: 0, arg: 'bar' },
{ id: 'onObjectEnd', text: '}', startLine: 2, startCharacter: 6 },
]);
assertVisit('/* g\n */ { "foo": //f\n"bar"\n}',
[
{ id: 'onObjectBegin', text: '{', startLine: 1, startCharacter: 4 },
{ id: 'onObjectProperty', text: '"foo"', startLine: 1, startCharacter: 6, arg: 'foo' },
{ id: 'onSeparator', text: ':', startLine: 1, startCharacter: 11, arg: ':' },
{ id: 'onLiteralValue', text: '"bar"', startLine: 2, startCharacter: 0, arg: 'bar' },
{ id: 'onObjectEnd', text: '}', startLine: 3, startCharacter: 0 },
],
[
{ error: ParseErrorCode.InvalidCommentToken, offset: 0, length: 8, startLine: 0, startCharacter: 0 },
{ error: ParseErrorCode.InvalidCommentToken, offset: 18, length: 3, startLine: 1, startCharacter: 13 },
],
true);
});

Expand Down

0 comments on commit b9ecc3b

Please sign in to comment.