Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[0.10] Clean up LoaderVisitor and IDL loading #287

Merged
merged 2 commits into from
Feb 24, 2020
Merged

[0.10] Clean up LoaderVisitor and IDL loading #287

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a3f30cd
Reduce forward reference lookups
mtdowling Feb 23, 2020
1e1e4fa
Move IDL specific loading logic to IDL
mtdowling Feb 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion smithy-model/src/main/java/software/amazon/smithy/model/loader/AstModelLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@ enum AstModelLoader {

void load(ObjectNode model, LoaderVisitor visitor) {
model.expectNoAdditionalProperties(TOP_LEVEL_PROPERTIES);
visitor.onOpenFile(model.getSourceLocation().getFilename());
loadMetadata(model, visitor);
loadShapes(model, visitor);
}
Expand Down
144 changes: 121 additions & 23 deletions smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
Expand All @@ -52,6 +53,8 @@
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import software.amazon.smithy.model.FromSourceLocation;
import software.amazon.smithy.model.SourceException;
import software.amazon.smithy.model.SourceLocation;
import software.amazon.smithy.model.node.ArrayNode;
import software.amazon.smithy.model.node.BooleanNode;
Expand Down Expand Up @@ -141,6 +144,9 @@ final class IdlModelLoader {
private final LoaderVisitor visitor;
private final List<Pair<String, Node>> pendingTraits = new ArrayList<>();

/** Map of shape aliases to their targets. */
private final Map<String, ShapeId> useShapes = new HashMap<>();

private Token current;
private DocComment pendingDocComment;
private String definedVersion;
Expand All @@ -152,7 +158,6 @@ private IdlModelLoader(String filename, SmithyModelLexer lexer, LoaderVisitor vi
this.filename = filename;
this.visitor = visitor;
this.lexer = lexer;
visitor.onOpenFile(filename);

while (!eof()) {
Token token = expect(UNQUOTED, ANNOTATION, CONTROL, DOC);
Expand Down Expand Up @@ -285,8 +290,7 @@ private void parseNamespace() {
throw syntax(format("Invalid namespace name `%s`", parsedNamespace));
}

visitor.onNamespace(parsedNamespace, current());
this.namespace = parsedNamespace;
namespace = parsedNamespace;
}

private void parseUseStatement() {
Expand All @@ -300,7 +304,13 @@ private void parseUseStatement() {

try {
Token namespaceToken = expect(UNQUOTED);
visitor.onUseShape(ShapeId.from(namespaceToken.lexeme), namespaceToken);
ShapeId target = ShapeId.from(namespaceToken.lexeme);
ShapeId previous = useShapes.put(target.getName(), target);
if (previous != null) {
String message = String.format("Cannot use name `%s` because it conflicts with `%s`",
target, previous);
throw new UseException(message, namespaceToken.getSourceLocation());
}
expectNewline();
} catch (ShapeIdSyntaxException e) {
throw syntax(e.getMessage());
Expand Down Expand Up @@ -385,7 +395,7 @@ private Pair<String, Node> parseTraitValue(Token token, TraitValueType type) {
requireNamespaceOrThrow();

// Resolve the trait name and ensure that the trait forms a syntactically valid value.
ShapeId.fromOptionalNamespace(visitor.getNamespace(), token.lexeme);
ShapeId.fromOptionalNamespace(namespace, token.lexeme);
Pair<String, Node> result = Pair.of(token.lexeme, parseTraitValueBody());

// `apply` doesn't require any specific token to follow.
Expand Down Expand Up @@ -481,11 +491,59 @@ private Node parseUnquotedNode(Token token) {
Pair<StringNode, Consumer<String>> pair = StringNode.createLazyString(
token.lexeme, token.getSourceLocation());
Consumer<String> consumer = pair.right;
visitor.onShapeTarget(token.lexeme, token, id -> consumer.accept(id.toString()));
onShapeTarget(token.lexeme, token, id -> consumer.accept(id.toString()));
return pair.left;
}
}

/**
* Resolve shape targets and tracks forward references.
*
* <p>Smithy models allow for forward references to shapes that have not
* yet been defined. Only after all shapes are loaded is the entire set
* of possible shape IDs known. This normally isn't a concern, but Smithy
* allows for public shapes defined in the prelude to be referenced by
* targets like members and resource identifiers without an absolute
* shape ID (for example, {@code String}). However, a relative prelude
* shape ID is only resolved for such a target if a shape of the same
* name was not defined in the same namespace in which the target
* was defined.
*
* <p>If a shape in the same namespace as the target has already been
* defined or if the target is absolute and cannot resolve to a prelude
* shape, the provided {@code resolver} is invoked immediately. Otherwise,
* the {@code resolver} is invoked inside of the {@link LoaderVisitor#onEnd}
* method only after all shapes have been declared.
*
* @param target Shape that is targeted.
* @param sourceLocation The location of where the target occurred.
* @param resolver The consumer to invoke once the shape ID is resolved.
*/
private void onShapeTarget(String target, FromSourceLocation sourceLocation, Consumer<ShapeId> resolver) {
// Account for aliased shapes.
if (useShapes.containsKey(target)) {
resolver.accept(useShapes.get(target));
return;
}

try {
// A namespace is not set when parsing metadata.
ShapeId expectedId = namespace == null
? ShapeId.from(target)
: ShapeId.fromOptionalNamespace(namespace, target);

if (isRealizedShapeId(expectedId, target)) {
// Account for previously seen shapes in this namespace, absolute shapes, and prelude namespaces
// always resolve to prelude shapes.
resolver.accept(expectedId);
} else {
visitor.addForwardReference(expectedId, resolver);
}
} catch (ShapeIdSyntaxException e) {
throw new SourceException("Error resolving shape target; " + e.getMessage(), sourceLocation, e);
}
}

private NumberNode parseNumber(Token token) {
if (token.lexeme.contains("e") || token.lexeme.contains(".")) {
return new NumberNode(Double.valueOf(token.lexeme), token.getSourceLocation());
Expand Down Expand Up @@ -546,14 +604,49 @@ private void parseSimpleShape(AbstractShapeBuilder builder) {
* @return Returns the parsed shape ID.
*/
private ShapeId parseShapeName() {
requireNamespaceOrThrow();
definedShapes = true;
Token nameToken = expect(UNQUOTED);
String name = nameToken.lexeme;
ShapeId id = visitor.onShapeDefName(name, nameToken);
pendingTraits.forEach(pair -> visitor.onTrait(id, pair.getLeft(), pair.getRight()));
pendingTraits.clear();
collectPendingDocString(id);
return id;

if (useShapes.containsKey(name)) {
String msg = String.format("shape name `%s` conflicts with imported shape `%s`",
name, useShapes.get(name));
throw new UseException(msg, nameToken);
}

try {
ShapeId id = ShapeId.fromRelative(namespace, name);
for (Pair<String, Node> pair : pendingTraits) {
onDeferredTrait(id, pair.getLeft(), pair.getRight());
}
pendingTraits.clear();
collectPendingDocString(id);
return id;
} catch (ShapeIdSyntaxException e) {
throw new ModelSyntaxException("Invalid shape name: " + name, nameToken);
}
}

/**
* Adds a trait to a shape after resolving all shape IDs.
*
* <p>Resolving the trait against a trait definition is deferred until
* the entire model is loaded. A namespace is required to have been set
* if the trait name is not absolute.
*
* @param target Shape to add the trait to.
* @param traitName Trait name to add.
* @param traitValue Trait value as a Node object.
*/
private void onDeferredTrait(ShapeId target, String traitName, Node traitValue) {
onShapeTarget(traitName, traitValue.getSourceLocation(), id -> visitor.onTrait(target, id, traitValue));
}

private boolean isRealizedShapeId(ShapeId expectedId, String target) {
return Objects.equals(namespace, Prelude.NAMESPACE)
|| visitor.hasDefinedShape(expectedId)
|| target.contains("#");
}

private void collectPendingDocString(ShapeId id) {
Expand Down Expand Up @@ -606,7 +699,7 @@ private void parseStructuredContents(String shapeType, ShapeId parent, Collectio
parseMember(memberId);
// Add the loaded traits on the member now that the ID is known.
for (Pair<String, Node> pair : memberTraits) {
visitor.onTrait(memberId, pair.getLeft(), pair.getRight());
onDeferredTrait(memberId, pair.getLeft(), pair.getRight());
}
memberTraits.clear();
collectPendingDocString(memberId);
Expand All @@ -630,7 +723,7 @@ private void parseMember(ShapeId memberId) {
MemberShape.Builder memberBuilder = MemberShape.builder().id(memberId).source(currentLocation());
visitor.onShape(memberBuilder);
Token targetToken = expect(UNQUOTED, QUOTED);
visitor.onShapeTarget(targetToken.lexeme, targetToken, memberBuilder::target);
onShapeTarget(targetToken.lexeme, targetToken, memberBuilder::target);
}

private void parseCollection(String shapeType, CollectionShape.Builder builder) {
Expand Down Expand Up @@ -658,12 +751,17 @@ private void parseMap() {
private void parseApply() {
requireNamespaceOrThrow();
// apply <ShapeName> @<trait>\n
String name = expect(UNQUOTED).lexeme;
ShapeId id = ShapeId.fromOptionalNamespace(visitor.getNamespace(), name);
Token nextToken = expect(UNQUOTED);
String name = nextToken.lexeme;
Token token = expect(ANNOTATION);
Pair<String, Node> trait = parseTraitValue(token, TraitValueType.APPLY);
visitor.onTrait(id, trait.getLeft(), trait.getRight());
expectNewline();

// First, resolve the targeted shape.
onShapeTarget(name, nextToken.getSourceLocation(), id -> {
// Next, resolve the trait ID.
onDeferredTrait(id, trait.getLeft(), trait.getRight());
});
}

/**
Expand Down Expand Up @@ -833,7 +931,7 @@ private void parseResource() {
for (Map.Entry<StringNode, Node> entry : ids.getMembers().entrySet()) {
String name = entry.getKey().getValue();
StringNode target = entry.getValue().expectStringNode();
visitor.onShapeTarget(target.getValue(), target, id -> builder.addIdentifier(name, id));
onShapeTarget(target.getValue(), target, id -> builder.addIdentifier(name, id));
}
});

Expand All @@ -853,14 +951,14 @@ private void parseOperation() {
ObjectNode node = parseObjectNode(opening.getSourceLocation(), RBRACE);
node.expectNoAdditionalProperties(OPERATION_PROPERTY_NAMES);
node.getStringMember("input").ifPresent(input -> {
visitor.onShapeTarget(input.getValue(), input, builder::input);
onShapeTarget(input.getValue(), input, builder::input);
});
node.getStringMember("output").ifPresent(output -> {
visitor.onShapeTarget(output.getValue(), output, builder::output);
onShapeTarget(output.getValue(), output, builder::output);
});
node.getArrayMember("errors").ifPresent(errors -> {
for (StringNode value : errors.getElementsAs(StringNode.class)) {
visitor.onShapeTarget(value.getValue(), value, builder::addError);
onShapeTarget(value.getValue(), value, builder::addError);
}
});
}
Expand All @@ -879,15 +977,15 @@ private void parseDeprecatedOperationSyntax(OperationShape.Builder builder) {

Token next = expect(RPAREN, UNQUOTED);
if (next.type == UNQUOTED) {
visitor.onShapeTarget(next.lexeme, next, builder::input);
onShapeTarget(next.lexeme, next, builder::input);
expect(RPAREN);
}

// Parse the optionally present return value.
if (test(RETURN)) {
expect(RETURN);
Token returnToken = expect(UNQUOTED);
visitor.onShapeTarget(returnToken.lexeme, returnToken, builder::output);
onShapeTarget(returnToken.lexeme, returnToken, builder::output);
}

// Parse the optionally present errors list.
Expand All @@ -897,7 +995,7 @@ private void parseDeprecatedOperationSyntax(OperationShape.Builder builder) {
if (!test(RBRACKET)) {
while (true) {
Token errorToken = expect(UNQUOTED);
visitor.onShapeTarget(errorToken.lexeme, errorToken, builder::addError);
onShapeTarget(errorToken.lexeme, errorToken, builder::addError);
if (test(RBRACKET)) {
break;
}
Expand Down
Loading