diff --git a/docs/source/1.0/spec/core/model.rst b/docs/source/1.0/spec/core/model.rst index 466585297a8..d5c4f4773cf 100644 --- a/docs/source/1.0/spec/core/model.rst +++ b/docs/source/1.0/spec/core/model.rst @@ -149,15 +149,31 @@ together to form a valid semantic model. Merging model files =================== -Implementations MUST take the following steps to merge models together to load -the semantic model: +Implementations MUST take the following steps when merging two or more +model files to form a semantic model: + +#. Merge the metadata objects of all model files using the steps defined in + :ref:`merging-metadata`. +#. Shapes defined in a single model file are added to the semantic model as-is. +#. Shapes with the same shape ID defined in multiple model files are + reconciled using the following rules: + + #. All conflicting shapes MUST have the same shape type. + #. Conflicting :ref:`aggregate shapes ` MUST contain the + same members that target the same shapes. + #. Conflicting :ref:`service shapes ` MUST contain the same + properties and target the same shapes. +#. Conflicting traits defined in shape definitions or through + :ref:`apply statements ` are reconciled using + :ref:`trait conflict resolution `. -#. Duplicate shape IDs, if found, MUST cause the model merge to fail. - See :ref:`shape-id-conflicts` for more information. -#. Merge any conflicting applied traits using - :ref:`trait conflict resolution `. -#. Merge the metadata objects of both models using the steps defined - in :ref:`merging-metadata`. +.. note:: + + *The following guidance is non-normative.* Because the Smithy IDL allows + forward references to shapes that have not yet been defined or shapes + that are defined in another model file, implementations likely need to + defer :ref:`resolving relative shape IDs ` to + absolute shape IDs until *all* model files are loaded. .. _metadata: @@ -485,6 +501,12 @@ To illustrate, ``com.Foo#baz`` and ``com.foo#BAZ`` are not allowed in the same semantic model. This restriction also extends to member names: ``com.foo#Baz$bar`` and ``com.foo#Baz$BAR`` are in conflict. +.. seealso:: + + :ref:`merging-models` for information on how conflicting shape + definitions for the same shape ID are handled when assembling the + semantic model from multiple model files. + .. _simple-types: @@ -2153,7 +2175,18 @@ immediately precede a shape. The following example applies the * Refer to the :ref:`JSON AST specification ` for a description of how traits are applied in the JSON AST. -.. rubric:: Applying traits externally +.. rubric:: Scope of member traits + +Traits that target :ref:`members ` apply only in the context of +the member shape and do not affect the shape targeted by the member. Traits +applied to a member supersede traits applied to the shape targeted by the +member and do not inherently conflict. + + +.. _apply-statements: + +Applying traits externally +-------------------------- Both the IDL and JSON AST model representations allow traits to be applied to shapes outside of a shape's definition. This is done using an @@ -2200,13 +2233,6 @@ The following example applies the :ref:`documentation-trait` and treated exactly the same as applying the trait inside of a shape definition. -.. rubric:: Scope of member traits - -Traits that target :ref:`members ` apply only in the context of -the member shape and do not affect the shape targeted by the member. Traits -applied to a member supersede traits applied to the shape targeted by the -member and do not inherently conflict. - .. _trait-conflict-resolution: diff --git a/smithy-jsonschema/src/test/resources/software/amazon/smithy/jsonschema/test-service.json b/smithy-jsonschema/src/test/resources/software/amazon/smithy/jsonschema/test-service.json index c5891d60a5a..0a9d34f6d33 100644 --- a/smithy-jsonschema/src/test/resources/software/amazon/smithy/jsonschema/test-service.json +++ b/smithy-jsonschema/src/test/resources/software/amazon/smithy/jsonschema/test-service.json @@ -126,7 +126,7 @@ "target": "example.rest#Map" }, "stringDateTime": { - "target": "smithy.api#StringDateTime" + "target": "smithy.example#StringDateTime" } } }, @@ -208,7 +208,7 @@ "example.rest#Timestamp": { "type": "timestamp" }, - "smithy.api#StringDateTime": { + "smithy.example#StringDateTime": { "type": "timestamp", "traits": { "smithy.api#timestampFormat": "date-time" diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/Model.java b/smithy-model/src/main/java/software/amazon/smithy/model/Model.java index 8b592438c88..9ffffacbfa5 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/Model.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/Model.java @@ -162,6 +162,15 @@ private TraitCache getTraitCache() { return cache; } + /** + * Gets the immutable set of {@code ShapeId} in the model. + * + * @return Returns the shape IDs. + */ + public Set getShapeIds() { + return shapeMap.keySet(); + } + /** * Gets a set of shapes in the model marked with a specific trait. * diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/AbstractMutableModelFile.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/AbstractMutableModelFile.java new file mode 100644 index 00000000000..4eb85d29c3e --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/AbstractMutableModelFile.java @@ -0,0 +1,162 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import software.amazon.smithy.model.SourceException; +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.shapes.AbstractShapeBuilder; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.ShapeType; +import software.amazon.smithy.model.traits.Trait; +import software.amazon.smithy.model.traits.TraitFactory; +import software.amazon.smithy.model.validation.ValidationEvent; + +/** + * Base class used for mutable model files. + */ +abstract class AbstractMutableModelFile implements ModelFile { + + protected final TraitContainer traitContainer; + + // A LinkedHashMap is used to maintain member order. + private final Map> shapes = new LinkedHashMap<>(); + private final List events = new ArrayList<>(); + private final MetadataContainer metadata = new MetadataContainer(events); + private final TraitFactory traitFactory; + + /** + * @param traitFactory Factory used to create traits when merging traits. + */ + AbstractMutableModelFile(TraitFactory traitFactory) { + this.traitFactory = Objects.requireNonNull(traitFactory, "traitFactory must not be null"); + traitContainer = new TraitContainer.TraitHashMap(traitFactory, events); + } + + /** + * Adds a shape to the ModelFile, checking for conflicts with other shapes. + * + * @param builder Shape builder to register. + */ + void onShape(AbstractShapeBuilder builder) { + if (shapes.containsKey(builder.getId())) { + AbstractShapeBuilder previous = shapes.get(builder.getId()); + // Duplicate shapes in the same model file are not allowed. + ValidationEvent event = LoaderUtils.onShapeConflict(builder.getId(), builder.getSourceLocation(), + previous.getSourceLocation()); + throw new SourceException(event.getMessage(), event.getSourceLocation()); + } + + shapes.put(builder.getId(), builder); + } + + /** + * Adds metadata to be reported by the ModelFile. + * + * @param key Metadata key to set. + * @param value Metadata value to set. + */ + final void putMetadata(String key, Node value) { + metadata.putMetadata(key, value); + } + + /** + * Invoked when a trait is to be reported by the ModelFile. + * + * @param target The shape the trait is applied to. + * @param trait The trait shape ID. + * @param value The node value of the trait. + */ + final void onTrait(ShapeId target, ShapeId trait, Node value) { + traitContainer.onTrait(target, trait, value); + } + + /** + * Invoked when a trait is to be reported by the ModelFile. + * + * @param target The shape the trait is applied to. + * @param trait The trait to apply to the shape. + */ + final void onTrait(ShapeId target, Trait trait) { + traitContainer.onTrait(target, trait); + } + + @Override + public final List events() { + return events; + } + + @Override + public final Map metadata() { + return metadata.getData(); + } + + @Override + public final Set shapeIds() { + return shapes.keySet(); + } + + @Override + public final Collection createShapes(TraitContainer resolvedTraits) { + List resolved = new ArrayList<>(shapes.size()); + + // Build members and add them to top-level shapes. + for (AbstractShapeBuilder builder : shapes.values()) { + if (builder instanceof MemberShape.Builder) { + ShapeId id = builder.getId(); + AbstractShapeBuilder container = shapes.get(id.withoutMember()); + if (container == null) { + throw new RuntimeException("Container shape not found for member: " + id); + } + for (Trait trait : resolvedTraits.getTraitsForShape(id).values()) { + builder.addTrait(trait); + } + container.addMember((MemberShape) builder.build()); + } + } + + // Build top-level shapes. + for (AbstractShapeBuilder builder : shapes.values()) { + if (!(builder instanceof MemberShape.Builder)) { + // Try/catch since shapes could have problems building, like an invalid Shape ID. + try { + for (Trait trait : resolvedTraits.getTraitsForShape(builder.getId()).values()) { + builder.addTrait(trait); + } + resolved.add(builder.build()); + } catch (SourceException e) { + events.add(ValidationEvent.fromSourceException(e).toBuilder() + .shapeId(builder.getId()).build()); + } + } + } + + return resolved; + } + + @Override + public final ShapeType getShapeType(ShapeId id) { + return shapes.containsKey(id) ? shapes.get(id).getShapeType() : null; + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/AstModelLoader.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/AstModelLoader.java index 8bbfd6be237..6249289e98b 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/AstModelLoader.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/AstModelLoader.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -50,6 +50,7 @@ import software.amazon.smithy.model.shapes.StructureShape; import software.amazon.smithy.model.shapes.TimestampShape; import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.TraitFactory; import software.amazon.smithy.model.validation.ValidationEvent; import software.amazon.smithy.utils.ListUtils; import software.amazon.smithy.utils.SetUtils; @@ -84,249 +85,252 @@ enum AstModelLoader { private static final Set SERVICE_PROPERTIES = SetUtils.of( TYPE, "version", "operations", "resources", TRAITS); - void load(ObjectNode model, LoaderVisitor visitor) { - visitor.checkForAdditionalProperties(model, null, TOP_LEVEL_PROPERTIES); - loadMetadata(model, visitor); - loadShapes(model, visitor); + ModelFile load(TraitFactory traitFactory, ObjectNode model) { + FullyResolvedModelFile modelFile = new FullyResolvedModelFile(traitFactory); + LoaderUtils.checkForAdditionalProperties(model, null, TOP_LEVEL_PROPERTIES, modelFile.events()); + loadMetadata(model, modelFile); + loadShapes(model, modelFile); + return modelFile; } - private void loadMetadata(ObjectNode model, LoaderVisitor visitor) { + private void loadMetadata(ObjectNode model, FullyResolvedModelFile modelFile) { try { model.getObjectMember(METADATA).ifPresent(metadata -> { for (Map.Entry entry : metadata.getStringMap().entrySet()) { - visitor.onMetadata(entry.getKey(), entry.getValue()); + modelFile.putMetadata(entry.getKey(), entry.getValue()); } }); } catch (SourceException e) { - visitor.onError(ValidationEvent.fromSourceException(e)); + modelFile.events().add(ValidationEvent.fromSourceException(e)); } } - private void loadShapes(ObjectNode model, LoaderVisitor visitor) { + private void loadShapes(ObjectNode model, FullyResolvedModelFile modelFile) { model.getObjectMember(SHAPES).ifPresent(shapes -> { for (Map.Entry entry : shapes.getMembers().entrySet()) { ShapeId id = entry.getKey().expectShapeId(); ObjectNode definition = entry.getValue().expectObjectNode(); String type = definition.expectStringMember(TYPE).getValue(); try { - loadShape(id, type, definition, visitor); + loadShape(id, type, definition, modelFile); } catch (SourceException e) { ValidationEvent event = ValidationEvent.fromSourceException(e).toBuilder().shapeId(id).build(); - visitor.onError(event); + modelFile.events().add(event); } } }); } - private void loadShape(ShapeId id, String type, ObjectNode value, LoaderVisitor visitor) { + private void loadShape(ShapeId id, String type, ObjectNode value, FullyResolvedModelFile modelFile) { switch (type) { case "blob": - loadSimpleShape(id, value, BlobShape.builder(), visitor); + loadSimpleShape(id, value, BlobShape.builder(), modelFile); break; case "boolean": - loadSimpleShape(id, value, BooleanShape.builder(), visitor); + loadSimpleShape(id, value, BooleanShape.builder(), modelFile); break; case "byte": - loadSimpleShape(id, value, ByteShape.builder(), visitor); + loadSimpleShape(id, value, ByteShape.builder(), modelFile); break; case "short": - loadSimpleShape(id, value, ShortShape.builder(), visitor); + loadSimpleShape(id, value, ShortShape.builder(), modelFile); break; case "integer": - loadSimpleShape(id, value, IntegerShape.builder(), visitor); + loadSimpleShape(id, value, IntegerShape.builder(), modelFile); break; case "long": - loadSimpleShape(id, value, LongShape.builder(), visitor); + loadSimpleShape(id, value, LongShape.builder(), modelFile); break; case "float": - loadSimpleShape(id, value, FloatShape.builder(), visitor); + loadSimpleShape(id, value, FloatShape.builder(), modelFile); break; case "double": - loadSimpleShape(id, value, DoubleShape.builder(), visitor); + loadSimpleShape(id, value, DoubleShape.builder(), modelFile); break; case "document": - loadSimpleShape(id, value, DocumentShape.builder(), visitor); + loadSimpleShape(id, value, DocumentShape.builder(), modelFile); break; case "bigDecimal": - loadSimpleShape(id, value, BigDecimalShape.builder(), visitor); + loadSimpleShape(id, value, BigDecimalShape.builder(), modelFile); break; case "bigInteger": - loadSimpleShape(id, value, BigIntegerShape.builder(), visitor); + loadSimpleShape(id, value, BigIntegerShape.builder(), modelFile); break; case "string": - loadSimpleShape(id, value, StringShape.builder(), visitor); + loadSimpleShape(id, value, StringShape.builder(), modelFile); break; case "timestamp": - loadSimpleShape(id, value, TimestampShape.builder(), visitor); + loadSimpleShape(id, value, TimestampShape.builder(), modelFile); break; case "list": - loadCollection(id, value, ListShape.builder(), visitor); + loadCollection(id, value, ListShape.builder(), modelFile); break; case "set": - loadCollection(id, value, SetShape.builder(), visitor); + loadCollection(id, value, SetShape.builder(), modelFile); break; case "map": - loadMap(id, value, visitor); + loadMap(id, value, modelFile); break; case "resource": - loadResource(id, value, visitor); + loadResource(id, value, modelFile); break; case "service": - loadService(id, value, visitor); + loadService(id, value, modelFile); break; case "structure": - loadStructure(id, value, visitor); + loadStructure(id, value, modelFile); break; case "union": - loadUnion(id, value, visitor); + loadUnion(id, value, modelFile); break; case "operation": - loadOperation(id, value, visitor); + loadOperation(id, value, modelFile); break; case "apply": - visitor.checkForAdditionalProperties(value, id, APPLY_PROPERTIES); - applyTraits(id, value.expectObjectMember(TRAITS), visitor); + LoaderUtils.checkForAdditionalProperties(value, id, APPLY_PROPERTIES, modelFile.events()); + applyTraits(id, value.expectObjectMember(TRAITS), modelFile); break; default: throw new SourceException("Invalid shape `type`: " + type, value); } } - private void applyTraits(ShapeId id, ObjectNode traits, LoaderVisitor visitor) { + private void applyTraits(ShapeId id, ObjectNode traits, FullyResolvedModelFile modelFile) { for (Map.Entry traitNode : traits.getMembers().entrySet()) { ShapeId traitId = traitNode.getKey().expectShapeId(); // JSON AST model traits are never considered annotation traits, meaning // that a null value provided in the AST is not coerced in the same way // as an omitted value in the IDL (e.g., "@foo"). - visitor.onTrait(id, traitId, traitNode.getValue()); + modelFile.onTrait(id, traitId, traitNode.getValue()); } } - private void applyShapeTraits(ShapeId id, ObjectNode node, LoaderVisitor visitor) { - node.getObjectMember(TRAITS).ifPresent(traits -> applyTraits(id, traits, visitor)); + private void applyShapeTraits(ShapeId id, ObjectNode node, FullyResolvedModelFile modelFile) { + node.getObjectMember(TRAITS).ifPresent(traits -> applyTraits(id, traits, modelFile)); } - private void loadMember(LoaderVisitor visitor, ShapeId id, ObjectNode targetNode) { - visitor.checkForAdditionalProperties(targetNode, id, MEMBER_PROPERTIES); + private void loadMember(FullyResolvedModelFile modelFile, ShapeId id, ObjectNode targetNode) { + LoaderUtils.checkForAdditionalProperties(targetNode, id, MEMBER_PROPERTIES, modelFile.events()); MemberShape.Builder builder = MemberShape.builder().source(targetNode.getSourceLocation()).id(id); ShapeId target = targetNode.expectStringMember(TARGET).expectShapeId(); builder.target(target); - applyShapeTraits(id, targetNode, visitor); - visitor.onShape(builder); + applyShapeTraits(id, targetNode, modelFile); + modelFile.onShape(builder); } private void loadCollection( ShapeId id, ObjectNode node, - CollectionShape.Builder builder, - LoaderVisitor visitor + CollectionShape.Builder builder, + FullyResolvedModelFile modelFile ) { - visitor.checkForAdditionalProperties(node, id, COLLECTION_PROPERTY_NAMES); - applyShapeTraits(id, node, visitor); - loadMember(visitor, id.withMember("member"), node.expectObjectMember("member")); - visitor.onShape(builder.id(id).source(node.getSourceLocation())); + LoaderUtils.checkForAdditionalProperties(node, id, COLLECTION_PROPERTY_NAMES, modelFile.events()); + applyShapeTraits(id, node, modelFile); + loadMember(modelFile, id.withMember("member"), node.expectObjectMember("member")); + modelFile.onShape(builder.id(id).source(node.getSourceLocation())); } - private void loadMap(ShapeId id, ObjectNode node, LoaderVisitor visitor) { - visitor.checkForAdditionalProperties(node, id, MAP_PROPERTY_NAMES); - loadMember(visitor, id.withMember("key"), node.expectObjectMember("key")); - loadMember(visitor, id.withMember("value"), node.expectObjectMember("value")); - applyShapeTraits(id, node, visitor); - visitor.onShape(MapShape.builder().id(id).source(node.getSourceLocation())); + private void loadMap(ShapeId id, ObjectNode node, FullyResolvedModelFile modelFile) { + LoaderUtils.checkForAdditionalProperties(node, id, MAP_PROPERTY_NAMES, modelFile.events()); + loadMember(modelFile, id.withMember("key"), node.expectObjectMember("key")); + loadMember(modelFile, id.withMember("value"), node.expectObjectMember("value")); + applyShapeTraits(id, node, modelFile); + modelFile.onShape(MapShape.builder().id(id).source(node.getSourceLocation())); } - private void loadOperation(ShapeId id, ObjectNode node, LoaderVisitor visitor) { - visitor.checkForAdditionalProperties(node, id, OPERATION_PROPERTY_NAMES); - applyShapeTraits(id, node, visitor); + private void loadOperation(ShapeId id, ObjectNode node, FullyResolvedModelFile modelFile) { + LoaderUtils.checkForAdditionalProperties(node, id, OPERATION_PROPERTY_NAMES, modelFile.events()); + applyShapeTraits(id, node, modelFile); OperationShape.Builder builder = OperationShape.builder() .id(id) .source(node.getSourceLocation()) - .addErrors(loadOptionalTargetList(visitor, id, node, "errors")); + .addErrors(loadOptionalTargetList(modelFile, id, node, "errors")); - loadOptionalTarget(visitor, id, node, "input").ifPresent(builder::input); - loadOptionalTarget(visitor, id, node, "output").ifPresent(builder::output); - visitor.onShape(builder); + loadOptionalTarget(modelFile, id, node, "input").ifPresent(builder::input); + loadOptionalTarget(modelFile, id, node, "output").ifPresent(builder::output); + modelFile.onShape(builder); } - private void loadResource(ShapeId id, ObjectNode node, LoaderVisitor visitor) { - visitor.checkForAdditionalProperties(node, id, RESOURCE_PROPERTIES); - applyShapeTraits(id, node, visitor); + private void loadResource(ShapeId id, ObjectNode node, FullyResolvedModelFile modelFile) { + LoaderUtils.checkForAdditionalProperties(node, id, RESOURCE_PROPERTIES, modelFile.events()); + applyShapeTraits(id, node, modelFile); ResourceShape.Builder builder = ResourceShape.builder().id(id).source(node.getSourceLocation()); - loadOptionalTarget(visitor, id, node, "put").ifPresent(builder::put); - loadOptionalTarget(visitor, id, node, "create").ifPresent(builder::create); - loadOptionalTarget(visitor, id, node, "read").ifPresent(builder::read); - loadOptionalTarget(visitor, id, node, "update").ifPresent(builder::update); - loadOptionalTarget(visitor, id, node, "delete").ifPresent(builder::delete); - loadOptionalTarget(visitor, id, node, "list").ifPresent(builder::list); - builder.operations(loadOptionalTargetList(visitor, id, node, "operations")); - builder.collectionOperations(loadOptionalTargetList(visitor, id, node, "collectionOperations")); - builder.resources(loadOptionalTargetList(visitor, id, node, "resources")); + loadOptionalTarget(modelFile, id, node, "put").ifPresent(builder::put); + loadOptionalTarget(modelFile, id, node, "create").ifPresent(builder::create); + loadOptionalTarget(modelFile, id, node, "read").ifPresent(builder::read); + loadOptionalTarget(modelFile, id, node, "update").ifPresent(builder::update); + loadOptionalTarget(modelFile, id, node, "delete").ifPresent(builder::delete); + loadOptionalTarget(modelFile, id, node, "list").ifPresent(builder::list); + builder.operations(loadOptionalTargetList(modelFile, id, node, "operations")); + builder.collectionOperations(loadOptionalTargetList(modelFile, id, node, "collectionOperations")); + builder.resources(loadOptionalTargetList(modelFile, id, node, "resources")); // Load identifiers and resolve forward references. node.getObjectMember("identifiers").ifPresent(ids -> { for (Map.Entry entry : ids.getMembers().entrySet()) { String name = entry.getKey().getValue(); - ShapeId target = loadReferenceBody(visitor, id, entry.getValue()); + ShapeId target = loadReferenceBody(modelFile, id, entry.getValue()); builder.addIdentifier(name, target); } }); - visitor.onShape(builder); + modelFile.onShape(builder); } - private void loadService(ShapeId id, ObjectNode node, LoaderVisitor visitor) { - visitor.checkForAdditionalProperties(node, id, SERVICE_PROPERTIES); - applyShapeTraits(id, node, visitor); + private void loadService(ShapeId id, ObjectNode node, FullyResolvedModelFile modelFile) { + LoaderUtils.checkForAdditionalProperties(node, id, SERVICE_PROPERTIES, modelFile.events()); + applyShapeTraits(id, node, modelFile); ServiceShape.Builder builder = new ServiceShape.Builder().id(id).source(node.getSourceLocation()); builder.version(node.expectStringMember("version").getValue()); - builder.operations(loadOptionalTargetList(visitor, id, node, "operations")); - builder.resources(loadOptionalTargetList(visitor, id, node, "resources")); - visitor.onShape(builder); + builder.operations(loadOptionalTargetList(modelFile, id, node, "operations")); + builder.resources(loadOptionalTargetList(modelFile, id, node, "resources")); + modelFile.onShape(builder); } private void loadSimpleShape( - ShapeId id, ObjectNode node, AbstractShapeBuilder builder, LoaderVisitor visitor) { - visitor.checkForAdditionalProperties(node, id, SIMPLE_PROPERTY_NAMES); - applyShapeTraits(id, node, visitor); - visitor.onShape(builder.id(id).source(node.getSourceLocation())); + ShapeId id, ObjectNode node, AbstractShapeBuilder builder, FullyResolvedModelFile modelFile) { + LoaderUtils.checkForAdditionalProperties(node, id, SIMPLE_PROPERTY_NAMES, modelFile.events()); + applyShapeTraits(id, node, modelFile); + modelFile.onShape(builder.id(id).source(node.getSourceLocation())); } - private void loadStructure(ShapeId id, ObjectNode node, LoaderVisitor visitor) { - visitor.checkForAdditionalProperties(node, id, STRUCTURE_AND_UNION_PROPERTY_NAMES); - visitor.onShape(StructureShape.builder().id(id).source(node.getSourceLocation())); - finishLoadingStructOrUnionMembers(id, node, visitor); + private void loadStructure(ShapeId id, ObjectNode node, FullyResolvedModelFile modelFile) { + LoaderUtils.checkForAdditionalProperties(node, id, STRUCTURE_AND_UNION_PROPERTY_NAMES, modelFile.events()); + modelFile.onShape(StructureShape.builder().id(id).source(node.getSourceLocation())); + finishLoadingStructOrUnionMembers(id, node, modelFile); } - private void loadUnion(ShapeId id, ObjectNode node, LoaderVisitor visitor) { - visitor.checkForAdditionalProperties(node, id, STRUCTURE_AND_UNION_PROPERTY_NAMES); - visitor.onShape(UnionShape.builder().id(id).source(node.getSourceLocation())); - finishLoadingStructOrUnionMembers(id, node, visitor); + private void loadUnion(ShapeId id, ObjectNode node, FullyResolvedModelFile modelFile) { + LoaderUtils.checkForAdditionalProperties(node, id, STRUCTURE_AND_UNION_PROPERTY_NAMES, modelFile.events()); + modelFile.onShape(UnionShape.builder().id(id).source(node.getSourceLocation())); + finishLoadingStructOrUnionMembers(id, node, modelFile); } - private void finishLoadingStructOrUnionMembers(ShapeId id, ObjectNode node, LoaderVisitor visitor) { - applyShapeTraits(id, node, visitor); + private void finishLoadingStructOrUnionMembers(ShapeId id, ObjectNode node, FullyResolvedModelFile modelFile) { + applyShapeTraits(id, node, modelFile); ObjectNode memberObject = node.getObjectMember(MEMBERS).orElse(Node.objectNode()); for (Map.Entry entry : memberObject.getStringMap().entrySet()) { - loadMember(visitor, id.withMember(entry.getKey()), entry.getValue().expectObjectNode()); + loadMember(modelFile, id.withMember(entry.getKey()), entry.getValue().expectObjectNode()); } } private Optional loadOptionalTarget( - LoaderVisitor visitor, ShapeId id, ObjectNode node, String member) { - return node.getObjectMember(member).map(r -> loadReferenceBody(visitor, id, r)); + FullyResolvedModelFile modelFile, ShapeId id, ObjectNode node, String member) { + return node.getObjectMember(member).map(r -> loadReferenceBody(modelFile, id, r)); } - private ShapeId loadReferenceBody(LoaderVisitor visitor, ShapeId id, Node reference) { + private ShapeId loadReferenceBody(FullyResolvedModelFile modelFile, ShapeId id, Node reference) { ObjectNode referenceObject = reference.expectObjectNode(); - visitor.checkForAdditionalProperties(referenceObject, id, REFERENCE_PROPERTIES); + LoaderUtils.checkForAdditionalProperties(referenceObject, id, REFERENCE_PROPERTIES, modelFile.events()); return referenceObject.expectStringMember(TARGET).expectShapeId(); } - private List loadOptionalTargetList(LoaderVisitor visitor, ShapeId id, ObjectNode node, String member) { + private List loadOptionalTargetList( + FullyResolvedModelFile modelFile, ShapeId id, ObjectNode node, String member) { return node.getArrayMember(member).map(array -> { List ids = new ArrayList<>(array.size()); for (Node element : array.getElements()) { - ids.add(loadReferenceBody(visitor, id, element)); + ids.add(loadReferenceBody(modelFile, id, element)); } return ids; }).orElseGet(Collections::emptyList); diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ForwardReferenceModelFile.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ForwardReferenceModelFile.java new file mode 100644 index 00000000000..70fa98dd48a --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ForwardReferenceModelFile.java @@ -0,0 +1,163 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import software.amazon.smithy.model.SourceLocation; +import software.amazon.smithy.model.shapes.AbstractShapeBuilder; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.ShapeType; +import software.amazon.smithy.model.traits.TraitFactory; +import software.amazon.smithy.utils.Pair; + +/** + * A ModelFile that contains forward references. + * + * @see IdlModelParser + */ +final class ForwardReferenceModelFile extends AbstractMutableModelFile { + + /** The nullable namespace. Is null until it's set. */ + private String namespace; + + /** A queue of forward references. A queue is used since references can be added during resolution. */ + private final Deque>>> forwardReferences + = new ArrayDeque<>(); + + private final Map> useShapes = new HashMap<>(); + + /** + * @param traitFactory Factory used to create traits when merging traits. + */ + ForwardReferenceModelFile(TraitFactory traitFactory) { + super(traitFactory); + } + + /** + * Get the currently set namespace. + * + * @return Returns the currently set namespace or {@code null} if not set. + */ + String namespace() { + return namespace; + } + + /** + * Sets the current namespace. + * + * @param namespace Namespace to set. + */ + void setNamespace(String namespace) { + this.namespace = namespace; + } + + /** + * Invoked when a shape is "used" in the model file. + * + * @param id Shape ID to use. + * @param location The source location of where this use occurred. + */ + void useShape(ShapeId id, SourceLocation location) { + // Duplicate use statements. + if (useShapes.containsKey(id.getName())) { + ShapeId previous = useShapes.get(id.getName()).left; + String message = String.format("Cannot use name `%s` because it conflicts with `%s`", id, previous); + throw new ModelSyntaxException(message, location); + } + + useShapes.put(id.getName(), Pair.of(id, location)); + } + + @Override + void onShape(AbstractShapeBuilder builder) { + if (useShapes.containsKey(builder.getId().getName())) { + ShapeId previous = useShapes.get(builder.getId().getName()).left; + String message = String.format("Shape name `%s` conflicts with imported shape `%s`", + builder.getId().getName(), previous); + throw new ModelSyntaxException(message, builder); + } + + super.onShape(builder); + } + + /** + * Adds a forward reference that will be resolved when + * {@link #resolveShapes} is called. + * + * @param name The name of the shape that needs to be resolved. + * @param consumer The consumer that receives the resolved shape ID. + */ + void addForwardReference(String name, Consumer consumer) { + forwardReferences.add(Pair.of(name, (id, typeProvider) -> consumer.accept(id))); + } + + /** + * Adds a forward reference that will be resolved when + * {@link #resolveShapes} is called. + * + *

This variant of {@code addForwardReference} issued when the consumer + * also needs to know the type of shape that is being resolved. For + * example, this is necessary in order to coerce annotation traits + * (traits that define no value) into the expected type for a shape + * (e.g., a list or object). + * + * @param name The name of the shape that needs to be resolved. + * @param consumer The consumer that receives the resolved shape ID. + */ + void addForwardReference(String name, BiConsumer> consumer) { + forwardReferences.add(Pair.of(name, consumer)); + } + + @Override + public TraitContainer resolveShapes(Set ids, Function typeProvider) { + while (!forwardReferences.isEmpty()) { + Pair>> pair = forwardReferences.pop(); + String name = pair.left; + BiConsumer> consumer = pair.right; + + ShapeId resolved; + // Use absolute IDs as-is. + if (name.contains("#")) { + resolved = ShapeId.from(name); + } else if (useShapes.containsKey(name)) { + // Check use statements. + resolved = useShapes.get(name).left; + } else { + // Check if there's a shape with this name in the current namespace. + resolved = ShapeId.from(namespace() + "#" + name); + + // If not defined in the namespace, then check the prelude. + if (!ids.contains(resolved)) { + ShapeId preludeTest = ShapeId.from(Prelude.NAMESPACE + '#' + name); + if (ids.contains(preludeTest)) { + resolved = preludeTest; + } + } + } + + consumer.accept(resolved, typeProvider); + } + + return traitContainer; + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/FullyResolvedModelFile.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/FullyResolvedModelFile.java new file mode 100644 index 00000000000..982c46085a0 --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/FullyResolvedModelFile.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import java.util.Collection; +import java.util.Set; +import java.util.function.Function; +import software.amazon.smithy.model.shapes.AbstractShapeBuilder; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.ShapeType; +import software.amazon.smithy.model.traits.Trait; +import software.amazon.smithy.model.traits.TraitFactory; + + +/** + * A model file used for models that do not need forward reference + * resolution (e.g., the JSON AST, manually loaded Nodes, pre-made + * {@link AbstractShapeBuilder} objects, etc). + * + * @see AstModelLoader + */ +final class FullyResolvedModelFile extends AbstractMutableModelFile { + + /** + * @param traitFactory Factory used to create traits when merging traits. + */ + FullyResolvedModelFile(TraitFactory traitFactory) { + super(traitFactory); + } + + /** + * Create a {@code FullyResolvedModelFile} from already built shapes. + * + * @param traitFactory Factory used to create traits when merging traits. + * @param shapes Shapes to convert into builders and treat as a ModelFile. + * @return Returns the create {@code FullyResolvedModelFile} containing the shapes. + */ + static FullyResolvedModelFile fromShapes(TraitFactory traitFactory, Collection shapes) { + FullyResolvedModelFile modelFile = new FullyResolvedModelFile(traitFactory); + for (Shape shape : shapes) { + // Convert the shape to a builder and remove all the traits. + // These traits are added to the trait container so that they + // can be merged correctly with any other model. + AbstractShapeBuilder builder = Shape.shapeToBuilder(shape).clearTraits(); + modelFile.onShape(builder); + // Add the traits that were present on the shape. + for (Trait trait : shape.getAllTraits().values()) { + modelFile.onTrait(shape.getId(), trait); + } + } + return modelFile; + } + + @Override + public TraitContainer resolveShapes(Set shapeIds, Function typeProvider) { + return traitContainer; + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelParser.java index eab82a3c978..c988a657b56 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelParser.java @@ -19,18 +19,16 @@ import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.StringJoiner; -import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Collectors; -import software.amazon.smithy.model.FromSourceLocation; import software.amazon.smithy.model.SourceLocation; +import software.amazon.smithy.model.node.ArrayNode; import software.amazon.smithy.model.node.Node; import software.amazon.smithy.model.node.NumberNode; import software.amazon.smithy.model.node.ObjectNode; @@ -62,6 +60,7 @@ import software.amazon.smithy.model.shapes.TimestampShape; import software.amazon.smithy.model.shapes.UnionShape; import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.model.traits.TraitFactory; import software.amazon.smithy.model.validation.Severity; import software.amazon.smithy.model.validation.ValidationEvent; import software.amazon.smithy.model.validation.ValidationUtils; @@ -105,15 +104,11 @@ final class IdlModelParser extends SimpleParser { } } + final ForwardReferenceModelFile modelFile; private final String filename; - private final LoaderVisitor visitor; - private String namespace; private String definedVersion; private TraitEntry pendingDocumentationComment; - /** Map of shape aliases to their targets. */ - private final Map useShapes = new HashMap<>(); - // A pending trait that also doesn't yet have a resolved trait shape ID. static final class TraitEntry { final String traitName; @@ -127,17 +122,18 @@ static final class TraitEntry { } } - IdlModelParser(String filename, String model, LoaderVisitor visitor) { + IdlModelParser(TraitFactory traitFactory, String filename, String model) { super(model, MAX_NESTING_LEVEL); this.filename = filename; - this.visitor = visitor; + this.modelFile = new ForwardReferenceModelFile(traitFactory); } - void parse() { + List parse() { ws(); parseControlSection(); parseMetadataSection(); parseShapeSection(); + return Collections.singletonList(modelFile); } /** @@ -188,7 +184,7 @@ private void parseControlSection() { if (key.equals("version")) { onVersion(value); } else { - visitor.onError(ValidationEvent.builder() + modelFile.events().add(ValidationEvent.builder() .id(Validator.MODEL_ERROR) .sourceLocation(value) .severity(Severity.WARNING) @@ -208,7 +204,7 @@ private void onVersion(Node value) { } String parsedVersion = value.expectStringNode().getValue(); - if (!visitor.isVersionSupported(parsedVersion)) { + if (!LoaderUtils.isVersionSupported(parsedVersion)) { throw syntax("Unsupported Smithy version number: " + parsedVersion); } @@ -230,7 +226,7 @@ private void parseMetadataSection() { ws(); expect('='); ws(); - visitor.onMetadata(key, IdlNodeParser.parseNode(this)); + modelFile.putMetadata(key, IdlNodeParser.parseNode(this)); br(); ws(); } @@ -252,7 +248,7 @@ private void parseShapeSection() { // Parse the namespace. int start = position(); ParserUtils.consumeNamespace(this); - namespace = sliceFrom(start); + modelFile.setNamespace(sliceFrom(start)); br(); // Clear out any erroneous documentation comments. @@ -277,6 +273,7 @@ private void parseUseSection() { ws(); int start = position(); + SourceLocation location = currentLocation(); ParserUtils.consumeNamespace(this); expect('#'); ParserUtils.consumeIdentifier(this); @@ -286,12 +283,7 @@ private void parseUseSection() { clearPendingDocs(); ws(); - ShapeId target = ShapeId.from(lexeme); - ShapeId previous = useShapes.put(target.getName(), target); - if (previous != null) { - throw syntax(String.format("Cannot use name `%s` because it conflicts with `%s`", - target, previous)); - } + modelFile.useShape(ShapeId.from(lexeme), location); } } @@ -435,26 +427,20 @@ private void parseShape(List traits) { private ShapeId parseShapeName() { String name = ParserUtils.parseIdentifier(this); - - if (useShapes.containsKey(name)) { - throw syntax(String.format( - "shape name `%s` conflicts with imported shape `%s`", name, useShapes.get(name))); - } - - return ShapeId.fromRelative(namespace, name); + return ShapeId.fromRelative(modelFile.namespace(), name); } private void parseSimpleShape(ShapeId id, SourceLocation location, AbstractShapeBuilder builder) { - visitor.onShape(builder.source(location).id(id)); + modelFile.onShape(builder.source(location).id(id)); } // See parseMap for information on why members are parsed before the - // list/set is registered with the LoaderVisitor. + // list/set is registered with the ModelFile. private void parseCollection(ShapeId id, SourceLocation location, CollectionShape.Builder builder) { ws(); builder.id(id).source(location); parseMembers(id, SetUtils.of("member")); - visitor.onShape(builder.id(id)); + modelFile.onShape(builder.id(id)); } private void parseMembers(ShapeId id, Set requiredMembers) { @@ -524,10 +510,9 @@ private String parseMember(ShapeId parent, Set required, Set def ws(); ShapeId memberId = parent.withMember(memberName); MemberShape.Builder memberBuilder = MemberShape.builder().id(memberId).source(memberLocation); - SourceLocation targetLocation = currentLocation(); String target = ParserUtils.parseShapeId(this); - visitor.onShape(memberBuilder); - onShapeTarget(target, targetLocation, memberBuilder::target); + modelFile.onShape(memberBuilder); + modelFile.addForwardReference(target, memberBuilder::target); addTraits(memberId, memberTraits); return memberName; @@ -535,14 +520,14 @@ private String parseMember(ShapeId parent, Set required, Set def private void parseMapStatement(ShapeId id, SourceLocation location) { // Parsing members of list/set/map before registering the shape with - // the LoaderVisitor ensures that the shape is only registered if it - // has all of its required members. Otherwise, the LoaderVisitor gives + // the ModelFile ensures that the shape is only registered if it + // has all of its required members. Otherwise, the validation gives // a cryptic message with no context about how a "member" wasn't set // on a builder. This does not suffer the same error messages as // structures/unions because list/set/map have a fixed and required // set of members that must be provided. parseMembers(id, SetUtils.of("key", "value")); - visitor.onShape(MapShape.builder().id(id).source(location)); + modelFile.onShape(MapShape.builder().id(id).source(location)); } private void parseStructuredShape(ShapeId id, SourceLocation location, AbstractShapeBuilder builder) { @@ -551,7 +536,7 @@ private void parseStructuredShape(ShapeId id, SourceLocation location, AbstractS // and still give useful error messages. Trying to parse members first // would otherwise result in cryptic error messages like: // "Member `foo.baz#Foo$Baz` cannot be added to software.amazon.smithy.model.shapes.OperationShape$Builder" - visitor.onShape(builder.id(id).source(location)); + modelFile.onShape(builder.id(id).source(location)); parseMembers(id, Collections.emptySet()); } @@ -559,17 +544,17 @@ private void parseOperationStatement(ShapeId id, SourceLocation location) { ws(); OperationShape.Builder builder = OperationShape.builder().id(id).source(location); ObjectNode node = IdlNodeParser.parseObjectNode(this); - visitor.checkForAdditionalProperties(node, id, OPERATION_PROPERTY_NAMES); - visitor.onShape(builder); + LoaderUtils.checkForAdditionalProperties(node, id, OPERATION_PROPERTY_NAMES, modelFile.events()); + modelFile.onShape(builder); node.getStringMember("input").ifPresent(input -> { - onShapeTarget(input.getValue(), input, builder::input); + modelFile.addForwardReference(input.getValue(), builder::input); }); node.getStringMember("output").ifPresent(output -> { - onShapeTarget(output.getValue(), output, builder::output); + modelFile.addForwardReference(output.getValue(), builder::output); }); node.getArrayMember("errors").ifPresent(errors -> { for (StringNode value : errors.getElementsAs(StringNode.class)) { - onShapeTarget(value.getValue(), value, builder::addError); + modelFile.addForwardReference(value.getValue(), builder::addError); } }); } @@ -578,9 +563,9 @@ private void parseServiceStatement(ShapeId id, SourceLocation location) { ws(); ServiceShape.Builder builder = new ServiceShape.Builder().id(id).source(location); ObjectNode shapeNode = IdlNodeParser.parseObjectNode(this); - visitor.checkForAdditionalProperties(shapeNode, id, SERVICE_PROPERTY_NAMES); + LoaderUtils.checkForAdditionalProperties(shapeNode, id, SERVICE_PROPERTY_NAMES, modelFile.events()); builder.version(shapeNode.expectStringMember(VERSION_KEY).getValue()); - visitor.onShape(builder); + modelFile.onShape(builder); optionalIdList(shapeNode, id.getNamespace(), OPERATIONS_KEY).forEach(builder::addOperation); optionalIdList(shapeNode, id.getNamespace(), RESOURCES_KEY).forEach(builder::addResource); } @@ -601,10 +586,10 @@ private static List optionalIdList(ObjectNode node, String namespace, S private void parseResourceStatement(ShapeId id, SourceLocation location) { ws(); ResourceShape.Builder builder = ResourceShape.builder().id(id).source(location); - visitor.onShape(builder); + modelFile.onShape(builder); ObjectNode shapeNode = IdlNodeParser.parseObjectNode(this); - visitor.checkForAdditionalProperties(shapeNode, id, RESOURCE_PROPERTY_NAMES); + LoaderUtils.checkForAdditionalProperties(shapeNode, id, RESOURCE_PROPERTY_NAMES, modelFile.events()); optionalId(shapeNode, id.getNamespace(), PUT_KEY).ifPresent(builder::put); optionalId(shapeNode, id.getNamespace(), CREATE_KEY).ifPresent(builder::create); optionalId(shapeNode, id.getNamespace(), READ_KEY).ifPresent(builder::read); @@ -621,7 +606,7 @@ private void parseResourceStatement(ShapeId id, SourceLocation location) { for (Map.Entry entry : ids.getMembers().entrySet()) { String name = entry.getKey().getValue(); StringNode target = entry.getValue().expectStringNode(); - onShapeTarget(target.getValue(), target, targetId -> builder.addIdentifier(name, targetId)); + modelFile.addForwardReference(target.getValue(), targetId -> builder.addIdentifier(name, targetId)); } }); } @@ -670,14 +655,13 @@ private void parseApplyStatement() { expect('y'); ws(); - SourceLocation location = currentLocation(); String name = ParserUtils.parseShapeId(this); ws(); TraitEntry traitEntry = IdlTraitParser.parseTraitValue(this); // First, resolve the targeted shape. - onShapeTarget(name, location, id -> { + modelFile.addForwardReference(name, id -> { // Next, resolve the trait ID. onDeferredTrait(id, traitEntry.traitName, traitEntry.value, traitEntry.isAnnotation); }); @@ -688,62 +672,6 @@ private void parseApplyStatement() { ws(); } - /** - * Resolve shape targets and tracks forward references. - * - *

Smithy models allow for forward references to shapes that have not - * yet been defined. Only after all shapes are loaded is the entire set - * of possible shape IDs known. This normally isn't a concern, but Smithy - * allows for public shapes defined in the prelude to be referenced by - * targets like members and resource identifiers without an absolute - * shape ID (for example, {@code String}). However, a relative prelude - * shape ID is only resolved for such a target if a shape of the same - * name was not defined in the same namespace in which the target - * was defined. - * - *

If a shape in the same namespace as the target has already been - * defined or if the target is absolute and cannot resolve to a prelude - * shape, the provided {@code resolver} is invoked immediately. Otherwise, - * the {@code resolver} is invoked inside of the {@link LoaderVisitor#onEnd} - * method only after all shapes have been declared. - * - * @param target Shape that is targeted. - * @param sourceLocation The location of where the target occurred. - * @param resolver The consumer to invoke once the shape ID is resolved. - */ - void onShapeTarget(String target, FromSourceLocation sourceLocation, Consumer resolver) { - // Account for aliased shapes. - if (useShapes.containsKey(target)) { - resolver.accept(useShapes.get(target)); - return; - } - - // A namespace is not set when parsing metadata. - ShapeId expectedId = namespace == null - ? ShapeId.from(target) - : ShapeId.fromOptionalNamespace(namespace, target); - if (isRealizedShapeId(expectedId, target)) { - // Account for previously seen shapes in this namespace, absolute shapes, and prelude namespaces - // always resolve to prelude shapes. - resolver.accept(expectedId); - } else { - visitor.addForwardReference(expectedId, resolver); - } - } - - /** - * Returns true if the shape ID does not need to be deferred. - * - * @param expectedId Shape ID that this ID probably references. - * @param target The target name. - * @return Returns true if this is a known shape ID. - */ - private boolean isRealizedShapeId(ShapeId expectedId, String target) { - return Objects.equals(namespace, Prelude.NAMESPACE) - || visitor.hasDefinedShape(expectedId) - || target.contains("#"); - } - private void addTraits(ShapeId id, List traits) { for (TraitEntry traitEntry : traits) { onDeferredTrait(id, traitEntry.traitName, traitEntry.value, traitEntry.isAnnotation); @@ -763,15 +691,27 @@ private void addTraits(ShapeId id, List traits) { * @param isAnnotation Set to true to indicate that the value for the trait was omitted. */ private void onDeferredTrait(ShapeId target, String traitName, Node traitValue, boolean isAnnotation) { - onShapeTarget(traitName, traitValue.getSourceLocation(), id -> { - if (isAnnotation) { - visitor.onAnnotationTrait(target, id, traitValue.expectNullNode()); - } else { - visitor.onTrait(target, id, traitValue); - } + modelFile.addForwardReference(traitName, (id, typeProvider) -> { + modelFile.onTrait(target, id, coerceTraitValue(id, traitValue, isAnnotation, typeProvider)); }); } + private Node coerceTraitValue(ShapeId traitId, Node value, boolean isAnnotation, + Function typeProvider) { + if (isAnnotation && value.isNullNode()) { + ShapeType targetType = typeProvider.apply(traitId); + if (targetType != null) { + if (targetType == ShapeType.STRUCTURE || targetType == ShapeType.MAP) { + return new ObjectNode(Collections.emptyMap(), value.getSourceLocation()); + } else if (targetType == ShapeType.LIST || targetType == ShapeType.SET) { + return new ArrayNode(Collections.emptyList(), value.getSourceLocation()); + } + } + } + + return value; + } + SourceLocation currentLocation() { return new SourceLocation(filename, line(), column()); } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java index baf6cfb3fa8..c4ecd44562a 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java @@ -84,7 +84,7 @@ static Node parseNodeTextWithKeywords(IdlModelParser parser, SourceLocation loca // not be able to be resolved until after the entire model is loaded. Pair> pair = StringNode.createLazyString(text, location); Consumer consumer = pair.right; - parser.onShapeTarget(text, location, id -> consumer.accept(id.toString())); + parser.modelFile.addForwardReference(text, id -> consumer.accept(id.toString())); return pair.left; } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ImmutablePreludeModelFile.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ImmutablePreludeModelFile.java new file mode 100644 index 00000000000..4b482a2c11e --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ImmutablePreludeModelFile.java @@ -0,0 +1,95 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.ShapeType; +import software.amazon.smithy.model.traits.Trait; +import software.amazon.smithy.model.validation.Severity; +import software.amazon.smithy.model.validation.ValidationEvent; +import software.amazon.smithy.model.validation.Validator; + +/** + * A model file that contains the immutable prelude. Traits cannot be added + * to the prelude outside of the prelude's definition. + * + * @see Prelude#getPreludeModel() + */ +final class ImmutablePreludeModelFile implements ModelFile { + private final Model prelude; + private final List events = new ArrayList<>(); + + ImmutablePreludeModelFile(Model prelude) { + this.prelude = prelude; + } + + @Override + public Set shapeIds() { + return prelude.getShapeIds(); + } + + @Override + public Map metadata() { + return prelude.getMetadata(); + } + + @Override + public TraitContainer resolveShapes(Set ids, Function typeProvider) { + return TraitContainer.EMPTY; + } + + @Override + public Collection createShapes(TraitContainer resolvedTraits) { + // Create error events for each trait applied outside of the prelude. + Map> invalidTraits = resolvedTraits.getTraitsAppliedToPrelude(); + + for (Map.Entry> entry : invalidTraits.entrySet()) { + for (Map.Entry trait : entry.getValue().entrySet()) { + String message = String.format( + "Cannot apply `%s` to an immutable prelude shape defined in `smithy.api`.", + trait.getKey()); + events.add(ValidationEvent.builder() + .severity(Severity.ERROR) + .id(Validator.MODEL_ERROR) + .sourceLocation(trait.getValue().getSourceLocation()) + .shapeId(entry.getKey()) + .message(message) + .build()); + } + } + + return prelude.toSet(); + } + + @Override + public List events() { + return events; + } + + @Override + public ShapeType getShapeType(ShapeId id) { + return prelude.getShape(id).map(Shape::getType).orElse(null); + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoaderUtils.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoaderUtils.java new file mode 100644 index 00000000000..45587335355 --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoaderUtils.java @@ -0,0 +1,106 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import software.amazon.smithy.model.SourceLocation; +import software.amazon.smithy.model.node.ExpectationNotMetException; +import software.amazon.smithy.model.node.ObjectNode; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.ShapeType; +import software.amazon.smithy.model.validation.Severity; +import software.amazon.smithy.model.validation.ValidationEvent; +import software.amazon.smithy.model.validation.Validator; + +final class LoaderUtils { + + private LoaderUtils() {} + + /** + * Checks if additional properties are in an object, and if so, emits a warning event. + * + * @param node Node to check. + * @param shape Shape to associate with the error. + * @param properties Properties to allow. + */ + static void checkForAdditionalProperties( + ObjectNode node, + ShapeId shape, Collection properties, + List events + ) { + try { + node.expectNoAdditionalProperties(properties); + } catch (ExpectationNotMetException e) { + ValidationEvent event = ValidationEvent.fromSourceException(e) + .toBuilder() + .shapeId(shape) + .severity(Severity.WARNING) + .build(); + events.add(event); + } + } + + /** + * Checks if the given version string is supported. + * + * @param versionString Version string to check (e.g., 1, 1.0). + * @return Returns true if this is a supported model version. + */ + static boolean isVersionSupported(String versionString) { + return versionString.equals("1") || versionString.equals("1.0"); + } + + /** + * Create a {@link ValidationEvent} for a shape conflict. + * + * @param id Shape ID in conflict. + * @param a The first location of this shape. + * @param b The second location of this shape. + * @return Returns the created validation event. + */ + static ValidationEvent onShapeConflict(ShapeId id, SourceLocation a, SourceLocation b) { + return ValidationEvent.builder() + .id(Validator.MODEL_ERROR) + .severity(Severity.ERROR) + .sourceLocation(b) + .shapeId(id) + .message(String.format("Conflicting shape definition for `%s` found at `%s` and `%s`", id, a, b)) + .build(); + } + + /** + * Iterates over ModelFiles to find the {@link ShapeType} of a shape. + * + *

The first found shape in a ModelFile wins. This is OK, since any + * kind of conflict is detected when the ModelFiles are merged together. + * + * @param modelFiles ModelFile instances to iterate over, searching for shapes by ID. + * @return Returns the found {@link ShapeType} or {@code null} if the shape does not exist. + */ + public static Function aggregateTypeProvider(List modelFiles) { + return id -> { + for (ModelFile modFile : modelFiles) { + ShapeType fileType = modFile.getShapeType(id); + if (fileType != null) { + return fileType; + } + } + return null; + }; + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoaderVisitor.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoaderVisitor.java deleted file mode 100644 index f6f84cb1aaf..00000000000 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoaderVisitor.java +++ /dev/null @@ -1,701 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.smithy.model.loader; - -import static java.lang.String.format; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Consumer; -import java.util.logging.Logger; -import software.amazon.smithy.model.FromSourceLocation; -import software.amazon.smithy.model.Model; -import software.amazon.smithy.model.SourceException; -import software.amazon.smithy.model.SourceLocation; -import software.amazon.smithy.model.node.ArrayNode; -import software.amazon.smithy.model.node.ExpectationNotMetException; -import software.amazon.smithy.model.node.Node; -import software.amazon.smithy.model.node.NullNode; -import software.amazon.smithy.model.node.ObjectNode; -import software.amazon.smithy.model.shapes.AbstractShapeBuilder; -import software.amazon.smithy.model.shapes.MemberShape; -import software.amazon.smithy.model.shapes.Shape; -import software.amazon.smithy.model.shapes.ShapeId; -import software.amazon.smithy.model.shapes.ShapeType; -import software.amazon.smithy.model.traits.DynamicTrait; -import software.amazon.smithy.model.traits.Trait; -import software.amazon.smithy.model.traits.TraitDefinition; -import software.amazon.smithy.model.traits.TraitFactory; -import software.amazon.smithy.model.validation.Severity; -import software.amazon.smithy.model.validation.ValidatedResult; -import software.amazon.smithy.model.validation.ValidationEvent; -import software.amazon.smithy.model.validation.Validator; -import software.amazon.smithy.utils.ListUtils; -import software.amazon.smithy.utils.SmithyBuilder; - -/** - * Visitor used to drive the creation of a Model. - * - *

The intent of this visitor is to decouple the serialized format of a - * Smithy model from the in-memory Model representation. This allows for - * deserialization code to focus on just extracting data from the model - * rather than logic around duplication detection, trait loading, etc. - */ -final class LoaderVisitor { - private static final Logger LOGGER = Logger.getLogger(LoaderVisitor.class.getName()); - private static final TraitDefinition.Provider TRAIT_DEF_PROVIDER = new TraitDefinition.Provider(); - - /** Factory used to create traits. */ - private final TraitFactory traitFactory; - - /** Property bag used to configure the LoaderVisitor. */ - private final Map properties; - - /** Validation events encountered while loading the model. */ - private final List events = new ArrayList<>(); - - /** Model metadata to assemble together. */ - private final Map metadata = new HashMap<>(); - - /** Map of shape IDs to a list of traits to apply to the shape once they're built. */ - private final Map> pendingTraits = new HashMap<>(); - - /** References that need to be resolved against a namespace or the prelude. */ - private final List forwardReferenceResolvers = new ArrayList<>(); - - /** Shapes that have yet to be built. */ - private final Map pendingShapes = new LinkedHashMap<>(); - - /** Built shapes to add to the Model. Keys are not allowed to conflict with pendingShapes. */ - private final Map builtShapes = new LinkedHashMap<>(); - - /** Built trait definitions. */ - private final Map builtTraitDefinitions = new HashMap<>(); - - /** The result that is populated when onEnd is called. */ - private ValidatedResult result; - - private static final class PendingTrait { - final ShapeId id; - final Node value; - final Trait trait; - final boolean isAnnotation; - - // A pending trait that needs to be created. - PendingTrait(ShapeId id, Node value, boolean isAnnotation) { - this.id = id; - this.value = value; - this.trait = null; - this.isAnnotation = isAnnotation; - } - - // A pending trait that's already created. - PendingTrait(ShapeId id, Trait trait) { - this.id = id; - this.trait = trait; - this.value = null; - isAnnotation = false; - } - } - - private static final class ForwardReferenceResolver { - final ShapeId expectedId; - final Consumer consumer; - - ForwardReferenceResolver(ShapeId expectedId, Consumer consumer) { - this.expectedId = expectedId; - this.consumer = consumer; - } - } - - /** - * @param traitFactory Trait factory to use when resolving traits. - */ - LoaderVisitor(TraitFactory traitFactory) { - this(traitFactory, Collections.emptyMap()); - } - - /** - * @param traitFactory Trait factory to use when resolving traits. - * @param properties Map of loader visitor properties. - */ - LoaderVisitor(TraitFactory traitFactory, Map properties) { - this.traitFactory = traitFactory; - this.properties = properties; - } - - /** - * Checks if the LoaderVisitor has defined a specific shape. - * - * @param id Shape ID to check. - * @return Returns true if the shape has been defined. - */ - public boolean hasDefinedShape(ShapeId id) { - return builtShapes.containsKey(id) || pendingShapes.containsKey(id); - } - - /** - * Checks if a specific property is set. - * - * @param property Name of the property to check. - * @return Returns true if the property is set. - */ - public boolean hasProperty(String property) { - return properties.containsKey(property); - } - - /** - * Gets a property from the loader visitor. - * - * @param property Name of the property to get. - * @return Returns the optionally found property. - */ - public Optional getProperty(String property) { - return Optional.ofNullable(properties.get(property)); - } - - /** - * Gets a property from the loader visitor of a specific type. - * - * @param property Name of the property to get. - * @param Type to convert the property to if found. - * @param type Type to convert the property to if found. - * @return Returns the optionally found property. - * @throws ClassCastException if a found property is not of the expected type. - */ - @SuppressWarnings("unchecked") - public Optional getProperty(String property, Class type) { - return getProperty(property).map(value -> { - if (!type.isInstance(value)) { - throw new ClassCastException(String.format( - "Expected `%s` property of the LoaderVisitor to be a `%s`, but found a `%s`", - property, type.getName(), value.getClass().getName())); - } - return (T) value; - }); - } - - /** - * Adds an error to the loader. - * - * @param event Validation event to add. - */ - public void onError(ValidationEvent event) { - events.add(Objects.requireNonNull(event)); - } - - /** - * Adds a shape to the loader. - * - * @param shapeBuilder Shape builder to add. - */ - public void onShape(AbstractShapeBuilder shapeBuilder) { - ShapeId id = SmithyBuilder.requiredState("id", shapeBuilder.getId()); - if (validateOnShape(id, shapeBuilder)) { - pendingShapes.put(id, shapeBuilder); - } - } - - /** - * Adds a shape to the loader. - * - * @param shape Built shape to add to the loader visitor. - */ - public void onShape(Shape shape) { - if (validateOnShape(shape.getId(), shape)) { - builtShapes.put(shape.getId(), shape); - } - - // Whether or not a shape defines a trait needs to be tracked when - // adding already built shapes (e.g., this happens with the prelude model). - shape.getTrait(TraitDefinition.class).ifPresent(def -> onTraitDefinition(shape.getId(), def)); - } - - private void onTraitDefinition(ShapeId target, TraitDefinition definition) { - builtTraitDefinitions.put(target, definition); - } - - private boolean validateOnShape(ShapeId id, FromSourceLocation source) { - if (!hasDefinedShape(id)) { - return true; - } else if (Prelude.isPreludeShape(id)) { - // Ignore prelude shape conflicts since it's such a common case of - // passing an already built model into a ModelAssembler. - return false; - } - - // The shape has been duplicated, so get the previously defined pending shape or built shape. - SourceLocation previous = Optional.ofNullable(pendingShapes.get(id)) - .orElseGet(() -> builtShapes.get(id)).getSourceLocation(); - - // Ignore duplicate shapes defined in the same file (this can happen - // when the same file is included multiple times in a model assembler). - if (previous != SourceLocation.NONE && previous.equals(source.getSourceLocation())) { - LOGGER.warning(() -> "Ignoring duplicate shape definition defined in the same file: " - + id + " defined at " + source.getSourceLocation()); - return false; - } else { - String message = String.format("Duplicate shape definition for `%s` found at `%s` and `%s`", - id, previous.getSourceLocation(), source.getSourceLocation()); - throw new SourceException(message, source); - } - } - - /** - * Adds a trait to a shape. - * - *

Resolving the trait against a trait definition is deferred until - * the entire model is loaded. A namespace is required to have been set - * if the trait name is not absolute. - * - * @param target Shape to add the trait to. - * @param trait Shape ID of the trait to add. - * @param traitValue Trait value as a Node object. - */ - public void onTrait(ShapeId target, ShapeId trait, Node traitValue) { - onTrait(target, trait, traitValue, false); - } - - /** - * Adds an annotation trait to a shape. - * - *

An annotation trait has no value, and it's value is coerced from - * null into an object or array. - * - * @param target Shape to add the trait to. - * @param trait Shape ID of the trait to add. - * @param traitValue Trait value as a Node object. - */ - public void onAnnotationTrait(ShapeId target, ShapeId trait, NullNode traitValue) { - onTrait(target, trait, traitValue, true); - } - - private void onTrait(ShapeId target, ShapeId trait, Node traitValue, boolean isAnnotation) { - // Special handling for the loading of trait definitions. These need to be - // loaded first before other traits can be resolved. - if (trait.equals(TraitDefinition.ID)) { - TraitDefinition traitDef = TRAIT_DEF_PROVIDER.createTrait(target, traitValue); - // Register this as a trait definition to resolve against pending traits. - onTraitDefinition(target, traitDef); - // Add the definition trait to the shape. - onTrait(target, traitDef); - } else { - PendingTrait pendingTrait = new PendingTrait(trait, traitValue, isAnnotation); - addPendingTrait(target, traitValue.getSourceLocation(), trait, pendingTrait); - } - } - - /** - * Adds a resolved and parsed trait to a shape. - * - * @param target Shape to add the trait to. - * @param trait Trait to add to the shape. - */ - public void onTrait(ShapeId target, Trait trait) { - PendingTrait pending = new PendingTrait(target, trait); - addPendingTrait(target, trait.getSourceLocation(), trait.toShapeId(), pending); - } - - private void addPendingTrait(ShapeId target, SourceLocation sourceLocation, ShapeId trait, PendingTrait pending) { - if (Prelude.isImmutablePublicPreludeShape(target)) { - onError(ValidationEvent.builder() - .severity(Severity.ERROR) - .id(Validator.MODEL_ERROR) - .sourceLocation(sourceLocation) - .shapeId(target) - .message(String.format( - "Cannot apply `%s` to an immutable prelude shape defined in `smithy.api`.", trait)) - .build()); - } else { - pendingTraits.computeIfAbsent(target, targetId -> new ArrayList<>()).add(pending); - } - } - - /** - * Adds a forward reference that is resolved once all shapes have been loaded. - * - * @param expectedId The shape ID that would be resolved in the current namespace. - * @param consumer The consumer that receives the resolved shape ID. - */ - void addForwardReference(ShapeId expectedId, Consumer consumer) { - forwardReferenceResolvers.add(new ForwardReferenceResolver(expectedId, consumer)); - } - - /** - * Adds metadata to the loader. - * - * @param key Metadata key to add. - * @param value Metadata value to add. - */ - public void onMetadata(String key, Node value) { - if (!metadata.containsKey(key)) { - metadata.put(key, value); - } else if (metadata.get(key).isArrayNode() && value.isArrayNode()) { - ArrayNode previous = metadata.get(key).expectArrayNode(); - List merged = new ArrayList<>(previous.getElements()); - merged.addAll(value.expectArrayNode().getElements()); - ArrayNode mergedArray = new ArrayNode(merged, value.getSourceLocation()); - metadata.put(key, mergedArray); - } else if (!metadata.get(key).equals(value)) { - onError(ValidationEvent.builder() - .id(Validator.MODEL_ERROR) - .severity(Severity.ERROR) - .sourceLocation(value) - .message(format( - "Metadata conflict for key `%s`. Defined in both `%s` and `%s`", - key, value.getSourceLocation(), metadata.get(key).getSourceLocation())) - .build()); - } else { - LOGGER.fine(() -> "Ignoring duplicate metadata definition of " + key); - } - } - - /** - * Called when the visitor has completed. - * - * @return Returns the validated model result. - */ - public ValidatedResult onEnd() { - if (result != null) { - return result; - } - - Model.Builder modelBuilder = Model.builder().metadata(metadata); - - finalizeShapeTargets(); - finalizePendingTraits(); - - // Ensure that shape builders are created for the container shapes of - // each modified member (builders were already created if a shape has - // a pending trait). This needs to be done before iterating over the - // pending shapes because a collection can't be modified while - // iterating. - List needsConversion = new ArrayList<>(); - for (AbstractShapeBuilder builder : pendingShapes.values()) { - if (builder instanceof MemberShape.Builder) { - needsConversion.add(builder.getId().withoutMember()); - } - } - needsConversion.forEach(this::resolveShapeBuilder); - - // Build members and add them to their containing shape builders. - for (AbstractShapeBuilder shape : pendingShapes.values()) { - if (shape.getClass() == MemberShape.Builder.class) { - MemberShape member = (MemberShape) buildShape(modelBuilder, shape); - if (member != null) { - AbstractShapeBuilder container = pendingShapes.get(shape.getId().withoutMember()); - if (container == null) { - // As of today, this can only happen when the loading process fails while - // parsing a container shape in the IDL. If the LoaderVisitor is externalized, - // then we may need an alternative approach (for example, something more - // heavyweight like pushing scopes when loading members that allows mutations - // to the LoaderVisitor to be undone when parsing the container fails). - LOGGER.warning(format("Member shape `%s` added to non-existent shape: %s", - member.getId(), member.getSourceLocation())); - } else { - container.addMember(member); - } - } - } - } - - // Now that members were built, build all non-members. - for (AbstractShapeBuilder shape : pendingShapes.values()) { - if (shape.getClass() != MemberShape.Builder.class) { - buildShape(modelBuilder, shape); - } - } - - // Add any remaining built shapes. - modelBuilder.addShapes(builtShapes.values()); - result = new ValidatedResult<>(modelBuilder.build(), events); - - return result; - } - - private void finalizeShapeTargets() { - // Run any finalizers used for things like forward reference resolution. - // Do this through a copy to handle forward references that generate more - // forward references, like when a Prelude trait is applied to a shape that - // is defined in another file. - List resolvers = ListUtils.copyOf(forwardReferenceResolvers); - forwardReferenceResolvers.clear(); - - for (ForwardReferenceResolver resolver : resolvers) { - // First, resolve to a shape in the current namespace if one exists. - if (!hasDefinedShape(resolver.expectedId)) { - // Next resolve to a prelude shape if one exists and is public. - ShapeId preludeId = resolver.expectedId.withNamespace(Prelude.NAMESPACE); - if (Prelude.isPublicPreludeShape(preludeId)) { - resolver.consumer.accept(preludeId); - continue; - } - // Finally, just default back to original namespace using a broken target. - } - resolver.consumer.accept(resolver.expectedId); - } - - // Resolve any new forward references created. - if (!forwardReferenceResolvers.isEmpty()) { - finalizeShapeTargets(); - } - } - - private void finalizePendingTraits() { - // Build trait nodes and add them to their shape builders. - for (Map.Entry> entry : pendingTraits.entrySet()) { - ShapeId target = entry.getKey(); - List pendingTraits = entry.getValue(); - AbstractShapeBuilder builder = resolveShapeBuilder(target); - if (builder == null) { - // The shape was not found, so emit a validation event for every trait applied to it. - emitErrorsForEachInvalidTraitTarget(target, pendingTraits); - continue; - } - - // Add already built traits to the shape. Note that these kinds of - // traits *could* be overwritten by traits defined in the model. - // However, that is only technically possible with the documentation - // trait since one is manually created for documentation comments. - for (PendingTrait pending : pendingTraits) { - if (pending.trait != null) { - builder.addTrait(pending.trait); - } - } - - // Compute the shapes to add and merge into the shape. - for (Map.Entry computedEntry : computeTraits(builder, pendingTraits).entrySet()) { - createAndApplyTraitToShape(builder, computedEntry.getKey(), computedEntry.getValue()); - } - } - } - - private AbstractShapeBuilder resolveShapeBuilder(ShapeId id) { - if (pendingShapes.containsKey(id)) { - return pendingShapes.get(id); - } else if (builtShapes.containsKey(id)) { - // If the shape is not a builder but rather a built shape, then convert into a builder. - // Once converted, the shape is removed from builtShapes and added into pendingShapes. - AbstractShapeBuilder builder = (AbstractShapeBuilder) Shape.shapeToBuilder(builtShapes.remove(id)); - pendingShapes.put(id, builder); - return builder; - } else { - return null; - } - } - - private void emitErrorsForEachInvalidTraitTarget(ShapeId target, List pendingTraits) { - for (PendingTrait pendingTrait : pendingTraits) { - onError(ValidationEvent.builder() - .id(Validator.MODEL_ERROR) - .severity(Severity.ERROR) - .sourceLocation(pendingTrait.value.getSourceLocation()) - .message(format("Trait `%s` applied to unknown shape `%s`", - Trait.getIdiomaticTraitName(pendingTrait.id), target)) - .build()); - } - } - - private Shape buildShape(Model.Builder modelBuilder, AbstractShapeBuilder shapeBuilder) { - try { - Shape result = (Shape) shapeBuilder.build(); - modelBuilder.addShape(result); - return result; - } catch (SourceException e) { - onError(ValidationEvent.fromSourceException(e).toBuilder().shapeId(shapeBuilder.getId()).build()); - return null; - } - } - - /** - * This method resolves the fully-qualified names of each pending trait - * (checking the namespace that contains the trait and the prelude), merges - * them if necessary, and returns the merged map of trait names to trait - * node values. - * - * Traits are added to a flat list of pending traits for a shape. We can - * only actually determine the resolved trait definition to apply once the - * entire model is loaded an all trait definitions are known. As such, - * the logic for merging traits and detecting duplicates is deferred until - * the end of the model is detected. - * - * @param shapeBuilder Shape that is being resolved. - * @param pending The list of pending traits to resolve. - * @return Returns the resolved map of pending traits. - */ - private Map computeTraits(AbstractShapeBuilder shapeBuilder, List pending) { - Map traits = new HashMap<>(); - for (PendingTrait trait : pending) { - // Already resolved traits don't need to be computed. - if (trait.trait != null) { - continue; - } - - TraitDefinition definition = builtTraitDefinitions.get(trait.id); - - if (definition == null) { - onUnresolvedTraitName(shapeBuilder, trait); - continue; - } - - ShapeId traitId = trait.id; - Node value = coerceTraitValue(trait); - Node previous = traits.get(traitId); - - if (previous == null) { - traits.put(traitId, value); - } else if (previous.isArrayNode() && value.isArrayNode()) { - // You can merge trait arrays. - traits.put(traitId, value.asArrayNode().get().merge(previous.asArrayNode().get())); - } else if (previous.equals(value)) { - LOGGER.fine(() -> String.format( - "Ignoring duplicate %s trait value on %s", traitId, shapeBuilder.getId())); - } else { - onDuplicateTrait(shapeBuilder.getId(), traitId, previous, value); - } - } - - return traits; - } - - /** - * Coerces a null annotation trait value for the given type. - * - *

Null values provided for traits are coerced in some cases to fit - * the type referenced by the shape. This is used in the .smithy format - * to make is so that you can write "@foo" rather than "@foo([])". - * - *

    - *
  • Structure and map traits are converted to an empty object.
    • - *
  • List and set traits are converted to an empty array.
    • - *
- * - * @param trait Trait to coerce. - * @return Returns the coerced value. - */ - private Node coerceTraitValue(PendingTrait trait) { - if (trait.isAnnotation && trait.value.isNullNode()) { - ShapeType targetType = determineTraitDefinitionType(trait.id); - if (targetType == ShapeType.STRUCTURE || targetType == ShapeType.MAP) { - return new ObjectNode(Collections.emptyMap(), trait.value.getSourceLocation()); - } else if (targetType == ShapeType.LIST || targetType == ShapeType.SET) { - return new ArrayNode(Collections.emptyList(), trait.value.getSourceLocation()); - } - } - - return trait.value; - } - - private ShapeType determineTraitDefinitionType(ShapeId traitId) { - assert pendingShapes.containsKey(traitId) || builtShapes.containsKey(traitId); - - if (pendingShapes.containsKey(traitId)) { - return pendingShapes.get(traitId).getShapeType(); - } else { - return builtShapes.get(traitId).getType(); - } - } - - private void onDuplicateTrait(ShapeId target, ShapeId traitName, FromSourceLocation previous, Node duplicate) { - onError(ValidationEvent.builder() - .id(Validator.MODEL_ERROR) - .severity(Severity.ERROR) - .sourceLocation(duplicate.getSourceLocation()) - .shapeId(target) - .message(format( - "Conflicting `%s` trait found on shape `%s`. The previous trait was defined at `%s`, " - + "and a conflicting trait was defined at `%s`.", - traitName, target, previous.getSourceLocation(), duplicate.getSourceLocation())) - .build()); - } - - private void onUnresolvedTraitName(AbstractShapeBuilder shapeBuilder, PendingTrait trait) { - Severity severity = getProperty(ModelAssembler.ALLOW_UNKNOWN_TRAITS, Boolean.class).orElse(false) - ? Severity.WARNING : Severity.ERROR; - - // Fail if the trait cannot be resolved. - onError(ValidationEvent.builder() - .id(Validator.MODEL_ERROR) - .severity(severity) - .sourceLocation(trait.value.getSourceLocation()) - .shapeId(shapeBuilder.getId()) - .message(format("Unable to resolve trait `%s`. If this is a custom trait, then it must be " - + "defined before it can be used in a model.", trait.id)) - .build()); - } - - /** - * Applies a trait to a shape. If a concrete class for the trait cannot - * be found, then a {@link DynamicTrait} is created. If the trait throws - * an exception while being created, it is caught and the validation - * event is logged. - * - * @param shapeBuilder Shape builder to update. - * @param traitId ID of the fully-qualified trait to add. - * @param traitValue The trait value to set. - */ - private void createAndApplyTraitToShape(AbstractShapeBuilder shapeBuilder, ShapeId traitId, Node traitValue) { - try { - // Create the trait using a factory, or default to an un-typed modeled trait. - Trait createdTrait = traitFactory.createTrait(traitId, shapeBuilder.getId(), traitValue) - .orElseGet(() -> new DynamicTrait(traitId, traitValue)); - shapeBuilder.addTrait(createdTrait); - } catch (SourceException e) { - events.add(ValidationEvent.fromSourceException(e, format("Error creating trait `%s`: ", - Trait.getIdiomaticTraitName(traitId))) - .toBuilder() - .shapeId(shapeBuilder.getId()) - .build()); - } - } - - /** - * Checks if additional properties are in an object, and if so, emits a warning event. - * - * @param node Node to check. - * @param shape Shape to associate with the error. - * @param properties Properties to allow. - */ - void checkForAdditionalProperties(ObjectNode node, ShapeId shape, Collection properties) { - try { - node.expectNoAdditionalProperties(properties); - } catch (ExpectationNotMetException e) { - ValidationEvent event = ValidationEvent.fromSourceException(e) - .toBuilder() - .shapeId(shape) - .severity(Severity.WARNING) - .build(); - onError(event); - } - } - - /** - * Checks if the given version string is supported. - * - * @param versionString Version string to check (e.g., 1, 1.0). - * @return Returns true if this is a supported model version. - */ - boolean isVersionSupported(String versionString) { - return versionString.equals("1") || versionString.equals("1.0"); - } -} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/MetadataContainer.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/MetadataContainer.java new file mode 100644 index 00000000000..fa1dec8be31 --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/MetadataContainer.java @@ -0,0 +1,100 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import static java.lang.String.format; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import software.amazon.smithy.model.node.ArrayNode; +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.validation.Severity; +import software.amazon.smithy.model.validation.ValidationEvent; +import software.amazon.smithy.model.validation.Validator; + +/** + * Captures and merges metadata during the model loading process. + */ +final class MetadataContainer { + private static final Logger LOGGER = Logger.getLogger(MetadataContainer.class.getName()); + + private final Map data = new LinkedHashMap<>(); + private final List events; + + /** + * @param events Mutable, by-reference list of validation events. + */ + MetadataContainer(List events) { + this.events = events; + } + + /** + * Put metadata into the map. + * + *

If the given key conflicts with another key, then the values are + * merged (that is, if both values are arrays, then combine them, if + * both values are equal then ignore the new value, or fail the merge + * and add a validation event). + * + * @param key Metadata key to set. + * @param value Value to set. + */ + void putMetadata(String key, Node value) { + if (!data.containsKey(key)) { + data.put(key, value); + } else if (data.get(key).isArrayNode() && value.isArrayNode()) { + ArrayNode previous = data.get(key).expectArrayNode(); + List merged = new ArrayList<>(previous.getElements()); + merged.addAll(value.expectArrayNode().getElements()); + ArrayNode mergedArray = new ArrayNode(merged, value.getSourceLocation()); + data.put(key, mergedArray); + } else if (!data.get(key).equals(value)) { + events.add(ValidationEvent.builder() + .id(Validator.MODEL_ERROR) + .severity(Severity.ERROR) + .sourceLocation(value) + .message(format( + "Metadata conflict for key `%s`. Defined in both `%s` and `%s`", + key, value.getSourceLocation(), data.get(key).getSourceLocation())) + .build()); + } else { + LOGGER.fine(() -> "Ignoring duplicate metadata definition of " + key); + } + } + + /** + * Merges another metadata container into this container. + * + * @param other Metadata container to merge into this container. + */ + void mergeWith(Map other) { + for (Map.Entry entry : other.entrySet()) { + putMetadata(entry.getKey(), entry.getValue()); + } + } + + /** + * Gets all of the metadata in the container. + * + * @return Returns the metadata. + */ + Map getData() { + return data; + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelAssembler.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelAssembler.java index 9722a09a4c9..08d7f1fc18d 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelAssembler.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelAssembler.java @@ -21,32 +21,37 @@ import java.io.UncheckedIOException; import java.net.URL; import java.net.URLConnection; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.FileVisitOption; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.TreeMap; +import java.util.function.Function; import java.util.function.Supplier; import java.util.logging.Logger; +import java.util.stream.Collectors; import software.amazon.smithy.model.Model; import software.amazon.smithy.model.SourceException; import software.amazon.smithy.model.node.Node; -import software.amazon.smithy.model.shapes.AbstractShapeBuilder; import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.ShapeType; import software.amazon.smithy.model.traits.Trait; import software.amazon.smithy.model.traits.TraitFactory; +import software.amazon.smithy.model.validation.Severity; import software.amazon.smithy.model.validation.ValidatedResult; import software.amazon.smithy.model.validation.ValidationEvent; import software.amazon.smithy.model.validation.Validator; import software.amazon.smithy.model.validation.ValidatorFactory; +import software.amazon.smithy.utils.FunctionalUtils; +import software.amazon.smithy.utils.Pair; /** * Assembles and validates a {@link Model} from documents, files, shapes, and @@ -84,38 +89,12 @@ public final class ModelAssembler { private TraitFactory traitFactory; private ValidatorFactory validatorFactory; private boolean disableValidation; - - /** - * A map of files to parse and load into the Model. - * - *

A {@code TreeMap} is used to ensure that JSON models are loaded - * before IDL models. This is mostly a performance optimization. JSON - * models always use absolute references and require no forward - * reference lookups. IDL models often require forward reference lookups. - * Loading JSON models first should make many of the forward lookups - * when loading IDL models occur immediately rather than needing to be - * resolved once all of the shapes are loaded. - */ - private final Map> inputStreamModels = new TreeMap<>((a, b) -> { - boolean isJsonA = a.endsWith(".json"); - boolean isJsonB = b.endsWith(".json"); - - if (isJsonA) { - if (!isJsonB) { - return -1; - } - } else if (isJsonB) { - return 1; - } - - return a.compareTo(b); - }); - + private final Map> inputStreamModels = new HashMap<>(); private final List validators = new ArrayList<>(); private final List documentNodes = new ArrayList<>(); private final List mergeModels = new ArrayList<>(); - private final List> shapes = new ArrayList<>(); - private final List pendingTraits = new ArrayList<>(); + private final List shapes = new ArrayList<>(); + private final List> pendingTraits = new ArrayList<>(); private final Map metadata = new HashMap<>(); private final Map properties = new HashMap<>(); private boolean disablePrelude; @@ -237,7 +216,7 @@ public ModelAssembler addValidator(Validator validator) { */ public ModelAssembler addUnparsedModel(String sourceLocation, String model) { inputStreamModels.put(sourceLocation, - () -> new ByteArrayInputStream(model.getBytes(Charset.forName("UTF-8")))); + () -> new ByteArrayInputStream(model.getBytes(StandardCharsets.UTF_8))); return this; } @@ -353,7 +332,7 @@ public ModelAssembler disablePrelude() { * @return Returns the assembler. */ public ModelAssembler addShape(Shape shape) { - this.shapes.add(Shape.shapeToBuilder(shape)); + this.shapes.add(shape); return this; } @@ -378,8 +357,7 @@ public ModelAssembler addShapes(Shape... shapes) { * @return Returns the assembler. */ public ModelAssembler addTrait(ShapeId target, Trait trait) { - PendingTrait pendingTrait = new PendingTrait(target, trait); - this.pendingTraits.add(pendingTrait); + this.pendingTraits.add(Pair.of(target, trait)); return this; } @@ -486,64 +464,208 @@ public ModelAssembler disableValidation() { /** * Assembles the model and returns the validated result. * + *

Implementation notes

+ * + *

Assembling models is a multi-step process that revolves around + * {@link ModelFile}s. ModelFiles are essentially files that contain + * localized definitions of shapes and metadata. Some model files use + * forward references and can't be fully resolved until all other model + * files have been loaded. To achieve this, the assembler first creates a + * model file that represents manually added shapes, traits, validators, + * etc., and then parses each given import. Parsing an import is used to + * create zero or more ModelFiles by parsing .json, .smithy, and + * .jar files. + * + *

After the parsing phase, each model file returns the metadata + * defined in the file using {@link ModelFile#metadata()} and the set of + * shape IDs that were defined in the file using {@link ModelFile#shapeIds()}. + * The metadata across each file is merged together using the rules + * defined in the Smithy specification. + * + *

Next, the assembler calls {@link ModelFile#resolveShapes} on each + * ModelFile which resolves any forward references, creates traits, and + * returns all of the traits created in the file. The assembler passes in + * the set of found shapes IDs along with a function that can be used to + * get the {@link ShapeType} of any defined shape (this is used to coerce + * annotation trait values into the appropriate type for a trait). + * The assembler aggregates and merges the traits applied across all model + * files using the merge rules defined in the Smithy specification. + * + *

Next, the assembler invokes {@link ModelFile#createShapes}, passing + * in all of the traits defined across every model file. This method + * causes a ModelFile to apply these traits to any shape in the ModelFile, + * build each shape, and return the built shapes. The assembler then + * aggregates all of the created traits, performs conflict resolution, + * and builds a {@link Model} from the shapes and loaded metadata. A shape + * is allowed to be defined in multiple model files if the conflicting + * shapes are equivalent after all traits have been applied to both + * shapes. + * * @return Returns the validated result that optionally contains a Model * and validation events. */ public ValidatedResult assemble() { - LoaderVisitor visitor = createLoaderVisitor(); + if (traitFactory == null) { + traitFactory = LazyTraitFactoryHolder.INSTANCE; + } + + // Create "model files" for the prelude, manually added shapes, imports, etc. + List modelFiles = createModelFiles(); try { - return doAssemble(visitor); + return doAssemble(modelFiles); } catch (SourceException e) { - ValidatedResult modelResult = visitor.onEnd(); - List events = new ArrayList<>(modelResult.getValidationEvents()); + List events = new ArrayList<>(); events.add(ValidationEvent.fromSourceException(e)); - return new ValidatedResult<>(modelResult.getResult().orElse(null), events); + for (ModelFile modelFile : modelFiles) { + events.addAll(modelFile.events()); + } + return ValidatedResult.fromErrors(events); } } - private LoaderVisitor createLoaderVisitor() { - if (traitFactory == null) { - traitFactory = LazyTraitFactoryHolder.INSTANCE; - } - - return new LoaderVisitor(traitFactory, properties); - } + private List createModelFiles() { + List modelFiles = new ArrayList<>(); - private ValidatedResult doAssemble(LoaderVisitor visitor) { if (!disablePrelude) { - mergeModelIntoVisitor(Prelude.getPreludeModel(), visitor); + modelFiles.add(new ImmutablePreludeModelFile(Prelude.getPreludeModel())); } - shapes.forEach(visitor::onShape); - metadata.forEach(visitor::onMetadata); - for (PendingTrait pendingTrait : pendingTraits) { - visitor.onTrait(pendingTrait.id, pendingTrait.trait); + // A modelFile is created for the assembler to capture anything that was manually added. + FullyResolvedModelFile assemblerModelFile = FullyResolvedModelFile.fromShapes(traitFactory, shapes); + modelFiles.add(assemblerModelFile); + metadata.forEach(assemblerModelFile::putMetadata); + for (Pair pendingTrait : pendingTraits) { + assemblerModelFile.onTrait(pendingTrait.left, pendingTrait.right); } + // Merge in fully-built models into the assembler. for (Model model : mergeModels) { - mergeModelIntoVisitor(model, visitor); + // Fully resolved models typically contain a prelude. This ensures that the prelude is not included + // in the assembler since it would cause pointless conflicts. + List nonPrelude = model.shapes() + .filter(FunctionalUtils.not(Prelude::isPreludeShape)) + .collect(Collectors.toList()); + FullyResolvedModelFile resolvedFile = FullyResolvedModelFile.fromShapes(traitFactory, nonPrelude); + model.getMetadata().forEach(resolvedFile::putMetadata); + modelFiles.add(resolvedFile); } + // Load parsed AST nodes and merge them into the assembler. for (Node node : documentNodes) { - ModelLoader.loadParsedNode(node, visitor); + try { + modelFiles.addAll(ModelLoader.loadParsedNode(traitFactory, node)); + } catch (SourceException e) { + assemblerModelFile.events().add(ValidationEvent.fromSourceException(e)); + } } + // Load model files and merge them into the assembler. for (Map.Entry> modelEntry : inputStreamModels.entrySet()) { - if (!ModelLoader.load(modelEntry.getKey(), modelEntry.getValue(), visitor)) { - LOGGER.warning(() -> "No ModelLoader was able to load " + modelEntry.getKey()); + try { + List loaded = ModelLoader.load( + traitFactory, properties, modelEntry.getKey(), modelEntry.getValue()); + if (loaded.isEmpty()) { + LOGGER.warning(() -> "No ModelLoader was able to load " + modelEntry.getKey()); + } else { + modelFiles.addAll(loaded); + } + } catch (SourceException e) { + assemblerModelFile.events().add(ValidationEvent.fromSourceException(e)); } } - ValidatedResult modelResult = visitor.onEnd(); - return !modelResult.getResult().isPresent() - ? modelResult - : validate(modelResult.getResult().get(), modelResult.getValidationEvents()); + return modelFiles; } - private static void mergeModelIntoVisitor(Model model, LoaderVisitor visitor) { - model.getMetadata().forEach(visitor::onMetadata); - model.shapes().forEach(visitor::onShape); + private ValidatedResult doAssemble(List modelFiles) { + List events = new ArrayList<>(); + Set ids = new HashSet<>(); + MetadataContainer metadata = new MetadataContainer(events); + + for (ModelFile modelFile : modelFiles) { + // Merge all metadata. + metadata.mergeWith(modelFile.metadata()); + // Collect all known shape IDs across all modelFiles. + ids.addAll(modelFile.shapeIds()); + } + + Function typeProvider = LoaderUtils.aggregateTypeProvider(modelFiles); + + // Merge all pending traits across every model. + TraitContainer.TraitHashMap traitValues = new TraitContainer.TraitHashMap(traitFactory, events); + for (ModelFile modelFile : modelFiles) { + traitValues.merge(modelFile.resolveShapes(ids, typeProvider)); + } + + // Merge all shapes together, resolve conflicts, and warn for acceptable conflicts. + Map shapes = new HashMap<>(); + for (ModelFile modelFile : modelFiles) { + for (Shape shape : modelFile.createShapes(traitValues)) { + Shape previous = shapes.get(shape.getId()); + if (previous == null) { + shapes.put(shape.getId(), shape); + } else if (!previous.equals(shape)) { + events.add(LoaderUtils.onShapeConflict(shape.getId(), shape.getSourceLocation(), + previous.getSourceLocation())); + } else { + LOGGER.warning(() -> "Ignoring duplicate but equivalent shape definition: " + previous.getId() + + " defined at " + shape.getSourceLocation() + " and " + + previous.getSourceLocation()); + } + } + } + + // Find traits applied to shapes that don't exist. + for (Map.Entry> entry : traitValues.traits().entrySet()) { + ShapeId target = entry.getKey(); + if (!ids.contains(target)) { + for (Trait trait : entry.getValue().values()) { + events.add(ValidationEvent.builder() + .id(Validator.MODEL_ERROR) + .severity(Severity.ERROR) + .sourceLocation(trait) + .message(String.format("Trait `%s` applied to unknown shape `%s`", + Trait.getIdiomaticTraitName(trait.toShapeId()), target)) + .build()); + } + } + } + + // Find trait values that weren't defined. + Severity severity = areUnknownTraitsAllowed() ? Severity.WARNING : Severity.ERROR; + for (Map.Entry> entry : traitValues.traits().entrySet()) { + ShapeId target = entry.getKey(); + for (Trait trait : entry.getValue().values()) { + if (!ids.contains(trait.toShapeId())) { + events.add(ValidationEvent.builder() + .id(Validator.MODEL_ERROR) + .severity(severity) + .sourceLocation(trait) + .shapeId(target) + .message(String.format( + "Unable to resolve trait `%s`. If this is a custom trait, then it must be " + + "defined before it can be used in a model.", trait.toShapeId())) + .build()); + } + } + } + + for (ModelFile modelFile : modelFiles) { + events.addAll(modelFile.events()); + } + + Model.Builder builder = Model.builder(); + builder.metadata(metadata.getData()); + builder.addShapes(shapes.values()); + Model model = builder.build(); + return validate(model, events); + } + + private boolean areUnknownTraitsAllowed() { + // Find trait values that weren't defined. + Object allowUnknown = properties.get(ModelAssembler.ALLOW_UNKNOWN_TRAITS); + return allowUnknown != null && (boolean) allowUnknown; } private ValidatedResult validate(Model model, List modelResultEvents) { @@ -567,15 +689,4 @@ private List assembleValidators() { copiedValidators.addAll(validators); return copiedValidators; } - - private static final class PendingTrait { - final ShapeId id; - final Trait trait; - - // A pending trait that's already created. - PendingTrait(ShapeId id, Trait trait) { - this.id = id; - this.trait = trait; - } - } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelFile.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelFile.java new file mode 100644 index 00000000000..a7b230c0817 --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelFile.java @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.ShapeType; +import software.amazon.smithy.model.validation.ValidationEvent; + +/** + * Represents a model file as defined in the Smithy specification. + * + *

A model file is used as a self-contained scope of shapes and + * metadata. Model files are created in isolation, then merged by + * a {@link ModelAssembler}. + */ +interface ModelFile { + + /** + * Gets the shape IDs that are defined in this ModelFile. + * + *

This is called before any other method in the ModelFile. + * + * @return Returns the shape IDs defined in this file. + */ + Set shapeIds(); + + /** + * Gets the {@link ShapeType} of a shape by ID. + * + *

This is used, for example, to coerce annotation traits into the + * appropriate type when parsing trait node values. + * + * @param id Shape ID to check. + * @return Returns the {@link ShapeType} if known, or {@code null} if not found. + */ + ShapeType getShapeType(ShapeId id); + + /** + * Get the metadata defined in the ModelFile. + * + * @return Returns the defined, non-null metadata. + */ + Map metadata(); + + /** + * Resolves any forward references and returns all of the traits that were + * applied to shapes in this ModelFile. + * + * @param ids All of the shape IDs found across all ModelFiles being assembled. + * @param typeProvider A function that can return type information about shapes. + * @return Returns a container of traits to apply to shapes. + */ + TraitContainer resolveShapes(Set ids, Function typeProvider); + + /** + * Finalizes and creates shapes in the ModelFile. + * + *

This is called after {@link #resolveShapes}. + * + * @param resolvedTraits Traits to apply to the shapes in the ModelFile. + * @return Returns the created shapes. + */ + Collection createShapes(TraitContainer resolvedTraits); + + /** + * Gets a mutable list of {@link ValidationEvent} objects encountered when + * loading this ModelFile. + * + * @return Returns the list of events. + */ + List events(); +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelLoader.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelLoader.java index aa4e2f4a1a3..a51525f7f35 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelLoader.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelLoader.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -19,6 +19,10 @@ import java.io.InputStream; import java.net.URL; import java.net.URLConnection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.function.Supplier; import java.util.logging.Logger; import software.amazon.smithy.model.SourceException; @@ -26,6 +30,7 @@ import software.amazon.smithy.model.node.Node; import software.amazon.smithy.model.node.ObjectNode; import software.amazon.smithy.model.node.StringNode; +import software.amazon.smithy.model.traits.TraitFactory; import software.amazon.smithy.utils.IoUtils; /** @@ -39,33 +44,39 @@ final class ModelLoader { private ModelLoader() {} /** - * Loads the contents of a model into a {@code LoaderVisitor}. + * Loads the contents of a model into a {@code ModelFile}. * *

The format contained in the supplied {@code InputStream} is * determined based on the file extension in the provided * {@code filename}. * + * @param traitFactory Factory used to create traits. + * @param properties Bag of loading properties. * @param filename Filename Filename to assign to the model. * @param contentSupplier The supplier that provides an InputStream. The * supplied {@code InputStream} is automatically closed when the loader * has finished reading from it. - * @param visitor The visitor to update while loading. - * @return Returns true if the file could be loaded, and false if not. + * @return Returns a non-empty list of {@code ModelFile}s if model(s) could be loaded. * @throws SourceException if there is an error reading from the contents. */ - static boolean load(String filename, Supplier contentSupplier, LoaderVisitor visitor) { + static List load( + TraitFactory traitFactory, + Map properties, + String filename, + Supplier contentSupplier + ) { if (filename.endsWith(".json")) { - return loadParsedNode(Node.parse(contentSupplier.get(), filename), visitor); + return loadParsedNode(traitFactory, Node.parse(contentSupplier.get(), filename)); } else if (filename.endsWith(".smithy")) { - new IdlModelParser(filename, IoUtils.toUtf8String(contentSupplier.get()), visitor).parse(); - return true; + String contents = IoUtils.toUtf8String(contentSupplier.get()); + return new IdlModelParser(traitFactory, filename, contents).parse(); } else if (filename.endsWith(".jar")) { - return loadJar(filename, visitor); + return loadJar(traitFactory, properties, filename); } else if (filename.equals(SourceLocation.NONE.getFilename())) { // Assume it's JSON if there's a N/A filename. - return loadParsedNode(Node.parse(contentSupplier.get(), filename), visitor); + return loadParsedNode(traitFactory, Node.parse(contentSupplier.get(), filename)); } else { - return false; + return Collections.emptyList(); } } @@ -75,13 +86,12 @@ static boolean load(String filename, Supplier contentSupplier, Load // Smithy JSON AST format. // // This loader supports version 1.0. Support for 0.5 and 0.4 was removed in 0.10. - static boolean loadParsedNode(Node node, LoaderVisitor visitor) { + static List loadParsedNode(TraitFactory traitFactory, Node node) { ObjectNode model = node.expectObjectNode("Smithy documents must be an object. Found {type}."); StringNode version = model.expectStringMember(SMITHY); - if (visitor.isVersionSupported(version.getValue())) { - AstModelLoader.INSTANCE.load(model, visitor); - return true; + if (LoaderUtils.isVersionSupported(version.getValue())) { + return Collections.singletonList(AstModelLoader.INSTANCE.load(traitFactory, model)); } else { throw new ModelSyntaxException("Unsupported Smithy version number: " + version.getValue(), version); } @@ -89,32 +99,34 @@ static boolean loadParsedNode(Node node, LoaderVisitor visitor) { // Allows importing JAR files by discovering models inside of a JAR file. // This is similar to model discovery, but done using an explicit import. - private static boolean loadJar(String filename, LoaderVisitor visitor) { + private static List loadJar(TraitFactory traitFactory, Map properties, String filename) { URL manifestUrl = ModelDiscovery.createSmithyJarManifestUrl(filename); LOGGER.fine(() -> "Loading Smithy model imports from JAR: " + manifestUrl); + List result = new ArrayList<>(); for (URL model : ModelDiscovery.findModels(manifestUrl)) { try { URLConnection connection = model.openConnection(); - if (visitor.hasProperty(ModelAssembler.DISABLE_JAR_CACHE)) { + if (properties.containsKey(ModelAssembler.DISABLE_JAR_CACHE)) { connection.setUseCaches(false); } - load(model.toExternalForm(), () -> { + List innerResult = load(traitFactory, properties, model.toExternalForm(), () -> { try { return connection.getInputStream(); } catch (IOException e) { throw throwIoJarException(model, e); } - }, visitor); + }); + result.addAll(innerResult); } catch (IOException e) { throw throwIoJarException(model, e); } } - return true; + return result; } private static ModelImportException throwIoJarException(URL model, Throwable e) { diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/Prelude.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/Prelude.java index e73913fd1b1..e7cb43c8435 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/Prelude.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/Prelude.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ import java.util.Set; import java.util.stream.Collectors; import software.amazon.smithy.model.Model; -import software.amazon.smithy.model.shapes.AbstractShapeBuilder; import software.amazon.smithy.model.shapes.BigDecimalShape; import software.amazon.smithy.model.shapes.BigIntegerShape; import software.amazon.smithy.model.shapes.BlobShape; @@ -89,6 +88,7 @@ import software.amazon.smithy.model.traits.TimestampFormatTrait; import software.amazon.smithy.model.traits.TitleTrait; import software.amazon.smithy.model.traits.TraitDefinition; +import software.amazon.smithy.model.traits.TraitFactory; import software.amazon.smithy.model.traits.UniqueItemsTrait; import software.amazon.smithy.model.traits.UnstableTrait; import software.amazon.smithy.model.traits.XmlAttributeTrait; @@ -113,27 +113,27 @@ public final class Prelude { /** The Smithy prelude namespace. */ public static final String NAMESPACE = "smithy.api"; - private static final List PUBLIC_PRELUDE_SHAPES = ListUtils.of( - StringShape.builder().id(NAMESPACE + "#String"), - BlobShape.builder().id(NAMESPACE + "#Blob"), - BigIntegerShape.builder().id(NAMESPACE + "#BigInteger"), - BigDecimalShape.builder().id(NAMESPACE + "#BigDecimal"), - TimestampShape.builder().id(NAMESPACE + "#Timestamp"), - DocumentShape.builder().id(NAMESPACE + "#Document"), - BooleanShape.builder().id(NAMESPACE + "#Boolean").addTrait(new BoxTrait()), - BooleanShape.builder().id(NAMESPACE + "#PrimitiveBoolean"), - ByteShape.builder().id(NAMESPACE + "#Byte").addTrait(new BoxTrait()), - ByteShape.builder().id(NAMESPACE + "#PrimitiveByte"), - ShortShape.builder().id(NAMESPACE + "#Short").addTrait(new BoxTrait()), - ShortShape.builder().id(NAMESPACE + "#PrimitiveShort"), - IntegerShape.builder().id(NAMESPACE + "#Integer").addTrait(new BoxTrait()), - IntegerShape.builder().id(NAMESPACE + "#PrimitiveInteger"), - LongShape.builder().id(NAMESPACE + "#Long").addTrait(new BoxTrait()), - LongShape.builder().id(NAMESPACE + "#PrimitiveLong"), - FloatShape.builder().id(NAMESPACE + "#Float").addTrait(new BoxTrait()), - FloatShape.builder().id(NAMESPACE + "#PrimitiveFloat"), - DoubleShape.builder().id(NAMESPACE + "#Double").addTrait(new BoxTrait()), - DoubleShape.builder().id(NAMESPACE + "#PrimitiveDouble")); + private static final List PUBLIC_PRELUDE_SHAPES = ListUtils.of( + StringShape.builder().id(NAMESPACE + "#String").build(), + BlobShape.builder().id(NAMESPACE + "#Blob").build(), + BigIntegerShape.builder().id(NAMESPACE + "#BigInteger").build(), + BigDecimalShape.builder().id(NAMESPACE + "#BigDecimal").build(), + TimestampShape.builder().id(NAMESPACE + "#Timestamp").build(), + DocumentShape.builder().id(NAMESPACE + "#Document").build(), + BooleanShape.builder().id(NAMESPACE + "#Boolean").addTrait(new BoxTrait()).build(), + BooleanShape.builder().id(NAMESPACE + "#PrimitiveBoolean").build(), + ByteShape.builder().id(NAMESPACE + "#Byte").addTrait(new BoxTrait()).build(), + ByteShape.builder().id(NAMESPACE + "#PrimitiveByte").build(), + ShortShape.builder().id(NAMESPACE + "#Short").addTrait(new BoxTrait()).build(), + ShortShape.builder().id(NAMESPACE + "#PrimitiveShort").build(), + IntegerShape.builder().id(NAMESPACE + "#Integer").addTrait(new BoxTrait()).build(), + IntegerShape.builder().id(NAMESPACE + "#PrimitiveInteger").build(), + LongShape.builder().id(NAMESPACE + "#Long").addTrait(new BoxTrait()).build(), + LongShape.builder().id(NAMESPACE + "#PrimitiveLong").build(), + FloatShape.builder().id(NAMESPACE + "#Float").addTrait(new BoxTrait()).build(), + FloatShape.builder().id(NAMESPACE + "#PrimitiveFloat").build(), + DoubleShape.builder().id(NAMESPACE + "#Double").addTrait(new BoxTrait()).build(), + DoubleShape.builder().id(NAMESPACE + "#PrimitiveDouble").build()); /** * This list of public prelude traits is manually maintained and must match the @@ -210,7 +210,7 @@ public final class Prelude { static { PUBLIC_PRELUDE_SHAPE_IDS = PUBLIC_PRELUDE_SHAPES.stream() - .map(AbstractShapeBuilder::getId) + .map(Shape::getId) .collect(SetUtils.toUnmodifiableSet()); } @@ -241,16 +241,6 @@ public static boolean isPublicPreludeShape(ToShapeId id) { return PUBLIC_PRELUDE_SHAPE_IDS.contains(toId) || PRELUDE_TRAITS.contains(toId); } - /** - * Checks if the given shape is an immutable public shape. - * - * @param id Shape to check. - * @return Returns true if the shape is immutable. - */ - static boolean isImmutablePublicPreludeShape(ToShapeId id) { - return PUBLIC_PRELUDE_SHAPE_IDS.contains(id.toShapeId()); - } - // Used by the ModelAssembler to load the prelude into another visitor. static Model getPreludeModel() { return PreludeHolder.PRELUDE; @@ -261,18 +251,17 @@ private static final class PreludeHolder { private static final Model PRELUDE = loadPrelude(); private static Model loadPrelude() { - LoaderVisitor visitor = new LoaderVisitor(ModelAssembler.LazyTraitFactoryHolder.INSTANCE); + TraitFactory traitFactory = ModelAssembler.LazyTraitFactoryHolder.INSTANCE; + ModelAssembler assembler = Model.assembler() + .disablePrelude() + .traitFactory(traitFactory) + .addImport(Prelude.class.getResource("prelude-traits.smithy")); - // Register prelude shape definitions. - for (AbstractShapeBuilder builder : PUBLIC_PRELUDE_SHAPES) { - visitor.onShape(builder); + for (Shape shape : PUBLIC_PRELUDE_SHAPES) { + assembler.addShape(shape); } - // Register prelude trait definitions. - String filename = "prelude-traits.smithy"; - - ModelLoader.load(filename, () -> Prelude.class.getResourceAsStream(filename), visitor); - Model preludeModel = visitor.onEnd().unwrap(); + Model preludeModel = assembler.assemble().unwrap(); // Sanity check to ensure that the prelude model and the tracked prelude traits are consistent. // TODO: Can this be moved to a build step in Gradle? diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/TraitContainer.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/TraitContainer.java new file mode 100644 index 00000000000..a9a675c1d20 --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/TraitContainer.java @@ -0,0 +1,229 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import static java.lang.String.format; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.logging.Logger; +import software.amazon.smithy.model.SourceException; +import software.amazon.smithy.model.node.ArrayNode; +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.traits.DynamicTrait; +import software.amazon.smithy.model.traits.Trait; +import software.amazon.smithy.model.traits.TraitFactory; +import software.amazon.smithy.model.validation.Severity; +import software.amazon.smithy.model.validation.ValidationEvent; +import software.amazon.smithy.model.validation.Validator; + +/** + * Aggregates, merges, and creates traits. + */ +public interface TraitContainer { + + /** Shared empty, immutable instance. */ + TraitContainer EMPTY = new TraitContainer() { + @Override + public Map> traits() { + return Collections.emptyMap(); + } + + @Override + public Map getTraitsForShape(ShapeId shape) { + return Collections.emptyMap(); + } + + @Override + public Map> getTraitsAppliedToPrelude() { + return Collections.emptyMap(); + } + + @Override + public void onTrait(ShapeId target, Trait value) { + throw new UnsupportedOperationException("Cannot add trait " + value.toShapeId() + " to " + target); + } + + @Override + public void onTrait(ShapeId target, ShapeId traitId, Node value) { + throw new UnsupportedOperationException("Cannot add trait " + traitId + " to " + target); + } + }; + + /** + * @return Gets all traits in the value map. + */ + Map> traits(); + + /** + * Gets the traits applied to a shape. + * + * @param shape Shape to get the traits of. + * @return Returns the traits of the shape. + */ + Map getTraitsForShape(ShapeId shape); + + /** + * Gets all traits applied to the prelude. + * + * @return Returns the traits applied to prelude shapes. + */ + Map> getTraitsAppliedToPrelude(); + + /** + * Add a trait. + * + * @param target Shape to add the trait to. + * @param value Trait to add. + */ + void onTrait(ShapeId target, Trait value); + + /** + * Create and add a trait. + * + * @param target Shape to add the trait to. + * @param traitId Trait shape ID to create. + * @param value The value to assign to the trait. + */ + void onTrait(ShapeId target, ShapeId traitId, Node value); + + /** + * The actual, mutable implementation used to aggregate traits. + */ + final class TraitHashMap implements TraitContainer { + private static final Logger LOGGER = Logger.getLogger(TraitContainer.class.getName()); + + private final Map> targetToTraits = new HashMap<>(); + private final Map> traitsAppliedToPrelude = new HashMap<>(); + private final TraitFactory traitFactory; + private final List events; + + /** + * @param traitFactory Factory used to create traits. + * @param events Mutable, by-reference validation event list. + */ + TraitHashMap(TraitFactory traitFactory, List events) { + this.traitFactory = Objects.requireNonNull(traitFactory, "Trait factory must not be null"); + this.events = Objects.requireNonNull(events, "events must not be null"); + } + + @Override + public Map> traits() { + return targetToTraits; + } + + @Override + public Map getTraitsForShape(ShapeId shape) { + return targetToTraits.getOrDefault(shape, Collections.emptyMap()); + } + + @Override + public Map> getTraitsAppliedToPrelude() { + return traitsAppliedToPrelude; + } + + @Override + public void onTrait(ShapeId target, Trait value) { + ShapeId traitId = value.toShapeId(); + Map traits = targetToTraits.computeIfAbsent(target, id -> new HashMap<>()); + + if (traits.containsKey(traitId)) { + Trait previousTrait = traits.get(traitId); + Node previous = previousTrait.toNode(); + Node updated = value.toNode(); + + if (previous.isArrayNode() && updated.isArrayNode()) { + // You can merge trait arrays. + ArrayNode merged = previous.expectArrayNode().merge(updated.expectArrayNode()); + value = createTrait(target, traitId, merged); + if (value == null) { + return; + } + } else if (previous.equals(updated)) { + LOGGER.fine(() -> String.format( + "Ignoring duplicate %s trait value on %s", traitId, target)); + return; + } else { + events.add(ValidationEvent.builder() + .id(Validator.MODEL_ERROR) + .severity(Severity.ERROR) + .sourceLocation(value.getSourceLocation()) + .shapeId(target) + .message(String.format( + "Conflicting `%s` trait found on shape `%s`. The previous trait was defined at " + + "`%s`, and a conflicting trait was defined at `%s`.", + traitId, target, previous.getSourceLocation(), value.getSourceLocation())) + .build()); + return; + } + } + + traits.put(traitId, value); + + if (target.getNamespace().equals(Prelude.NAMESPACE)) { + traitsAppliedToPrelude.computeIfAbsent(target, id -> new HashMap<>()).put(traitId, value); + } + } + + @Override + public void onTrait(ShapeId target, ShapeId traitId, Node value) { + Trait trait = createTrait(target, traitId, value); + if (trait != null) { + onTrait(target, trait); + } + } + + /** + * Merges the given {@code other} value into this value. + * + * @param other TraitValues to merge into this TraitValues. + */ + void merge(TraitContainer other) { + for (Map.Entry> entry : other.traits().entrySet()) { + ShapeId target = entry.getKey(); + for (Map.Entry appliedEntry : entry.getValue().entrySet()) { + onTrait(target, appliedEntry.getValue()); + } + } + } + + /** + * Creates a trait and returns null if it can't be created. + * + * @param target Shape to apply the trait to. + * @param traitId Trait shape ID being created. + * @param traitValue The value to assign to the trait. + * @return Returns the created trait on success, or null on failure. + */ + private Trait createTrait(ShapeId target, ShapeId traitId, Node traitValue) { + try { + return traitFactory.createTrait(traitId, target, traitValue) + .orElseGet(() -> new DynamicTrait(traitId, traitValue)); + } catch (SourceException e) { + String message = format("Error creating trait `%s`: ", Trait.getIdiomaticTraitName(traitId)); + events.add(ValidationEvent.fromSourceException(e, message) + .toBuilder() + .shapeId(target) + .build()); + return null; + } + } + } +} diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlTextParserTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlTextParserTest.java index b3338639807..fe023b93a02 100644 --- a/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlTextParserTest.java +++ b/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlTextParserTest.java @@ -8,12 +8,13 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import software.amazon.smithy.model.node.StringNode; +import software.amazon.smithy.model.traits.TraitFactory; public class IdlTextParserTest { @ParameterizedTest @MethodSource("validTextProvider") public void parsesText(String input, String lexeme) { - IdlModelParser parser = new IdlModelParser("/foo", input, null); + IdlModelParser parser = new IdlModelParser(TraitFactory.createServiceFactory(), "/foo", input); StringNode result = IdlNodeParser.parseNode(parser).expectStringNode(); assertThat(result.getValue(), equalTo(lexeme)); } diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/LoaderVisitorTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/LoaderVisitorTest.java deleted file mode 100644 index 38d8073331f..00000000000 --- a/smithy-model/src/test/java/software/amazon/smithy/model/loader/LoaderVisitorTest.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.smithy.model.loader; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; - -import java.net.URL; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import software.amazon.smithy.model.Model; -import software.amazon.smithy.model.SourceException; -import software.amazon.smithy.model.node.Node; -import software.amazon.smithy.model.selector.Selector; -import software.amazon.smithy.model.shapes.MemberShape; -import software.amazon.smithy.model.shapes.Shape; -import software.amazon.smithy.model.shapes.ShapeId; -import software.amazon.smithy.model.shapes.StringShape; -import software.amazon.smithy.model.shapes.StructureShape; -import software.amazon.smithy.model.traits.DocumentationTrait; -import software.amazon.smithy.model.traits.DynamicTrait; -import software.amazon.smithy.model.traits.TraitDefinition; -import software.amazon.smithy.model.traits.TraitFactory; -import software.amazon.smithy.model.validation.ValidationEvent; -import software.amazon.smithy.utils.MapUtils; - -public class LoaderVisitorTest { - private static final TraitFactory FACTORY = TraitFactory.createServiceFactory(); - - @Test - public void callingOnEndTwiceIsIdempotent() { - LoaderVisitor visitor = new LoaderVisitor(FACTORY); - - assertThat(visitor.onEnd(), is(visitor.onEnd())); - } - - @Test - public void cannotDuplicateTraitDefs() { - Assertions.assertThrows(SourceException.class, () -> { - LoaderVisitor visitor = new LoaderVisitor(FACTORY); - StringShape def1 = StringShape.builder() - .id("foo.baz#Bar") - .addTrait(TraitDefinition.builder().build()) - .build(); - StringShape def2 = StringShape.builder() - .id("foo.baz#Bar") - .addTrait(TraitDefinition.builder().selector(Selector.parse("string")).build()) - .build(); - - visitor.onShape(def1); - visitor.onShape(def2); - visitor.onEnd(); - }); - } - - @Test - public void ignoresDuplicateTraitDefsFromPrelude() { - LoaderVisitor visitor = new LoaderVisitor(FACTORY); - Shape def1 = StructureShape.builder() - .id("smithy.api#deprecated") - .addTrait(TraitDefinition.builder().build()) - .build(); - Shape def2 = StructureShape.builder() - .id("smithy.api#deprecated") - .addTrait(TraitDefinition.builder().build()) - .build(); - - visitor.onShape(def1); - visitor.onShape(def2); - List events = visitor.onEnd().getValidationEvents(); - - assertThat(events, empty()); - } - - @Test - public void cannotDuplicateNonPreludeTraitDefs() { - Assertions.assertThrows(SourceException.class, () -> { - LoaderVisitor visitor = new LoaderVisitor(FACTORY); - Shape def1 = StructureShape.builder() - .id("smithy.example#deprecated") - .addTrait(TraitDefinition.builder().build()) - .build(); - Shape def2 = StructureShape.builder() - .id("smithy.example#deprecated") - .addTrait(TraitDefinition.builder().build()) - .build(); - - visitor.onShape(def1); - visitor.onShape(def2); - visitor.onEnd(); - }); - } - - @Test - public void cannotDuplicateTraits() { - LoaderVisitor visitor = new LoaderVisitor(FACTORY); - ShapeId id = ShapeId.from("foo.bam#Boo"); - visitor.onShape(StringShape.builder().id(id)); - visitor.onTrait(id, DocumentationTrait.ID, Node.from("abc")); - visitor.onTrait(id, DocumentationTrait.ID, Node.from("def")); - List events = visitor.onEnd().getValidationEvents(); - - assertThat(events, not(empty())); - } - - @Test - public void createsDynamicTraitWhenTraitFactoryReturnsEmpty() { - LoaderVisitor visitor = new LoaderVisitor(FACTORY); - Shape def = StructureShape.builder() - .id("foo.baz#Bar") - .addTrait(TraitDefinition.builder().build()) - .build(); - visitor.onShape(def); - ShapeId id = ShapeId.from("foo.bam#Boo"); - visitor.onShape(StringShape.builder().id(id)); - visitor.onTrait(id, ShapeId.from("foo.baz#Bar"), Node.from(true)); - Model model = visitor.onEnd().unwrap(); - - assertThat(model.expectShape(id).findTrait("foo.baz#Bar").get(), - instanceOf(DynamicTrait.class)); - } - - @Test - public void failsWhenTraitNotFound() { - LoaderVisitor visitor = new LoaderVisitor(FACTORY); - ShapeId id = ShapeId.from("foo.bam#Boo"); - visitor.onShape(StringShape.builder().id(id)); - visitor.onTrait(id, ShapeId.from("foo.baz#Bar"), Node.from(true)); - List events = visitor.onEnd().getValidationEvents(); - - assertThat(events, not(empty())); - } - - @Test - public void supportsCustomProperties() { - Map properties = MapUtils.of("a", true, "b", new HashMap<>()); - LoaderVisitor visitor = new LoaderVisitor(TraitFactory.createServiceFactory(), properties); - - assertThat(visitor.getProperty("a").get(), equalTo(true)); - assertThat(visitor.getProperty("b").get(), equalTo(new HashMap<>())); - assertThat(visitor.getProperty("a", Boolean.class).get(), equalTo(true)); - assertThat(visitor.getProperty("b", Map.class).get(), equalTo(new HashMap<>())); - } - - @Test - public void assertsCorrectPropertyType() { - Assertions.assertThrows(ClassCastException.class, () -> { - Map properties = MapUtils.of("a", true); - LoaderVisitor visitor = new LoaderVisitor(TraitFactory.createServiceFactory(), properties); - - visitor.getProperty("a", Integer.class).get(); - }); - } - - /** - * Members are only ever added to non-existent shapes when parsing a containing - * shape fails. Normally, the members are added to the containing builder at - * the end of the loading process. Members are added to the LoaderVisitor first, - * followed by the containing shape. If there's a syntax error or a duplicate - * shape error when loading the containing shape, then the members are present - * in the LoaderVisitor but the containing shape is not. In this event, the - * LoaderVisitor just logs and continues. The loading process will eventually - * fail with the syntax error. - */ - @Test - public void ignoresAddingMemberToNonExistentShape() { - LoaderVisitor visitor = new LoaderVisitor(FACTORY); - visitor.onShape(MemberShape.builder().id("foo.baz#Bar$bam").target("foo.baz#Bam")); - visitor.onEnd().unwrap(); - } - - @Test - public void errorWhenShapesConflict() { - Assertions.assertThrows(SourceException.class, () -> { - LoaderVisitor visitor = new LoaderVisitor(FACTORY); - Shape shape = StringShape.builder().id("smithy.foo#Baz").build(); - visitor.onShape(shape); - visitor.onShape(shape); - visitor.onEnd(); - }); - } - - @Test - public void ignoresDuplicateFiles() { - URL file = getClass().getResource("valid/trait-definitions.smithy"); - Model model = Model.assembler().addImport(file).assemble().unwrap(); - Model.assembler().addModel(model).addImport(file).assemble().unwrap(); - } -} diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/ModelAssemblerTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/ModelAssemblerTest.java index 623ac9d8064..6f0bd766214 100644 --- a/smithy-model/src/test/java/software/amazon/smithy/model/loader/ModelAssemblerTest.java +++ b/smithy-model/src/test/java/software/amazon/smithy/model/loader/ModelAssemblerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.startsWith; @@ -54,7 +55,10 @@ import software.amazon.smithy.model.shapes.ShapeId; import software.amazon.smithy.model.shapes.ShapeType; import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.traits.DeprecatedTrait; import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.model.traits.DynamicTrait; import software.amazon.smithy.model.traits.MediaTypeTrait; import software.amazon.smithy.model.traits.SensitiveTrait; import software.amazon.smithy.model.traits.SuppressTrait; @@ -452,7 +456,6 @@ public void canIgnoreUnknownTraits() { .putProperty(ModelAssembler.ALLOW_UNKNOWN_TRAITS, true) .assemble(); - System.out.println(result.getValidationEvents()); assertThat(result.getValidationEvents(), not(empty())); assertFalse(result.isBroken()); } @@ -472,20 +475,6 @@ public void canLoadModelsFromJar() { } } - @Test - public void gracefullyParsesPartialDocuments() { - String document = "namespace foo.baz\n" - + "@required\n" // < this trait is invalid, but that's not validated due to the syntax error - + "string MyString\n" - + "str"; // < syntax error here - ValidatedResult result = new ModelAssembler().addUnparsedModel("foo.smithy", document).assemble(); - - assertTrue(result.isBroken()); - assertThat(result.getValidationEvents(Severity.ERROR), hasSize(1)); - assertTrue(result.getResult().isPresent()); - assertTrue(result.getResult().get().getShape(ShapeId.from("foo.baz#MyString")).isPresent()); - } - @Test public void canDisableValidation() { String document = "namespace foo.baz\n" @@ -510,4 +499,121 @@ public void canHandleBadSuppressions() { .assemble(); assertTrue(result.isBroken()); } + + @Test + public void detectsDuplicateTraitsWithDifferentTypes() { + // This test ensures that traits with different types are detected too. + // This is picked up by normal trait collision, but this test is a good + // regression test to ensure that trait specific stuff like coercion + // is handled correctly. + String document1 = "namespace foo.baz\n" + + "@trait\n" + + "structure myTrait {}\n"; + String document2 = "namespace foo.baz\n" + + "@trait\n" + + "integer myTrait\n"; + String document3 = "namespace foo.baz\n" + + "@myTrait(10)\n" + + "string MyShape\n"; + ValidatedResult result = new ModelAssembler() + .addUnparsedModel("1.smithy", document1) + .addUnparsedModel("2.smithy", document2) + .addUnparsedModel("3.smithy", document3) + .assemble(); + + assertTrue(result.isBroken()); + } + + @Test + public void allowsConflictingShapesThatAreEqual() { + // While these two shapes have different traits, the traits merge. + // Since they are equivalent the conflicts are allowed. + String document1 = "namespace foo.baz\n" + + "@deprecated\n" + + "structure Foo {\n" + + " foo: String," + + "}\n"; + String document2 = "namespace foo.baz\n" + + "structure Foo {\n" + + " @sensitive\n" + + " foo: String,\n" + + "}\n"; + ValidatedResult result = new ModelAssembler() + .addUnparsedModel("1.smithy", document1) + .addUnparsedModel("2.smithy", document2) + .assemble(); + + assertFalse(result.isBroken()); + + // Ensure that traits across each duplicate are all merged together. + StructureShape shape = result.unwrap().expectShape(ShapeId.from("foo.baz#Foo"), StructureShape.class); + assertTrue(shape.hasTrait(DeprecatedTrait.class)); + assertTrue(shape.getMember("foo").isPresent()); + assertTrue(shape.getMember("foo").get().hasTrait(SensitiveTrait.class)); + } + + @Test + public void detectsConflictingDuplicateAggregates() { + // Aggregate shapes have to have the same exact members. + String document1 = "namespace foo.baz\n" + + "structure Foo {\n" + + " foo: String," + + "}\n"; + String document2 = "namespace foo.baz\n" + + "structure Foo {}\n"; + ValidatedResult result = new ModelAssembler() + .addUnparsedModel("1.smithy", document1) + .addUnparsedModel("2.smithy", document2) + .assemble(); + + assertTrue(result.isBroken()); + } + + @Test + public void allowsDuplicateEquivalentMetadata() { + String document = "metadata foo = 10\n"; + ValidatedResult result = new ModelAssembler() + .addUnparsedModel("1.smithy", document) + .addUnparsedModel("2.smithy", document) + .assemble(); + + assertFalse(result.isBroken()); + assertThat(result.unwrap().getMetadata().get("foo"), equalTo(Node.from(10))); + } + + @Test + public void mergesConflictingMetadataArrays() { + String document = "metadata foo = [\"a\"]\n"; + ValidatedResult result = new ModelAssembler() + .addUnparsedModel("1.smithy", document) + .addUnparsedModel("2.smithy", document) + .assemble(); + + assertFalse(result.isBroken()); + assertThat(result.unwrap().getMetadata().get("foo"), equalTo(Node.fromStrings("a", "a"))); + } + + @Test + public void createsDynamicTraitWhenTraitFactoryReturnsEmpty() { + ShapeId id = ShapeId.from("ns.foo#Test"); + ShapeId traitId = ShapeId.from("smithy.foo#baz"); + String document = "{\n" + + "\"smithy\": \"" + Model.MODEL_VERSION + "\",\n" + + " \"shapes\": {\n" + + " \"" + id + "\": {\n" + + " \"type\": \"string\",\n" + + " \"traits\": {\"" + traitId + "\": true}\n" + + " }\n" + + " }\n" + + "}"; + Model model = new ModelAssembler() + .addUnparsedModel(SourceLocation.NONE.getFilename(), document) + .putProperty(ModelAssembler.ALLOW_UNKNOWN_TRAITS, true) + .assemble() + .unwrap(); + + + assertTrue(model.expectShape(id).findTrait(traitId).isPresent()); + assertThat(model.expectShape(id).findTrait(traitId).get(), instanceOf(DynamicTrait.class)); + } } diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/shapes/SmithyIdlModelSerializerTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/shapes/SmithyIdlModelSerializerTest.java index 11224d49591..0dd089da1a1 100644 --- a/smithy-model/src/test/java/software/amazon/smithy/model/shapes/SmithyIdlModelSerializerTest.java +++ b/smithy-model/src/test/java/software/amazon/smithy/model/shapes/SmithyIdlModelSerializerTest.java @@ -14,6 +14,7 @@ import java.nio.file.Paths; import java.util.Map; import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; @@ -41,7 +42,7 @@ public void testConversion(Path path) { } String serializedString = serialized.entrySet().iterator().next().getValue(); - assertThat(serializedString, equalTo(IoUtils.readUtf8File(path))); + Assertions.assertEquals(serializedString, IoUtils.readUtf8File(path)); } @Test diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/detects-duplicates-with-members.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/detects-duplicates-with-members.errors index 597be60aa3f..8f0f08466a8 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/detects-duplicates-with-members.errors +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/detects-duplicates-with-members.errors @@ -1 +1 @@ -[ERROR] -: Duplicate shape definition for `foo.baz#Foo` found at | Model +[ERROR] -: Conflicting shape definition for `foo.baz#Foo` found at | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/metadata-conflict.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/metadata-conflict.errors new file mode 100644 index 00000000000..25a0c4e5bee --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/metadata-conflict.errors @@ -0,0 +1,3 @@ +[ERROR] -: Metadata conflict for key `foo`. Defined in both | Model +[ERROR] -: Metadata conflict for key `baz`. Defined in both | Model +[ERROR] -: Metadata conflict for key `bar`. Defined in both | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/metadata-conflict.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/metadata-conflict.smithy new file mode 100644 index 00000000000..3c0092dcb6c --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/metadata-conflict.smithy @@ -0,0 +1,6 @@ +metadata foo = 10 +metadata foo = ["hi"] +metadata baz = [10] +metadata baz = "hi" +metadata bar = 10 +metadata bar = "hi" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/unsupported-json-version.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/unsupported-json-version.errors new file mode 100644 index 00000000000..5cdc4139e03 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/unsupported-json-version.errors @@ -0,0 +1 @@ +[ERROR] -: Unsupported Smithy version number: 999 | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/unsupported-json-version.json b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/unsupported-json-version.json new file mode 100644 index 00000000000..af9a865827f --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/unsupported-json-version.json @@ -0,0 +1,3 @@ +{ + "smithy": "999" +} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/conflicting-shape-def.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/conflicting-shape-def.smithy index 7a2969f42cf..77107ef0a3a 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/conflicting-shape-def.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/conflicting-shape-def.smithy @@ -1,4 +1,4 @@ -// shape name `String` conflicts with imported shape `smithy.api#String` +// Shape name `String` conflicts with imported shape `smithy.api#String` namespace smithy.example use smithy.api#String diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/conflicting-trait-def.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/conflicting-trait-def.smithy index ebe9e7601f3..340c578aabc 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/conflicting-trait-def.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/conflicting-trait-def.smithy @@ -1,4 +1,4 @@ -// shape name `deprecated` conflicts with imported shape `smithy.api#deprecated` +// Shape name `deprecated` conflicts with imported shape `smithy.api#deprecated` namespace smithy.example use smithy.api#deprecated