Add Maven dependency resolution to the CLI

.github/workflows/ci.yml

@@ -26,7 +26,11 @@ jobs:
           distribution: 'zulu'
 
       - name: Clean and build
-        run: ./gradlew clean build -Plog-tests
+        run: ./gradlew clean build -Plog-tests --stacktrace
+
+      - name: CLI integration tests
+        if: matrix.java >= 17
+        run: ./gradlew :smithy-cli:integ -Plog-tests --stacktrace
 
       - uses: actions/upload-artifact@v3
         if: failure()

config/checkstyle/checkstyle.xml

@@ -163,8 +163,12 @@
         <module name="RightCurly">
             <property name="id" value="RightCurlyAlone"/>
             <property name="option" value="alone"/>
-            <property name="tokens"
-                      value="CLASS_DEF, METHOD_DEF, LITERAL_FOR, LITERAL_WHILE, STATIC_INIT, INSTANCE_INIT"/>
+            <property name="tokens" value="LITERAL_FOR, LITERAL_WHILE, STATIC_INIT, INSTANCE_INIT"/>
+        </module>
+        <module name="RightCurly">
+            <property name="id" value="RightCurlyAloneOrSingle"/>
+            <property name="option" value="alone_or_singleline"/>
+            <property name="tokens" value="CLASS_DEF, METHOD_DEF"/>
         </module>
 
         <!-- Checks for common coding problems               -->

config/spotbugs/filter.xml

@@ -148,4 +148,15 @@
         <Class name="software.amazon.smithy.model.shapes.ShapeId$ShapeIdFactory"/>
         <Bug pattern="RV_RETURN_VALUE_OF_PUTIFABSENT_IGNORED"/>
     </Match>
+
+    <!-- The FileWriter API that specifies a Charset is not available in Java 8 -->
+    <Match>
+        <Class name="software.amazon.smithy.cli.commands.WarmupCommand"/>
+        <Bug pattern="DM_DEFAULT_ENCODING"/>
+    </Match>
+
+    <!-- SecurityManager is deprecated and scheduled for removal, so this isn't a good check. -->
+    <Match>
+        <Bug pattern="DP_CREATE_CLASSLOADER_INSIDE_DO_PRIVILEGED"/>
+    </Match>
 </FindBugsFilter>

docs/source-2.0/guides/building-models/build-config.rst

@@ -33,14 +33,23 @@ The configuration file accepts the following properties:
         projection will create a subdirectory named after the projection, and
         the artifacts from the projection, including a ``model.json`` file,
         will be placed in the directory.
+    * - sources
+      - ``[string]``
+      - Provides a list of relative files or directories that contain the
+        models that are considered the source models of the build. When a
+        directory is encountered, all files in the entire directory tree are
+        added as sources. Sources are relative to the configuration file.
     * - imports
-      - ``string[]``
-      - Provides a list of relative imports to combine into a single model.
-        When a directory is encountered, all files and all files within all
-        subdirectories are imported. Note that build systems MAY choose to rely
-        on other mechanisms for importing models and forming a composite model.
-        These imports are used in every projection. Note: imports are relative
-        to the configuration file.
+      - ``[string]``
+      - Provides a list of model files and directories to load when validating
+        and building the model. Imports are a local dependency: they are not
+        considered part of model package being built, but are required to build
+        the model package. Models added through ``imports`` are not present in
+        the output of the built-in ``sources`` plugin.
+
+        When a directory is encountered, all files in the entire directory
+        tree are imported. Imports defined at the top-level are used in every
+        projection. Imports are relative to the configuration file.
     * - projections
       - ``map<string, object>``
       - A map of projection names to projection configurations.
@@ -54,6 +63,11 @@ The configuration file accepts the following properties:
       - If a plugin can't be found, Smithy will by default fail the build. This
         setting can be set to ``true`` to allow the build to progress even if
         a plugin can't be found on the classpath.
+    * - maven
+      - :ref:`maven-configuration` structure
+      - Defines Java Maven dependencies needed to build the model.
+        Dependencies are used to bring in model imports, build plugins,
+        validators, transforms, and other extensions.
 
 The following is an example ``smithy-build.json`` configuration:
 
@@ -62,7 +76,13 @@ The following is an example ``smithy-build.json`` configuration:
     {
         "version": "1.0",
         "outputDirectory": "build/output",
+        "sources": ["model"],
         "imports": ["foo.json", "some/directory"],
+        "maven": {
+            "dependencies": [
+                "software.amazon.smithy:smithy-aws-traits:__smithy_version__"
+            ]
+        },
         "projections": {
             "my-abstract-projection": {
                 "abstract": true
@@ -100,6 +120,166 @@ The following is an example ``smithy-build.json`` configuration:
     }
 
 
+.. _maven-configuration:
+
+Maven configuration
+===================
+
+Maven dependencies and repositories can be defined in smithy-build.json files,
+and the Smithy CLI will automatically resolve these dependencies using the
+`Apache Maven`_ dependency resolver.
+
+The ``maven`` property accepts the following configuration:
+
+.. list-table::
+    :header-rows: 1
+    :widths: 10 20 70
+
+    * - Property
+      - Type
+      - Description
+    * - dependencies
+      - ``[string]``
+      - A list of Maven dependency coordinates in the form of
+        ``groupId:artifactId:version``. The Smithy CLI will search each
+        registered Maven repository for the dependency.
+    * - repositories
+      - ``[`` :ref:`maven-repositories` ``]``
+      - A list of Maven repositories to search for dependencies. If no
+        repositories are defined and the :ref:`SMITHY_MAVEN_REPOS environment variable <SMITHY_MAVEN_REPOS>`
+        is not defined, then this value defaults to `Maven Central`_.
+
+
+Dependency versions
+-------------------
+
+Maven dependencies are defined using GAV coordinates
+(``groupId:artifactId:version``). The version of a dependency can specify
+*version requirements* that are used to control how versions are resolved.
+Requirements can be given as *soft requirements*, meaning the version can be
+replaced by other versions found in the dependency graph. Hard requirements
+can be used to mandate a particular version and override soft requirements.
+Maven picks the highest version of each project that satisfies all the hard
+requirements of the dependencies on that project. If no version satisfies
+all the hard requirements, dependency resolution fails.
+
+The following table demonstrates version requirement syntax as defined in
+the `official Maven documentation`_:
+
+.. list-table:: Dependency version syntax
+    :header-rows: 1
+    :widths: 20 80
+
+    * - Version
+      - Description
+    * - ``1.0``
+      - Soft requirement for 1.0. Use 1.0 if no other version appears earlier
+        in the dependency tree.
+    * - ``[1.0]``
+      - Hard requirement for 1.0. Use 1.0 and only 1.0.
+    * - ``(,1.0]``
+      - Hard requirement for any version <= 1.0.
+    * - ``[1.2,1.3]``
+      - Hard requirement for any version between 1.2 and 1.3 inclusive.
+    * - ``[1.0,2.0)``
+      - 1.0 <= x < 2.0; Hard requirement for any version between 1.0 inclusive
+        and 2.0 exclusive.
+    * - ``[1.5,)``
+      - Hard requirement for any version greater than or equal to 1.5.
+    * - ``(,1.0],[1.2,)``
+      - Multiple requirements are separated by commas. This requirement
+        forbids version 1.1 by adding a hard requirement for any version less
+        than or equal to 1.0 or greater than or equal to 1.2.
+    * - ``(,1.1),(1.1,)``
+      - Hard requirement for any version except 1.1 (for example, if 1.1
+        has a critical vulnerability).
+
+
+Unsupported version requirements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* LATEST, SNAPSHOT, RELEASE, latest-status, and latest.* versions are not
+  supported.
+* Gradle style ``+`` versions are not supported.
+
+
+.. _maven-repositories:
+
+Maven Repositories
+------------------
+
+The ``repositories`` property accepts a list of structures that each accept
+the following configuration:
+
+.. list-table::
+    :header-rows: 1
+    :widths: 10 20 70
+
+    * - Property
+      - Type
+      - Description
+    * - url
+      - ``string``
+      - The URL of the repository (for example, ``https://repo.maven.apache.org/maven2``).
+    * - httpCredentials
+      - ``string``
+      - HTTP basic or digest credentials to use with the repository.
+        Credentials are provided in the form of "username:password".
+
+        .. warning::
+
+            Credentials SHOULD NOT be defined statically in a smithy-build.json
+            file. Instead, use :ref:`environment variables <build_envars>` to
+            keep credentials out of source control.
+
+.. code-block:: json
+
+    {
+        "version": "1.0",
+        "maven": {
+            "repositories": [
+                {
+                    "url": "https://my_domain-111122223333.d.codeartifact.region.amazonaws.com/maven/my_repo/",
+                    "httpCredentials": "aws:${CODEARTIFACT_AUTH_TOKEN}"
+                }
+            ],
+            "dependencies": [
+                "software.amazon.smithy:smithy-aws-traits:__smithy_version__"
+            ]
+        }
+    }
+
+
+.. _SMITHY_MAVEN_REPOS:
+
+SMITHY_MAVEN_REPOS environment variable
+---------------------------------------
+
+When using the Smithy CLI, the ``SMITHY_MAVEN_REPOS`` environment variable can
+be used to configure Maven repositories automatically. The
+``SMITHY_MAVEN_REPOS`` environment variable is a pipe-delimited value (``|``)
+that contains the URL of each repository to use.
+
+.. code-block::
+
+    SMITHY_MAVEN_REPOS="https://repo.maven.apache.org/maven2|https://example.repo.com/maven"
+
+Credentials can be provided in the URL. For example:
+
+.. code-block::
+
+    SMITHY_MAVEN_REPOS='https://user:[email protected]/maven'
+
+When repositories are provided through the ``SMITHY_MAVEN_REPOS`` environment
+variable, no default repositories are assumed when resolving the
+``maven.repositories`` setting.
+
+.. important::
+
+    Repositories defined in ``SMITHY_MAVEN_REPOS`` take precedence over
+    repositories defined through smithy-build.json configuration.
+
+
 .. _projections:
 
 Projections
@@ -129,11 +309,12 @@ A projection accepts the following configuration:
         Smithy will not build artifacts for abstract projections. Abstract
         projections must not define ``imports`` or ``plugins``.
     * - imports
-      - ``string[]``
+      - ``[string]``
       - Provides a list of relative imports to include when building this
-        specific projection. When a directory is encountered, all files and
-        all files within all subdirectories are imported. Note: imports are
-        relative to the configuration file.
+        specific projection (in addition to any imports defined at the
+        top-level). When a directory is encountered, all files in the
+        directory tree are imported. Note: imports are relative to the
+        configuration file.
     * - transforms
       - ``list<Transforms>``
       - Defines the transformations to apply to the projection.
@@ -1401,3 +1582,6 @@ ignored. A Smithy manifest file is stored in a JAR as ``META-INF/smithy/manifest
 All model files referenced by the manifest are relative to ``META-INF/smithy/``.
 
 .. _Java SPI: https://docs.oracle.com/javase/tutorial/sound/SPI-intro.html
+.. _Apache Maven: https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html
+.. _Maven Central: https://search.maven.org
+.. _official Maven documentation: https://maven.apache.org/pom.html#dependency-version-requirement-specification

smithy-aws-protocol-tests/build.gradle

@@ -25,7 +25,7 @@ ext {
 }
 
 dependencies {
-    implementation project(":smithy-cli")
+    implementation project(path: ":smithy-cli", configuration: "shadow")
     implementation project(":smithy-protocol-test-traits")
     implementation project(":smithy-aws-traits")
     api project(":smithy-validation-model")

smithy-build/src/main/java/software/amazon/smithy/build/SmithyBuild.java

@@ -92,6 +92,15 @@ public static SmithyBuild create(ClassLoader classLoader, Supplier<ModelAssemble
                 .pluginClassLoader(classLoader);
     }
 
+    /**
+     * Gets the default directory where smithy-build artifacts are written.
+     *
+     * @return Returns the build output path.
+     */
+    public static Path getDefaultOutputDirectory() {
+        return DefaultPathHolder.DEFAULT_PATH;
+    }
+
     /**
      * Builds the model and applies all projections.
      *
@@ -169,9 +178,19 @@ public void build(Consumer<ProjectionResult> resultCallback, BiConsumer<String,
      */
     public SmithyBuild config(SmithyBuildConfig config) {
         this.config = config;
+        for (String source : config.getSources()) {
+            addSource(Paths.get(source));
+        }
         return this;
     }
 
+    // Add a source path using absolute paths to better de-conflict source files. ModelAssembler also
+    // de-conflicts imports with absolute paths, but this ensures the same file doesn't appear twice in
+    // the build plugin output (though it does not use realpath to de-conflict based on symlinks).
+    private void addSource(Path path) {
+        sources.add(path.toAbsolutePath());
+    }
+
     /**
      * Sets the <strong>required</strong> configuration object used to
      * build the model.
@@ -365,7 +384,9 @@ public SmithyBuild pluginClassLoader(ClassLoader pluginClassLoader) {
      * @return Returns the builder.
      */
     public SmithyBuild registerSources(Path... pathToSources) {
-        Collections.addAll(sources, pathToSources);
+        for (Path path : pathToSources) {
+            addSource(path);
+        }
         return this;
     }
 
@@ -392,4 +413,13 @@ public SmithyBuild pluginFilter(Predicate<String> pluginFilter) {
         this.pluginFilter = Objects.requireNonNull(pluginFilter);
         return this;
     }
+
+    // Lazy initialization holder class idiom.
+    private static final class DefaultPathHolder {
+        private static final Path DEFAULT_PATH = resolveDefaultPath();
+
+        private static Path resolveDefaultPath() {
+            return Paths.get(".").toAbsolutePath().normalize().resolve("build").resolve("smithy");
+        }
+    }
 }

smithy-build/src/main/java/software/amazon/smithy/build/SmithyBuildImpl.java

@@ -92,7 +92,7 @@ final class SmithyBuildImpl {
             outputDirectory = Paths.get(config.getOutputDirectory().get());
         } else {
             // Default the output directory to the current working directory + "./build/smithy"
-            outputDirectory = Paths.get(".").toAbsolutePath().normalize().resolve("build").resolve("smithy");
+            outputDirectory = SmithyBuild.getDefaultOutputDirectory();
         }
 
         // Create the transformers for each projection.