Add generic depdency graph implementation to smithy-utils and use it for handling topological sorting

.changes/next-release/feature-5918b18d6bad7f0ad2c8061f9910f70dccadce2c.json

@@ -0,0 +1,7 @@
+{
+  "type": "feature",
+  "description": "Updated `LoaderShapeMap` to use `DependencyGraph` to sort shapes\ninstead of `TopologicalShapeSort`. This results in some error messages\nchanging:\n\n* There is no longer a special message for missing transitive mixins.\n* The message for genrically missing mixins is now applied by\n  ApplyMixin, which has a slightly different wording and which will\n  trigger for each mising mixin.\n* The error message for detected cycles will now include the entire\n  cycle that a shape is part of, rather than just the edges that it\n  is directly connected to.",
+  "pull_requests": [
+    "[#2774](https://github.com/smithy-lang/smithy/pull/2774)"
+  ]
+}

.changes/next-release/feature-9d99ada83fb4eaabb1f0d929c0da87d73134dd3e.json

@@ -0,0 +1,7 @@
+{
+  "type": "feature",
+  "description": "Add the ability for smithy build plugins to declare that they must be run before or after other plugins. These dependencies are soft, so missing dependencies will be logged and ignored.",
+  "pull_requests": [
+    "[#2774](https://github.com/smithy-lang/smithy/pull/2774)"
+  ]
+}

.changes/next-release/feature-ca86aa5a954b100820845d1b67a3491c40b8058c.json

@@ -0,0 +1,7 @@
+{
+  "type": "feature",
+  "description": "Updated `ReplaceShapes` to use `DependencyGraph` to sort shapes.",
+  "pull_requests": [
+    "[#2774](https://github.com/smithy-lang/smithy/pull/2774)"
+  ]
+}

.changes/next-release/feature-d368237cdeb1655c7c10969e36c6aa3ca3b3034a.json

@@ -0,0 +1,7 @@
+{
+  "type": "feature",
+  "description": "Add a generic dependency graph to smithy-utils to be used for sorting various dependent objects, such as integrations and plugins.",
+  "pull_requests": [
+    "[#2774](https://github.com/smithy-lang/smithy/pull/2774)"
+  ]
+}

.changes/next-release/feature-e0f8e99e0b8a04946095a8711bd3e921405d9a53.json

@@ -0,0 +1,7 @@
+{
+  "type": "feature",
+  "description": "Deprecated `TopologicalShapeSort` in favor of `DependencyGraph`.",
+  "pull_requests": [
+    "[#2774](https://github.com/smithy-lang/smithy/pull/2774)"
+  ]
+}

smithy-build/src/main/java/software/amazon/smithy/build/SmithyBuildImpl.java

@@ -33,6 +33,8 @@
 import software.amazon.smithy.model.node.ObjectNode;
 import software.amazon.smithy.model.transform.ModelTransformer;
 import software.amazon.smithy.model.validation.ValidatedResult;
+import software.amazon.smithy.utils.CycleException;
+import software.amazon.smithy.utils.DependencyGraph;
 import software.amazon.smithy.utils.Pair;
 import software.amazon.smithy.utils.SmithyBuilder;
 
@@ -216,7 +218,8 @@ void applyAllProjections(
     private List<ResolvedPlugin> resolvePlugins(String projectionName, ProjectionConfig config) {
         // Ensure that no two plugins use the same artifact name.
         Set<String> seenArtifactNames = new HashSet<>();
-        List<ResolvedPlugin> resolvedPlugins = new ArrayList<>();
+        DependencyGraph<PluginId> dependencyGraph = new DependencyGraph<>();
+        Map<PluginId, ResolvedPlugin> resolvedPlugins = new HashMap<>();
 
         for (Map.Entry<String, ObjectNode> pluginEntry : getCombinedPlugins(config).entrySet()) {
             PluginId id = PluginId.from(pluginEntry.getKey());
@@ -226,12 +229,45 @@ private List<ResolvedPlugin> resolvePlugins(String projectionName, ProjectionCon
                         id.getArtifactName(),
                         projectionName));
             }
+
             createPlugin(projectionName, id).ifPresent(plugin -> {
-                resolvedPlugins.add(new ResolvedPlugin(id, plugin, pluginEntry.getValue()));
+                dependencyGraph.add(id);
+                for (String dependency : plugin.runAfter()) {
+                    dependencyGraph.addDependency(id, PluginId.from(dependency));
+                }
+                for (String dependant : plugin.runBefore()) {
+                    dependencyGraph.addDependency(PluginId.from(dependant), id);
+                }
+                resolvedPlugins.put(id, new ResolvedPlugin(id, plugin, pluginEntry.getValue()));
             });
         }
 
-        return resolvedPlugins;
+        List<PluginId> sorted;
+        try {
+            sorted = dependencyGraph.toSortedList();
+        } catch (CycleException e) {
+            throw new SmithyBuildException(e.getMessage(), e);
+        }
+
+        List<ResolvedPlugin> result = new ArrayList<>();
+        for (PluginId id : sorted) {
+            ResolvedPlugin resolvedPlugin = resolvedPlugins.get(id);
+            if (resolvedPlugin != null) {
+                result.add(resolvedPlugin);
+                continue;
+            }
+
+            // If the plugin wasn't resolved, that's either because it was declared but not
+            // available on the classpath or not declared at all. In the former case we
+            // already have a log message that covers it, including a default build failure.
+            // If the plugin was seen, it was declared, and we don't need to log a second
+            // time.
+            if (!seenArtifactNames.contains(id.getArtifactName())) {
+                logMissingPluginDependency(dependencyGraph, id);
+            }
+        }
+
+        return result;
     }
 
     private Map<String, ObjectNode> getCombinedPlugins(ProjectionConfig projection) {
@@ -259,6 +295,33 @@ private Optional<SmithyBuildPlugin> createPlugin(String projectionName, PluginId
         throw new SmithyBuildException(message);
     }
 
+    private void logMissingPluginDependency(DependencyGraph<PluginId> dependencyGraph, PluginId name) {
+        StringBuilder message = new StringBuilder("Could not find plugin named '");
+        message.append(name).append('\'');
+        if (!dependencyGraph.getDirectDependants(name).isEmpty()) {
+            message.append(" that was supposed to run before plugins [");
+            message.append(dependencyGraph.getDirectDependants(name)
+                    .stream()
+                    .map(PluginId::toString)
+                    .collect(Collectors.joining(", ")));
+            message.append("]");
+        }
+        if (!dependencyGraph.getDirectDependencies(name).isEmpty()) {
+            if (!dependencyGraph.getDirectDependants(name).isEmpty()) {
+                message.append(" and ");
+            } else {
+                message.append(" that ");
+            }
+            message.append("was supposed to run after plugins [");
+            message.append(dependencyGraph.getDirectDependencies(name)
+                    .stream()
+                    .map(PluginId::toString)
+                    .collect(Collectors.joining(", ")));
+            message.append("]");
+        }
+        LOGGER.warning(message.toString());
+    }
+
     private boolean areAnyResolvedPluginsSerial(List<ResolvedPlugin> resolvedPlugins) {
         for (ResolvedPlugin plugin : resolvedPlugins) {
             if (plugin.plugin.isSerial()) {

smithy-build/src/main/java/software/amazon/smithy/build/SmithyBuildPlugin.java

@@ -5,6 +5,7 @@
 package software.amazon.smithy.build;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.ServiceLoader;
@@ -104,4 +105,28 @@ static Function<String, Optional<SmithyBuildPlugin>> createServiceFactory() {
     static Function<String, Optional<SmithyBuildPlugin>> createServiceFactory(ClassLoader classLoader) {
         return createServiceFactory(ServiceLoader.load(SmithyBuildPlugin.class, classLoader));
     }
+
+    /**
+     * Gets the names of plugins that this plugin must come before.
+     *
+     * <p>Dependencies are soft. Dependencies on plugin names that cannot be found
+     * log a warning and are ignored.
+     *
+     * @return Returns the plugin names this must come before.
+     */
+    default List<String> runBefore() {
+        return Collections.emptyList();
+    }
+
+    /**
+     * Gets the name of the plugins that this plugin must come after.
+     *
+     * <p>Dependencies are soft. Dependencies on plugin names that cannot be found
+     * log a warning and are ignored.
+     *
+     * @return Returns the plugins names this must come after.
+     */
+    default List<String> runAfter() {
+        return Collections.emptyList();
+    }
 }

smithy-build/src/test/java/software/amazon/smithy/build/CyclicPlugin1.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package software.amazon.smithy.build;
+
+import java.util.List;
+import software.amazon.smithy.utils.ListUtils;
+
+public class CyclicPlugin1 implements SmithyBuildPlugin {
+    @Override
+    public String getName() {
+        return "cyclicplugin1";
+    }
+
+    @Override
+    public void execute(PluginContext context) {}
+
+    @Override
+    public List<String> runBefore() {
+        return ListUtils.of("cyclicplugin2");
+    }
+}

smithy-build/src/test/java/software/amazon/smithy/build/CyclicPlugin2.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package software.amazon.smithy.build;
+
+import java.util.List;
+import software.amazon.smithy.utils.ListUtils;
+
+public class CyclicPlugin2 implements SmithyBuildPlugin {
+    @Override
+    public String getName() {
+        return "cyclicplugin2";
+    }
+
+    @Override
+    public void execute(PluginContext context) {}
+
+    @Override
+    public List<String> runBefore() {
+        return ListUtils.of("cyclicplugin1");
+    }
+}

smithy-build/src/test/java/software/amazon/smithy/build/SmithyBuildTest.java

@@ -15,6 +15,7 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.File;
@@ -25,6 +26,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -582,6 +584,92 @@ private void assertPluginPresent(String pluginName, String outputFileName, Proje
         }
     }
 
+    @Test
+    public void topologicallySortsPlugins() throws Exception {
+        Map<String, SmithyBuildPlugin> plugins = MapUtils.of(
+                "timestampPlugin1",
+                new TimestampPlugin1(),
+                "timestampPlugin2",
+                new TimestampPlugin2(),
+                "timestampPlugin3",
+                new TimestampPlugin3());
+        Function<String, Optional<SmithyBuildPlugin>> factory = SmithyBuildPlugin.createServiceFactory();
+        Function<String, Optional<SmithyBuildPlugin>> composed = name -> OptionalUtils.or(
+                Optional.ofNullable(plugins.get(name)),
+                () -> factory.apply(name));
+
+        SmithyBuild builder = new SmithyBuild().pluginFactory(composed);
+        builder.fileManifestFactory(MockManifest::new);
+        builder.config(SmithyBuildConfig.builder()
+                .load(Paths.get(getClass().getResource("topologically-sorts-plugins.json").toURI()))
+                .outputDirectory("/foo")
+                .build());
+
+        SmithyBuildResult results = builder.build();
+        ProjectionResult source = results.getProjectionResult("source").get();
+        MockManifest manifest1 = (MockManifest) source.getPluginManifest("timestampPlugin1").get();
+        MockManifest manifest2 = (MockManifest) source.getPluginManifest("timestampPlugin2").get();
+        MockManifest manifest3 = (MockManifest) source.getPluginManifest("timestampPlugin3").get();
+
+        Instant instant1 = Instant.ofEpochMilli(Long.parseLong(manifest1.getFileString("timestamp").get()));
+        Instant instant2 = Instant.ofEpochMilli(Long.parseLong(manifest2.getFileString("timestamp").get()));
+        Instant instant3 = Instant.ofEpochMilli(Long.parseLong(manifest3.getFileString("timestamp").get()));
+
+        assertTrue(instant2.isBefore(instant1));
+        assertTrue(instant3.isAfter(instant1));
+    }
+
+    @Test
+    public void dependenciesAreSoft() throws Exception {
+        Map<String, SmithyBuildPlugin> plugins = MapUtils.of(
+                "timestampPlugin2",
+                new TimestampPlugin2(),
+                "timestampPlugin3",
+                new TimestampPlugin3());
+        Function<String, Optional<SmithyBuildPlugin>> factory = SmithyBuildPlugin.createServiceFactory();
+        Function<String, Optional<SmithyBuildPlugin>> composed = name -> OptionalUtils.or(
+                Optional.ofNullable(plugins.get(name)),
+                () -> factory.apply(name));
+
+        SmithyBuild builder = new SmithyBuild().pluginFactory(composed);
+        builder.fileManifestFactory(MockManifest::new);
+        builder.config(SmithyBuildConfig.builder()
+                .load(Paths.get(getClass().getResource("soft-plugin-dependencies.json").toURI()))
+                .outputDirectory("/foo")
+                .build());
+
+        SmithyBuildResult results = builder.build();
+        ProjectionResult source = results.getProjectionResult("source").get();
+        MockManifest manifest2 = (MockManifest) source.getPluginManifest("timestampPlugin2").get();
+        MockManifest manifest3 = (MockManifest) source.getPluginManifest("timestampPlugin3").get();
+
+        assertTrue(manifest2.hasFile("timestamp"));
+        assertTrue(manifest3.hasFile("timestamp"));
+    }
+
+    @Test
+    public void detectsPluginCycles() throws Exception {
+        Map<String, SmithyBuildPlugin> plugins = MapUtils.of(
+                "cyclicplugin1",
+                new CyclicPlugin1(),
+                "cyclicplugin2",
+                new CyclicPlugin2());
+        Function<String, Optional<SmithyBuildPlugin>> factory = SmithyBuildPlugin.createServiceFactory();
+        Function<String, Optional<SmithyBuildPlugin>> composed = name -> OptionalUtils.or(
+                Optional.ofNullable(plugins.get(name)),
+                () -> factory.apply(name));
+
+        SmithyBuild builder = new SmithyBuild().pluginFactory(composed);
+        builder.fileManifestFactory(MockManifest::new);
+        builder.config(SmithyBuildConfig.builder()
+                .load(Paths.get(getClass().getResource("detects-plugin-cycles.json").toURI()))
+                .outputDirectory("/foo")
+                .build());
+
+        SmithyBuildException e = assertThrows(SmithyBuildException.class, builder::build);
+        assertThat(e.getMessage(), containsString("Cycle(s) detected"));
+    }
+
     @Test
     public void buildCanOverrideConfigOutputDirectory() throws Exception {
         Path outputDirectory = Paths.get("/custom/foo");

smithy-build/src/test/java/software/amazon/smithy/build/TimestampPlugin1.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package software.amazon.smithy.build;
+
+import java.time.Instant;
+
+public class TimestampPlugin1 implements SmithyBuildPlugin {
+    @Override
+    public String getName() {
+        return "timestampPlugin1";
+    }
+
+    @Override
+    public void execute(PluginContext context) {
+        context.getFileManifest().writeFile("timestamp", String.valueOf(Instant.now().toEpochMilli()));
+        try {
+            Thread.sleep(1);
+        } catch (InterruptedException ignored) {}
+    }
+
+    // This is made serial to protect against test failures if we decide later to
+    // have plugins run in parallel. These plugins MUST run serially with respect
+    // to each other to function.
+    @Override
+    public boolean isSerial() {
+        return true;
+    }
+}