diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java new file mode 100644 index 0000000000000..438e4d5c8b753 --- /dev/null +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java @@ -0,0 +1,84 @@ +package com.microsoft.azure; + +import java.util.ArrayList; +import java.util.List; + +/** + * The type representing node in a {@link DAGraph}. + * + * @param the type of the data stored in the node + */ +public class DAGNode extends Node { + private List dependentKeys; + private int toBeResolved; + + /** + * Creates a DAG node. + * + * @param key unique id of the node + * @param data data to be stored in the node + */ + public DAGNode(String key, T data) { + super(key, data); + dependentKeys = new ArrayList<>(); + } + + /** + * @return a list of keys of nodes in {@link DAGraph} those are dependents on this node + */ + List dependentKeys() { + return this.dependentKeys; + } + + /** + * mark the node identified by the given key as dependent of this node. + * + * @param key the id of the dependent node + */ + public void addDependent(String key) { + this.dependentKeys.add(key); + } + + /** + * @return a list of keys of nodes in {@link DAGraph} that this node depends on + */ + public List dependencyKeys() { + return this.children(); + } + + /** + * mark the node identified by the given key as this node's dependency. + * + * @param dependencyKey the id of the dependency node + */ + public void addDependency(String dependencyKey) { + toBeResolved++; + super.addChild(dependencyKey); + } + + /** + * @return true if this node has any dependency + */ + public boolean hasDependencies() { + return this.hasChildren(); + } + + /** + * @return true if all dependencies of this node are ready to be consumed + */ + boolean hasAllResolved() { + return toBeResolved == 0; + } + + /** + * Reports that one of this node's dependency has been resolved and ready to be consumed. + * + * @param dependencyKey the id of the dependency node + */ + void reportResolved(String dependencyKey) { + if (toBeResolved == 0) { + throw new RuntimeException("invalid state - " + this.key() + ": The dependency '" + dependencyKey + "' is already reported or there is no such dependencyKey"); + } + toBeResolved--; + } +} diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java new file mode 100644 index 0000000000000..388d5599c59c5 --- /dev/null +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java @@ -0,0 +1,156 @@ +package com.microsoft.azure; + +import java.util.ArrayDeque; +import java.util.Map; +import java.util.Queue; + +/** + * Type representing a DAG (directed acyclic graph). + *

+ * each node in a DAG is represented by {@link DAGNode} + * + * @param the type of the data stored in the graph nodes + * @param the type of the nodes in the graph + */ +public class DAGraph> extends Graph { + private Queue queue; + private boolean hasParent; + private U rootNode; + + /** + * Creates a new DAG. + * + * @param rootNode the root node of this DAG + */ + public DAGraph(U rootNode) { + this.rootNode = rootNode; + this.queue = new ArrayDeque<>(); + this.addNode(rootNode); + } + + /** + * @return true if this DAG is merged with another DAG and hence has a parent + */ + public boolean hasParent() { + return hasParent; + } + + /** + * Checks whether the given node is root node of this DAG. + * + * @param node the node {@link DAGNode} to be checked + * @return true if the given node is root node + */ + public boolean isRootNode(U node) { + return this.rootNode == node; + } + + /** + * Merge this DAG with another DAG. + *

+ * this will mark this DAG as a child DAG, the dependencies of nodes in this DAG will be merged + * with (copied to) the parent DAG + * + * @param parent the parent DAG + */ + public void merge(DAGraph parent) { + this.hasParent = true; + parent.rootNode.addDependency(this.rootNode.key()); + this.rootNode.addDependent(parent.rootNode.key()); + for (Map.Entry entry: graph.entrySet()) { + String key = entry.getKey(); + if (!parent.graph.containsKey(key)) { + parent.graph.put(key, entry.getValue()); + } + } + } + + /** + * Prepares this DAG for traversal using getNext method, each call to getNext returns next node + * in the DAG with no dependencies. + */ + public void prepare() { + initializeQueue(); + if (queue.isEmpty()) { + throw new RuntimeException("Found circular dependency"); + } + } + + /** + * Gets next node in the DAG which has no dependency or all of it's dependencies are resolved and + * ready to be consumed. + *

+ * null will be returned when all the nodes are explored + * + * @return next node + */ + public U getNext() { + return graph.get(queue.poll()); + } + + /** + * Gets the data stored in a graph node with a given key. + * + * @param key the key of the node + * @return the value stored in the node + */ + public T getNodeData(String key) { + return graph.get(key).data(); + } + + /** + * Reports that a node is resolved hence other nodes depends on it can consume it. + * + * @param completed the node ready to be consumed + */ + public void reportedCompleted(U completed) { + String dependency = completed.key(); + for (String dependentKey : graph.get(dependency).dependentKeys()) { + DAGNode dependent = graph.get(dependentKey); + dependent.reportResolved(dependency); + if (dependent.hasAllResolved()) { + queue.add(dependent.key()); + } + } + } + + /** + * populate dependents of all nodes. + *

+ * the DAG will be explored in DFS order and all node's dependents will be identified, + * this prepares the DAG for traversal using getNext method, each call to getNext returns next node + * in the DAG with no dependencies. + */ + public void populateDependentKeys() { + this.queue.clear(); + visit(new Visitor() { + @Override + public void visit(U node) { + if (node.dependencyKeys().isEmpty()) { + queue.add(node.key()); + return; + } + + String dependentKey = node.key(); + for (String dependencyKey : node.dependencyKeys()) { + graph.get(dependencyKey) + .dependentKeys() + .add(dependentKey); + } + } + }); + } + + /** + * Initializes the queue that tracks the next set of nodes with no dependencies or + * whose dependencies are resolved. + */ + private void initializeQueue() { + this.queue.clear(); + for (Map.Entry entry: graph.entrySet()) { + if (!entry.getValue().hasDependencies()) { + this.queue.add(entry.getKey()); + } + } + } +} diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/Graph.java b/azure-client-runtime/src/main/java/com/microsoft/azure/Graph.java new file mode 100644 index 0000000000000..a4f2c03cde59f --- /dev/null +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/Graph.java @@ -0,0 +1,78 @@ +package com.microsoft.azure; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Type representing a directed graph data structure. + *

+ * each node in a graph is represented by {@link Node} + * + * @param the type of the data stored in the graph's nodes + * @param the type of the nodes in the graph + */ +public class Graph> { + protected Map graph; + private Set visited; + + /** + * Creates a directed graph. + */ + public Graph() { + this.graph = new HashMap<>(); + this.visited = new HashSet<>(); + } + + /** + * Adds a node to this graph. + * + * @param node the node + */ + public void addNode(U node) { + graph.put(node.key(), node); + } + + /** + * Represents a visitor to be implemented by the consumer who want to visit the + * graph's nodes in DFS order. + * + * @param the type of the node + */ + interface Visitor { + /** + * visit a node. + * + * @param node the node to visited + */ + void visit(U node); + } + + /** + * Perform DFS visit in this graph. + *

+ * The directed graph will be traversed in DFS order and the visitor will be notified as + * search explores each node + * + * @param visitor the graph visitor + */ + public void visit(Visitor visitor) { + for (Map.Entry> item : graph.entrySet()) { + if (!visited.contains(item.getKey())) { + this.dfs(visitor, item.getValue()); + } + } + visited.clear(); + } + + private void dfs(Visitor visitor, Node node) { + visitor.visit(node); + visited.add(node.key()); + for (String childKey : node.children()) { + if (!visited.contains(childKey)) { + this.dfs(visitor, this.graph.get(childKey)); + } + } + } +} diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/Node.java b/azure-client-runtime/src/main/java/com/microsoft/azure/Node.java new file mode 100644 index 0000000000000..e7c4a83768b01 --- /dev/null +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/Node.java @@ -0,0 +1,62 @@ +package com.microsoft.azure; + +import java.util.ArrayList; +import java.util.List; + +/** + * Type represents a node in a {@link Graph}. + * + * @param the type of the data stored in the node + */ +public class Node { + private String key; + private T data; + private List children; + + /** + * Creates a graph node. + * + * @param key unique id of the node + * @param data data to be stored in the node + */ + public Node(String key, T data) { + this.key = key; + this.data = data; + this.children = new ArrayList<>(); + } + + /** + * @return this node's unique id + */ + public String key() { + return this.key; + } + + /** + * @return data stored in this node + */ + public T data() { + return data; + } + + /** + * @return true if this node has any children + */ + public boolean hasChildren() { + return !this.children.isEmpty(); + } + + /** + * @return children (neighbours) of this node + */ + public List children() { + return this.children; + } + + /** + * @param childKey add a child (neighbour) of this node + */ + public void addChild(String childKey) { + this.children.add(childKey); + } +} diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/Resource.java b/azure-client-runtime/src/main/java/com/microsoft/azure/Resource.java index b679fc29897e7..2f98e586a960d 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/Resource.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/Resource.java @@ -84,9 +84,11 @@ public String location() { * Set the location value. * * @param location the location value to set + * @return the resource itself */ - public void withLocation(String location) { + public Resource withLocation(String location) { this.location = location; + return this; } /** @@ -102,8 +104,10 @@ public Map getTags() { * Set the tags value. * * @param tags the tags value to set + * @return the resource itself */ - public void withTags(Map tags) { + public Resource withTags(Map tags) { this.tags = tags; + return this; } } \ No newline at end of file diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/SubResource.java b/azure-client-runtime/src/main/java/com/microsoft/azure/SubResource.java index 4f89c0afb8be7..12bedc30c57fc 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/SubResource.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/SubResource.java @@ -29,8 +29,10 @@ public String id() { * Set the id value. * * @param id the id value to set + * @return the sub resource itself */ - public void setId(String id) { + public SubResource withId(String id) { this.id = id; + return this; } } \ No newline at end of file diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java new file mode 100644 index 0000000000000..cf522afe9e928 --- /dev/null +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java @@ -0,0 +1,53 @@ +package com.microsoft.azure; + +/** + * Represents a group of related tasks. + *

+ * each task in a group is represented by {@link TaskItem} + * + * @param the type of result of tasks in the group + * @param the task type + */ +public interface TaskGroup> { + /** + * Gets underlying directed acyclic graph structure that stores tasks in the group and + * dependency information between them. + * + * @return the dag + */ + DAGraph> dag(); + + /** + * @return true if this is a root (parent) task group composing other task groups. + */ + boolean isRoot(); + + /** + * Merges this task group with parent task group. + *

+ * once merged, calling execute in the parent group will executes the task in this + * group as well. + * + * @param parentTaskGroup task group + */ + void merge(TaskGroup parentTaskGroup); + + /** + * Executes the tasks in the group. + *

+ * the order of execution of tasks ensure that a task gets selected for execution only after + * the execution of all the tasks it depends on + * @throws Exception the exception + */ + void execute() throws Exception; + + /** + * Gets the result of execution of a task in the group. + *

+ * this method can null if the task has not yet been executed + * + * @param taskId the task id + * @return the task result + */ + T taskResult(String taskId); +} diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java new file mode 100644 index 0000000000000..e939e9ae2666c --- /dev/null +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java @@ -0,0 +1,71 @@ +package com.microsoft.azure; + +/** + * The base implementation of TaskGroup interface. + * + * @param the result type of the tasks in the group + * @param type representing task in the group + */ +public abstract class TaskGroupBase> + implements TaskGroup { + private DAGraph> dag; + + /** + * Creates TaskGroupBase. + * + * @param rootTaskItemId the id of the root task in this task group + * @param rootTaskItem the root task + */ + public TaskGroupBase(String rootTaskItemId, U rootTaskItem) { + this.dag = new DAGraph<>(new DAGNode<>(rootTaskItemId, rootTaskItem)); + } + + @Override + public DAGraph> dag() { + return dag; + } + + @Override + public boolean isRoot() { + return !dag.hasParent(); + } + + @Override + public void merge(TaskGroup parentTaskGroup) { + dag.merge(parentTaskGroup.dag()); + } + + @Override + public void execute() throws Exception { + if (isRoot()) { + dag.prepare(); + DAGNode nextNode = dag.getNext(); + while (nextNode != null) { + if (dag.isRootNode(nextNode)) { + executeRootTask(nextNode.data()); + } else { + nextNode.data().execute(); + } + dag.reportedCompleted(nextNode); + nextNode = dag.getNext(); + } + } + } + + @Override + public T taskResult(String taskId) { + return dag.getNodeData(taskId).result(); + } + + /** + * executes the root task in this group. + *

+ * this method will be invoked when all the task dependencies of the root task are finished + * executing, at this point root task can be executed by consuming the result of tasks it + * depends on. + * + * @param task the root task in this group + * @throws Exception the exception + */ + public abstract void executeRootTask(U task) throws Exception; +} diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java new file mode 100644 index 0000000000000..bd40a810a5fe9 --- /dev/null +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java @@ -0,0 +1,21 @@ +package com.microsoft.azure; + +/** + * Type representing a task in a task group {@link TaskGroup}. + * + * @param the task result type + */ +public interface TaskItem { + /** + * @return the result of the task execution + */ + U result(); + + /** + * Executes the task. + *

+ * once executed the result will be available through result getter + * @throws Exception exception + */ + void execute() throws Exception; +} diff --git a/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTest.java b/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTest.java new file mode 100644 index 0000000000000..8c1e12dd7011d --- /dev/null +++ b/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTest.java @@ -0,0 +1,145 @@ +package com.microsoft.azure; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class DAGraphTest { + @Test + public void testDAGraphGetNext() { + /** + * |-------->[D]------>[B]-----------[A] + * | ^ ^ + * | | | + * [F]------->[E]-------| | + * | | | + * | |------->[G]----->[C]---- + * | + * |-------->[H]-------------------->[I] + */ + List expectedOrder = new ArrayList<>(); + expectedOrder.add("A"); expectedOrder.add("I"); + expectedOrder.add("B"); expectedOrder.add("C"); expectedOrder.add("H"); + expectedOrder.add("D"); expectedOrder.add("G"); + expectedOrder.add("E"); + expectedOrder.add("F"); + + DAGNode nodeA = new DAGNode<>("A", "dataA"); + DAGNode nodeI = new DAGNode<>("I", "dataI"); + + DAGNode nodeB = new DAGNode<>("B", "dataB"); + nodeB.addDependency(nodeA.key()); + + DAGNode nodeC = new DAGNode<>("C", "dataC"); + nodeC.addDependency(nodeA.key()); + + DAGNode nodeH = new DAGNode<>("H", "dataH"); + nodeH.addDependency(nodeI.key()); + + DAGNode nodeG = new DAGNode<>("G", "dataG"); + nodeG.addDependency(nodeC.key()); + + DAGNode nodeE = new DAGNode<>("E", "dataE"); + nodeE.addDependency(nodeB.key()); + nodeE.addDependency(nodeG.key()); + + DAGNode nodeD = new DAGNode<>("D", "dataD"); + nodeD.addDependency(nodeB.key()); + + + DAGNode nodeF = new DAGNode<>("F", "dataF"); + nodeF.addDependency(nodeD.key()); + nodeF.addDependency(nodeE.key()); + nodeF.addDependency(nodeH.key()); + + DAGraph> dag = new DAGraph<>(nodeF); + dag.addNode(nodeA); + dag.addNode(nodeB); + dag.addNode(nodeC); + dag.addNode(nodeD); + dag.addNode(nodeE); + dag.addNode(nodeG); + dag.addNode(nodeH); + dag.addNode(nodeI); + + dag.populateDependentKeys(); + DAGNode nextNode = dag.getNext(); + int i = 0; + while (nextNode != null) { + Assert.assertEquals(nextNode.key(), expectedOrder.get(i)); + dag.reportedCompleted(nextNode); + nextNode = dag.getNext(); + i++; + } + + System.out.println("done"); + } + + @Test + public void testGraphMerge() { + /** + * |-------->[D]------>[B]-----------[A] + * | ^ ^ + * | | | + * [F]------->[E]-------| | + * | | | + * | |------->[G]----->[C]---- + * | + * |-------->[H]-------------------->[I] + */ + List expectedOrder = new ArrayList<>(); + expectedOrder.add("A"); expectedOrder.add("I"); + expectedOrder.add("B"); expectedOrder.add("C"); expectedOrder.add("H"); + expectedOrder.add("D"); expectedOrder.add("G"); + expectedOrder.add("E"); + expectedOrder.add("F"); + + DAGraph> graphA = createGraph("A"); + DAGraph> graphI = createGraph("I"); + + DAGraph> graphB = createGraph("B"); + graphA.merge(graphB); + + DAGraph> graphC = createGraph("C"); + graphA.merge(graphC); + + DAGraph> graphH = createGraph("H"); + graphI.merge(graphH); + + DAGraph> graphG = createGraph("G"); + graphC.merge(graphG); + + DAGraph> graphE = createGraph("E"); + graphB.merge(graphE); + graphG.merge(graphE); + + DAGraph> graphD = createGraph("D"); + graphB.merge(graphD); + + DAGraph> graphF = createGraph("F"); + graphD.merge(graphF); + graphE.merge(graphF); + graphH.merge(graphF); + + DAGraph> dag = graphF; + dag.prepare(); + + DAGNode nextNode = dag.getNext(); + int i = 0; + while (nextNode != null) { + Assert.assertEquals(expectedOrder.get(i), nextNode.key()); + // Process the node + dag.reportedCompleted(nextNode); + nextNode = dag.getNext(); + i++; + } + } + + private DAGraph> createGraph(String resourceName) { + DAGNode node = new DAGNode<>(resourceName, "data" + resourceName); + DAGraph> graph = new DAGraph<>(node); + return graph; + } +}