feat: directed acyclic graph (DAG)

.gitignore

@@ -10,6 +10,9 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
+# Graphvis files
+*.gv
+
 # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
 .glide/
 

cmd/task/task.go

@@ -178,7 +178,7 @@ func run() error {
 
 	globals.Set("CLI_ARGS", ast.Var{Value: cliArgs})
 	globals.Set("CLI_FORCE", ast.Var{Value: flags.Force || flags.ForceAll})
-	e.Taskfile.Vars.Merge(globals)
+	e.Taskfile.Vars.Merge(globals, nil)
 
 	if !flags.Watch {
 		e.InterceptInterruptSignals()

errors/errors.go

@@ -19,6 +19,8 @@ const (
 	CodeTaskfileCacheNotFound
 	CodeTaskfileVersionCheckError
 	CodeTaskfileNetworkTimeout
+	CodeTaskfileDuplicateInclude
+	CodeTaskfileCycle
 )
 
 // Task related exit codes

errors/errors_taskfile.go

@@ -3,6 +3,7 @@ package errors
 import (
 	"fmt"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/Masterminds/semver/v3"
@@ -174,3 +175,37 @@ func (err *TaskfileNetworkTimeoutError) Error() string {
 func (err *TaskfileNetworkTimeoutError) Code() int {
 	return CodeTaskfileNetworkTimeout
 }
+
+type TaskfileDuplicateIncludeError struct {
+	URI         string
+	IncludedURI string
+	Namespaces  []string
+}
+
+func (err *TaskfileDuplicateIncludeError) Error() string {
+	return fmt.Sprintf(
+		`task: Taskfile %q attempted to include %q multiple times with namespaces: %s`, err.URI, err.IncludedURI, strings.Join(err.Namespaces, ", "),
+	)
+}
+
+func (err *TaskfileDuplicateIncludeError) Code() int {
+	return CodeTaskfileDuplicateInclude
+}
+
+// TaskfileCycleError is returned when we detect that a Taskfile includes a
+// set of Taskfiles that include each other in a cycle.
+type TaskfileCycleError struct {
+	Source      string
+	Destination string
+}
+
+func (err TaskfileCycleError) Error() string {
+	return fmt.Sprintf("task: include cycle detected between %s <--> %s",
+		err.Source,
+		err.Destination,
+	)
+}
+
+func (err TaskfileCycleError) Code() int {
+	return CodeTaskfileCycle
+}

go.mod

@@ -5,6 +5,7 @@ go 1.21
 require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/davecgh/go-spew v1.1.1
+	github.com/dominikbraun/graph v0.23.0
 	github.com/fatih/color v1.16.0
 	github.com/go-task/slim-sprig/v3 v3.0.0
 	github.com/joho/godotenv v1.5.1

go.sum

@@ -4,6 +4,8 @@ github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
 github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo=
+github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=

setup.go

@@ -63,8 +63,7 @@ func (e *Executor) getRootNode() (taskfile.Node, error) {
 }
 
 func (e *Executor) readTaskfile(node taskfile.Node) error {
-	var err error
-	e.Taskfile, err = taskfile.Read(
+	reader := taskfile.NewReader(
 		node,
 		e.Insecure,
 		e.Download,
@@ -73,9 +72,13 @@ func (e *Executor) readTaskfile(node taskfile.Node) error {
 		e.TempDir,
 		e.Logger,
 	)
+	graph, err := reader.Read()
 	if err != nil {
 		return err
 	}
+	if e.Taskfile, err = graph.Merge(); err != nil {
+		return err
+	}
 	return nil
 }
 

task_test.go

@@ -981,8 +981,8 @@ func TestIncludes(t *testing.T) {
 			"included_directory.txt":                    "included_directory",
 			"included_directory_without_dir.txt":        "included_directory_without_dir",
 			"included_taskfile_without_dir.txt":         "included_taskfile_without_dir",
-			"./module2/included_directory_with_dir.txt": "included_directory_with_dir",
-			"./module2/included_taskfile_with_dir.txt":  "included_taskfile_with_dir",
+			"./module3/included_taskfile_with_dir.txt":  "included_taskfile_with_dir",
+			"./module4/included_directory_with_dir.txt": "included_directory_with_dir",
 			"os_include.txt":                            "os",
 		},
 	}
@@ -1199,15 +1199,15 @@ func TestIncludesInterpolation(t *testing.T) {
 		expectedErr    bool
 		expectedOutput string
 	}{
-		{"include", "include", false, "includes_interpolation\n"},
-		{"include with dir", "include-with-dir", false, "included\n"},
+		{"include", "include", false, "include\n"},
+		{"include_with_dir", "include-with-dir", false, "included\n"},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			var buff bytes.Buffer
 			e := task.Executor{
-				Dir:    dir,
+				Dir:    filepath.Join(dir, test.name),
 				Stdout: &buff,
 				Stderr: &buff,
 				Silent: true,

taskfile/ast/graph.go

@@ -0,0 +1,124 @@
+package ast
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/dominikbraun/graph"
+	"github.com/dominikbraun/graph/draw"
+	"golang.org/x/sync/errgroup"
+)
+
+type TaskfileGraph struct {
+	graph.Graph[string, *TaskfileVertex]
+}
+
+// A TaskfileVertex is a vertex on the Taskfile DAG.
+type TaskfileVertex struct {
+	URI      string
+	Taskfile *Taskfile
+}
+
+func taskfileHash(vertex *TaskfileVertex) string {
+	return vertex.URI
+}
+
+func NewTaskfileGraph() *TaskfileGraph {
+	return &TaskfileGraph{
+		graph.New(taskfileHash,
+			graph.Directed(),
+			graph.PreventCycles(),
+			graph.Rooted(),
+		),
+	}
+}
+
+func (tfg *TaskfileGraph) Visualize(filename string) error {
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return draw.DOT(tfg.Graph, f)
+}
+
+func (tfg *TaskfileGraph) Merge() (*Taskfile, error) {
+	hashes, err := graph.TopologicalSort(tfg.Graph)
+	if err != nil {
+		return nil, err
+	}
+
+	predecessorMap, err := tfg.PredecessorMap()
+	if err != nil {
+		return nil, err
+	}
+
+	// Loop over each vertex in reverse topological order except for the root vertex.
+	// This gives us a loop over every included Taskfile in an order which is safe to merge.
+	for i := len(hashes) - 1; i > 0; i-- {
+		hash := hashes[i]
+
+		// Get the included vertex
+		includedVertex, err := tfg.Vertex(hash)
+		if err != nil {
+			return nil, err
+		}
+
+		// Create an error group to wait for all the included Taskfiles to be merged with all its parents
+		var g errgroup.Group
+
+		// Loop over edge that leads to a vertex that includes the current vertex
+		for _, edge := range predecessorMap[hash] {
+
+			// Start a goroutine to process each included Taskfile
+			g.Go(func() error {
+				// Get the base vertex
+				vertex, err := tfg.Vertex(edge.Source)
+				if err != nil {
+					return err
+				}
+
+				// Get the merge options
+				include, ok := edge.Properties.Data.(Include)
+				if !ok {
+					return fmt.Errorf("task: Failed to get merge options")
+				}
+
+				// Merge the included Taskfile into the parent Taskfile
+				if err := vertex.Taskfile.Merge(
+					includedVertex.Taskfile,
+					&include,
+				); err != nil {
+					return err
+				}
+
+				return nil
+			})
+			if err := g.Wait(); err != nil {
+				return nil, err
+			}
+		}
+
+		// Wait for all the go routines to finish
+		if err := g.Wait(); err != nil {
+			return nil, err
+		}
+	}
+
+	// Get the root vertex
+	rootVertex, err := tfg.Vertex(hashes[0])
+	if err != nil {
+		return nil, err
+	}
+
+	_ = rootVertex.Taskfile.Tasks.Range(func(name string, task *Task) error {
+		if task == nil {
+			task = &Task{}
+			rootVertex.Taskfile.Tasks.Set(name, task)
+		}
+		task.Task = name
+		return nil
+	})
+
+	return rootVertex.Taskfile, nil
+}

taskfile/ast/taskfile.go

@@ -1,6 +1,7 @@
 package ast
 
 import (
+	"errors"
 	"fmt"
 	"time"
 
@@ -13,6 +14,9 @@ const NamespaceSeparator = ":"
 
 var V3 = semver.MustParse("3")
 
+// ErrIncludedTaskfilesCantHaveDotenvs is returned when a included Taskfile contains dotenvs
+var ErrIncludedTaskfilesCantHaveDotenvs = errors.New("task: Included Taskfiles can't have dotenv declarations. Please, move the dotenv declaration to the main Taskfile")
+
 // Taskfile is the abstract syntax tree for a Taskfile
 type Taskfile struct {
 	Location string
@@ -36,6 +40,9 @@ func (t1 *Taskfile) Merge(t2 *Taskfile, include *Include) error {
 	if !t1.Version.Equal(t2.Version) {
 		return fmt.Errorf(`task: Taskfiles versions should match. First is "%s" but second is "%s"`, t1.Version, t2.Version)
 	}
+	if len(t2.Dotenv) > 0 {
+		return ErrIncludedTaskfilesCantHaveDotenvs
+	}
 	if t2.Output.IsSet() {
 		t1.Output = t2.Output
 	}
@@ -45,9 +52,9 @@ func (t1 *Taskfile) Merge(t2 *Taskfile, include *Include) error {
 	if t1.Env == nil {
 		t1.Env = &Vars{}
 	}
-	t1.Vars.Merge(t2.Vars)
-	t1.Env.Merge(t2.Env)
-	t1.Tasks.Merge(t2.Tasks, include)
+	t1.Vars.Merge(t2.Vars, include)
+	t1.Env.Merge(t2.Env, include)
+	t1.Tasks.Merge(t2.Tasks, include, t1.Vars)
 	return nil
 }
 

taskfile/ast/tasks.go

@@ -6,6 +6,7 @@ import (
 
 	"gopkg.in/yaml.v3"
 
+	"github.com/go-task/task/v3/internal/filepathext"
 	"github.com/go-task/task/v3/internal/omap"
 )
 
@@ -44,7 +45,7 @@ func (t *Tasks) FindMatchingTasks(call *Call) []*MatchingTask {
 	return matchingTasks
 }
 
-func (t1 *Tasks) Merge(t2 Tasks, include *Include) {
+func (t1 *Tasks) Merge(t2 Tasks, include *Include, includedTaskfileVars *Vars) {
 	_ = t2.Range(func(k string, v *Task) error {
 		// We do a deep copy of the task struct here to ensure that no data can
 		// be changed elsewhere once the taskfile is merged.
@@ -54,20 +55,25 @@ func (t1 *Tasks) Merge(t2 Tasks, include *Include) {
 		// taskfile are marked as internal
 		task.Internal = task.Internal || (include != nil && include.Internal)
 
-		// Add namespaces to dependencies, commands and aliases
+		// Add namespaces to task dependencies
 		for _, dep := range task.Deps {
 			if dep != nil && dep.Task != "" {
 				dep.Task = taskNameWithNamespace(dep.Task, include.Namespace)
 			}
 		}
+
+		// Add namespaces to task commands
 		for _, cmd := range task.Cmds {
 			if cmd != nil && cmd.Task != "" {
 				cmd.Task = taskNameWithNamespace(cmd.Task, include.Namespace)
 			}
 		}
+
+		// Add namespaces to task aliases
 		for i, alias := range task.Aliases {
 			task.Aliases[i] = taskNameWithNamespace(alias, include.Namespace)
 		}
+
 		// Add namespace aliases
 		if include != nil {
 			for _, namespaceAlias := range include.Aliases {
@@ -78,6 +84,15 @@ func (t1 *Tasks) Merge(t2 Tasks, include *Include) {
 			}
 		}
 
+		if include.AdvancedImport {
+			task.Dir = filepathext.SmartJoin(include.Dir, task.Dir)
+			if task.IncludeVars == nil {
+				task.IncludeVars = &Vars{}
+			}
+			task.IncludeVars.Merge(include.Vars, nil)
+			task.IncludedTaskfileVars = includedTaskfileVars
+		}
+
 		// Add the task to the merged taskfile
 		taskNameWithNamespace := taskNameWithNamespace(k, include.Namespace)
 		task.Task = taskNameWithNamespace

taskfile/ast/var.go

@@ -45,11 +45,17 @@ func (vs *Vars) Range(f func(k string, v Var) error) error {
 }
 
 // Wrapper around OrderedMap.Merge to ensure we don't get nil pointer errors
-func (vs *Vars) Merge(other *Vars) {
+func (vs *Vars) Merge(other *Vars, include *Include) {
 	if vs == nil || other == nil {
 		return
 	}
-	vs.OrderedMap.Merge(other.OrderedMap)
+	_ = other.Range(func(key string, value Var) error {
+		if include != nil && include.AdvancedImport {
+			value.Dir = include.Dir
+		}
+		vs.Set(key, value)
+		return nil
+	})
 }
 
 // Wrapper around OrderedMap.Len to ensure we don't get nil pointer errors

taskfile/node.go

@@ -17,7 +17,6 @@ type Node interface {
 	Parent() Node
 	Location() string
 	Dir() string
-	Optional() bool
 	Remote() bool
 	ResolveEntrypoint(entrypoint string) (string, error)
 	ResolveDir(dir string) (string, error)