Merge branch 'main' into hotrealoading

README.md

@@ -79,4 +79,5 @@ Multiline `sh` and `bash` commands are entirely possible, powered by [mvdan/sh](
 
 # Comparisons
 * [Dagger vs. bob](https://medium.com/benchkram/dagger-vs-bob-2e917cd185d3)
+* [Bazel vs. bob](https://bob.build/blog/vs-bazel)
 * [Mage vs. bob](https://medium.com/benchkram/build-system-comparison-mage-vs-bob-aaf4665e3d5c)

bob.yaml

@@ -1,4 +1,5 @@
 project: bob.build/benchkram/bob
+nixpkgs: https://github.com/NixOS/nixpkgs/archive/nixos-22.11.tar.gz
 
 variables:
   VERSION: v0.0.0
@@ -14,6 +15,7 @@ build:
     cmd: go mod tidy
 
   lint:
+    input: "*"
     cmd: CGO_ENABLED=0 golangci-lint run --timeout=10m0s
 
   test:
@@ -29,10 +31,9 @@ build:
 dependencies:
   - nix
   - git
-  - go_1_18
+  - go_1_19
   - golangci-lint
   - docker
   - gcc
   - protobuf
-  - protoc-gen-go
-
+  - protoc-gen-go

bob/aggregate.go

@@ -156,6 +156,7 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) {
 	// Assure tasks are correctly initialised.
 	for i, task := range aggregate.BTasks {
 		task.WithLocalstore(b.local)
+		task.WithEnvStore(b.nix.EnvStore())
 		task.WithBuildinfoStore(b.buildInfoStore)
 		task.WithDockerRegistryClient(b.dockerRegistryClient)
 
@@ -223,6 +224,7 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) {
 	err = aggregate.BTasks.IgnoreChildTargets()
 	errz.Fatal(err)
 
+	// Filter input must run before any work is done.
 	err = aggregate.BTasks.FilterInputs()
 	errz.Fatal(err)
 

bob/aggregate_test.go

@@ -2,7 +2,6 @@ package bob
 
 import (
 	"fmt"
-	"io/ioutil"
 	_ "net/http/pprof"
 	"os"
 	"testing"
@@ -20,7 +19,7 @@ func BenchmarkAggregateOnPlayground(b *testing.B) {
 	b.StopTimer() // Stop during initialization
 
 	// Create playground
-	dir, err := ioutil.TempDir("", "bob-test-benchmark-playground-*")
+	dir, err := os.MkdirTemp("", "bob-test-benchmark-playground-*")
 	assert.Nil(b, err)
 
 	defer os.RemoveAll(dir)
@@ -65,7 +64,7 @@ func benchmarkAggregate(b *testing.B, ignoredMultiplier int) {
 	b.StopTimer() // Stop during initialization
 
 	// Create playground
-	dir, err := ioutil.TempDir("", "bob-test-benchmark-*")
+	dir, err := os.MkdirTemp("", "bob-test-benchmark-*")
 	assert.Nil(b, err)
 
 	defer os.RemoveAll(dir)
@@ -182,7 +181,7 @@ func createIgnoreFileSturcture(dir string, multiplier int) error {
 
 func TestEmptyProjectName(t *testing.T) {
 	// Create playground
-	dir, err := ioutil.TempDir("", "bob-test-aggregate-*")
+	dir, err := os.MkdirTemp("", "bob-test-aggregate-*")
 	assert.Nil(t, err)
 
 	defer os.RemoveAll(dir)
@@ -205,7 +204,7 @@ func TestEmptyProjectName(t *testing.T) {
 
 func TestProjectName(t *testing.T) {
 	// Create playground
-	dir, err := ioutil.TempDir("", "bob-test-aggregate-*")
+	dir, err := os.MkdirTemp("", "bob-test-aggregate-*")
 	assert.Nil(t, err)
 
 	defer os.RemoveAll(dir)
@@ -232,7 +231,7 @@ func TestProjectName(t *testing.T) {
 
 func TestInvalidProjectName(t *testing.T) {
 	// Create playground
-	dir, err := ioutil.TempDir("", "bob-test-aggregate-*")
+	dir, err := os.MkdirTemp("", "bob-test-aggregate-*")
 	assert.Nil(t, err)
 
 	defer os.RemoveAll(dir)
@@ -260,7 +259,7 @@ func TestInvalidProjectName(t *testing.T) {
 //
 // func TestDuplicateProjectNameSimple(t *testing.T) {
 // 	// Create playground
-// 	dir, err := ioutil.TempDir("", "bob-test-aggregate-*")
+// 	dir, err :=os.MkdirTemp("", "bob-test-aggregate-*")
 // 	assert.Nil(t, err)
 
 // 	defer os.RemoveAll(dir)
@@ -294,7 +293,7 @@ func TestInvalidProjectName(t *testing.T) {
 //
 // func TestDuplicateProjectNameComplex(t *testing.T) {
 // 	// Create playground
-// 	dir, err := ioutil.TempDir("", "bob-test-aggregate-*")
+// 	dir, err :=os.MkdirTemp("", "bob-test-aggregate-*")
 // 	assert.Nil(t, err)
 
 // 	defer os.RemoveAll(dir)
@@ -325,7 +324,7 @@ func TestInvalidProjectName(t *testing.T) {
 
 func TestMultiLevelBobfileSameProjectName(t *testing.T) {
 	// Create playground
-	dir, err := ioutil.TempDir("", "bob-test-aggregate-*")
+	dir, err := os.MkdirTemp("", "bob-test-aggregate-*")
 	assert.Nil(t, err)
 
 	defer os.RemoveAll(dir)

bob/bob.go

@@ -1,10 +1,10 @@
 package bob
 
 import (
-	"io/ioutil"
 	"os"
 	"runtime"
 
+	nixbuilder "github.com/benchkram/bob/bob/nix-builder"
 	"github.com/benchkram/bob/pkg/auth"
 	"github.com/benchkram/bob/pkg/dockermobyutil"
 	"github.com/benchkram/bob/pkg/usererror"
@@ -57,7 +57,7 @@ type B struct {
 	enablePull bool
 
 	// nix builds dependencies for tasks
-	nix *NixBuilder
+	nix *nixbuilder.NB
 
 	// authStore is used to store authentication credentials for remote store
 	authStore *auth.Store
@@ -122,6 +122,12 @@ func BobWithBaseStoreDir(baseStoreDir string, opts ...Option) (*B, error) {
 	}
 	bob.authStore = authStore
 
+	nixBuilder, err := NixBuilder(baseStoreDir)
+	if err != nil {
+		return nil, err
+	}
+	bob.nix = nixBuilder
+
 	for _, opt := range opts {
 		if opt == nil {
 			continue
@@ -164,7 +170,7 @@ func Bob(opts ...Option) (*B, error) {
 	}
 
 	if bob.nix == nil {
-		nix, err := DefaultNix()
+		nix, err := DefaultNixBuilder()
 		if err != nil {
 			return nil, err
 		}
@@ -186,7 +192,7 @@ func (b *B) Dir() string {
 	return b.dir
 }
 
-func (b *B) Nix() *NixBuilder {
+func (b *B) Nix() *nixbuilder.NB {
 	return b.nix
 }
 
@@ -197,7 +203,7 @@ func (b *B) write() (err error) {
 	errz.Fatal(err)
 
 	const mode = 0644
-	return ioutil.WriteFile(b.WorkspaceFilePath(), bin, mode)
+	return os.WriteFile(b.WorkspaceFilePath(), bin, mode)
 }
 
 func (b *B) read() (err error) {
@@ -209,7 +215,7 @@ func (b *B) read() (err error) {
 		errz.Fatal(err)
 	}
 
-	bin, err := ioutil.ReadFile(b.WorkspaceFilePath())
+	bin, err := os.ReadFile(b.WorkspaceFilePath())
 	errz.Fatal(err)
 
 	err = yaml.Unmarshal(bin, b)

bob/bob_stores.go → bob/bob_defaults.go

@@ -7,8 +7,10 @@ import (
 	"github.com/benchkram/errz"
 
 	"github.com/benchkram/bob/bob/global"
+	nixbuilder "github.com/benchkram/bob/bob/nix-builder"
 	"github.com/benchkram/bob/pkg/auth"
 	"github.com/benchkram/bob/pkg/buildinfostore"
+	"github.com/benchkram/bob/pkg/nix"
 	"github.com/benchkram/bob/pkg/store"
 	"github.com/benchkram/bob/pkg/store/filestore"
 )
@@ -46,10 +48,10 @@ func DefaultBuildinfoStore() (s buildinfostore.Store, err error) {
 	return BuildinfoStore(home)
 }
 
-func BuildinfoStore(dir string) (s buildinfostore.Store, err error) {
+func BuildinfoStore(baseDir string) (s buildinfostore.Store, err error) {
 	defer errz.Recover(&err)
 
-	storeDir := filepath.Join(dir, global.BobCacheBuildinfoDir)
+	storeDir := filepath.Join(baseDir, global.BobCacheBuildinfoDir)
 	err = os.MkdirAll(storeDir, 0775)
 	errz.Fatal(err)
 
@@ -66,10 +68,10 @@ func (b *B) Localstore() store.Store {
 	return b.local
 }
 
-func AuthStore(dir string) (s *auth.Store, err error) {
+func AuthStore(baseDir string) (s *auth.Store, err error) {
 	defer errz.Recover(&err)
 
-	storeDir := filepath.Join(dir, global.BobAuthStoreDir)
+	storeDir := filepath.Join(baseDir, global.BobAuthStoreDir)
 	err = os.MkdirAll(storeDir, 0775)
 	errz.Fatal(err)
 
@@ -84,3 +86,37 @@ func DefaultAuthStore() (s *auth.Store, err error) {
 
 	return AuthStore(home)
 }
+
+// NixBuilder initialises a new nix builder object with the cache setup
+// in the given location.
+//
+// It's save to use the same base dir as for BuildinfoStore(),
+// Filestore() and AuthStore().
+func NixBuilder(baseDir string) (_ *nixbuilder.NB, err error) {
+
+	cacheDir := filepath.Join(baseDir, global.BobCacheNixFileName)
+
+	err = os.MkdirAll(filepath.Dir(cacheDir), 0775)
+	errz.Fatal(err)
+
+	nixCache, err := nix.NewCacheStore(nix.WithPath(cacheDir))
+	errz.Fatal(err)
+
+	shellCache := nix.NewShellCache(filepath.Join(baseDir, global.BobCacheNixShellCacheDir))
+
+	nb := nixbuilder.New(
+		nixbuilder.WithCache(nixCache),
+		nixbuilder.WithShellCache(shellCache),
+	)
+
+	return nb, nil
+}
+
+func DefaultNixBuilder() (_ *nixbuilder.NB, err error) {
+	defer errz.Recover(&err)
+
+	home, err := os.UserHomeDir()
+	errz.Fatal(err)
+
+	return NixBuilder(home)
+}

bob/bobfile/bobfile.go

@@ -3,11 +3,12 @@ package bobfile
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"net/url"
+	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/benchkram/bob/pkg/dockermobyutil"
 	"github.com/benchkram/bob/pkg/nix"
 	storeclient "github.com/benchkram/bob/pkg/store-client"
 
@@ -114,7 +115,7 @@ func bobfileRead(dir string) (_ *Bobfile, err error) {
 	if !file.Exists(bobfilePath) {
 		return nil, usererror.Wrap(ErrBobfileNotFound)
 	}
-	bin, err := ioutil.ReadFile(bobfilePath)
+	bin, err := os.ReadFile(bobfilePath)
 	errz.Fatal(err)
 
 	bobfile := &Bobfile{
@@ -138,6 +139,9 @@ func bobfileRead(dir string) (_ *Bobfile, err error) {
 		bobfile.RTasks = bobrun.RunMap{}
 	}
 
+	// a shared registry clients for all tasks.
+	dockerRegistryClient := dockermobyutil.NewRegistryClient()
+
 	// Assure tasks are initialized with their defaults
 	for key, task := range bobfile.BTasks {
 		task.SetDir(bobfile.dir)
@@ -149,6 +153,7 @@ func bobfileRead(dir string) (_ *Bobfile, err error) {
 		// This means switching to pointer types for most members.
 		task.SetEnv([]string{})
 		task.SetRebuildStrategy(bobtask.RebuildOnChange)
+		task.WithDockerRegistryClient(dockerRegistryClient)
 
 		// initialize docker registry for task
 		task.SetDependencies(initializeDependencies(dir, task.DependenciesDirty, bobfile))
@@ -341,7 +346,7 @@ func (b *Bobfile) BobfileSave(dir, name string) (err error) {
 	err = encoder.Encode(b)
 	errz.Fatal(err)
 
-	return ioutil.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0664)
+	return os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0664)
 }
 
 func (b *Bobfile) Dir() string {

bob/bobfile/playbook.go

@@ -7,17 +7,33 @@ import (
 
 func (b *Bobfile) Playbook(taskName string, opts ...playbook.Option) (*playbook.Playbook, error) {
 
+	var idCounter int
 	pb := playbook.New(
 		taskName,
+		idCounter,
 		opts...,
 	)
+	idCounter++
 
 	err := b.BTasks.Walk(taskName, "", func(tn string, task bobtask.Task, err error) error {
 		if err != nil {
 			return err
 		}
+		if taskName == tn {
+			// The root task already has an id
+			statusTask := playbook.NewStatus(&task)
+			pb.Tasks[tn] = statusTask
+			pb.TasksOptimized = append(pb.TasksOptimized, statusTask)
+			return nil
+		}
+
+		task.TaskID = idCounter
+		statusTask := playbook.NewStatus(&task)
+
+		pb.Tasks[tn] = statusTask
+		pb.TasksOptimized = append(pb.TasksOptimized, statusTask)
 
-		pb.Tasks[tn] = playbook.NewStatus(&task)
+		idCounter++
 
 		return nil
 	})

bob/build.go

@@ -37,8 +37,6 @@ func (b *B) Build(ctx context.Context, taskName string) (err error) {
 		playbook.WithLocalStore(b.local),
 		playbook.WithPushEnabled(b.enablePush),
 		playbook.WithPullEnabled(b.enablePull),
-		playbook.WitNixCache(b.Nix().cache),
-		playbook.WitNixShellCache(b.Nix().shellCache),
 	)
 	errz.Fatal(err)
 

bob/builder.go

@@ -4,22 +4,23 @@ import (
 	"context"
 
 	"github.com/benchkram/bob/bob/bobfile"
+	nixbuilder "github.com/benchkram/bob/bob/nix-builder"
 	"github.com/benchkram/bob/pkg/ctl"
 )
 
 var _ ctl.Builder = (*builder)(nil)
 
-type BuildFunc func(_ context.Context, runname string, aggregate *bobfile.Bobfile, nix *NixBuilder) error
+type BuildFunc func(_ context.Context, runname string, aggregate *bobfile.Bobfile, nix *nixbuilder.NB) error
 
 // builder holds all dependencies to build a build task
 type builder struct {
 	task      string
 	aggregate *bobfile.Bobfile
 	f         BuildFunc
-	nix       *NixBuilder
+	nix       *nixbuilder.NB
 }
 
-func NewBuilder(task string, aggregate *bobfile.Bobfile, f BuildFunc, nix *NixBuilder) ctl.Builder {
+func NewBuilder(task string, aggregate *bobfile.Bobfile, f BuildFunc, nix *nixbuilder.NB) ctl.Builder {
 	builder := &builder{
 		task:      task,
 		aggregate: aggregate,