use a in memory env cache

bob/aggregate.go

@@ -156,6 +156,7 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) {
 	// Assure tasks are correctly initialised.
 	for i, task := range aggregate.BTasks {
 		task.WithLocalstore(b.local)
+		task.WithEnvStore(b.envStore)
 		task.WithBuildinfoStore(b.buildInfoStore)
 		task.WithDockerRegistryClient(b.dockerRegistryClient)
 

bob/bob.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/benchkram/bob/pkg/auth"
 	"github.com/benchkram/bob/pkg/dockermobyutil"
+	"github.com/benchkram/bob/pkg/envutil"
 	"github.com/benchkram/bob/pkg/usererror"
 
 	"github.com/hashicorp/go-version"
@@ -35,6 +36,9 @@ type B struct {
 	// remotestore the place to store artifacts remotly
 	remote store.Store
 
+	// envStore the place to store environments in memory
+	envStore envutil.Store
+
 	// buildInfoStore stores build infos for tasks.
 	buildInfoStore buildinfostore.Store
 
@@ -82,6 +86,8 @@ func newBob(opts ...Option) *B {
 		allowInsecure: false,
 		maxParallel:   runtime.NumCPU(),
 
+		envStore: envutil.NewStore(),
+
 		dockerRegistryClient: dockermobyutil.NewRegistryClient(),
 	}
 
@@ -164,7 +170,7 @@ func Bob(opts ...Option) (*B, error) {
 	}
 
 	if bob.nix == nil {
-		nix, err := DefaultNix()
+		nix, err := DefaultNix(bob.envStore)
 		if err != nil {
 			return nil, err
 		}

bob/nix_builder.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/benchkram/bob/pkg/envutil"
 	"github.com/benchkram/errz"
-	"github.com/sanity-io/litter"
 
 	"github.com/benchkram/bob/bob/bobfile"
 	"github.com/benchkram/bob/pkg/nix"
@@ -17,8 +16,13 @@ import (
 type NixBuilder struct {
 	// cache allows caching the dependency to store path
 	cache *nix.Cache
+
 	// shellCache allows caching of the nix-shell --command='env' output
 	shellCache *nix.ShellCache
+
+	// envStore is filled by NixBuilder with the environment
+	// used by tasks.
+	envStore envutil.Store
 }
 
 type NixOption func(n *NixBuilder)
@@ -35,9 +39,17 @@ func WithShellCache(cache *nix.ShellCache) NixOption {
 	}
 }
 
+func WithEnvironmentStore(store envutil.Store) NixOption {
+	return func(n *NixBuilder) {
+		n.envStore = store
+	}
+}
+
 // NewNixBuilder instantiates a new Nix builder instance
 func NewNixBuilder(opts ...NixOption) *NixBuilder {
-	n := &NixBuilder{}
+	n := &NixBuilder{
+		envStore: envutil.NewStore(),
+	}
 
 	for _, opt := range opts {
 		if opt == nil {
@@ -73,7 +85,7 @@ func (n *NixBuilder) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipel
 	}
 
 	// maps nix dependencies to nixShellEnv
-	environmentCache := make(map[string][]string)
+	//environmentCache := make(map[string][]string)
 
 	// Resolve nix storePaths from dependencies
 	// and rewrite the affected tasks.
@@ -90,21 +102,21 @@ func (n *NixBuilder) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipel
 		hash, err := nix.HashDependencies(deps)
 		errz.Fatal(err)
 
-		if _, ok := environmentCache[hash]; !ok {
+		if _, ok := n.envStore[envutil.Hash(hash)]; !ok {
 			nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs)
 			errz.Fatal(err)
-			environmentCache[hash] = nixShellEnv
+			n.envStore[envutil.Hash(hash)] = nixShellEnv
 		}
-		t.SetEnv(envutil.Merge(environmentCache[hash], t.Env()))
+		t.SetEnvID(envutil.Hash(hash))
 
 		ag.BTasks[name] = t
 	}
 
-	println(len(environmentCache))
-	for k, e := range environmentCache {
-		println(k)
-		litter.Dump(e)
-	}
+	// println(len(environmentCache))
+	// for k, e := range environmentCache {
+	// 	println(k)
+	// 	litter.Dump(e)
+	// }
 
 	for _, name := range runTasksInPipeline {
 		t := ag.RTasks[name]
@@ -119,12 +131,13 @@ func (n *NixBuilder) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipel
 		hash, err := nix.HashDependencies(deps)
 		errz.Fatal(err)
 
-		if _, ok := environmentCache[hash]; !ok {
+		if _, ok := n.envStore[envutil.Hash(hash)]; !ok {
 			nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs)
 			errz.Fatal(err)
-			environmentCache[hash] = nixShellEnv
+			n.envStore[envutil.Hash(hash)] = nixShellEnv
 		}
-		t.SetEnv(envutil.Merge(environmentCache[hash], t.Env()))
+		// TODO: also implement for run tasks
+		//t.SetEnvID(envutil.Hash(hash))
 
 		ag.RTasks[name] = t
 	}

bob/nix_builder_defaults.go

@@ -7,10 +7,11 @@ import (
 	"github.com/benchkram/errz"
 
 	"github.com/benchkram/bob/bob/global"
+	"github.com/benchkram/bob/pkg/envutil"
 	"github.com/benchkram/bob/pkg/nix"
 )
 
-func DefaultNix() (_ *NixBuilder, err error) {
+func DefaultNix(envStore envutil.Store) (_ *NixBuilder, err error) {
 	defer errz.Recover(&err)
 
 	home, err := os.UserHomeDir()
@@ -26,5 +27,11 @@ func DefaultNix() (_ *NixBuilder, err error) {
 
 	shellCache := nix.NewShellCache(filepath.Join(home, global.BobCacheNixShellCacheDir))
 
-	return NewNixBuilder(WithCache(nixCache), WithShellCache(shellCache)), nil
+	nb := NewNixBuilder(
+		WithCache(nixCache),
+		WithShellCache(shellCache),
+		WithEnvironmentStore(envStore),
+	)
+
+	return nb, nil
 }

bob/options.go

@@ -3,6 +3,7 @@ package bob
 import (
 	"github.com/benchkram/bob/pkg/auth"
 	"github.com/benchkram/bob/pkg/buildinfostore"
+	"github.com/benchkram/bob/pkg/envutil"
 	"github.com/benchkram/bob/pkg/store"
 )
 
@@ -46,6 +47,12 @@ func WithBuildinfoStore(store buildinfostore.Store) Option {
 	}
 }
 
+func WithEnvStore(store envutil.Store) Option {
+	return func(b *B) {
+		b.envStore = store
+	}
+}
+
 func WithCachingEnabled(enabled bool) Option {
 	return func(b *B) {
 		b.enableCaching = enabled

bobtask/options.go

@@ -7,3 +7,9 @@ func WithEnvironment(envs []string) TaskOption {
 		t.env = append(t.env, envs...)
 	}
 }
+
+// func WithEnvId(envs []string) TaskOption {
+// 	return func(t *Task) {
+// 		t.env = append(t.env, envs...)
+// 	}
+// }

bobtask/run.go

@@ -22,12 +22,19 @@ import (
 func (t *Task) Run(ctx context.Context, namePad int, nixCache *nix.Cache, shellCache *nix.ShellCache) (err error) {
 	defer errz.Recover(&err)
 
-	if len(t.Env()) == 0 {
-		nixShellEnv, err := nix.BuildEnvironment(t.dependencies, t.nixpkgs, nixCache, shellCache)
-		errz.Fatal(err)
-		t.SetEnv(envutil.Merge(nixShellEnv, t.env))
+	// if len(t.Env()) == 0 {
+	// 	nixShellEnv, err := nix.BuildEnvironment(t.dependencies, t.nixpkgs, nixCache, shellCache)
+	// 	errz.Fatal(err)
+	// 	t.SetEnv(envutil.Merge(nixShellEnv, t.env))
+	// }
+
+	nixEnv, ok := t.envStore[t.envID]
+	if !ok {
+		return fmt.Errorf("missing nix environment in envStore")
 	}
 
+	env := envutil.Merge(nixEnv, t.env)
+
 	for _, run := range t.cmds {
 		p, err := syntax.NewParser().Parse(strings.NewReader(run), "")
 		if err != nil {
@@ -60,7 +67,7 @@ func (t *Task) Run(ctx context.Context, namePad int, nixCache *nix.Cache, shellC
 		r, err := interp.New(
 			interp.Params("-e"),
 			interp.Dir(t.dir),
-			interp.Env(expand.ListEnviron(t.Env()...)),
+			interp.Env(expand.ListEnviron(env...)),
 			interp.StdIO(os.Stdin, pw, pw),
 		)
 

bobtask/task.go

@@ -4,6 +4,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/benchkram/bob/pkg/envutil"
 	"github.com/benchkram/bob/pkg/nix"
 	"github.com/logrusorgru/aurora"
 
@@ -72,6 +73,11 @@ type Task struct {
 	// when the task is executed.
 	env []string
 
+	// envID is used to retrieve the environment
+	// from a environment store. This is used to
+	// optimize garbage collection.
+	envID envutil.Hash
+
 	// hashIn stores the `In` has for reuse
 	hashIn *hash.In
 
@@ -81,6 +87,10 @@ type Task struct {
 	// remote store for artifacts
 	remote store.Store
 
+	// envStore is the global store  used to
+	// manage environments.
+	envStore envutil.Store
+
 	// buildInfoStore stores buildinfos.
 	buildInfoStore buildinfostore.Store
 

bobtask/task_get_set.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/benchkram/bob/pkg/buildinfostore"
 	"github.com/benchkram/bob/pkg/dockermobyutil"
+	"github.com/benchkram/bob/pkg/envutil"
 	"github.com/benchkram/bob/pkg/nix"
 	"github.com/benchkram/bob/pkg/store"
 	"github.com/logrusorgru/aurora"
@@ -51,6 +52,10 @@ func (t *Task) SetEnv(env []string) {
 	t.env = env
 }
 
+func (t *Task) SetEnvID(envID envutil.Hash) {
+	t.envID = envID
+}
+
 func (t *Task) Dependencies() []nix.Dependency {
 	return t.dependencies
 }
@@ -105,3 +110,8 @@ func (t *Task) WithDockerRegistryClient(c dockermobyutil.RegistryClient) *Task {
 	t.dockerRegistryClient = c
 	return t
 }
+
+func (t *Task) WithEnvStore(s envutil.Store) *Task {
+	t.envStore = s
+	return t
+}

test/e2e/target-symlink/symlink_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/benchkram/bob/bob"
 	"github.com/benchkram/bob/pkg/file"
+	"github.com/benchkram/errz"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -27,6 +28,7 @@ var _ = Describe("Testing that symlink targets are preserved", func() {
 
 		It("should build the task", func() {
 			err := b.Build(ctx, "build")
+			errz.Log(err)
 			Expect(err).NotTo(HaveOccurred())
 
 			dirContents, err := readDir(".")