diff --git a/clients/lighthouse-bn/hive.yaml b/clients/lighthouse-bn/hive.yaml index acd66b31e6..3b6c636f72 100644 --- a/clients/lighthouse-bn/hive.yaml +++ b/clients/lighthouse-bn/hive.yaml @@ -1,5 +1,2 @@ roles: - beacon -build_targets: - - mainnet - - minimal diff --git a/clients/lighthouse-bn/minimal.Dockerfile b/clients/lighthouse-bn/minimal.Dockerfile deleted file mode 100644 index 931d212f34..0000000000 --- a/clients/lighthouse-bn/minimal.Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -ARG tag=latest -ARG baseimage=sigp/lighthouse_minimal - -# TODO: either special upstream build, or clone + build minimal version here in dockerfile. -FROM $baseimage:$tag - -ADD lighthouse_bn.sh /lighthouse_bn.sh -RUN chmod +x /lighthouse_bn.sh - -# TODO: output accurate client version -RUN echo "latest" > /version.txt - -ENTRYPOINT ["/lighthouse_bn.sh"] - diff --git a/clients/lodestar-bn/hive.yaml b/clients/lodestar-bn/hive.yaml index eeb8c815de..7924591ec5 100644 --- a/clients/lodestar-bn/hive.yaml +++ b/clients/lodestar-bn/hive.yaml @@ -1,4 +1,3 @@ roles: - beacon -build_targets: - - mainnet + diff --git a/clients/nimbus-bn/hive.yaml b/clients/nimbus-bn/hive.yaml index f299a5e002..3b6c636f72 100644 --- a/clients/nimbus-bn/hive.yaml +++ b/clients/nimbus-bn/hive.yaml @@ -1,4 +1,2 @@ roles: - beacon -build_targets: - - mainnet \ No newline at end of file diff --git a/clients/prysm-bn/hive.yaml b/clients/prysm-bn/hive.yaml index acd66b31e6..3b6c636f72 100644 --- a/clients/prysm-bn/hive.yaml +++ b/clients/prysm-bn/hive.yaml @@ -1,5 +1,2 @@ roles: - beacon -build_targets: - - mainnet - - minimal diff --git a/clients/prysm-bn/minimal.Dockerfile b/clients/prysm-bn/minimal.Dockerfile deleted file mode 100644 index 23e5917c7c..0000000000 --- a/clients/prysm-bn/minimal.Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -ARG baseimage=gcr.io/prysmaticlabs/prysm/beacon-chain -ARG tag=latest - -# TODO: either special upstream build, or clone + build minimal version here in dockerfile. -FROM $baseimage:$tag - -ADD prysm_bn.sh /prysm_bn.sh -RUN chmod +x /prysm_bn.sh - -# TODO: output accurate client version -RUN echo "latest" > /version.txt - -ENTRYPOINT ["/prysm_bn.sh"] - diff --git a/internal/fakes/builder.go b/internal/fakes/builder.go index b19f27d2f9..dc79c9dd59 100644 --- a/internal/fakes/builder.go +++ b/internal/fakes/builder.go @@ -12,7 +12,6 @@ type BuilderHooks struct { BuildClientImage func(context.Context, libhive.ClientDesignator) (string, error) BuildSimulatorImage func(context.Context, string) (string, error) ReadFile func(ctx context.Context, image string, file string) ([]byte, error) - ReadClientMetadata func(client libhive.ClientDesignator) (*libhive.ClientMetadata, error) } // fakeBuilder implements Backend without docker. @@ -47,14 +46,6 @@ func (b *fakeBuilder) BuildImage(ctx context.Context, name string, fsys fs.FS) e return nil } -func (b *fakeBuilder) ReadClientMetadata(client libhive.ClientDesignator) (*libhive.ClientMetadata, error) { - if b.hooks.ReadClientMetadata != nil { - return b.hooks.ReadClientMetadata(client) - } - m := libhive.ClientMetadata{Roles: []string{"eth1"}} - return &m, nil -} - func (b *fakeBuilder) ReadFile(ctx context.Context, image, file string) ([]byte, error) { if b.hooks.ReadFile != nil { return b.hooks.ReadFile(ctx, image, file) diff --git a/internal/libdocker/builder.go b/internal/libdocker/builder.go index 648b84001a..f0bbebb5eb 100644 --- a/internal/libdocker/builder.go +++ b/internal/libdocker/builder.go @@ -13,7 +13,6 @@ import ( docker "github.com/fsouza/go-dockerclient" "gopkg.in/inconshreveable/log15.v2" - "gopkg.in/yaml.v3" "github.com/ethereum/hive/internal/libhive" ) @@ -39,26 +38,6 @@ func NewBuilder(client *docker.Client, cfg *Config, auth Authenticator) *Builder return b } -// ReadClientMetadata reads metadata of the given client. -func (b *Builder) ReadClientMetadata(client libhive.ClientDesignator) (*libhive.ClientMetadata, error) { - dir := b.config.Inventory.ClientDirectory(client) - f, err := os.Open(filepath.Join(dir, "hive.yaml")) - if err != nil { - if os.IsNotExist(err) { - // Eth1 client by default. - return &libhive.ClientMetadata{Roles: []string{"eth1"}}, nil - } else { - return nil, fmt.Errorf("failed to read hive metadata file in '%s': %v", dir, err) - } - } - defer f.Close() - var out libhive.ClientMetadata - if err := yaml.NewDecoder(f).Decode(&out); err != nil { - return nil, fmt.Errorf("failed to decode hive metadata file in '%s': %v", dir, err) - } - return &out, nil -} - // BuildClientImage builds a docker image of the given client. func (b *Builder) BuildClientImage(ctx context.Context, client libhive.ClientDesignator) (string, error) { dir := b.config.Inventory.ClientDirectory(client) diff --git a/internal/libhive/dockerface.go b/internal/libhive/dockerface.go index 7212c97794..a5ae1dab8b 100644 --- a/internal/libhive/dockerface.go +++ b/internal/libhive/dockerface.go @@ -80,7 +80,6 @@ type ContainerInfo struct { // Builder can build docker images of clients and simulators. type Builder interface { - ReadClientMetadata(client ClientDesignator) (*ClientMetadata, error) BuildClientImage(ctx context.Context, client ClientDesignator) (string, error) BuildSimulatorImage(ctx context.Context, name string) (string, error) BuildImage(ctx context.Context, name string, fsys fs.FS) error diff --git a/internal/libhive/inventory.go b/internal/libhive/inventory.go index e5af59cba1..931bfc1ec9 100644 --- a/internal/libhive/inventory.go +++ b/internal/libhive/inventory.go @@ -15,6 +15,170 @@ import ( "gopkg.in/yaml.v3" ) +// Inventory keeps names of clients and simulators. +type Inventory struct { + BaseDir string + Clients map[string]InventoryClient + Simulators map[string]struct{} +} + +type InventoryClient struct { + Dockerfiles []string + Meta ClientMetadata +} + +// ClientDirectory returns the directory containing the given client's Dockerfile. +// The client name may contain a branch specifier. +func (inv Inventory) ClientDirectory(client ClientDesignator) string { + return filepath.Join(inv.BaseDir, "clients", filepath.FromSlash(client.Client)) +} + +// SimulatorDirectory returns the directory of containing the given simulator's Dockerfile. +func (inv Inventory) SimulatorDirectory(name string) string { + return filepath.Join(inv.BaseDir, "simulators", filepath.FromSlash(name)) +} + +// AddClient ensures the given client name is known to the inventory. +// This method exists for unit testing purposes only. +func (inv *Inventory) AddClient(name string, ic *InventoryClient) { + if inv.Clients == nil { + inv.Clients = make(map[string]InventoryClient) + } + var icv InventoryClient + if ic != nil { + icv = *ic + } + inv.Clients[name] = icv +} + +// AddSimulator ensures the given simulator name is known to the inventory. +// This method exists for unit testing purposes only. +func (inv *Inventory) AddSimulator(name string) { + if inv.Simulators == nil { + inv.Simulators = make(map[string]struct{}) + } + inv.Simulators[name] = struct{}{} +} + +// MatchSimulators returns matching simulator names. +func (inv *Inventory) MatchSimulators(expr string) ([]string, error) { + expr = strings.TrimSpace(expr) + if expr == "" { + return nil, nil + } + re, err := regexp.Compile(expr) + if err != nil { + return nil, err + } + var result []string + for sim := range inv.Simulators { + if re.MatchString(sim) { + result = append(result, sim) + } + } + sort.Strings(result) + return result, nil +} + +// LoadInventory finds all clients and simulators in basedir. +func LoadInventory(basedir string) (Inventory, error) { + var err error + inv := Inventory{BaseDir: basedir} + inv.Clients, err = findClients(filepath.Join(basedir, "clients")) + if err != nil { + return inv, err + } + inv.Simulators, err = findSimulators(filepath.Join(basedir, "simulators")) + return inv, err +} + +func findSimulators(dir string) (map[string]struct{}, error) { + names := make(map[string]struct{}) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + name := info.Name() + // If we hit a dockerfile, add the parent and stop looking in this directory. + if name == "Dockerfile" { + rel, _ := filepath.Rel(dir, filepath.Dir(path)) + name := filepath.ToSlash(rel) + names[name] = struct{}{} + return filepath.SkipDir + } + return nil + }) + return names, err +} + +func findClients(dir string) (map[string]InventoryClient, error) { + clients := make(map[string]InventoryClient) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil || path == dir { + return err + } + rel, err := filepath.Rel(dir, filepath.Dir(path)) + if err != nil { + return err + } + clientName := filepath.ToSlash(rel) + + // Skip client sub-directories. + if info.IsDir() && path != dir { + if _, ok := clients[clientName]; ok { + return filepath.SkipDir + } + } + // Add Dockerfiles. + file := info.Name() + switch { + case file == "Dockerfile": + clients[clientName] = InventoryClient{ + Meta: ClientMetadata{ + Roles: []string{"eth1"}, // default role + }, + } + case strings.HasPrefix(file, "Dockerfile."): + client, ok := clients[clientName] + if !ok { + log15.Warn(fmt.Sprintf("found %s in directory without Dockerfile", file), "path", filepath.Dir(path)) + return nil + } + client.Dockerfiles = append(client.Dockerfiles, strings.TrimPrefix(file, "Dockerfile.")) + clients[clientName] = client + case file == "hive.yaml": + client, ok := clients[clientName] + if !ok { + log15.Warn("found hive.yaml in directory without Dockerfile", "path", filepath.Dir(path)) + return nil + } + md, err := loadClientMetadata(path) + if err != nil { + return err + } + client.Meta = md + clients[clientName] = client + } + return nil + }) + return clients, err +} + +func loadClientMetadata(path string) (m ClientMetadata, err error) { + f, err := os.Open(path) + if err != nil { + return m, err + } + defer f.Close() + + dec := yaml.NewDecoder(f) + dec.KnownFields(true) + if err := dec.Decode(&m); err != nil { + return m, fmt.Errorf("error in %s: %v", path, err) + } + return m, nil +} + // ClientDesignator specifies a client and build parameters for it. type ClientDesignator struct { // Client is the client name. @@ -186,153 +350,6 @@ func validateClients(inv *Inventory, list []ClientDesignator) error { return nil } -// Inventory keeps names of clients and simulators. -type Inventory struct { - BaseDir string - Clients map[string]InventoryClient - Simulators map[string]struct{} -} - -type InventoryClient struct { - Dockerfiles []string -} - -// HasClient returns true if the inventory contains the given client. -func (inv Inventory) HasClient(client ClientDesignator) bool { - ic, ok := inv.Clients[client.Client] - if !ok { - return false - } - if client.DockerfileExt != "" { - return slices.Contains(ic.Dockerfiles, client.DockerfileExt) - } - return ok -} - -// ClientDirectory returns the directory containing the given client's Dockerfile. -// The client name may contain a branch specifier. -func (inv Inventory) ClientDirectory(client ClientDesignator) string { - return filepath.Join(inv.BaseDir, "clients", filepath.FromSlash(client.Client)) -} - -// HasSimulator returns true if the inventory contains the given simulator. -func (inv Inventory) HasSimulator(name string) bool { - _, ok := inv.Simulators[name] - return ok -} - -// SimulatorDirectory returns the directory of containing the given simulator's Dockerfile. -func (inv Inventory) SimulatorDirectory(name string) string { - return filepath.Join(inv.BaseDir, "simulators", filepath.FromSlash(name)) -} - -// AddClient ensures the given client name is known to the inventory. -// This method exists for unit testing purposes only. -func (inv *Inventory) AddClient(name string, ic *InventoryClient) { - if inv.Clients == nil { - inv.Clients = make(map[string]InventoryClient) - } - var icv InventoryClient - if ic != nil { - icv = *ic - } - inv.Clients[name] = icv -} - -// AddSimulator ensures the given simulator name is known to the inventory. -// This method exists for unit testing purposes only. -func (inv *Inventory) AddSimulator(name string) { - if inv.Simulators == nil { - inv.Simulators = make(map[string]struct{}) - } - inv.Simulators[name] = struct{}{} -} - -// MatchSimulators returns matching simulator names. -func (inv *Inventory) MatchSimulators(expr string) ([]string, error) { - expr = strings.TrimSpace(expr) - if expr == "" { - return nil, nil - } - re, err := regexp.Compile(expr) - if err != nil { - return nil, err - } - var result []string - for sim := range inv.Simulators { - if re.MatchString(sim) { - result = append(result, sim) - } - } - sort.Strings(result) - return result, nil -} - -// LoadInventory finds all clients and simulators in basedir. -func LoadInventory(basedir string) (Inventory, error) { - var err error - inv := Inventory{BaseDir: basedir} - inv.Clients, err = findClients(filepath.Join(basedir, "clients")) - if err != nil { - return inv, err - } - inv.Simulators, err = findSimulators(filepath.Join(basedir, "simulators")) - return inv, err -} - -func findClients(dir string) (map[string]InventoryClient, error) { - clients := make(map[string]InventoryClient) - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil || path == dir { - return err - } - rel, err := filepath.Rel(dir, filepath.Dir(path)) - if err != nil { - return err - } - clientName := filepath.ToSlash(rel) - - // Skip client sub-directories. - if info.IsDir() && path != dir { - if _, ok := clients[clientName]; ok { - return filepath.SkipDir - } - } - // Add Dockerfiles. - file := info.Name() - if file == "Dockerfile" || strings.HasPrefix(file, "Dockerfile.") { - if file == "Dockerfile" { - clients[clientName] = InventoryClient{} - } else { - client := clients[clientName] - client.Dockerfiles = append(client.Dockerfiles, strings.TrimPrefix(file, "Dockerfile.")) - clients[clientName] = client - } - } - return nil - }) - return clients, err -} - -func findSimulators(dir string) (map[string]struct{}, error) { - names := make(map[string]struct{}) - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - name := info.Name() - // If we hit a dockerfile, add the parent and stop looking in this directory. - if name == "Dockerfile" { - rel, _ := filepath.Rel(dir, filepath.Dir(path)) - name := filepath.ToSlash(rel) - names[name] = struct{}{} - return filepath.SkipDir - } - return nil - }) - return names, err -} - type set[X comparable] map[X]struct{} func (s set[X]) add(x X) set[X] { diff --git a/internal/libhive/inventory_test.go b/internal/libhive/inventory_test.go index ed7c9c44a9..95a125d38f 100644 --- a/internal/libhive/inventory_test.go +++ b/internal/libhive/inventory_test.go @@ -172,38 +172,14 @@ func TestParseClientListYAML(t *testing.T) { } } -func TestInventory(t *testing.T) { +// This test ensures the real hive client definitions can be loaded. +func TestLoadInventory(t *testing.T) { basedir := filepath.FromSlash("../..") inv, err := LoadInventory(basedir) if err != nil { t.Fatal(err) } - spew.Dump(inv) - t.Run("HasClient", func(t *testing.T) { - clientInfo, err := ParseClientList(&inv, "go-ethereum,go-ethereum_latest,lighthouse-vc") - if err != nil { - t.Fatal(err) - } - if len(clientInfo) != 3 { - t.Fatal("wrong number of clients") - } - if !inv.HasClient(clientInfo[0]) { - t.Error("can't find go-ethereum client") - } - if !inv.HasClient(clientInfo[1]) { - t.Error("can't find go-ethereum_latest client") - } - if inv.HasClient(ClientDesignator{Client: "supereth3000"}) { - t.Error("returned true for unknown client") - } - }) - t.Run("HasSimulator", func(t *testing.T) { - if !inv.HasSimulator("smoke/genesis") { - t.Error("can't find smoke/genesis simulator") - } - if inv.HasSimulator("unknown simulator name") { - t.Error("returned true for unknown simulator name") - } - }) + t.Log("clients:", spew.Sdump(inv.Clients)) + t.Log("simulators:", inv.Simulators) } diff --git a/internal/libhive/run.go b/internal/libhive/run.go index 237dad69f6..37e0f8bd11 100644 --- a/internal/libhive/run.go +++ b/internal/libhive/run.go @@ -63,13 +63,6 @@ func (r *Runner) buildClients(ctx context.Context, clientList []ClientDesignator var anyBuilt bool log15.Info(fmt.Sprintf("building %d clients...", len(clientList))) for _, client := range clientList { - if !r.inv.HasClient(client) { - return fmt.Errorf("unknown client %q", client) - } - meta, err := r.builder.ReadClientMetadata(client) - if err != nil { - return err - } image, err := r.builder.BuildClientImage(ctx, client) if err != nil { continue @@ -83,7 +76,7 @@ func (r *Runner) buildClients(ctx context.Context, clientList []ClientDesignator Name: client.Name(), Version: strings.TrimSpace(string(version)), Image: image, - Meta: *meta, + Meta: r.inv.Clients[client.Client].Meta, } } if !anyBuilt {