libhive,libdocker: Allow more args to docker image building

clients/go-ethereum/Dockerfile

@@ -1,21 +1,9 @@
-## By default, the geth is pulled from Docker Hub.
+## By default, geth is pulled from Docker Hub.
 
+ARG user=ethereum
+ARG repo=client-go
 ARG branch=latest
-FROM ethereum/client-go:$branch as builder
-
-## ----
-## Uncomment the steps below (and comment out the steps above!) to build go-ethereum
-## from local sources in the ./go-ethereum directory.
-
-# FROM golang:1-alpine as builder
-# ARG branch=master
-# ADD go-ethereum /go-ethereum
-# WORKDIR /go-ethereum
-# RUN apk add --update bash curl jq git make
-# RUN make geth
-# RUN mv ./build/bin/geth /usr/local/bin/geth
-
-## ----
+FROM $user/$repo:$branch as builder
 
 FROM alpine:latest
 RUN apk add --update bash curl jq

clients/go-ethereum/Dockerfile.git

@@ -0,0 +1,52 @@
+## Pulls geth from a git repository and builds it from source.
+
+FROM alpine:latest as builder
+ARG user=ethereum
+ARG repo=go-ethereum
+ARG branch=master
+
+RUN \
+  apk add --update bash curl jq go git make gcc musl-dev         \
+        ca-certificates linux-headers                         && \
+  git clone --depth 1 --branch $branch                           \
+      https://github.com/$user/$repo                          && \
+  (cd go-ethereum && make geth)                               && \
+  (cd go-ethereum                                             && \
+  echo "{}"                                                      \
+  | jq ".+ {\"repo\":\"$(git config --get remote.origin.url)\"}" \
+  | jq ".+ {\"branch\":\"$(git rev-parse --abbrev-ref HEAD)\"}"  \
+  | jq ".+ {\"commit\":\"$(git rev-parse HEAD)\"}"               \
+  > /version.json)                                            && \
+  cp go-ethereum/build/bin/geth /usr/local/bin/geth           && \
+  apk del go git make gcc musl-dev linux-headers              && \
+  rm -rf /go-ethereum && rm -rf /var/cache/apk/*
+
+FROM alpine:latest
+RUN apk add --update bash curl jq
+COPY --from=builder /usr/local/bin/geth /usr/local/bin/geth
+
+# Generate the version.txt file.
+RUN /usr/local/bin/geth console --exec 'console.log(admin.nodeInfo.name)' --maxpeers=0 --nodiscover --dev 2>/dev/null | head -1 > /version.txt
+
+# Inject the startup script.
+ADD geth.sh /geth.sh
+ADD mapper.jq /mapper.jq
+RUN chmod +x /geth.sh
+
+# Inject the enode id retriever script.
+RUN mkdir /hive-bin
+ADD enode.sh /hive-bin/enode.sh
+RUN chmod +x /hive-bin/enode.sh
+
+# Add a default genesis file.
+ADD genesis.json /genesis.json
+
+# Export the usual networking ports to allow outside access to the node
+EXPOSE 8545 8546 8547 8551 30303 30303/udp
+
+# Generate the ethash verification caches
+RUN \
+ /usr/local/bin/geth makecache     1 ~/.ethereum/geth/ethash && \
+ /usr/local/bin/geth makecache 30001 ~/.ethereum/geth/ethash
+
+ENTRYPOINT ["/geth.sh"]

clients/go-ethereum/Dockerfile.local

@@ -0,0 +1,38 @@
+## Build geth from source from a local directory called go-ethereum.
+
+FROM golang:1-alpine as builder
+ADD go-ethereum /go-ethereum
+WORKDIR /go-ethereum
+RUN apk add --update bash curl jq git make
+RUN make geth
+RUN mv ./build/bin/geth /usr/local/bin/geth
+
+FROM alpine:latest
+RUN apk add --update bash curl jq
+COPY --from=builder /usr/local/bin/geth /usr/local/bin/geth
+
+# Generate the version.txt file.
+RUN /usr/local/bin/geth console --exec 'console.log(admin.nodeInfo.name)' --maxpeers=0 --nodiscover --dev 2>/dev/null | head -1 > /version.txt
+
+# Inject the startup script.
+ADD geth.sh /geth.sh
+ADD mapper.jq /mapper.jq
+RUN chmod +x /geth.sh
+
+# Inject the enode id retriever script.
+RUN mkdir /hive-bin
+ADD enode.sh /hive-bin/enode.sh
+RUN chmod +x /hive-bin/enode.sh
+
+# Add a default genesis file.
+ADD genesis.json /genesis.json
+
+# Export the usual networking ports to allow outside access to the node
+EXPOSE 8545 8546 8547 8551 30303 30303/udp
+
+# Generate the ethash verification caches
+RUN \
+ /usr/local/bin/geth makecache     1 ~/.ethereum/geth/ethash && \
+ /usr/local/bin/geth makecache 30001 ~/.ethereum/geth/ethash
+
+ENTRYPOINT ["/geth.sh"]

docs/clients.md

@@ -20,6 +20,54 @@ name like:
 
     ./hive --sim my-simulation --client go-ethereum_v1.9.23,go_ethereum_v1.9.22
 
+Other arguments to the docker image building process of the client can be specified by
+using a single letter and colon prefix.
+
+Supported prefixes are:
+
+	f: - docker file name (used to build the client image)
+	u: - user name (owner of git repository)
+	r: - repository name
+	b: - branch or tag name
+
+Examples:
+
+	besu_nightly -> client: besu, branch: nightly
+	besu_u:hyperledger_b:master -> client: besu, user: hyperledger, branch: master
+	go-ethereum_f:git -> client: go-ethereum, dockerfile: Dockerfile.git
+
+Client configuration can also be specified as a YAML or JSON file.
+
+    ./hive --sim my-simulation --client clients.yaml
+
+```yaml
+- name: go-ethereum
+  dockerfile: git
+- name: nethermind
+  user: nethermindeth
+  repo: hive
+  branch: latest
+```
+
+    ./hive --sim my-simulation --client clients.json
+
+```json
+[
+	{
+		"name": "go-ethereum",
+		"dockerfile": "git"
+	},
+	{
+		"name": "nethermind",
+		"user": "nethermindeth",
+		"repo": "hive",
+		"branch": "latest",
+	}
+]
+```
+
+The client name is required, but all other fields are optional.
+
 See the [go-ethereum client definition][geth-docker] for an example of a client
 Dockerfile.
 
@@ -35,7 +83,7 @@ list:
 
 The role list is available to simulators and can be used to differentiate between clients
 based on features. Declaring a client role also signals that the client supports certain
-role-specific environment variables and files. If `hive.yml` is missing or doesn't declare
+role-specific environment variables and files. If `hive.yaml` is missing or doesn't declare
 roles, the `eth1` role is assumed.
 
 ### /version.txt

hive.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"os/signal"
 	"regexp"
-	"strings"
 	"time"
 
 	"github.com/ethereum/hive/internal/libdocker"
@@ -38,7 +37,8 @@ func main() {
 			"just the client name, or a client_branch specifier. If a branch name is supplied,\n"+
 			"the client image will use the given git branch or docker tag. Multiple instances of\n"+
 			"a single client type may be requested with different branches.\n"+
-			"Example: \"besu_latest,besu_20.10.2\"")
+			"Example: \"besu_latest,besu_20.10.2\"\n"+
+			"If a valid file is specified instead, the client configuration will be parsed from this file as YALM or JSON.\n")
 		clientTimeout = flag.Duration("client.checktimelimit", 3*time.Minute, "The `timeout` of waiting for clients to open up the RPC port.\n"+
 			"If a very long chain is imported, this timeout may need to be quite large.\n"+
 			"A lower value means that hive won't wait as long in case the node crashes and\n"+
@@ -111,7 +111,10 @@ func main() {
 		ClientStartTimeout: *clientTimeout,
 	}
 	runner := libhive.NewRunner(inv, builder, cb)
-	clientList := splitAndTrim(*clients, ",")
+	clientList, err := parseClients(*clients)
+	if err != nil {
+		fatal(err)
+	}
 
 	if err := runner.Build(ctx, clientList, simList); err != nil {
 		fatal(err)
@@ -146,10 +149,18 @@ func fatal(args ...interface{}) {
 	os.Exit(1)
 }
 
-func splitAndTrim(input, sep string) []string {
-	list := strings.Split(input, sep)
-	for i := range list {
-		list[i] = strings.TrimSpace(list[i])
+func parseClients(input string) ([]libhive.ClientBuildInfo, error) {
+	// First check if the input is a path to a file.
+	if _, err := os.Stat(input); err == nil {
+		// Read the file as yaml/json and try to parse the list of client info.
+		f, err := os.Open(input)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		return libhive.ClientsBuildInfoFromFile(f)
+	} else {
+		// Otherwise, parse the input as a comma-separated list of client info.
+		return libhive.ClientsBuildInfoFromString(input)
 	}
-	return list
 }

internal/fakes/builder.go

@@ -9,10 +9,10 @@ import (
 
 // BuilderHooks can be used to override the behavior of the fake builder.
 type BuilderHooks struct {
-	BuildClientImage    func(context.Context, string) (string, error)
+	BuildClientImage    func(context.Context, libhive.ClientBuildInfo) (string, error)
 	BuildSimulatorImage func(context.Context, string) (string, error)
 	ReadFile            func(ctx context.Context, image string, file string) ([]byte, error)
-	ReadClientMetadata  func(name string) (*libhive.ClientMetadata, error)
+	ReadClientMetadata  func(client libhive.ClientBuildInfo) (*libhive.ClientMetadata, error)
 }
 
 // fakeBuilder implements Backend without docker.
@@ -29,11 +29,11 @@ func NewBuilder(hooks *BuilderHooks) libhive.Builder {
 	return b
 }
 
-func (b *fakeBuilder) BuildClientImage(ctx context.Context, client string) (string, error) {
+func (b *fakeBuilder) BuildClientImage(ctx context.Context, client libhive.ClientBuildInfo) (string, error) {
 	if b.hooks.BuildClientImage != nil {
 		return b.hooks.BuildClientImage(ctx, client)
 	}
-	return "fakebuild/client/" + client + ":latest", nil
+	return "fakebuild/client/" + client.Name + ":latest", nil
 }
 
 func (b *fakeBuilder) BuildSimulatorImage(ctx context.Context, sim string) (string, error) {
@@ -47,9 +47,9 @@ func (b *fakeBuilder) BuildImage(ctx context.Context, name string, fsys fs.FS) e
 	return nil
 }
 
-func (b *fakeBuilder) ReadClientMetadata(name string) (*libhive.ClientMetadata, error) {
+func (b *fakeBuilder) ReadClientMetadata(client libhive.ClientBuildInfo) (*libhive.ClientMetadata, error) {
 	if b.hooks.ReadClientMetadata != nil {
-		return b.hooks.ReadClientMetadata(name)
+		return b.hooks.ReadClientMetadata(client)
 	}
 	m := libhive.ClientMetadata{Roles: []string{"eth1"}}
 	return &m, nil

internal/libdocker/builder.go

@@ -40,8 +40,8 @@ func NewBuilder(client *docker.Client, cfg *Config, auth Authenticator) *Builder
 }
 
 // ReadClientMetadata reads metadata of the given client.
-func (b *Builder) ReadClientMetadata(name string) (*libhive.ClientMetadata, error) {
-	dir := b.config.Inventory.ClientDirectory(name)
+func (b *Builder) ReadClientMetadata(client libhive.ClientBuildInfo) (*libhive.ClientMetadata, error) {
+	dir := b.config.Inventory.ClientDirectory(client)
 	f, err := os.Open(filepath.Join(dir, "hive.yaml"))
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -60,11 +60,25 @@ func (b *Builder) ReadClientMetadata(name string) (*libhive.ClientMetadata, erro
 }
 
 // BuildClientImage builds a docker image of the given client.
-func (b *Builder) BuildClientImage(ctx context.Context, name string) (string, error) {
-	dir := b.config.Inventory.ClientDirectory(name)
-	_, branch := libhive.SplitClientName(name)
-	tag := fmt.Sprintf("hive/clients/%s:latest", name)
-	err := b.buildImage(ctx, dir, "Dockerfile", branch, tag)
+func (b *Builder) BuildClientImage(ctx context.Context, client libhive.ClientBuildInfo) (string, error) {
+	dir := b.config.Inventory.ClientDirectory(client)
+	tag := fmt.Sprintf("hive/clients/%s:latest", client.String())
+	dockerFile := "Dockerfile"
+	if client.DockerFile != "" {
+		// Custom Dockerfile.
+		dockerFile += "." + client.DockerFile
+	}
+	buildArgs := make([]docker.BuildArg, 0)
+	if client.User != "" {
+		buildArgs = append(buildArgs, docker.BuildArg{Name: "user", Value: client.User})
+	}
+	if client.Repo != "" {
+		buildArgs = append(buildArgs, docker.BuildArg{Name: "repo", Value: client.Repo})
+	}
+	if client.TagBranch != "" {
+		buildArgs = append(buildArgs, docker.BuildArg{Name: "branch", Value: client.TagBranch})
+	}
+	err := b.buildImage(ctx, dir, dockerFile, tag, buildArgs...)
 	return tag, err
 }
 
@@ -86,7 +100,7 @@ func (b *Builder) BuildSimulatorImage(ctx context.Context, name string) (string,
 		}
 	}
 	tag := fmt.Sprintf("hive/simulators/%s:latest", name)
-	err := b.buildImage(ctx, buildContextPath, buildDockerfile, "", tag)
+	err := b.buildImage(ctx, buildContextPath, buildDockerfile, tag)
 	return tag, err
 }
 
@@ -230,7 +244,7 @@ func (b *Builder) ReadFile(ctx context.Context, image, path string) ([]byte, err
 
 // buildImage builds a single docker image from the specified context.
 // branch specifes a build argument to use a specific base image branch or github source branch.
-func (b *Builder) buildImage(ctx context.Context, contextDir, dockerFile, branch, imageTag string) error {
+func (b *Builder) buildImage(ctx context.Context, contextDir, dockerFile, imageTag string, buildArgs ...docker.BuildArg) error {
 	logger := b.logger.New("image", imageTag)
 	context, err := filepath.Abs(contextDir)
 	if err != nil {
@@ -242,9 +256,11 @@ func (b *Builder) buildImage(ctx context.Context, contextDir, dockerFile, branch
 	opts.ContextDir = context
 	opts.Dockerfile = dockerFile
 	logctx := []interface{}{"dir", contextDir, "nocache", opts.NoCache, "pull", opts.Pull}
-	if branch != "" {
-		logctx = append(logctx, "branch", branch)
-		opts.BuildArgs = []docker.BuildArg{{Name: "branch", Value: branch}}
+	if len(buildArgs) > 0 {
+		for _, arg := range buildArgs {
+			logctx = append(logctx, "buildarg", fmt.Sprintf("%s=%s", arg.Name, arg.Value))
+		}
+		opts.BuildArgs = buildArgs
 	}
 
 	logger.Info("building image", logctx...)

internal/libhive/dockerface.go

@@ -80,8 +80,8 @@ type ContainerInfo struct {
 
 // Builder can build docker images of clients and simulators.
 type Builder interface {
-	ReadClientMetadata(name string) (*ClientMetadata, error)
-	BuildClientImage(ctx context.Context, name string) (string, error)
+	ReadClientMetadata(client ClientBuildInfo) (*ClientMetadata, error)
+	BuildClientImage(ctx context.Context, client ClientBuildInfo) (string, error)
 	BuildSimulatorImage(ctx context.Context, name string) (string, error)
 	BuildImage(ctx context.Context, name string, fsys fs.FS) error