diff --git a/ethereum.go b/ethereum.go index 0eef4af..bd22fda 100644 --- a/ethereum.go +++ b/ethereum.go @@ -17,7 +17,7 @@ const ( DefaultPackageRepository = "github.com/ethpandaops/ethereum-package" // DefaultPackageVersion is the pinned version of ethereum-package // See https://github.com/ethpandaops/ethereum-package/pull/1013 (v6.0.0). - DefaultPackageVersion = "release-please--branches--main--components--ethereum-package" + DefaultPackageVersion = "main" ) // RunOption configures how the Ethereum network is started diff --git a/examples/extra_files/configs/sample.yaml b/examples/extra_files/configs/sample.yaml new file mode 100644 index 0000000..d0b2927 --- /dev/null +++ b/examples/extra_files/configs/sample.yaml @@ -0,0 +1,6 @@ +# Sample validator configuration +graffiti: "ethereum-package-go extra_files example" +metrics: + enabled: true + address: 0.0.0.0 + port: 8008 \ No newline at end of file diff --git a/examples/extra_files/main.go b/examples/extra_files/main.go new file mode 100644 index 0000000..40b56a9 --- /dev/null +++ b/examples/extra_files/main.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/ethpandaops/ethereum-package-go" + "github.com/ethpandaops/ethereum-package-go/pkg/client" + "github.com/ethpandaops/ethereum-package-go/pkg/config" +) + +func main() { + ctx := context.Background() + + // Method 1: Using ExtraFilesHelper for convenience + helper := config.NewExtraFilesHelper() + + // Add inline content + helper.AddFile("beacon-config.yaml", ` +# Beacon node configuration +metrics: + enabled: true + port: 8080 +logging: + level: info +`) + + // Add from file (if it exists) + if err := helper.AddFileFromPath("validator-config.yaml", "./configs/sample.yaml"); err != nil { + log.Printf("Note: sample.yaml not found, using inline example: %v", err) + helper.AddFile("validator-config.yaml", "graffiti: 'ethereum-package-go example'") + } + + // Add JSON config + jsonConfig := map[string]interface{}{ + "version": "1.0", + "features": map[string]bool{ + "metrics": true, + "tracing": false, + }, + } + helper.AddJSON("features.json", jsonConfig) + + // Method 2: Direct inline definition + participant := config.NewParticipantBuilder(). + WithEL(client.Geth). + WithCL(client.Lighthouse). + WithCLExtraMounts(map[string]string{ + "/configs/beacon.yaml": "beacon-config.yaml", + "/configs/features.json": "features.json", + }). + WithCLExtraParams([]string{ + "--config-file=/configs/beacon.yaml", + }). + WithVCExtraMounts(map[string]string{ + "/configs/validator.yaml": "validator-config.yaml", + }). + Build() + + // Build network configuration with extra files at root level + networkConfig, err := config.NewConfigBuilder(). + WithParticipant(participant). + WithNetworkParams(&config.NetworkParams{ + Network: "kurtosis", + SecondsPerSlot: 12, + NumValidatorKeysPerNode: 32, + }). + WithExtraFiles(helper.Build()). + Build() + + if err != nil { + log.Fatalf("Failed to build config: %v", err) + } + + fmt.Printf("Starting network with %d extra files...\n", helper.Count()) + + // Start the network + network, err := ethereum.Run(ctx, ethereum.WithConfig(networkConfig)) + if err != nil { + log.Fatalf("Failed to start network: %v", err) + } + + fmt.Println("Network started successfully!") + fmt.Println("Extra files have been mounted into the containers.") + + // Display network info + execClients := network.ExecutionClients() + consClients := network.ConsensusClients() + fmt.Printf("\nExecution clients: %d\n", len(execClients.All())) + fmt.Printf("Consensus clients: %d\n", len(consClients.All())) + + // Wait for user input + fmt.Println("\nPress Enter to stop the network...") + var input string + fmt.Scanln(&input) + + // Cleanup + fmt.Println("Stopping network...") + if err := network.Cleanup(ctx); err != nil { + log.Fatalf("Failed to cleanup: %v", err) + } + + fmt.Println("Network stopped successfully!") +} diff --git a/pkg/config/builder.go b/pkg/config/builder.go index 7760c34..eefe1de 100644 --- a/pkg/config/builder.go +++ b/pkg/config/builder.go @@ -84,6 +84,21 @@ func (b *ConfigBuilder) WithDockerCacheParams(dockerCache *DockerCacheParams) *C return b } +// WithExtraFile adds a single extra file to the configuration +func (b *ConfigBuilder) WithExtraFile(name, content string) *ConfigBuilder { + if b.config.ExtraFiles == nil { + b.config.ExtraFiles = make(map[string]string) + } + b.config.ExtraFiles[name] = content + return b +} + +// WithExtraFiles sets all extra files at once +func (b *ConfigBuilder) WithExtraFiles(files map[string]string) *ConfigBuilder { + b.config.ExtraFiles = files + return b +} + // WithEthereumMetricsExporterEnabled sets the ethereum metrics exporter enabled func (b *ConfigBuilder) WithEthereumMetricsExporterEnabled(enabled bool) *ConfigBuilder { b.config.EthereumMetricsExporterEnabled = &enabled diff --git a/pkg/config/extra_files.go b/pkg/config/extra_files.go new file mode 100644 index 0000000..eaaddb6 --- /dev/null +++ b/pkg/config/extra_files.go @@ -0,0 +1,87 @@ +package config + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" +) + +// ExtraFilesHelper provides utilities for working with extra files +type ExtraFilesHelper struct { + files map[string]string +} + +// NewExtraFilesHelper creates a new helper for managing extra files +func NewExtraFilesHelper() *ExtraFilesHelper { + return &ExtraFilesHelper{ + files: make(map[string]string), + } +} + +// AddFile adds a file with inline content +func (h *ExtraFilesHelper) AddFile(name, content string) *ExtraFilesHelper { + h.files[name] = content + return h +} + +// AddFileFromPath reads a file from disk and adds it +func (h *ExtraFilesHelper) AddFileFromPath(name, path string) error { + content, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read file %s: %w", path, err) + } + h.files[name] = string(content) + return nil +} + +// AddFileFromReader reads content from an io.Reader +func (h *ExtraFilesHelper) AddFileFromReader(name string, r io.Reader) error { + content, err := io.ReadAll(r) + if err != nil { + return fmt.Errorf("failed to read content: %w", err) + } + h.files[name] = string(content) + return nil +} + +// AddJSON marshals an object to JSON and adds it as a file +func (h *ExtraFilesHelper) AddJSON(name string, v interface{}) error { + content, err := json.MarshalIndent(v, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal JSON: %w", err) + } + h.files[name] = string(content) + return nil +} + +// AddFilesFromDirectory adds all files from a directory +func (h *ExtraFilesHelper) AddFilesFromDirectory(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return fmt.Errorf("failed to read directory %s: %w", dir, err) + } + + for _, entry := range entries { + if !entry.IsDir() { + path := filepath.Join(dir, entry.Name()) + content, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read file %s: %w", path, err) + } + h.files[entry.Name()] = string(content) + } + } + return nil +} + +// Build returns the files map for use in NetworkParams +func (h *ExtraFilesHelper) Build() map[string]string { + return h.files +} + +// Count returns the number of files +func (h *ExtraFilesHelper) Count() int { + return len(h.files) +} diff --git a/pkg/config/extra_files_test.go b/pkg/config/extra_files_test.go new file mode 100644 index 0000000..df6f7a7 --- /dev/null +++ b/pkg/config/extra_files_test.go @@ -0,0 +1,353 @@ +package config + +import ( + "bytes" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestExtraFilesHelper(t *testing.T) { + t.Run("AddFile", func(t *testing.T) { + helper := NewExtraFilesHelper() + helper.AddFile("test.txt", "content") + + files := helper.Build() + if files["test.txt"] != "content" { + t.Errorf("expected content, got %s", files["test.txt"]) + } + }) + + t.Run("AddJSON", func(t *testing.T) { + helper := NewExtraFilesHelper() + data := map[string]string{"key": "value"} + + err := helper.AddJSON("config.json", data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + files := helper.Build() + if !strings.Contains(files["config.json"], `"key": "value"`) { + t.Errorf("JSON not properly marshaled: %s", files["config.json"]) + } + }) + + t.Run("AddFileFromPath", func(t *testing.T) { + // Create a temporary file + tmpDir := t.TempDir() + testFile := filepath.Join(tmpDir, "test.txt") + testContent := "test file content" + + err := os.WriteFile(testFile, []byte(testContent), 0644) + if err != nil { + t.Fatalf("failed to create test file: %v", err) + } + + helper := NewExtraFilesHelper() + err = helper.AddFileFromPath("loaded.txt", testFile) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + files := helper.Build() + if files["loaded.txt"] != testContent { + t.Errorf("expected %q, got %q", testContent, files["loaded.txt"]) + } + }) + + t.Run("AddFileFromPath_NonExistent", func(t *testing.T) { + helper := NewExtraFilesHelper() + err := helper.AddFileFromPath("test.txt", "/non/existent/file.txt") + if err == nil { + t.Error("expected error for non-existent file") + } + }) + + t.Run("AddFileFromReader", func(t *testing.T) { + helper := NewExtraFilesHelper() + content := "content from reader" + reader := strings.NewReader(content) + + err := helper.AddFileFromReader("from-reader.txt", reader) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + files := helper.Build() + if files["from-reader.txt"] != content { + t.Errorf("expected %q, got %q", content, files["from-reader.txt"]) + } + }) + + t.Run("AddFileFromReader_BytesBuffer", func(t *testing.T) { + helper := NewExtraFilesHelper() + content := []byte("binary content") + reader := bytes.NewBuffer(content) + + err := helper.AddFileFromReader("binary.dat", reader) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + files := helper.Build() + if files["binary.dat"] != string(content) { + t.Errorf("expected %q, got %q", string(content), files["binary.dat"]) + } + }) + + t.Run("AddFileFromReader_ErrorCase", func(t *testing.T) { + helper := NewExtraFilesHelper() + // Create a reader that always returns an error + reader := &errorReader{} + + err := helper.AddFileFromReader("error.txt", reader) + if err == nil { + t.Error("expected error from failing reader") + } + if !strings.Contains(err.Error(), "failed to read content") { + t.Errorf("unexpected error message: %v", err) + } + }) + + t.Run("AddFilesFromDirectory", func(t *testing.T) { + // Create a temporary directory with test files + tmpDir := t.TempDir() + + // Create test files + files := map[string]string{ + "file1.txt": "content 1", + "file2.yaml": "key: value", + "file3.json": `{"test": true}`, + } + + for name, content := range files { + err := os.WriteFile(filepath.Join(tmpDir, name), []byte(content), 0644) + if err != nil { + t.Fatalf("failed to create test file %s: %v", name, err) + } + } + + // Create a subdirectory (should be ignored) + subDir := filepath.Join(tmpDir, "subdir") + err := os.Mkdir(subDir, 0755) + if err != nil { + t.Fatalf("failed to create subdirectory: %v", err) + } + err = os.WriteFile(filepath.Join(subDir, "ignored.txt"), []byte("should not be included"), 0644) + if err != nil { + t.Fatalf("failed to create file in subdirectory: %v", err) + } + + helper := NewExtraFilesHelper() + err = helper.AddFilesFromDirectory(tmpDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + result := helper.Build() + + // Verify all files were added + for name, expectedContent := range files { + if result[name] != expectedContent { + t.Errorf("file %s: expected %q, got %q", name, expectedContent, result[name]) + } + } + + // Verify subdirectory file was not included + if _, exists := result["ignored.txt"]; exists { + t.Error("subdirectory file should not be included") + } + + // Verify count + if helper.Count() != len(files) { + t.Errorf("expected %d files, got %d", len(files), helper.Count()) + } + }) + + t.Run("AddFilesFromDirectory_Empty", func(t *testing.T) { + tmpDir := t.TempDir() + + helper := NewExtraFilesHelper() + err := helper.AddFilesFromDirectory(tmpDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if helper.Count() != 0 { + t.Errorf("expected 0 files from empty directory, got %d", helper.Count()) + } + }) + + t.Run("AddFilesFromDirectory_NonExistent", func(t *testing.T) { + helper := NewExtraFilesHelper() + err := helper.AddFilesFromDirectory("/non/existent/directory") + if err == nil { + t.Error("expected error for non-existent directory") + } + if !strings.Contains(err.Error(), "failed to read directory") { + t.Errorf("unexpected error message: %v", err) + } + }) + + t.Run("AddFilesFromDirectory_UnreadableFile", func(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("skipping test when running as root") + } + + tmpDir := t.TempDir() + + // Create a file that we can't read + unreadableFile := filepath.Join(tmpDir, "unreadable.txt") + err := os.WriteFile(unreadableFile, []byte("content"), 0000) + if err != nil { + t.Fatalf("failed to create test file: %v", err) + } + + helper := NewExtraFilesHelper() + err = helper.AddFilesFromDirectory(tmpDir) + if err == nil { + t.Error("expected error for unreadable file") + } + if !strings.Contains(err.Error(), "failed to read file") { + t.Errorf("unexpected error message: %v", err) + } + }) + + t.Run("Count", func(t *testing.T) { + helper := NewExtraFilesHelper() + helper.AddFile("file1.txt", "content1") + helper.AddFile("file2.txt", "content2") + + if helper.Count() != 2 { + t.Errorf("expected 2 files, got %d", helper.Count()) + } + }) + + t.Run("ChainedOperations", func(t *testing.T) { + tmpDir := t.TempDir() + testFile := filepath.Join(tmpDir, "test.txt") + err := os.WriteFile(testFile, []byte("file content"), 0644) + if err != nil { + t.Fatalf("failed to create test file: %v", err) + } + + helper := NewExtraFilesHelper() + + // Chain multiple operations + helper.AddFile("inline.txt", "inline content") + + err = helper.AddFileFromPath("from-path.txt", testFile) + if err != nil { + t.Fatalf("AddFileFromPath failed: %v", err) + } + + err = helper.AddFileFromReader("from-reader.txt", strings.NewReader("reader content")) + if err != nil { + t.Fatalf("AddFileFromReader failed: %v", err) + } + + err = helper.AddJSON("config.json", map[string]bool{"enabled": true}) + if err != nil { + t.Fatalf("AddJSON failed: %v", err) + } + + // Verify all files are present + files := helper.Build() + if len(files) != 4 { + t.Errorf("expected 4 files, got %d", len(files)) + } + + expectedContents := map[string]string{ + "inline.txt": "inline content", + "from-path.txt": "file content", + "from-reader.txt": "reader content", + } + + for name, expected := range expectedContents { + if files[name] != expected { + t.Errorf("file %s: expected %q, got %q", name, expected, files[name]) + } + } + + // Check JSON file contains expected content + if !strings.Contains(files["config.json"], `"enabled": true`) { + t.Errorf("JSON file doesn't contain expected content: %s", files["config.json"]) + } + }) +} + +// errorReader is a mock io.Reader that always returns an error +type errorReader struct{} + +func (e *errorReader) Read(p []byte) (n int, err error) { + return 0, io.ErrUnexpectedEOF +} + +func TestEthereumPackageConfigExtraFiles(t *testing.T) { + t.Run("Validation", func(t *testing.T) { + cfg := DefaultValidConfig() + cfg.ExtraFiles = map[string]string{ + "valid.txt": "content", + "": "empty name", // Should fail + } + + cfg.ApplyDefaults() + err := cfg.Validate() + if err == nil { + t.Error("expected validation error for empty file name") + } + }) + + t.Run("PathSeparatorValidation", func(t *testing.T) { + cfg := DefaultValidConfig() + cfg.ExtraFiles = map[string]string{ + "path/to/file.txt": "content", // Should fail + } + + cfg.ApplyDefaults() + err := cfg.Validate() + if err == nil { + t.Error("expected validation error for path separator in name") + } + }) +} + +func TestConfigBuilderExtraFiles(t *testing.T) { + t.Run("WithExtraFile", func(t *testing.T) { + builder := NewConfigBuilder() + builder.WithParticipants(DefaultValidConfig().Participants) + builder.WithExtraFile("test.txt", "content") + + cfg, err := builder.Build() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if cfg.ExtraFiles["test.txt"] != "content" { + t.Error("extra file not set correctly") + } + }) + + t.Run("WithExtraFiles", func(t *testing.T) { + files := map[string]string{ + "file1.txt": "content1", + "file2.txt": "content2", + } + + builder := NewConfigBuilder() + builder.WithParticipants(DefaultValidConfig().Participants) + builder.WithExtraFiles(files) + + cfg, err := builder.Build() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(cfg.ExtraFiles) != 2 { + t.Errorf("expected 2 files, got %d", len(cfg.ExtraFiles)) + } + }) +} diff --git a/pkg/config/types.go b/pkg/config/types.go index 8b865e5..3c45086 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -395,6 +395,10 @@ type EthereumPackageConfig struct { // Checkpoint sync configuration (root level) CheckpointSyncEnabled bool `yaml:"checkpoint_sync_enabled"` CheckpointSyncURL string `yaml:"checkpoint_sync_url,omitempty"` + + // ExtraFiles defines file contents that can be referenced in extra_mounts + // Keys are file names (used in extra_mounts), values are file contents + ExtraFiles map[string]string `yaml:"extra_files,omitempty"` } // Validate validates the EthereumPackageConfig @@ -473,6 +477,16 @@ func (c *EthereumPackageConfig) Validate() error { return fmt.Errorf("checkpoint sync URL must start with http:// or https://") } + // Validate ExtraFiles + for name := range c.ExtraFiles { + if name == "" { + return fmt.Errorf("extra_files cannot have empty names") + } + if strings.Contains(name, "/") { + return fmt.Errorf("extra_files name '%s' cannot contain path separators", name) + } + } + return nil } @@ -496,6 +510,11 @@ func (c *EthereumPackageConfig) ApplyDefaults() { if c.PortPublisher != nil { c.PortPublisher.ApplyDefaults() } + + // Initialize ExtraFiles if nil + if c.ExtraFiles == nil { + c.ExtraFiles = make(map[string]string) + } } // ConfigSource represents the source of configuration diff --git a/pkg/config/yaml_test.go b/pkg/config/yaml_test.go index 99be40c..00bb98e 100644 --- a/pkg/config/yaml_test.go +++ b/pkg/config/yaml_test.go @@ -745,3 +745,120 @@ func TestExtraConfigOmitEmpty(t *testing.T) { assert.NotContains(t, yamlStr, "vc_extra_env_vars:") assert.NotContains(t, yamlStr, "vc_extra_labels:") } + +func TestToYAMLWithExtraFiles(t *testing.T) { + config := &EthereumPackageConfig{ + Participants: []ParticipantConfig{ + { + ELType: client.Geth, + CLType: client.Lighthouse, + Count: 1, + CLExtraMounts: map[string]string{ + "/configs/beacon.yaml": "beacon-config.yaml", + }, + }, + }, + ExtraFiles: map[string]string{ + "beacon-config.yaml": "metrics:\n enabled: true\n port: 8080", + "validator.yaml": "graffiti: test", + }, + } + + yamlStr, err := ToYAML(config) + require.NoError(t, err) + assert.NotEmpty(t, yamlStr) + + // Check that extra_files is present at root level + assert.Contains(t, yamlStr, "extra_files:") + assert.Contains(t, yamlStr, "beacon-config.yaml:") + assert.Contains(t, yamlStr, "validator.yaml:") + assert.Contains(t, yamlStr, "graffiti: test") +} + +func TestFromYAMLWithExtraFiles(t *testing.T) { + yamlContent := ` +participants: + - el_type: geth + cl_type: lighthouse + cl_extra_mounts: + /configs/beacon.yaml: beacon-config.yaml + +extra_files: + beacon-config.yaml: | + metrics: + enabled: true + port: 8080 + validator.yaml: "graffiti: test" +` + + config, err := FromYAML(yamlContent) + require.NoError(t, err) + + // Check participants + assert.Len(t, config.Participants, 1) + assert.Equal(t, client.Geth, config.Participants[0].ELType) + assert.Equal(t, client.Lighthouse, config.Participants[0].CLType) + + // Check extra files + require.NotNil(t, config.ExtraFiles) + assert.Len(t, config.ExtraFiles, 2) + assert.Contains(t, config.ExtraFiles["beacon-config.yaml"], "metrics:") + assert.Equal(t, "graffiti: test", config.ExtraFiles["validator.yaml"]) + + // Check extra mounts reference extra files + assert.Equal(t, "beacon-config.yaml", config.Participants[0].CLExtraMounts["/configs/beacon.yaml"]) +} + +func TestExtraFilesRoundTrip(t *testing.T) { + // Create a config with extra files + original := &EthereumPackageConfig{ + Participants: []ParticipantConfig{ + { + ELType: client.Geth, + CLType: client.Prysm, + Count: 1, + CLExtraMounts: map[string]string{ + "/etc/config.yaml": "config.yaml", + "/etc/features.json": "features.json", + }, + }, + }, + ExtraFiles: map[string]string{ + "config.yaml": "key: value\nother: data", + "features.json": `{"feature1": true, "feature2": false}`, + }, + } + + // Convert to YAML + yamlStr, err := ToYAML(original) + require.NoError(t, err) + + // Parse back from YAML + parsed, err := FromYAML(yamlStr) + require.NoError(t, err) + + // Verify extra files match + require.NotNil(t, parsed.ExtraFiles) + assert.Equal(t, len(original.ExtraFiles), len(parsed.ExtraFiles)) + for key, value := range original.ExtraFiles { + assert.Equal(t, value, parsed.ExtraFiles[key]) + } +} + +func TestExtraFilesOmitEmpty(t *testing.T) { + // Create a config without extra files + config := &EthereumPackageConfig{ + Participants: []ParticipantConfig{ + { + ELType: client.Geth, + CLType: client.Lighthouse, + }, + }, + } + + yamlStr, err := ToYAML(config) + require.NoError(t, err) + + // Check that extra_files is omitted when empty + assert.NotContains(t, yamlStr, "extra_files:") +}