From 9c0c242fda77e5d125f721aa1d4036a86c8526c6 Mon Sep 17 00:00:00 2001 From: Olivier Pinon Date: Tue, 8 Apr 2025 18:32:33 +0200 Subject: [PATCH 1/6] Add WithOrderedInitScripts for Postgres testcontainers --- modules/postgres/postgres.go | 36 +++++++++--- modules/postgres/postgres_test.go | 56 +++++++++++++++++++ .../postgres/testdata/aaaa-insert-user.sql | 5 ++ 3 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 modules/postgres/testdata/aaaa-insert-user.sql diff --git a/modules/postgres/postgres.go b/modules/postgres/postgres.go index 1f54add722..ce95857fc2 100644 --- a/modules/postgres/postgres.go +++ b/modules/postgres/postgres.go @@ -94,24 +94,42 @@ func WithDatabase(dbName string) testcontainers.CustomizeRequestOption { } } -// WithInitScripts sets the init scripts to be run when the container starts +// WithInitScripts sets the init scripts to be run when the container starts. +// These init scripts will be executed in sorted name order as defined by the current locale, which defaults to en_US.utf8. +// If you need to run your scripts in a specific order, consider using `WithOrderedInitScripts` instead. func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { - initScripts := []testcontainers.ContainerFile{} for _, script := range scripts { - cf := testcontainers.ContainerFile{ - HostFilePath: script, - ContainerFilePath: "/docker-entrypoint-initdb.d/" + filepath.Base(script), - FileMode: 0o755, - } - initScripts = append(initScripts, cf) + filename := filepath.Base(script) + appendInitScript(req, script, filename) } - req.Files = append(req.Files, initScripts...) + return nil + } +} +// WithOrderedInitScripts sets the init scripts to be run when the container starts. +// The scripts will be run in the order that they are provided in this function. +func WithOrderedInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + for idx, script := range scripts { + filename := filepath.Base(script) + containerFilePath := fmt.Sprintf("%06d-%s", idx, filename) + appendInitScript(req, script, containerFilePath) + } return nil } } +// Adds an initialization script to the Postgres container +func appendInitScript(req *testcontainers.GenericContainerRequest, hostFilePath string, containerFilePath string) { + cf := testcontainers.ContainerFile{ + HostFilePath: hostFilePath, + ContainerFilePath: "/docker-entrypoint-initdb.d/" + containerFilePath, + FileMode: 0o755, + } + req.Files = append(req.Files, cf) +} + // WithPassword sets the initial password of the user to be created when the container starts // It is required for you to use the PostgreSQL image. It must not be empty or undefined. // This environment variable sets the superuser password for PostgreSQL. diff --git a/modules/postgres/postgres_test.go b/modules/postgres/postgres_test.go index e83b8e1454..4484159d9f 100644 --- a/modules/postgres/postgres_test.go +++ b/modules/postgres/postgres_test.go @@ -5,8 +5,10 @@ import ( "database/sql" "errors" "fmt" + "io" "os" "path/filepath" + "strings" "testing" "time" @@ -19,6 +21,7 @@ import ( "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" + tcexec "github.com/testcontainers/testcontainers-go/exec" "github.com/testcontainers/testcontainers-go/modules/postgres" "github.com/testcontainers/testcontainers-go/wait" ) @@ -288,6 +291,59 @@ func TestWithInitScript(t *testing.T) { require.NotNil(t, result) } +func TestWithOrderedInitScript(t *testing.T) { + ctx := context.Background() + + ctr, err := postgres.Run(ctx, + "postgres:15.2-alpine", + // Executes first the init-user-db shell-script, then the do-insert-user SQL script + // Using WithInitScripts, this would not work as + postgres.WithOrderedInitScripts( + filepath.Join("testdata", "init-user-db.sh"), + filepath.Join("testdata", "aaaa-insert-user.sql"), + ), + postgres.WithDatabase(dbname), + postgres.WithUsername(user), + postgres.WithPassword(password), + postgres.BasicWaitStrategies(), + ) + testcontainers.CleanupContainer(t, ctr) + require.NoError(t, err) + + // Test that init scripts have been correctly renamed + c, reader, err := ctr.Exec(ctx, []string{"ls", "-l", "/docker-entrypoint-initdb.d"}, tcexec.Multiplexed()) + require.NoError(t, err) + require.Equal(t, 0, c, "Expected to read init scripts from the container") + + buf := new(strings.Builder) + _, err = io.Copy(buf, reader) + require.NoError(t, err) + + initScripts := buf.String() + strings.Contains(initScripts, "000000-init-user-db.sh") + strings.Contains(initScripts, "000001-aaaa-insert-user.sql") + + // explicitly set sslmode=disable because the container is not configured to use TLS + connStr, err := ctr.ConnectionString(ctx, "sslmode=disable") + require.NoError(t, err) + + db, err := sql.Open("postgres", connStr) + require.NoError(t, err) + require.NotNil(t, db) + defer db.Close() + + // database created in init script. See testdata/init-user-db.sh + rows, err := db.Query("SELECT COUNT(*) FROM testdb;") + require.NoError(t, err) + require.NotNil(t, rows) + for rows.Next() { + var count int + err := rows.Scan(&count) + require.NoError(t, err) + require.Equal(t, 2, count) + } +} + func TestSnapshot(t *testing.T) { tests := []struct { name string diff --git a/modules/postgres/testdata/aaaa-insert-user.sql b/modules/postgres/testdata/aaaa-insert-user.sql new file mode 100644 index 0000000000..687937f374 --- /dev/null +++ b/modules/postgres/testdata/aaaa-insert-user.sql @@ -0,0 +1,5 @@ +-- Do not rename this file, it is named like this +-- to test correct ordering behavior. +-- +-- See TestWithOrderedInitScripts. +INSERT INTO testdb (id, name) VALUES (2, 'second user') From 12fb595259cbf69d90c1e7268198922f9e19a379 Mon Sep 17 00:00:00 2001 From: Olivier Pinon Date: Fri, 11 Apr 2025 08:53:12 +0200 Subject: [PATCH 2/6] Cleanup: Apply review comments --- modules/postgres/postgres.go | 8 +++----- modules/postgres/postgres_test.go | 7 ++++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/modules/postgres/postgres.go b/modules/postgres/postgres.go index ce95857fc2..7a1707c94c 100644 --- a/modules/postgres/postgres.go +++ b/modules/postgres/postgres.go @@ -95,13 +95,12 @@ func WithDatabase(dbName string) testcontainers.CustomizeRequestOption { } // WithInitScripts sets the init scripts to be run when the container starts. -// These init scripts will be executed in sorted name order as defined by the current locale, which defaults to en_US.utf8. +// These init scripts will be executed in sorted name order as defined by the container's current locale, which defaults to en_US.utf8. // If you need to run your scripts in a specific order, consider using `WithOrderedInitScripts` instead. func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { for _, script := range scripts { - filename := filepath.Base(script) - appendInitScript(req, script, filename) + appendInitScript(req, script, filepath.Base(script)) } return nil } @@ -112,8 +111,7 @@ func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { func WithOrderedInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { for idx, script := range scripts { - filename := filepath.Base(script) - containerFilePath := fmt.Sprintf("%06d-%s", idx, filename) + containerFilePath := fmt.Sprintf("%03d-%s", idx, filepath.Base(script)) appendInitScript(req, script, containerFilePath) } return nil diff --git a/modules/postgres/postgres_test.go b/modules/postgres/postgres_test.go index 4484159d9f..88ecda8d30 100644 --- a/modules/postgres/postgres_test.go +++ b/modules/postgres/postgres_test.go @@ -297,7 +297,8 @@ func TestWithOrderedInitScript(t *testing.T) { ctr, err := postgres.Run(ctx, "postgres:15.2-alpine", // Executes first the init-user-db shell-script, then the do-insert-user SQL script - // Using WithInitScripts, this would not work as + // Using WithInitScripts, this would not work. + // This is because aaaa-insert-user would get executed first, but requires init-user-db to be executed before. postgres.WithOrderedInitScripts( filepath.Join("testdata", "init-user-db.sh"), filepath.Join("testdata", "aaaa-insert-user.sql"), @@ -320,8 +321,8 @@ func TestWithOrderedInitScript(t *testing.T) { require.NoError(t, err) initScripts := buf.String() - strings.Contains(initScripts, "000000-init-user-db.sh") - strings.Contains(initScripts, "000001-aaaa-insert-user.sql") + strings.Contains(initScripts, "000-init-user-db.sh") + strings.Contains(initScripts, "001-aaaa-insert-user.sql") // explicitly set sslmode=disable because the container is not configured to use TLS connStr, err := ctr.ConnectionString(ctx, "sslmode=disable") From 59f7cf71a205ac2a110ec028cd9a3ccea3745f5d Mon Sep 17 00:00:00 2001 From: Olivier Pinon Date: Wed, 16 Apr 2025 09:23:23 +0200 Subject: [PATCH 3/6] Use testcontainers.WithFiles to inject init script file --- modules/postgres/postgres.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/modules/postgres/postgres.go b/modules/postgres/postgres.go index 7a1707c94c..fd00877429 100644 --- a/modules/postgres/postgres.go +++ b/modules/postgres/postgres.go @@ -99,10 +99,12 @@ func WithDatabase(dbName string) testcontainers.CustomizeRequestOption { // If you need to run your scripts in a specific order, consider using `WithOrderedInitScripts` instead. func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { + containerFiles := []testcontainers.ContainerFile{} for _, script := range scripts { - appendInitScript(req, script, filepath.Base(script)) + initScript := buildInitScriptContainerFile(script, filepath.Base(script)) + containerFiles = append(containerFiles, initScript) } - return nil + return testcontainers.WithFiles(containerFiles...)(req) } } @@ -110,22 +112,23 @@ func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { // The scripts will be run in the order that they are provided in this function. func WithOrderedInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { + containerFiles := []testcontainers.ContainerFile{} for idx, script := range scripts { containerFilePath := fmt.Sprintf("%03d-%s", idx, filepath.Base(script)) - appendInitScript(req, script, containerFilePath) + initScript := buildInitScriptContainerFile(script, containerFilePath) + containerFiles = append(containerFiles, initScript) } - return nil + return testcontainers.WithFiles(containerFiles...)(req) } } -// Adds an initialization script to the Postgres container -func appendInitScript(req *testcontainers.GenericContainerRequest, hostFilePath string, containerFilePath string) { - cf := testcontainers.ContainerFile{ +// Adds the file located at hostFilePath as an init script that postgres will launch +func buildInitScriptContainerFile(hostFilePath string, containerFilePath string) testcontainers.ContainerFile { + return testcontainers.ContainerFile{ HostFilePath: hostFilePath, ContainerFilePath: "/docker-entrypoint-initdb.d/" + containerFilePath, FileMode: 0o755, } - req.Files = append(req.Files, cf) } // WithPassword sets the initial password of the user to be created when the container starts From bdef121ca7db1ad100eff53c9ca55102ee1e255b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 24 Apr 2025 11:57:43 +0200 Subject: [PATCH 4/6] chore: no need to extract to function --- modules/postgres/postgres.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/modules/postgres/postgres.go b/modules/postgres/postgres.go index fd00877429..19c93ba479 100644 --- a/modules/postgres/postgres.go +++ b/modules/postgres/postgres.go @@ -101,7 +101,11 @@ func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { containerFiles := []testcontainers.ContainerFile{} for _, script := range scripts { - initScript := buildInitScriptContainerFile(script, filepath.Base(script)) + initScript := testcontainers.ContainerFile{ + HostFilePath: script, + ContainerFilePath: "/docker-entrypoint-initdb.d/" + filepath.Base(script), + FileMode: 0o755, + } containerFiles = append(containerFiles, initScript) } return testcontainers.WithFiles(containerFiles...)(req) @@ -114,23 +118,17 @@ func WithOrderedInitScripts(scripts ...string) testcontainers.CustomizeRequestOp return func(req *testcontainers.GenericContainerRequest) error { containerFiles := []testcontainers.ContainerFile{} for idx, script := range scripts { - containerFilePath := fmt.Sprintf("%03d-%s", idx, filepath.Base(script)) - initScript := buildInitScriptContainerFile(script, containerFilePath) + initScript := testcontainers.ContainerFile{ + HostFilePath: script, + ContainerFilePath: "/docker-entrypoint-initdb.d/" + fmt.Sprintf("%03d-%s", idx, filepath.Base(script)), + FileMode: 0o755, + } containerFiles = append(containerFiles, initScript) } return testcontainers.WithFiles(containerFiles...)(req) } } -// Adds the file located at hostFilePath as an init script that postgres will launch -func buildInitScriptContainerFile(hostFilePath string, containerFilePath string) testcontainers.ContainerFile { - return testcontainers.ContainerFile{ - HostFilePath: hostFilePath, - ContainerFilePath: "/docker-entrypoint-initdb.d/" + containerFilePath, - FileMode: 0o755, - } -} - // WithPassword sets the initial password of the user to be created when the container starts // It is required for you to use the PostgreSQL image. It must not be empty or undefined. // This environment variable sets the superuser password for PostgreSQL. From 2c81b564c792fb56f13dfd6a661b5be59205a5e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 24 Apr 2025 12:04:45 +0200 Subject: [PATCH 5/6] docs: document the new option --- docs/modules/postgres.md | 8 ++++++++ modules/postgres/postgres_test.go | 2 ++ 2 files changed, 10 insertions(+) diff --git a/docs/modules/postgres.md b/docs/modules/postgres.md index 09736f6279..a3af88e259 100644 --- a/docs/modules/postgres.md +++ b/docs/modules/postgres.md @@ -70,6 +70,14 @@ An example of a `*.sh` script that creates a user and database is shown below: [Init script content](../../modules/postgres/testdata/init-user-db.sh) +#### Ordered Init Scripts + +If you would like to run the init scripts in a specific order, you can use the `WithOrderedInitScripts` function, which copies the given scripts in the order they are provided to the container, prefixed with the order number so that Postgres executes them in the correct order. + + +[Ordered init scripts](../../modules/postgres/postgres_test.go) inside_block:orderedInitScripts + + #### Database configuration In the case you have a custom config file for Postgres, it's possible to copy that file into the container before it's started, using the `WithConfigFile(cfgPath string)` function. diff --git a/modules/postgres/postgres_test.go b/modules/postgres/postgres_test.go index 88ecda8d30..a389d68bff 100644 --- a/modules/postgres/postgres_test.go +++ b/modules/postgres/postgres_test.go @@ -296,6 +296,7 @@ func TestWithOrderedInitScript(t *testing.T) { ctr, err := postgres.Run(ctx, "postgres:15.2-alpine", + // orderedInitScripts { // Executes first the init-user-db shell-script, then the do-insert-user SQL script // Using WithInitScripts, this would not work. // This is because aaaa-insert-user would get executed first, but requires init-user-db to be executed before. @@ -303,6 +304,7 @@ func TestWithOrderedInitScript(t *testing.T) { filepath.Join("testdata", "init-user-db.sh"), filepath.Join("testdata", "aaaa-insert-user.sql"), ), + // } postgres.WithDatabase(dbname), postgres.WithUsername(user), postgres.WithPassword(password), From 9231855803e53be173ee23a7c3ea402750e56267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 24 Apr 2025 12:12:42 +0200 Subject: [PATCH 6/6] chore: simplify even more --- modules/postgres/postgres.go | 38 +++++++++++++++++------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/modules/postgres/postgres.go b/modules/postgres/postgres.go index 19c93ba479..865fa2c94b 100644 --- a/modules/postgres/postgres.go +++ b/modules/postgres/postgres.go @@ -98,35 +98,33 @@ func WithDatabase(dbName string) testcontainers.CustomizeRequestOption { // These init scripts will be executed in sorted name order as defined by the container's current locale, which defaults to en_US.utf8. // If you need to run your scripts in a specific order, consider using `WithOrderedInitScripts` instead. func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { - return func(req *testcontainers.GenericContainerRequest) error { - containerFiles := []testcontainers.ContainerFile{} - for _, script := range scripts { - initScript := testcontainers.ContainerFile{ - HostFilePath: script, - ContainerFilePath: "/docker-entrypoint-initdb.d/" + filepath.Base(script), - FileMode: 0o755, - } - containerFiles = append(containerFiles, initScript) + containerFiles := []testcontainers.ContainerFile{} + for _, script := range scripts { + initScript := testcontainers.ContainerFile{ + HostFilePath: script, + ContainerFilePath: "/docker-entrypoint-initdb.d/" + filepath.Base(script), + FileMode: 0o755, } - return testcontainers.WithFiles(containerFiles...)(req) + containerFiles = append(containerFiles, initScript) } + + return testcontainers.WithFiles(containerFiles...) } // WithOrderedInitScripts sets the init scripts to be run when the container starts. // The scripts will be run in the order that they are provided in this function. func WithOrderedInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { - return func(req *testcontainers.GenericContainerRequest) error { - containerFiles := []testcontainers.ContainerFile{} - for idx, script := range scripts { - initScript := testcontainers.ContainerFile{ - HostFilePath: script, - ContainerFilePath: "/docker-entrypoint-initdb.d/" + fmt.Sprintf("%03d-%s", idx, filepath.Base(script)), - FileMode: 0o755, - } - containerFiles = append(containerFiles, initScript) + containerFiles := []testcontainers.ContainerFile{} + for idx, script := range scripts { + initScript := testcontainers.ContainerFile{ + HostFilePath: script, + ContainerFilePath: "/docker-entrypoint-initdb.d/" + fmt.Sprintf("%03d-%s", idx, filepath.Base(script)), + FileMode: 0o755, } - return testcontainers.WithFiles(containerFiles...)(req) + containerFiles = append(containerFiles, initScript) } + + return testcontainers.WithFiles(containerFiles...) } // WithPassword sets the initial password of the user to be created when the container starts