Skip to content

Commit 1cf0a29

Browse files
raulbraulb
authored
feat: add conduit run (#2038)
* wip * use latest ecdysis * fix merge * fix go mod * fix lint
1 parent 3268d78 commit 1cf0a29

File tree

9 files changed

+214
-95
lines changed

9 files changed

+214
-95
lines changed

Diff for: ‎cmd/conduit/root/config.go renamed to ‎cmd/conduit/root/config/config.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,14 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
package root
15+
package config
1616

1717
import (
1818
"context"
1919
"fmt"
2020
"reflect"
2121

22+
"github.com/conduitio/conduit/cmd/conduit/root/run"
2223
"github.com/conduitio/ecdysis"
2324
)
2425

@@ -30,15 +31,15 @@ var (
3031
)
3132

3233
type ConfigCommand struct {
33-
rootCmd *RootCommand
34+
RunCmd *run.RunCommand
3435
}
3536

3637
func (c *ConfigCommand) Config() ecdysis.Config {
37-
return c.rootCmd.Config()
38+
return c.RunCmd.Config()
3839
}
3940

4041
func (c *ConfigCommand) Flags() []ecdysis.Flag {
41-
return c.rootCmd.Flags()
42+
return c.RunCmd.Flags()
4243
}
4344

4445
func (c *ConfigCommand) Docs() ecdysis.Docs {
@@ -83,6 +84,6 @@ func printStruct(v reflect.Value, parentPath string) {
8384
func (c *ConfigCommand) Usage() string { return "config" }
8485

8586
func (c ConfigCommand) Execute(_ context.Context) error {
86-
printStruct(reflect.ValueOf(c.rootCmd.cfg), "")
87+
printStruct(reflect.ValueOf(c.RunCmd.Cfg), "")
8788
return nil
8889
}

Diff for: ‎cmd/conduit/root/config_test.go renamed to ‎cmd/conduit/root/config/config_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
package root
15+
package config
1616

1717
import (
1818
"bytes"

Diff for: ‎cmd/conduit/root/init.go renamed to ‎cmd/conduit/root/initialize/init.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
package root
15+
package initialize
1616

1717
import (
1818
"bytes"
@@ -41,12 +41,12 @@ type InitFlags struct {
4141

4242
type InitCommand struct {
4343
flags InitFlags
44-
cfg *conduit.Config
44+
Cfg *conduit.Config
4545
}
4646

4747
func (c *InitCommand) Flags() []ecdysis.Flag {
4848
flags := ecdysis.BuildFlags(&c.flags)
49-
flags.SetDefault("path", filepath.Dir(c.cfg.ConduitCfgPath))
49+
flags.SetDefault("path", filepath.Dir(c.Cfg.ConduitCfgPath))
5050
return flags
5151
}
5252

@@ -83,7 +83,7 @@ func (c *InitCommand) createDirs() error {
8383

8484
func (c *InitCommand) createConfigYAML() error {
8585
cfgYAML := internal.NewYAMLTree()
86-
processConfigStruct(reflect.ValueOf(c.cfg).Elem(), "", cfgYAML)
86+
processConfigStruct(reflect.ValueOf(c.Cfg).Elem(), "", cfgYAML)
8787

8888
// Create encoder with custom indentation
8989
var buf bytes.Buffer
@@ -139,7 +139,7 @@ func processConfigStruct(v reflect.Value, parentPath string, cfgYAML *internal.Y
139139
}
140140
}
141141

142-
func (c *InitCommand) Execute(ctx context.Context) error {
142+
func (c *InitCommand) Execute(_ context.Context) error {
143143
err := c.createDirs()
144144
if err != nil {
145145
return err

Diff for: ‎cmd/conduit/root/root.go

+14-54
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,13 @@ import (
1818
"context"
1919
"fmt"
2020
"os"
21-
"path/filepath"
2221

22+
"github.com/conduitio/conduit/cmd/conduit/root/config"
23+
"github.com/conduitio/conduit/cmd/conduit/root/initialize"
2324
"github.com/conduitio/conduit/cmd/conduit/root/pipelines"
25+
"github.com/conduitio/conduit/cmd/conduit/root/run"
2426
"github.com/conduitio/conduit/cmd/conduit/root/version"
2527
"github.com/conduitio/conduit/pkg/conduit"
26-
"github.com/conduitio/conduit/pkg/foundation/cerrors"
2728
"github.com/conduitio/ecdysis"
2829
)
2930

@@ -32,77 +33,33 @@ var (
3233
_ ecdysis.CommandWithExecute = (*RootCommand)(nil)
3334
_ ecdysis.CommandWithDocs = (*RootCommand)(nil)
3435
_ ecdysis.CommandWithSubCommands = (*RootCommand)(nil)
35-
_ ecdysis.CommandWithConfig = (*RootCommand)(nil)
3636
)
3737

3838
type RootFlags struct {
3939
Version bool `long:"version" short:"v" usage:"show the current Conduit version"`
40-
conduit.Config
4140
}
4241

4342
type RootCommand struct {
4443
flags RootFlags
45-
cfg conduit.Config
4644
}
4745

48-
func (c *RootCommand) Execute(_ context.Context) error {
46+
func (c *RootCommand) Execute(ctx context.Context) error {
4947
if c.flags.Version {
5048
_, _ = fmt.Fprintf(os.Stdout, "%s\n", conduit.Version(true))
5149
return nil
5250
}
5351

54-
e := &conduit.Entrypoint{}
55-
e.Serve(c.cfg)
56-
return nil
57-
}
58-
59-
func (c *RootCommand) Config() ecdysis.Config {
60-
path := filepath.Dir(c.flags.ConduitCfgPath)
61-
62-
return ecdysis.Config{
63-
EnvPrefix: "CONDUIT",
64-
Parsed: &c.cfg,
65-
Path: c.flags.ConduitCfgPath,
66-
DefaultValues: conduit.DefaultConfigWithBasePath(path),
52+
if cmd := ecdysis.CobraCmdFromContext(ctx); cmd != nil {
53+
return cmd.Help()
6754
}
55+
56+
return nil
6857
}
6958

7059
func (c *RootCommand) Usage() string { return "conduit" }
7160

7261
func (c *RootCommand) Flags() []ecdysis.Flag {
73-
flags := ecdysis.BuildFlags(&c.flags)
74-
75-
currentPath, err := os.Getwd()
76-
if err != nil {
77-
panic(cerrors.Errorf("failed to get current working directory: %w", err))
78-
}
79-
80-
c.cfg = conduit.DefaultConfigWithBasePath(currentPath)
81-
flags.SetDefault("config.path", c.cfg.ConduitCfgPath)
82-
flags.SetDefault("db.type", c.cfg.DB.Type)
83-
flags.SetDefault("db.badger.path", c.cfg.DB.Badger.Path)
84-
flags.SetDefault("db.postgres.connection-string", c.cfg.DB.Postgres.ConnectionString)
85-
flags.SetDefault("db.postgres.table", c.cfg.DB.Postgres.Table)
86-
flags.SetDefault("db.sqlite.path", c.cfg.DB.SQLite.Path)
87-
flags.SetDefault("db.sqlite.table", c.cfg.DB.SQLite.Table)
88-
flags.SetDefault("api.enabled", c.cfg.API.Enabled)
89-
flags.SetDefault("http.address", c.cfg.API.HTTP.Address)
90-
flags.SetDefault("grpc.address", c.cfg.API.GRPC.Address)
91-
flags.SetDefault("log.level", c.cfg.Log.Level)
92-
flags.SetDefault("log.format", c.cfg.Log.Format)
93-
flags.SetDefault("connectors.path", c.cfg.Connectors.Path)
94-
flags.SetDefault("processors.path", c.cfg.Processors.Path)
95-
flags.SetDefault("pipelines.path", c.cfg.Pipelines.Path)
96-
flags.SetDefault("pipelines.exit-on-degraded", c.cfg.Pipelines.ExitOnDegraded)
97-
flags.SetDefault("pipelines.error-recovery.min-delay", c.cfg.Pipelines.ErrorRecovery.MinDelay)
98-
flags.SetDefault("pipelines.error-recovery.max-delay", c.cfg.Pipelines.ErrorRecovery.MaxDelay)
99-
flags.SetDefault("pipelines.error-recovery.backoff-factor", c.cfg.Pipelines.ErrorRecovery.BackoffFactor)
100-
flags.SetDefault("pipelines.error-recovery.max-retries", c.cfg.Pipelines.ErrorRecovery.MaxRetries)
101-
flags.SetDefault("pipelines.error-recovery.max-retries-window", c.cfg.Pipelines.ErrorRecovery.MaxRetriesWindow)
102-
flags.SetDefault("schema-registry.type", c.cfg.SchemaRegistry.Type)
103-
flags.SetDefault("schema-registry.confluent.connection-string", c.cfg.SchemaRegistry.Confluent.ConnectionString)
104-
flags.SetDefault("preview.pipeline-arch-v2", c.cfg.Preview.PipelineArchV2)
105-
return flags
62+
return ecdysis.BuildFlags(&c.flags)
10663
}
10764

10865
func (c *RootCommand) Docs() ecdysis.Docs {
@@ -113,10 +70,13 @@ func (c *RootCommand) Docs() ecdysis.Docs {
11370
}
11471

11572
func (c *RootCommand) SubCommands() []ecdysis.Command {
73+
runCmd := &run.RunCommand{}
74+
11675
return []ecdysis.Command{
117-
&ConfigCommand{rootCmd: c},
118-
&InitCommand{cfg: &c.cfg},
76+
&config.ConfigCommand{RunCmd: runCmd},
77+
&initialize.InitCommand{Cfg: &runCmd.Cfg},
11978
&version.VersionCommand{},
12079
&pipelines.PipelinesCommand{},
80+
&run.RunCommand{},
12181
}
12282
}

Diff for: ‎cmd/conduit/root/root_test.go

-27
Original file line numberDiff line numberDiff line change
@@ -31,34 +31,7 @@ func TestRootCommandFlags(t *testing.T) {
3131
usage string
3232
persistent bool
3333
}{
34-
{longName: "config.path", usage: "global conduit configuration file"},
3534
{longName: "version", shortName: "v", usage: "show the current Conduit version"},
36-
{longName: "db.type", usage: "database type; accepts badger,postgres,inmemory,sqlite"},
37-
{longName: "db.badger.path", usage: "path to badger DB"},
38-
{longName: "db.postgres.connection-string", usage: "postgres connection string, may be a database URL or in PostgreSQL keyword/value format"},
39-
{longName: "db.postgres.table", usage: "postgres table in which to store data (will be created if it does not exist)"},
40-
{longName: "db.sqlite.path", usage: "path to sqlite3 DB"},
41-
{longName: "db.sqlite.table", usage: "sqlite3 table in which to store data (will be created if it does not exist)"},
42-
{longName: "api.enabled", usage: "enable HTTP and gRPC API"},
43-
{longName: "http.address", usage: "address for serving the HTTP API"},
44-
{longName: "grpc.address", usage: "address for serving the gRPC API"},
45-
{longName: "log.level", usage: "sets logging level; accepts debug, info, warn, error, trace"},
46-
{longName: "log.format", usage: "sets the format of the logging; accepts json, cli"},
47-
{longName: "connectors.path", usage: "path to standalone connectors' directory"},
48-
{longName: "processors.path", usage: "path to standalone processors' directory"},
49-
{longName: "pipelines.path", usage: "path to pipelines' directory"},
50-
{longName: "pipelines.exit-on-degraded", usage: "exit Conduit if a pipeline is degraded"},
51-
{longName: "pipelines.error-recovery.min-delay", usage: "minimum delay before restart"},
52-
{longName: "pipelines.error-recovery.max-delay", usage: "maximum delay before restart"},
53-
{longName: "pipelines.error-recovery.backoff-factor", usage: "backoff factor applied to the last delay"},
54-
{longName: "pipelines.error-recovery.max-retries", usage: "maximum number of retries"},
55-
{longName: "pipelines.error-recovery.max-retries-window", usage: "amount of time running without any errors after which a pipeline is considered healthy"},
56-
{longName: "schema-registry.type", usage: "schema registry type; accepts builtin,confluent"},
57-
{longName: "schema-registry.confluent.connection-string", usage: "confluent schema registry connection string"},
58-
{longName: "preview.pipeline-arch-v2", usage: "enables experimental pipeline architecture v2 (note that the new architecture currently supports only 1 source and 1 destination per pipeline)"},
59-
{longName: "dev.cpuprofile", usage: "write CPU profile to file"},
60-
{longName: "dev.memprofile", usage: "write memory profile to file"},
61-
{longName: "dev.blockprofile", usage: "write block profile to file"},
6235
}
6336

6437
e := ecdysis.New()

Diff for: ‎cmd/conduit/root/run/run.go

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
// Copyright © 2024 Meroxa, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package run
16+
17+
import (
18+
"context"
19+
"os"
20+
"path/filepath"
21+
22+
"github.com/conduitio/conduit/pkg/conduit"
23+
"github.com/conduitio/conduit/pkg/foundation/cerrors"
24+
"github.com/conduitio/ecdysis"
25+
)
26+
27+
var (
28+
_ ecdysis.CommandWithFlags = (*RunCommand)(nil)
29+
_ ecdysis.CommandWithExecute = (*RunCommand)(nil)
30+
_ ecdysis.CommandWithDocs = (*RunCommand)(nil)
31+
_ ecdysis.CommandWithConfig = (*RunCommand)(nil)
32+
)
33+
34+
type RunFlags struct {
35+
conduit.Config
36+
}
37+
38+
type RunCommand struct {
39+
flags RunFlags
40+
Cfg conduit.Config
41+
}
42+
43+
func (c *RunCommand) Execute(_ context.Context) error {
44+
e := &conduit.Entrypoint{}
45+
e.Serve(c.Cfg)
46+
return nil
47+
}
48+
49+
func (c *RunCommand) Config() ecdysis.Config {
50+
path := filepath.Dir(c.flags.ConduitCfgPath)
51+
52+
return ecdysis.Config{
53+
EnvPrefix: "CONDUIT",
54+
Parsed: &c.Cfg,
55+
Path: c.flags.ConduitCfgPath,
56+
DefaultValues: conduit.DefaultConfigWithBasePath(path),
57+
}
58+
}
59+
60+
func (c *RunCommand) Usage() string { return "run" }
61+
62+
func (c *RunCommand) Flags() []ecdysis.Flag {
63+
flags := ecdysis.BuildFlags(&c.flags)
64+
65+
currentPath, err := os.Getwd()
66+
if err != nil {
67+
panic(cerrors.Errorf("failed to get current working directory: %w", err))
68+
}
69+
70+
c.Cfg = conduit.DefaultConfigWithBasePath(currentPath)
71+
flags.SetDefault("config.path", c.Cfg.ConduitCfgPath)
72+
flags.SetDefault("db.type", c.Cfg.DB.Type)
73+
flags.SetDefault("db.badger.path", c.Cfg.DB.Badger.Path)
74+
flags.SetDefault("db.postgres.connection-string", c.Cfg.DB.Postgres.ConnectionString)
75+
flags.SetDefault("db.postgres.table", c.Cfg.DB.Postgres.Table)
76+
flags.SetDefault("db.sqlite.path", c.Cfg.DB.SQLite.Path)
77+
flags.SetDefault("db.sqlite.table", c.Cfg.DB.SQLite.Table)
78+
flags.SetDefault("api.enabled", c.Cfg.API.Enabled)
79+
flags.SetDefault("http.address", c.Cfg.API.HTTP.Address)
80+
flags.SetDefault("grpc.address", c.Cfg.API.GRPC.Address)
81+
flags.SetDefault("log.level", c.Cfg.Log.Level)
82+
flags.SetDefault("log.format", c.Cfg.Log.Format)
83+
flags.SetDefault("connectors.path", c.Cfg.Connectors.Path)
84+
flags.SetDefault("processors.path", c.Cfg.Processors.Path)
85+
flags.SetDefault("pipelines.path", c.Cfg.Pipelines.Path)
86+
flags.SetDefault("pipelines.exit-on-degraded", c.Cfg.Pipelines.ExitOnDegraded)
87+
flags.SetDefault("pipelines.error-recovery.min-delay", c.Cfg.Pipelines.ErrorRecovery.MinDelay)
88+
flags.SetDefault("pipelines.error-recovery.max-delay", c.Cfg.Pipelines.ErrorRecovery.MaxDelay)
89+
flags.SetDefault("pipelines.error-recovery.backoff-factor", c.Cfg.Pipelines.ErrorRecovery.BackoffFactor)
90+
flags.SetDefault("pipelines.error-recovery.max-retries", c.Cfg.Pipelines.ErrorRecovery.MaxRetries)
91+
flags.SetDefault("pipelines.error-recovery.max-retries-window", c.Cfg.Pipelines.ErrorRecovery.MaxRetriesWindow)
92+
flags.SetDefault("schema-registry.type", c.Cfg.SchemaRegistry.Type)
93+
flags.SetDefault("schema-registry.confluent.connection-string", c.Cfg.SchemaRegistry.Confluent.ConnectionString)
94+
flags.SetDefault("preview.pipeline-arch-v2", c.Cfg.Preview.PipelineArchV2)
95+
return flags
96+
}
97+
98+
func (c *RunCommand) Docs() ecdysis.Docs {
99+
return ecdysis.Docs{
100+
Short: "Run Conduit",
101+
Long: `Starts the Conduit server and runs the configured pipelines.`,
102+
}
103+
}

0 commit comments

Comments
 (0)