-
Notifications
You must be signed in to change notification settings - Fork 31
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore(sidecar): Refactor sidecar (#70)
- Loading branch information
Showing
16 changed files
with
878 additions
and
685 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,7 +6,6 @@ linters: | |
| enable: | ||
| - deadcode | ||
| - errcheck | ||
| - gofumpt | ||
| - gosimple | ||
| - govet | ||
| - misspell | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| package sidecar | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "net/http" | ||
| "os" | ||
| "strconv" | ||
| "time" | ||
|
|
||
| dfv1 "github.com/argoproj-labs/argo-dataflow/api/v1alpha1" | ||
| "github.com/prometheus/client_golang/prometheus" | ||
| "github.com/prometheus/client_golang/prometheus/promauto" | ||
| ) | ||
|
|
||
| func connectIn(ctx context.Context, sink func([]byte) error) (func(context.Context, []byte) error, error) { | ||
| inFlight := promauto.NewGauge(prometheus.GaugeOpts{ | ||
| Subsystem: "input", | ||
| Name: "inflight", | ||
| Help: "Number of in-flight messages, see https://github.com/argoproj-labs/argo-dataflow/blob/main/docs/METRICS.md#input_inflight", | ||
| ConstLabels: map[string]string{"replica": strconv.Itoa(replica)}, | ||
| }) | ||
| messageTimeSeconds := promauto.NewHistogram(prometheus.HistogramOpts{ | ||
| Subsystem: "input", | ||
| Name: "message_time_seconds", | ||
| Help: "Message time, see https://github.com/argoproj-labs/argo-dataflow/blob/main/docs/METRICS.md#input_message_time_seconds", | ||
| ConstLabels: map[string]string{"replica": strconv.Itoa(replica)}, | ||
| }) | ||
| in := step.Spec.GetIn() | ||
| if in == nil { | ||
| logger.Info("no in interface configured") | ||
| return func(context.Context, []byte) error { | ||
| return fmt.Errorf("no in interface configured") | ||
| }, nil | ||
| } else if in.FIFO { | ||
| logger.Info("opened input FIFO") | ||
| fifo, err := os.OpenFile(dfv1.PathFIFOIn, os.O_WRONLY, os.ModeNamedPipe) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to open input FIFO: %w", err) | ||
| } | ||
| afterClosers = append(afterClosers, func(ctx context.Context) error { | ||
| logger.Info("closing FIFO") | ||
| return fifo.Close() | ||
| }) | ||
| return func(ctx context.Context, data []byte) error { | ||
| inFlight.Inc() | ||
| defer inFlight.Dec() | ||
| if _, err := fifo.Write(data); err != nil { | ||
| return fmt.Errorf("failed to send to main: %w", err) | ||
| } | ||
| if _, err := fifo.Write([]byte("\n")); err != nil { | ||
| return fmt.Errorf("failed to send to main: %w", err) | ||
| } | ||
| return nil | ||
| }, nil | ||
| } else if in.HTTP != nil { | ||
| logger.Info("HTTP in interface configured") | ||
| if err := waitReady(ctx); err != nil { | ||
| return nil, err | ||
| } | ||
| afterClosers = append(afterClosers, func(ctx context.Context) error { | ||
| return waitUnready(ctx) | ||
| }) | ||
| return func(ctx context.Context, data []byte) error { | ||
| inFlight.Inc() | ||
| defer inFlight.Dec() | ||
| start := time.Now() | ||
| defer func() { messageTimeSeconds.Observe(time.Since(start).Seconds()) }() | ||
| if resp, err := http.Post("http://localhost:8080/messages", "application/octet-stream", bytes.NewBuffer(data)); err != nil { | ||
| return fmt.Errorf("failed to send to main: %w", err) | ||
| } else { | ||
| body, _ := ioutil.ReadAll(resp.Body) | ||
| defer func() { _ = resp.Body.Close() }() | ||
| if resp.StatusCode >= 300 { | ||
| return fmt.Errorf("failed to send to main: %q %q", resp.Status, body) | ||
| } | ||
| if resp.StatusCode == 201 { | ||
| return sink(body) | ||
| } | ||
| } | ||
| return nil | ||
| }, nil | ||
| } else { | ||
| return nil, fmt.Errorf("in interface misconfigured") | ||
| } | ||
| } | ||
|
|
||
| func waitReady(ctx context.Context) error { | ||
| logger.Info("waiting for HTTP in interface to be ready") | ||
| for { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return ctx.Err() | ||
| default: | ||
| if resp, err := http.Get("http://localhost:8080/ready"); err == nil && resp.StatusCode < 300 { | ||
| logger.Info("HTTP in interface ready") | ||
| return nil | ||
| } | ||
| time.Sleep(3 * time.Second) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func waitUnready(ctx context.Context) error { | ||
| logger.Info("waiting for HTTP in interface to be unready") | ||
| for { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return ctx.Err() | ||
| default: | ||
| if resp, err := http.Get("http://localhost:8080/ready"); err != nil || resp.StatusCode >= 300 { | ||
| logger.Info("HTTP in interface unready") | ||
| return nil | ||
| } | ||
| time.Sleep(3 * time.Second) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| package sidecar | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| apierr "k8s.io/apimachinery/pkg/api/errors" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| v1 "k8s.io/client-go/kubernetes/typed/core/v1" | ||
|
|
||
| "github.com/Shopify/sarama" | ||
| dfv1 "github.com/argoproj-labs/argo-dataflow/api/v1alpha1" | ||
| runnerutil "github.com/argoproj-labs/argo-dataflow/runner/util" | ||
| corev1 "k8s.io/api/core/v1" | ||
| ) | ||
|
|
||
| func init() { | ||
| sarama.Logger = runnerutil.NewSaramaStdLogger(logger) | ||
| } | ||
|
|
||
| func kafkaFromSecret(k *dfv1.Kafka, secret *corev1.Secret) { | ||
| k.Brokers = dfv1.StringsOr(k.Brokers, strings.Split(string(secret.Data["brokers"]), ",")) | ||
| k.Version = dfv1.StringOr(k.Version, string(secret.Data["version"])) | ||
| if _, ok := secret.Data["net.tls"]; ok { | ||
| k.NET = &dfv1.KafkaNET{TLS: &dfv1.TLS{}} | ||
| } | ||
| } | ||
|
|
||
| func newKafkaConfig(k *dfv1.Kafka) (*sarama.Config, error) { | ||
| x := sarama.NewConfig() | ||
| x.ClientID = dfv1.CtrSidecar | ||
| if k.Version != "" { | ||
| v, err := sarama.ParseKafkaVersion(k.Version) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to parse kafka version %q: %w", k.Version, err) | ||
| } | ||
| x.Version = v | ||
| } | ||
| if k.NET != nil { | ||
| if k.NET.TLS != nil { | ||
| x.Net.TLS.Enable = true | ||
| } | ||
| } | ||
| return x, nil | ||
| } | ||
|
|
||
| func enrichKafka(ctx context.Context, secrets v1.SecretInterface, x *dfv1.Kafka) error { | ||
| secret, err := secrets.Get(ctx, "dataflow-kafka-"+x.Name, metav1.GetOptions{}) | ||
| if err != nil { | ||
| if !apierr.IsNotFound(err) { | ||
| return err | ||
| } | ||
| } else { | ||
| kafkaFromSecret(x, secret) | ||
| } | ||
| return nil | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package sidecar | ||
|
|
||
| import ( | ||
| "context" | ||
| "sync" | ||
| "time" | ||
| ) | ||
|
|
||
| var ( | ||
| preStopCh = make(chan bool, 16) | ||
| beforeClosers []func(ctx context.Context) error // should be closed before main container exits | ||
| afterClosers []func(ctx context.Context) error // should be close after the main container exits | ||
| preStopMu = sync.Mutex{} | ||
| ) | ||
|
|
||
| func preStop() { | ||
| logger.Info("pre-stop") | ||
| preStopMu.Lock() | ||
| defer preStopMu.Unlock() | ||
| closeClosers(beforeClosers) | ||
| beforeClosers = nil | ||
| preStopCh <- true | ||
| logger.Info("pre-stop done") | ||
| } | ||
|
|
||
| func stop() { | ||
| closeClosers(afterClosers) | ||
| } | ||
|
|
||
| func closeClosers(closers []func(ctx context.Context) error) { | ||
| logger.Info("closing closers", "len", len(closers)) | ||
| ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | ||
| defer cancel() | ||
| for i := len(closers) - 1; i >= 0; i-- { | ||
| logger.Info("closing", "i", i) | ||
| if err := closers[i](ctx); err != nil { | ||
| logger.Error(err, "failed to close", "i", i) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| package sidecar | ||
|
|
||
| import "sync" | ||
|
|
||
| var mu = sync.Mutex{} | ||
|
|
||
| func withLock(f func()) { | ||
| mu.Lock() | ||
| defer mu.Unlock() | ||
| f() | ||
| } |
Oops, something went wrong.