Skip to content

Commit

Permalink
feat: git ssh private key
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexec
alexec committed Jul 23, 2021
1 parent 9c1e918 commit bb3fcaa
Show file tree
Hide file tree
Showing 14 changed files with 468 additions and 227 deletions.
22 changes: 11 additions & 11 deletions api/v1alpha1/const.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,17 @@ const (
CtrMain = "main"
CtrSidecar = "sidecar"
// env vars
EnvBearerToken = "ARGO_DATAFLOW_BEARER_TOKEN"
EnvImagePrefix = "ARGO_DATAFLOW_IMAGE_PREFIX" // default "quay.io/argoproj"
EnvDeletionDelay = "ARGO_DATAFLOW_DELETION_DELAY" // default "720h" ~= "30d"
EnvNamespace = "ARGO_DATAFLOW_NAMESPACE"
EnvPipelineName = "ARGO_DATAFLOW_PIPELINE_NAME"
EnvReplica = "ARGO_DATAFLOW_REPLICA"
EnvStep = "ARGO_DATAFLOW_STEP"
EnvPeekDelay = "ARGO_DATAFLOW_PEEK_DELAY" // how long between peeking (default 4m)
EnvPullPolicy = "ARGO_DATAFLOW_PULL_POLICY" // default ""
EnvScalingDelay = "ARGO_DATAFLOW_SCALING_DELAY" // how long to wait between any scaling events (including peeking) default "4m"
EnvUpdateInterval = "ARGO_DATAFLOW_UPDATE_INTERVAL" // default "1m"
EnvBearerToken = "ARGO_DATAFLOW_BEARER_TOKEN"
EnvImagePrefix = "ARGO_DATAFLOW_IMAGE_PREFIX" // default "quay.io/argoproj"
EnvDeletionDelay = "ARGO_DATAFLOW_DELETION_DELAY" // default "720h" ~= "30d"
EnvNamespace = "ARGO_DATAFLOW_NAMESPACE"
EnvPipelineName = "ARGO_DATAFLOW_PIPELINE_NAME"
EnvReplica = "ARGO_DATAFLOW_REPLICA"
EnvStep = "ARGO_DATAFLOW_STEP"
EnvPeekDelay = "ARGO_DATAFLOW_PEEK_DELAY" // how long between peeking (default 4m)
EnvPullPolicy = "ARGO_DATAFLOW_PULL_POLICY" // default ""
EnvScalingDelay = "ARGO_DATAFLOW_SCALING_DELAY" // how long to wait between any scaling events (including peeking) default "4m"
EnvUpdateInterval = "ARGO_DATAFLOW_UPDATE_INTERVAL" // default "1m"
// label/annotation keys
KeyDefaultContainer = "kubectl.kubernetes.io/default-container"
KeyDescription = "dataflow.argoproj.io/description"
Expand Down
469 changes: 262 additions & 207 deletions api/v1alpha1/generated.pb.go
View file

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions api/v1alpha1/generated.proto
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 4 additions & 3 deletions api/v1alpha1/git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@ import (
)

type Git struct {
Image string `json:"image" protobuf:"bytes,1,opt,name=image"`
Command []string `json:"command,omitempty" protobuf:"bytes,6,rep,name=command"`
URL string `json:"url" protobuf:"bytes,2,opt,name=url"`
Image string `json:"image" protobuf:"bytes,1,opt,name=image"`
Command []string `json:"command,omitempty" protobuf:"bytes,6,rep,name=command"`
URL string `json:"url" protobuf:"bytes,2,opt,name=url"`
SSHPrivateKey *corev1.SecretKeySelector `json:"sshPrivateKey,omitempty" protobuf:"bytes,7,opt,name=sshPrivateKey"`
// +kubebuilder:default=.
Path string `json:"path,omitempty" protobuf:"bytes,3,opt,name=path"`
// +kubebuilder:default=main
Expand Down
5 changes: 5 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

30 changes: 30 additions & 0 deletions config/ci.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,21 @@ spec:
path:
default: .
type: string
sshPrivateKey:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
optional:
description: Specify whether the Secret or its key must be defined
type: boolean
required:
- key
type: object
url:
type: string
required:
Expand Down Expand Up @@ -2808,6 +2823,21 @@ spec:
path:
default: .
type: string
sshPrivateKey:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
optional:
description: Specify whether the Secret or its key must be defined
type: boolean
required:
- key
type: object
url:
type: string
required:
Expand Down
18 changes: 18 additions & 0 deletions config/crd/bases/dataflow.argoproj.io_pipelines.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1039,6 +1039,24 @@ spec:
path:
default: .
type: string
sshPrivateKey:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must
be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
optional:
description: Specify whether the Secret or its key must
be defined
type: boolean
required:
- key
type: object
url:
type: string
required:
Expand Down
18 changes: 18 additions & 0 deletions config/crd/bases/dataflow.argoproj.io_steps.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -998,6 +998,24 @@ spec:
path:
default: .
type: string
sshPrivateKey:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must be
a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
optional:
description: Specify whether the Secret or its key must be
defined
type: boolean
required:
- key
type: object
url:
type: string
required:
Expand Down
30 changes: 30 additions & 0 deletions config/default.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,21 @@ spec:
path:
default: .
type: string
sshPrivateKey:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
optional:
description: Specify whether the Secret or its key must be defined
type: boolean
required:
- key
type: object
url:
type: string
required:
Expand Down Expand Up @@ -2808,6 +2823,21 @@ spec:
path:
default: .
type: string
sshPrivateKey:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
optional:
description: Specify whether the Secret or its key must be defined
type: boolean
required:
- key
type: object
url:
type: string
required:
Expand Down
30 changes: 30 additions & 0 deletions config/dev.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,21 @@ spec:
path:
default: .
type: string
sshPrivateKey:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
optional:
description: Specify whether the Secret or its key must be defined
type: boolean
required:
- key
type: object
url:
type: string
required:
Expand Down Expand Up @@ -2808,6 +2823,21 @@ spec:
path:
default: .
type: string
sshPrivateKey:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
optional:
description: Specify whether the Secret or its key must be defined
type: boolean
required:
- key
type: object
url:
type: string
required:
Expand Down
30 changes: 30 additions & 0 deletions config/quick-start.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,21 @@ spec:
path:
default: .
type: string
sshPrivateKey:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
optional:
description: Specify whether the Secret or its key must be defined
type: boolean
required:
- key
type: object
url:
type: string
required:
Expand Down Expand Up @@ -2808,6 +2823,21 @@ spec:
path:
default: .
type: string
sshPrivateKey:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
optional:
description: Specify whether the Secret or its key must be defined
type: boolean
required:
- key
type: object
url:
type: string
required:
Expand Down
30 changes: 26 additions & 4 deletions runner/init/init.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
package init

import (
"context"
"fmt"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"io"
"io/ioutil"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"os"
"path/filepath"
ctrl "sigs.k8s.io/controller-runtime"
"syscall"

dfv1 "github.com/argoproj-labs/argo-dataflow/api/v1alpha1"
Expand All @@ -19,7 +25,7 @@ var logger = sharedutil.NewLogger()

// due to main container crashing, the init container may be started many times, so each operation we perform should be
// idempontent, i.e. if we copy a file to shared volume, and it already exists, we should ignore that error
func Exec() error {
func Exec(ctx context.Context) error {
for _, name := range []string{dfv1.PathKill, dfv1.PathPreStop} {
logger.Info("copying binary", "name", name)
a := filepath.Join("/bin", filepath.Base(name))
Expand Down Expand Up @@ -51,12 +57,28 @@ func Exec() error {
}
if g := step.Spec.Git; g != nil {
logger.Info("cloning", "url", g.URL, "checkout", dfv1.PathCheckout)
var auth transport.AuthMethod

if k := g.SSHPrivateKey; k != nil {
logger.Info("getting secret for auth", "sshPrivateKey", k)
secretInterface := kubernetes.NewForConfigOrDie(ctrl.GetConfigOrDie()).CoreV1().Secrets(os.Getenv(dfv1.EnvNamespace))
if secret, err := secretInterface.Get(ctx, k.Name, metav1.GetOptions{}); err != nil {
return fmt.Errorf("failed to get secret %q: %w", k.Name, err)
} else {
if v, err := ssh.NewPublicKeys("git", secret.Data[k.Key], ""); err != nil {
return fmt.Errorf("failed to get create public keys: %w", err)
} else {
auth = v
}
}
}
if _, err := git.PlainClone(dfv1.PathCheckout, false, &git.CloneOptions{
URL: g.URL,
Auth: auth,
Depth: 1, // checkout faster
Progress: os.Stdout,
SingleBranch: true, // checkout faster
Depth: 1, // checkout faster
ReferenceName: plumbing.NewBranchReferenceName(g.Branch),
SingleBranch: true, // checkout faster
URL: g.URL,
}); IgnoreErrRepositoryAlreadyExists(err) != nil {
return fmt.Errorf("failed to clone repo: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion runner/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func main() {
case "group":
return group.Exec(ctx, os.Args[2], os.Args[3], dfv1.GroupFormat(os.Args[4]))
case "init":
return _init.Exec()
return _init.Exec(ctx)
case "map":
return _map.Exec(ctx, os.Args[2])
case "sidecar":
Expand Down
2 changes: 1 addition & 1 deletion runner/sidecar/out.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func connectOut(toSinks func([]byte) error) {
func connectOutHTTP(f func([]byte) error) {
logger.Info("HTTP out interface configured")
http.HandleFunc("/messages", func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer "+os.Getenv(dfv1.EnvDataflowBearerToken) {
if r.Header.Get("Authorization") != "Bearer "+os.Getenv(dfv1.EnvBearerToken) {
w.WriteHeader(403)
return
}
Expand Down

0 comments on commit bb3fcaa

Please sign in to comment.