Add GitHub webhook handling to fast-track controller application reprocessing

Procfile

@@ -1,4 +1,4 @@
-controller: go run ./cmd/argocd-application-controller/main.go --app-resync 10
+controller: go run ./cmd/argocd-application-controller/main.go
 api-server: go run ./cmd/argocd-server/main.go --insecure --disable-auth
 repo-server: go run ./cmd/argocd-repo-server/main.go
 dex: sh -c "go run ./cmd/argocd-util/main.go gendexcfg -o `pwd`/dist/dex.yaml && docker run --rm -p 5556:5556 -p 5557:5557 -v `pwd`/dist/dex.yaml:/dex.yaml quay.io/coreos/dex:v2.10.0 serve /dex.yaml"

common/common.go

@@ -37,6 +37,11 @@ var (
 
 	// LabelApplicationName is the label which indicates that resource belongs to application with the specified name
 	LabelApplicationName = application.ApplicationFullName + "/app-name"
+
+	// AnnotationKeyRefresh is the annotation key in the application which is updated with an
+	// arbitrary value (i.e. timestamp) on a git event, to  force the controller to wake up and
+	// re-evaluate the application
+	AnnotationKeyRefresh = application.ApplicationFullName + "/refresh"
 )
 
 // ArgoCDManagerServiceAccount is the name of the service account for managing a cluster

controller/controller.go

@@ -4,9 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"time"
-
 	"sync"
+	"time"
 
 	"github.com/argoproj/argo-cd/common"
 	appv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"

install/manifests/02a_argocd-cm.yaml

@@ -31,8 +31,8 @@ data:
       id: github
       name: GitHub
       config:
-        clientID: $github.clientID
-        clientSecret: $github.clientSecret
+        clientID: aabbccddeeff00112233
+        clientSecret: $dex.github.clientSecret
         orgs:
         - name: your-github-org
 
@@ -42,8 +42,8 @@ data:
       name: Acme GitHub
       config:
         hostName: github.acme.com
-        clientID: $acme.clientID
-        clientSecret: $acme.clientSecret
+        clientID: abcdefghijklmnopqrst
+        clientSecret: $dex.acme.clientSecret
         orgs:
         - name: your-github-org
 
@@ -53,5 +53,5 @@ data:
       name: Okta
       config:
         issuer: https://dev-123456.oktapreview.com
-        clientID: $okta.clientID
-        clientSecret: $okta.clientSecret
+        clientID: aaaabbbbccccddddeee
+        clientSecret: $dex.okta.clientSecret

install/manifests/02b_argocd-secret.yaml

@@ -12,13 +12,13 @@ stringData:
   # random server signature key for session validation
   server.secretkey: aEDvv73vv70F77+9CRBSNu+/vTYQ77+9EUFh77+9LzFyJ++/vXfLsO+/vWRbeu+/ve+/vQ==
 
+  # these keys hold the shared secret for authenticating GitHub/GitLab webhook events
+  github.webhook.secret: shhhh! it's a github secret
+  gitlab.webhook.secret: shhhh! it's a gitlab secret
+  bitbucket.webhook.uuid: your-bitbucket-uuid
+
   # the following of user defined keys which are referenced in the example argocd-cm configmap
   # as pat of SSO configuration.
-  github.clientID: aabbccddeeff00112233
-  github.clientSecret: nv1vx8w4gw5byrflujfkxww6ykh85yq818aorvwy
-
-  acme.clientID: abcdefghijklmnopqrst
-  acme.clientSecret: 5pp7dyre3d5nyk0ree1tr0gd68k18xn94x8lfae9
-
-  okta.clientID: aaaabbbbccccddddeee
-  okta.clientSecret: x41ztv6ufyf07oyoopc6f62p222c00mox2ciquvt
+  dex.github.clientSecret: nv1vx8w4gw5byrflujfkxww6ykh85yq818aorvwy
+  dex.acme.clientSecret: 5pp7dyre3d5nyk0ree1tr0gd68k18xn94x8lfae9
+  dex.okta.clientSecret: x41ztv6ufyf07oyoopc6f62p222c00mox2ciquvt

server/server.go

@@ -28,6 +28,7 @@ import (
 	util_session "github.com/argoproj/argo-cd/util/session"
 	settings_util "github.com/argoproj/argo-cd/util/settings"
 	tlsutil "github.com/argoproj/argo-cd/util/tls"
+	"github.com/argoproj/argo-cd/util/webhook"
 	golang_proto "github.com/golang/protobuf/proto"
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
@@ -280,8 +281,13 @@ func (a *ArgoCDServer) newHTTPServer(ctx context.Context) *http.Server {
 	mustRegisterGWHandler(session.RegisterSessionServiceHandlerFromEndpoint, ctx, gwmux, endpoint, dOpts)
 	mustRegisterGWHandler(settings.RegisterSettingsServiceHandlerFromEndpoint, ctx, gwmux, endpoint, dOpts)
 
+	// Dex reverse proxy and client app and OAuth2 login/callback
 	a.registerDexHandlers(mux)
 
+	// Webhook handler for git events
+	acdWebhookHandler := webhook.NewHandler(a.Namespace, a.AppClientset, a.settings)
+	mux.HandleFunc("/api/webhook", acdWebhookHandler.Handler)
+
 	if a.StaticAssetsDir != "" {
 		mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
 			acceptHTML := false

util/argo/argo.go

@@ -1,8 +1,17 @@
 package argo
 
 import (
+	"encoding/json"
+	"time"
+
+	"github.com/argoproj/argo-cd/common"
 	appv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
+	argoappv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
+	"github.com/argoproj/argo-cd/pkg/client/clientset/versioned/typed/application/v1alpha1"
 	"github.com/argoproj/argo-cd/reposerver/repository"
+	log "github.com/sirupsen/logrus"
+	apierr "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 // ResolveServerNamespace resolves server and namespace to use given an application spec,
@@ -21,3 +30,33 @@ func ResolveServerNamespace(destination *appv1.ApplicationDestination, manifestI
 	}
 	return server, namespace
 }
+
+// RefreshApp updates the refresh annotation of an application to coerce the controller to process it
+func RefreshApp(appIf v1alpha1.ApplicationInterface, name string) (*argoappv1.Application, error) {
+	refreshString := time.Now().UTC().Format(time.RFC3339)
+	metadata := map[string]interface{}{
+		"metadata": map[string]interface{}{
+			"annotations": map[string]string{
+				common.AnnotationKeyRefresh: refreshString,
+			},
+		},
+	}
+	var err error
+	patch, err := json.Marshal(metadata)
+	if err != nil {
+		return nil, err
+	}
+	for attempt := 0; attempt < 5; attempt++ {
+		app, err := appIf.Patch(name, types.MergePatchType, patch)
+		if err != nil {
+			if !apierr.IsConflict(err) {
+				return nil, err
+			}
+		} else {
+			log.Infof("Refreshed app '%s' for controller reprocessing (%s)", name, refreshString)
+			return app, nil
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return nil, err
+}

util/argo/argo_test.go

@@ -0,0 +1,23 @@
+package argo
+
+import (
+	"testing"
+
+	argoappv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
+	appclientset "github.com/argoproj/argo-cd/pkg/client/clientset/versioned/fake"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRefreshApp(t *testing.T) {
+	var testApp argoappv1.Application
+	testApp.Name = "test-app"
+	testApp.Namespace = "default"
+	appClientset := appclientset.NewSimpleClientset(&testApp)
+	appIf := appClientset.ArgoprojV1alpha1().Applications("default")
+	_, err := RefreshApp(appIf, "test-app")
+	assert.Nil(t, err)
+	// For some reason, the fake Application inferface doesn't reflect the patch status after Patch(),
+	// so can't verify it was set in unit tests.
+	//_, ok := newApp.Annotations[common.AnnotationKeyRefresh]
+	//assert.True(t, ok)
+}

util/settings/settings.go

@@ -19,19 +19,21 @@ type ArgoCDSettings struct {
 	// URL is the externally facing URL users will visit to reach ArgoCD.
 	// The value here is used when configuring SSO. Omitting this value will disable SSO.
 	URL string
-
 	// DexConfig is contains portions of a dex config yaml
 	DexConfig string
-
 	// LocalUsers holds users local to (stored on) the server.  This is to be distinguished from any potential alternative future login providers (LDAP, SAML, etc.) that might ever be added.
 	LocalUsers map[string]string
-
 	// ServerSignature holds the key used to generate JWT tokens.
 	ServerSignature []byte
-
 	// Certificate holds the certificate/private key for the ArgoCD API server.
 	// If nil, will run insecure without TLS.
 	Certificate *tls.Certificate
+	// WebhookGitLabSecret holds the shared secret for authenticating GitHub webhook events
+	WebhookGitHubSecret string
+	// WebhookGitLabSecret holds the shared secret for authenticating GitLab webhook events
+	WebhookGitLabSecret string
+	// WebhookBitbucketUUID holds the UUID for authenticating Bitbucket webhook events
+	WebhookBitbucketUUID string
 }
 
 const (
@@ -47,6 +49,12 @@ const (
 	settingURLKey = "url"
 	// settingDexConfigKey designates the key for the dex config
 	settingDexConfigKey = "dex.config"
+	// settingsWebhookGitHubSecret is the key for the GitHub shared webhook secret
+	settingsWebhookGitHubSecretKey = "webhook.github.secret"
+	// settingsWebhookGitLabSecret is the key for the GitLab shared webhook secret
+	settingsWebhookGitLabSecretKey = "webhook.gitlab.secret"
+	// settingsWebhookBitbucketUUID is the key for Bitbucket webhook UUID
+	settingsWebhookBitbucketUUIDKey = "webhook.bitbucket.uuid"
 )
 
 // SettingsManager holds config info for a new manager with which to access Kubernetes ConfigMaps.
@@ -82,6 +90,15 @@ func (mgr *SettingsManager) GetSettings() (*ArgoCDSettings, error) {
 		return nil, fmt.Errorf("server secret key not found")
 	}
 	settings.ServerSignature = secretKey
+	if githubWebhookSecret := argoCDSecret.Data[settingsWebhookGitHubSecretKey]; len(githubWebhookSecret) > 0 {
+		settings.WebhookGitHubSecret = string(githubWebhookSecret)
+	}
+	if gitlabWebhookSecret := argoCDSecret.Data[settingsWebhookGitLabSecretKey]; len(gitlabWebhookSecret) > 0 {
+		settings.WebhookGitLabSecret = string(gitlabWebhookSecret)
+	}
+	if bitbucketWebhookUUID := argoCDSecret.Data[settingsWebhookBitbucketUUIDKey]; len(bitbucketWebhookUUID) > 0 {
+		settings.WebhookBitbucketUUID = string(bitbucketWebhookUUID)
+	}
 
 	serverCert, certOk := argoCDSecret.Data[settingServerCertificate]
 	serverKey, keyOk := argoCDSecret.Data[settingServerPrivateKey]
@@ -141,6 +158,15 @@ func (mgr *SettingsManager) SaveSettings(settings *ArgoCDSettings) error {
 	argoCDSecret.StringData = make(map[string]string)
 	argoCDSecret.StringData[settingServerSignatureKey] = string(settings.ServerSignature)
 	argoCDSecret.StringData[settingAdminPasswordKey] = settings.LocalUsers[common.ArgoCDAdminUsername]
+	if settings.WebhookGitHubSecret != "" {
+		argoCDSecret.StringData[settingsWebhookGitHubSecretKey] = settings.WebhookGitHubSecret
+	}
+	if settings.WebhookGitLabSecret != "" {
+		argoCDSecret.StringData[settingsWebhookGitLabSecretKey] = settings.WebhookGitLabSecret
+	}
+	if settings.WebhookBitbucketUUID != "" {
+		argoCDSecret.StringData[settingsWebhookBitbucketUUIDKey] = settings.WebhookBitbucketUUID
+	}
 	if settings.Certificate != nil {
 		certBytes, keyBytes := tlsutil.EncodeX509KeyPair(*settings.Certificate)
 		argoCDSecret.StringData[settingServerCertificate] = string(certBytes)