Oauth pull

configurator/backend/go.mod

@@ -71,6 +71,7 @@ require (
 	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/go-redsync/redsync/v4 v4.3.0 // indirect
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/gobeam/stringy v0.0.5 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect

configurator/backend/go.sum

@@ -459,6 +459,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobeam/stringy v0.0.5 h1:TvxQGSAqr/qF0SBVxa8Q67WWIo7bCWS0bM101WOd52g=
+github.com/gobeam/stringy v0.0.5/go.mod h1:W3620X9dJHf2FSZF5fRnWekHcHQjwmCz8ZQ2d1qloqE=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/goccy/go-json v0.9.6/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=

go.mod

@@ -1,3 +1,5 @@
 module github.com/jitsucom/jitsu
 
 go 1.17
+
+require github.com/gobeam/stringy v0.0.5 // indirect

go.sum

@@ -0,0 +1,2 @@
+github.com/gobeam/stringy v0.0.5 h1:TvxQGSAqr/qF0SBVxa8Q67WWIo7bCWS0bM101WOd52g=
+github.com/gobeam/stringy v0.0.5/go.mod h1:W3620X9dJHf2FSZF5fRnWekHcHQjwmCz8ZQ2d1qloqE=

server/drivers/base/config.go

@@ -8,7 +8,6 @@ import (
 	"github.com/jitsucom/jitsu/server/logging"
 	"github.com/jitsucom/jitsu/server/oauth"
 	"github.com/jitsucom/jitsu/server/timestamp"
-	"github.com/spf13/viper"
 )
 
 const ConfigSignatureSuffix = "_JITSU_config"
@@ -100,14 +99,14 @@ func StreamIdentifier(namespace, name string) string {
 }
 
 func FillPreconfiguredOauth(sourceType string, config interface{}) {
-	oathFields, ok := oauth.Fields[sourceType]
+	oathFields, ok := oauth.Get(sourceType)
 	if ok {
 		sourceConnectorConfig, ok := config.(map[string]interface{})
 		if ok {
 			for k, v := range oathFields {
 				cf, ok := sourceConnectorConfig[k]
-				if (!ok || cf == "") && viper.GetString(v) != "" {
-					sourceConnectorConfig[k] = viper.GetString(v)
+				if (!ok || cf == "") && v.Provided {
+					sourceConnectorConfig[k] = v.Value
 				}
 			}
 		}

server/drivers/base/types.go

@@ -5,14 +5,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/jitsucom/jitsu/server/logging"
-	"github.com/jitsucom/jitsu/server/oauth"
-	"github.com/jitsucom/jitsu/server/schema"
-	"github.com/spf13/viper"
 	"io"
 	"io/ioutil"
 	"strings"
 	"time"
+
+	"github.com/jitsucom/jitsu/server/logging"
+	"github.com/jitsucom/jitsu/server/oauth"
+	"github.com/jitsucom/jitsu/server/schema"
 )
 
 const (
@@ -78,13 +78,13 @@ func (gac *GoogleAuthConfig) FillPreconfiguredOauth(sourceType string) {
 	if gac == nil || gac.Type != GoogleOAuthAuthorizationType {
 		return
 	}
-	oathFields, ok := oauth.Fields[sourceType]
+	oauthConfig, ok := oauth.Get(sourceType)
 	if ok {
-		if clientId, ok := oathFields["client_id"]; gac.ClientID == "" && ok {
-			gac.ClientID = viper.GetString(clientId)
+		if clientId, ok := oauthConfig["client_id"]; gac.ClientID == "" && ok {
+			gac.ClientID = clientId.Value
 		}
-		if clientSecret, ok := oathFields["client_secret"]; gac.ClientSecret == "" && ok {
-			gac.ClientSecret = viper.GetString(clientSecret)
+		if clientSecret, ok := oauthConfig["client_secret"]; gac.ClientSecret == "" && ok {
+			gac.ClientSecret = clientSecret.Value
 		}
 	}
 }

server/drivers/google_ads/config.go

@@ -2,12 +2,13 @@ package google_ads
 
 import (
 	"errors"
+	"strconv"
+	"time"
+
 	"github.com/jitsucom/jitsu/server/adapters"
 	"github.com/jitsucom/jitsu/server/drivers/base"
 	"github.com/jitsucom/jitsu/server/oauth"
 	"github.com/spf13/viper"
-	"strconv"
-	"time"
 )
 
 //googleAdsHTTPConfiguration contains default amplitude HTTP timeouts/retry/delays,etc
@@ -32,10 +33,10 @@ type GoogleAdsCollectionConfig struct {
 }
 
 func (gac *GoogleAdsConfig) FillPreconfiguredOauth(sourceType string) {
-	oathFields, ok := oauth.Fields[sourceType]
+	oauthConfig, ok := oauth.Get(sourceType)
 	if ok {
-		if developerToken, ok := oathFields["developer_token"]; gac.DeveloperToken == "" && ok {
-			gac.DeveloperToken = viper.GetString(developerToken)
+		if developerToken, ok := oauthConfig["developer_token"]; gac.DeveloperToken == "" && ok {
+			gac.DeveloperToken = developerToken.Value
 		}
 		//backward compatibility with previous versions config
 		if gac.DeveloperToken == "" && viper.GetString("google-ads.developer-token") != "" {

server/go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/go-redsync/redsync/v4 v4.3.0
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/gomodule/redigo v1.8.2
-	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/go-cmp v0.5.6
 	github.com/google/go-github/v32 v32.1.0
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/martian v2.1.0+incompatible
@@ -69,6 +69,7 @@ require (
 )
 
 require (
+	github.com/gobeam/stringy v0.0.5
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/joomcode/errorx v1.1.0
 )

server/go.sum

@@ -369,6 +369,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobeam/stringy v0.0.5 h1:TvxQGSAqr/qF0SBVxa8Q67WWIo7bCWS0bM101WOd52g=
+github.com/gobeam/stringy v0.0.5/go.mod h1:W3620X9dJHf2FSZF5fRnWekHcHQjwmCz8ZQ2d1qloqE=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=

server/handlers/airbyte.go

@@ -5,6 +5,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"sort"
+	"time"
+
 	"github.com/gin-gonic/gin"
 	"github.com/jitsucom/jitsu/server/airbyte"
 	"github.com/jitsucom/jitsu/server/drivers/base"
@@ -13,12 +18,6 @@ import (
 	"github.com/jitsucom/jitsu/server/oauth"
 	"github.com/jitsucom/jitsu/server/runner"
 	"github.com/jitsucom/jitsu/server/utils"
-	"github.com/spf13/viper"
-	"io/ioutil"
-	"net/http"
-	"sort"
-	"strings"
-	"time"
 )
 
 const (
@@ -123,7 +122,7 @@ func (ah *AirbyteHandler) SpecHandler(c *gin.Context) {
 }
 
 func enrichOathFields(dockerImage string, spec interface{}) {
-	oathFields, ok := oauth.Fields[dockerImage]
+	oauthConfig, ok := oauth.Get(dockerImage)
 	if ok {
 		props, err := utils.ExtractObject(spec, "connectionSpecification", "properties")
 		if err != nil {
@@ -143,7 +142,7 @@ func enrichOathFields(dockerImage string, spec interface{}) {
 			return
 		}
 		provided := make(map[string]bool)
-		for k, v := range oathFields {
+		for k, v := range oauthConfig {
 			pr, ok := propsMap[k]
 			if !ok {
 				continue
@@ -153,11 +152,10 @@ func enrichOathFields(dockerImage string, spec interface{}) {
 				logging.Errorf("cannot convert property %s to map[string]interface{} from: %T", k, pr)
 				continue
 			}
-			prov := viper.GetString(v) != ""
-			prMap["env_name"] = strings.ReplaceAll(strings.ToUpper(v), ".", "_")
-			prMap["yaml_path"] = v
-			prMap["provided"] = prov
-			provided[k] = prov
+			prMap["env_name"] = v.EnvName
+			prMap["yaml_path"] = v.YAMLPath
+			prMap["provided"] = v.Provided
+			provided[k] = v.Provided
 		}
 		newReq := make([]interface{}, 0, len(required)-len(provided))
 		for _, v := range required {

server/handlers/sources.go

@@ -2,6 +2,9 @@ package handlers
 
 import (
 	"fmt"
+	"net/http"
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/hashicorp/go-multierror"
 	"github.com/jitsucom/jitsu/server/adapters"
@@ -15,9 +18,6 @@ import (
 	"github.com/jitsucom/jitsu/server/runner"
 	"github.com/jitsucom/jitsu/server/schema"
 	"github.com/jitsucom/jitsu/server/sources"
-	"github.com/spf13/viper"
-	"net/http"
-	"strings"
 )
 
 //ClearCacheRequest is a dto for ClearCache endpoint
@@ -172,13 +172,13 @@ func (sh *SourcesHandler) TestSourcesHandler(c *gin.Context) {
 func (sh *SourcesHandler) OauthFields(c *gin.Context) {
 	res := make(map[string]interface{})
 	sourceType := c.Param("sourceType")
-	oathFields, ok := oauth.Fields[sourceType]
+	oauthConfig, ok := oauth.Get(sourceType)
 	if ok {
-		for k, v := range oathFields {
+		for k, v := range oauthConfig {
 			fieldObject := make(map[string]interface{})
-			fieldObject["env_name"] = strings.ReplaceAll(strings.ToUpper(v), ".", "_")
-			fieldObject["yaml_path"] = v
-			fieldObject["provided"] = viper.GetString(v) != ""
+			fieldObject["env_name"] = v.EnvName
+			fieldObject["yaml_path"] = v.YAMLPath
+			fieldObject["provided"] = v.Provided
 			res[k] = fieldObject
 		}
 	}

server/main.go

@@ -18,6 +18,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/jitsucom/jitsu/server/oauth"
 	"github.com/jitsucom/jitsu/server/script/node"
 	"github.com/jitsucom/jitsu/server/templates"
 
@@ -220,6 +221,17 @@ func main() {
 		notifications.SystemErrorf("Panic:\n%s\n%s", value, string(debug.Stack()))
 	}
 
+	if oauthSecretsURL := viper.GetString("oauth_secrets"); oauthSecretsURL != "" {
+		oauthService, err := oauth.NewReloadableService(oauthSecretsURL)
+		if err != nil {
+			logging.Errorf("Failed to load OAuth secrets from %s, falling back to YAML configuration: %v",
+				oauthSecretsURL, err)
+		} else {
+			appconfig.Instance.ScheduleClosing(oauthService)
+			oauth.Set(oauthService)
+		}
+	}
+
 	clusterID := metaStorage.GetOrCreateClusterID(uuid.New())
 	systemInfo := runtime.GetInfo()
 	telemetry.EnrichSystemInfo(clusterID, systemInfo)

server/oauth/config_service.go

@@ -0,0 +1,74 @@
+package oauth
+
+import (
+	"strings"
+
+	"github.com/spf13/viper"
+)
+
+var Fields = map[string]map[string]string{
+	"source-github": {
+		"client_id":     "github.client_id",
+		"client_secret": "github.client_secret",
+	},
+	"source-bing-ads": {
+		"client_id":       "bing_ads.client_id",
+		"client_secret":   "bing_ads.client_secret",
+		"developer_token": "bing_ads.developer_token",
+	},
+	"tap-adroll": {
+		"client_id":     "adroll.client_id",
+		"client_secret": "adroll.client_secret",
+	},
+	"google_analytics": {
+		"client_id":     "google_analytics.client_id",
+		"client_secret": "google_analytics.client_secret",
+	},
+	"google_ads": {
+		"client_id":       "google_ads.client_id",
+		"client_secret":   "google_ads.client_secret",
+		"developer_token": "google_ads.developer_token",
+	},
+	"google_play": {
+		"client_id":     "google_play.client_id",
+		"client_secret": "google_play.client_secret",
+	},
+	"tap-google-sheets": {
+		"client_id":     "google_sheets.client_id",
+		"client_secret": "google_sheets.client_secret",
+	},
+	"firebase": {
+		"client_id":     "firebase.client_id",
+		"client_secret": "firebase.client_secret",
+	},
+	"tap-helpscout": {
+		"client_id":     "helpscout.client_id",
+		"client_secret": "helpscout.client_secret",
+	},
+	"tap-xero": {
+		"client_id":     "xero.client_id",
+		"client_secret": "xero.client_secret",
+	},
+}
+
+type ConfigService struct{}
+
+func (s *ConfigService) Get(id string) (Secrets, bool) {
+	fields, ok := Fields[id]
+	if !ok {
+		return nil, false
+	}
+
+	secret := make(Secrets)
+	for key, configKey := range fields {
+		value := viper.GetString(configKey)
+		secret[key] = SecretValue{
+			Value:    value,
+			EnvName:  strings.ReplaceAll(strings.ToUpper(configKey), ".", "_"),
+			YAMLPath: configKey,
+			Provided: value != "",
+		}
+	}
+
+	return secret, true
+}

server/oauth/oauth.go

@@ -0,0 +1,24 @@
+package oauth
+
+type SecretValue struct {
+	Value    string
+	EnvName  string
+	YAMLPath string
+	Provided bool
+}
+
+type Secrets = map[string]SecretValue
+
+type Interface interface {
+	Get(id string) (Secrets, bool)
+}
+
+var instance Interface = new(ConfigService)
+
+func Set(service Interface) {
+	instance = service
+}
+
+func Get(id string) (Secrets, bool) {
+	return instance.Get(id)
+}