From 73b3114f96213cb333b17fabce26492bbf12bea7 Mon Sep 17 00:00:00 2001 From: Gavin Frazar Date: Fri, 19 Aug 2022 12:31:38 -0700 Subject: [PATCH 1/2] Azure mysql postgres auto discovery configuration (#15629) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Azure auto-discovery configuration fields * Init databases if azure matchers are in config * Use AzureMatchers in db service * Use all azure subscriptions/resource groups if omitted in matcher * Add azure config tests * Update lib/services/matchers.go Co-authored-by: Krzysztof Skrzętnicki * Update lib/config/fileconf.go Co-authored-by: Marek Smoliński * Update lib/config/fileconf.go Co-authored-by: Marek Smoliński * Update lib/services/matchers.go Co-authored-by: Marek Smoliński * Remove superfluous cmp option for diffing azure matcher * Rename AzureMatchers Tags to ResourceTags * Deduplicate subscription/resource groups and add tests * Remove azure matcher config fixup Co-authored-by: Krzysztof Skrzętnicki Co-authored-by: Marek Smoliński --- lib/config/configuration.go | 10 ++++ lib/config/configuration_test.go | 87 +++++++++++++++++++++++++++++++- lib/config/fileconf.go | 16 ++++++ lib/config/testdata_test.go | 17 +++++++ lib/service/cfg.go | 2 + lib/service/db.go | 4 +- lib/services/matchers.go | 18 +++++++ lib/srv/db/server.go | 2 + 8 files changed, 154 insertions(+), 2 deletions(-) diff --git a/lib/config/configuration.go b/lib/config/configuration.go index 27371c708f82b..5e9fcb9c9713e 100644 --- a/lib/config/configuration.go +++ b/lib/config/configuration.go @@ -1117,6 +1117,16 @@ func applyDatabasesConfig(fc *FileConfig, cfg *service.Config) error { Tags: matcher.Tags, }) } + for _, matcher := range fc.Databases.AzureMatchers { + cfg.Databases.AzureMatchers = append(cfg.Databases.AzureMatchers, + services.AzureMatcher{ + Subscriptions: matcher.Subscriptions, + ResourceGroups: matcher.ResourceGroups, + Types: matcher.Types, + Regions: matcher.Regions, + ResourceTags: matcher.ResourceTags, + }) + } for _, database := range fc.Databases.Databases { staticLabels := make(map[string]string) if database.StaticLabels != nil { diff --git a/lib/config/configuration_test.go b/lib/config/configuration_test.go index 7a81a5c91f824..38e96193bcd20 100644 --- a/lib/config/configuration_test.go +++ b/lib/config/configuration_test.go @@ -290,7 +290,7 @@ func TestConfigReading(t *testing.T) { require.True(t, conf.SSH.Enabled()) require.False(t, conf.Kube.Enabled()) - // static config + // good config conf, err = ReadFromFile(testConfigs.configFile) require.NoError(t, err) require.Empty(t, cmp.Diff(conf, &FileConfig{ @@ -419,6 +419,35 @@ func TestConfigReading(t *testing.T) { }, }, }, + AzureMatchers: []AzureMatcher{ + { + Subscriptions: []string{"sub1", "sub2"}, + ResourceGroups: []string{"rg1", "rg2"}, + Types: []string{"mysql"}, + Regions: []string{"eastus", "westus"}, + ResourceTags: map[string]apiutils.Strings{ + "a": {"b"}, + }, + }, + { + Subscriptions: []string{"sub3", "sub4"}, + ResourceGroups: []string{"rg3", "rg4"}, + Types: []string{"postgres"}, + Regions: []string{"centralus"}, + ResourceTags: map[string]apiutils.Strings{ + "c": {"d"}, + }, + }, + { + Subscriptions: nil, + ResourceGroups: nil, + Types: []string{"mysql", "postgres"}, + Regions: []string{"centralus"}, + ResourceTags: map[string]apiutils.Strings{ + "e": {"f"}, + }, + }, + }, }, Metrics: Metrics{ Service: Service{ @@ -756,6 +785,29 @@ SREzU8onbBsjMg9QDiSf5oJLKvd/Ren+zGY7 require.Equal(t, 1, *cfg.Auth.KeyStore.SlotNumber) require.Equal(t, "example_pin", cfg.Auth.KeyStore.Pin) require.ElementsMatch(t, []string{"ca-pin-from-string", "ca-pin-from-file1", "ca-pin-from-file2"}, cfg.CAPins) + + require.True(t, cfg.Databases.Enabled) + require.Empty(t, cmp.Diff(cfg.Databases.AzureMatchers, + []services.AzureMatcher{ + { + Subscriptions: []string{"sub1", "sub2"}, + ResourceGroups: []string{"group1", "group2"}, + Types: []string{"postgres", "mysql"}, + Regions: []string{"eastus", "centralus"}, + ResourceTags: map[string]apiutils.Strings{ + "a": {"b"}, + }, + }, + { + Subscriptions: nil, + ResourceGroups: nil, + Types: []string{"postgres", "mysql"}, + Regions: []string{"westus"}, + ResourceTags: map[string]apiutils.Strings{ + "c": {"d"}, + }, + }, + })) } // TestApplyConfigNoneEnabled makes sure that if a section is not enabled, @@ -1276,6 +1328,33 @@ func makeConfigFixture() string { Tags: map[string]apiutils.Strings{"c": {"d"}}, }, } + conf.Databases.AzureMatchers = []AzureMatcher{ + { + Subscriptions: []string{"sub1", "sub2"}, + ResourceGroups: []string{"rg1", "rg2"}, + Types: []string{"mysql"}, + Regions: []string{"eastus", "westus"}, + ResourceTags: map[string]apiutils.Strings{ + "a": {"b"}, + }, + }, + { + Subscriptions: []string{"sub3", "sub4"}, + ResourceGroups: []string{"rg3", "rg4"}, + Types: []string{"postgres"}, + Regions: []string{"centralus"}, + ResourceTags: map[string]apiutils.Strings{ + "c": {"d"}, + }, + }, + { + Types: []string{"mysql", "postgres"}, + Regions: []string{"centralus"}, + ResourceTags: map[string]apiutils.Strings{ + "e": {"f"}, + }, + }, + } // Metrics service. conf.Metrics.EnabledFlag = "yes" @@ -1934,6 +2013,12 @@ db_service: regions: ["us-east-1", "us-west-1"] tags: '*': '*' + azure: + - subscriptions: ["foo", "bar"] + types: ["mysql", "postgres"] + regions: ["eastus", "westus"] + tags: + '*': '*' databases: - name: foo protocol: postgres diff --git a/lib/config/fileconf.go b/lib/config/fileconf.go index d6fb7aaf36290..733eb5c9f841a 100644 --- a/lib/config/fileconf.go +++ b/lib/config/fileconf.go @@ -1174,6 +1174,8 @@ type Databases struct { ResourceMatchers []ResourceMatcher `yaml:"resources,omitempty"` // AWSMatchers match AWS hosted databases. AWSMatchers []AWSMatcher `yaml:"aws,omitempty"` + // AzureMatchers match Azure hosted databases. + AzureMatchers []AzureMatcher `yaml:"azure,omitempty"` } // ResourceMatcher matches cluster resources. @@ -1193,6 +1195,20 @@ type AWSMatcher struct { Tags map[string]apiutils.Strings `yaml:"tags,omitempty"` } +// AzureMatcher matches Azure databases. +type AzureMatcher struct { + // Subscriptions are Azure subscriptions to query for resources. + Subscriptions []string `yaml:"subscriptions,omitempty"` + // ResourceGroups are Azure resource groups to query for resources. + ResourceGroups []string `yaml:"resource_groups,omitempty"` + // Types are Azure database types to match: "mysql", "postgres" + Types []string `yaml:"types,omitempty"` + // Regions are Azure locations to match for databases. + Regions []string `yaml:"regions,omitempty"` + // ResourceTags are Azure tags on resources to match. + ResourceTags map[string]apiutils.Strings `yaml:"tags,omitempty"` +} + // Database represents a single database proxied by the service. type Database struct { // Name is the name for the database proxy service. diff --git a/lib/config/testdata_test.go b/lib/config/testdata_test.go index 7102de2be43e5..ddc895c74fd1e 100644 --- a/lib/config/testdata_test.go +++ b/lib/config/testdata_test.go @@ -163,6 +163,23 @@ proxy_service: mysql_public_addr: mysql.example:3306 mongo_listen_addr: webhost:27017 mongo_public_addr: mongo.example:27017 + +db_service: + enabled: yes + resources: + - labels: + "*": "*" + azure: + - subscriptions: ["sub1", "sub2"] + resource_groups: ["group1", "group2"] + types: ["postgres", "mysql"] + regions: ["eastus", "centralus"] + tags: + "a": "b" + - types: ["postgres", "mysql"] + regions: ["westus"] + tags: + "c": "d" ` // NoServicesConfigString is a configuration file with no services enabled diff --git a/lib/service/cfg.go b/lib/service/cfg.go index 3d787729cf126..79ccf45ee2410 100644 --- a/lib/service/cfg.go +++ b/lib/service/cfg.go @@ -667,6 +667,8 @@ type DatabasesConfig struct { ResourceMatchers []services.ResourceMatcher // AWSMatchers match AWS hosted databases. AWSMatchers []services.AWSMatcher + // AzureMatchers match Azure hosted databases. + AzureMatchers []services.AzureMatcher // Limiter limits the connection and request rates. Limiter limiter.Config } diff --git a/lib/service/db.go b/lib/service/db.go index b008e9be251a5..dc99118e91180 100644 --- a/lib/service/db.go +++ b/lib/service/db.go @@ -34,7 +34,8 @@ func (process *TeleportProcess) shouldInitDatabases() bool { databasesCfg := len(process.Config.Databases.Databases) > 0 resourceMatchersCfg := len(process.Config.Databases.ResourceMatchers) > 0 awsMatchersCfg := len(process.Config.Databases.AWSMatchers) > 0 - anyCfg := databasesCfg || resourceMatchersCfg || awsMatchersCfg + azureMatchersCfg := len(process.Config.Databases.AzureMatchers) > 0 + anyCfg := databasesCfg || resourceMatchersCfg || awsMatchersCfg || azureMatchersCfg return process.Config.Databases.Enabled && anyCfg } @@ -215,6 +216,7 @@ func (process *TeleportProcess) initDatabaseService() (retErr error) { CloudLabels: process.cloudLabels, ResourceMatchers: process.Config.Databases.ResourceMatchers, AWSMatchers: process.Config.Databases.AWSMatchers, + AzureMatchers: process.Config.Databases.AzureMatchers, OnHeartbeat: process.onHeartbeat(teleport.ComponentDatabase), LockWatcher: lockWatcher, ConnectedProxyGetter: proxyGetter, diff --git a/lib/services/matchers.go b/lib/services/matchers.go index 207c6a690fc5b..de1bb647cb151 100644 --- a/lib/services/matchers.go +++ b/lib/services/matchers.go @@ -39,6 +39,20 @@ type AWSMatcher struct { Tags types.Labels } +// AzureMatcher matches Azure databases. +type AzureMatcher struct { + // Subscriptions are Azure subscriptions to query for resources. + Subscriptions []string + // ResourceGroups are Azure resource groups to query for resources. + ResourceGroups []string + // Types are Azure resource types to match, for example "mysql" or "postgres". + Types []string + // Regions are Azure regions to query for databases. + Regions []string + // ResourceTags are Azure tags to match. + ResourceTags types.Labels +} + // MatchResourceLabels returns true if any of the provided selectors matches the provided database. func MatchResourceLabels(matchers []ResourceMatcher, resource types.ResourceWithLabels) bool { for _, matcher := range matchers { @@ -227,4 +241,8 @@ const ( AWSMatcherElastiCache = "elasticache" // AWSMatcherMemoryDB is the AWS matcher type for MemoryDB databases. AWSMatcherMemoryDB = "memorydb" + // AzureMatcherMySQL is the Azure matcher type for Azure MySQL databases. + AzureMatcherMySQL = "mysql" + // AzureMatcherPostgres is the Azure matcher type for Azure Postgres databases. + AzureMatcherPostgres = "postgres" ) diff --git a/lib/srv/db/server.go b/lib/srv/db/server.go index d0ae25653db9b..6c264e85b3ccc 100644 --- a/lib/srv/db/server.go +++ b/lib/srv/db/server.go @@ -87,6 +87,8 @@ type Config struct { ResourceMatchers []services.ResourceMatcher // AWSMatchers is a list of AWS databases matchers. AWSMatchers []services.AWSMatcher + // AzureMatchers is a list of Azure databases matchers. + AzureMatchers []services.AzureMatcher // Databases is a list of proxied databases from static configuration. Databases types.Databases // CloudLabels is a service that imports labels from a cloud provider. The labels are shared From a5f32aa2422a8c9df3aadb8c3492860536dd0e66 Mon Sep 17 00:00:00 2001 From: Gavin Frazar Date: Fri, 19 Aug 2022 13:36:20 -0700 Subject: [PATCH 2/2] Azure mysql postgres auto discovery config create (#15630) * Add Azure auto-discovery configuration fields * Init databases if azure matchers are in config * Use AzureMatchers in db service * Use all azure subscriptions/resource groups if omitted in matcher * Add azure config tests * Add config create flags for azure matchers * Add config create tests for azure * Move discovery flags for azure below aws * Fixup merge --- lib/config/database.go | 40 ++++++++++++++++++++++++++++++++ lib/config/database_test.go | 22 ++++++++++++++++++ tool/teleport/common/teleport.go | 2 ++ 3 files changed, 64 insertions(+) diff --git a/lib/config/database.go b/lib/config/database.go index 91a492a098bee..e0bee3ee7e9f2 100644 --- a/lib/config/database.go +++ b/lib/config/database.go @@ -115,6 +115,40 @@ db_service: tags: "*": "*" {{- end }} + {{- if or .AzureMySQLDiscoveryRegions .AzurePostgresDiscoveryRegions }} + # Matchers for registering Azure-hosted databases. + azure: + {{- end }} + {{- if or .AzureMySQLDiscoveryRegions }} + # Azure MySQL databases auto-discovery. + # For more information about Azure MySQL auto-discovery: https://goteleport.com/docs/database-access/guides/azure-postgres-mysql/ + - subscriptions: ["*"] + resource_groups: ["*"] + types: ["mysql"] + # Azure regions to register databases from. + regions: + {{- range .AzureMySQLDiscoveryRegions }} + - {{ . }} + {{- end }} + # Azure resource tags to match when registering databases. + tags: + "*": "*" + {{- end }} + {{- if or .AzurePostgresDiscoveryRegions }} + # Azure Postgres databases auto-discovery. + # For more information about Azure Postgres auto-discovery: https://goteleport.com/docs/database-access/guides/azure-postgres-mysql/ + - subscriptions: ["*"] + resource_groups: ["*"] + types: ["postgres"] + # Azure regions to register databases from. + regions: + {{- range .AzurePostgresDiscoveryRegions }} + - {{ . }} + {{- end }} + # Azure resource tags to match when registering databases. + tags: + "*": "*" + {{- end }} # Lists statically registered databases proxied by this agent. {{- if .StaticDatabaseName }} databases: @@ -294,6 +328,12 @@ type DatabaseSampleFlags struct { AuthToken string // CAPins are the SKPI hashes of the CAs used to verify the Auth Server. CAPins []string + // AzureMySQLDiscoveryRegions is a list of regions Azure auto-discovery is + // configured to discover MySQL servers in. + AzureMySQLDiscoveryRegions []string + // AzurePostgresDiscoveryRegions is a list of regions Azure auto-discovery is + // configured to discover Postgres servers in. + AzurePostgresDiscoveryRegions []string // RDSDiscoveryRegions is a list of regions the RDS auto-discovery is // configured. RDSDiscoveryRegions []string diff --git a/lib/config/database_test.go b/lib/config/database_test.go index 25e5cfa54a0d1..c9b2c5c56c6c9 100644 --- a/lib/config/database_test.go +++ b/lib/config/database_test.go @@ -67,6 +67,28 @@ func TestMakeDatabaseConfig(t *testing.T) { require.ElementsMatch(t, flags.RedshiftDiscoveryRegions, databases.AWSMatchers[0].Regions) }) + t.Run("AzureMySQLAutoDiscovery", func(t *testing.T) { + flags := DatabaseSampleFlags{ + AzureMySQLDiscoveryRegions: []string{"eastus", "eastus2"}, + } + + databases := generateAndParseConfig(t, flags) + require.Len(t, databases.AzureMatchers, 1) + require.ElementsMatch(t, []string{"mysql"}, databases.AzureMatchers[0].Types) + require.ElementsMatch(t, flags.AzureMySQLDiscoveryRegions, databases.AzureMatchers[0].Regions) + }) + + t.Run("AzurePostgresAutoDiscovery", func(t *testing.T) { + flags := DatabaseSampleFlags{ + AzurePostgresDiscoveryRegions: []string{"eastus", "eastus2"}, + } + + databases := generateAndParseConfig(t, flags) + require.Len(t, databases.AzureMatchers, 1) + require.ElementsMatch(t, []string{"postgres"}, databases.AzureMatchers[0].Types) + require.ElementsMatch(t, flags.AzurePostgresDiscoveryRegions, databases.AzureMatchers[0].Regions) + }) + t.Run("StaticDatabase", func(t *testing.T) { flags := DatabaseSampleFlags{ StaticDatabaseName: "sample", diff --git a/tool/teleport/common/teleport.go b/tool/teleport/common/teleport.go index e407f09a3ab38..7c251d0015689 100644 --- a/tool/teleport/common/teleport.go +++ b/tool/teleport/common/teleport.go @@ -238,6 +238,8 @@ func Run(options Options) (app *kingpin.Application, executedCommand string, con dbConfigureCreate.Flag("redshift-discovery", "List of AWS regions in which the agent will discover Redshift instances.").StringsVar(&dbConfigCreateFlags.RedshiftDiscoveryRegions) dbConfigureCreate.Flag("elasticache-discovery", "List of AWS regions in which the agent will discover ElastiCache Redis clusters.").StringsVar(&dbConfigCreateFlags.ElastiCacheDiscoveryRegions) dbConfigureCreate.Flag("memorydb-discovery", "List of AWS regions in which the agent will discover MemoryDB clusters.").StringsVar(&dbConfigCreateFlags.MemoryDBDiscoveryRegions) + dbConfigureCreate.Flag("azure-mysql-discovery", "List of Azure regions in which the agent will discover MySQL servers.").StringsVar(&dbConfigCreateFlags.AzureMySQLDiscoveryRegions) + dbConfigureCreate.Flag("azure-postgres-discovery", "List of Azure regions in which the agent will discover Postgres servers.").StringsVar(&dbConfigCreateFlags.AzurePostgresDiscoveryRegions) dbConfigureCreate.Flag("ca-pin", "CA pin to validate the auth server (can be repeated for multiple pins).").StringsVar(&dbConfigCreateFlags.CAPins) dbConfigureCreate.Flag("name", "Name of the proxied database.").StringVar(&dbConfigCreateFlags.StaticDatabaseName) dbConfigureCreate.Flag("protocol", fmt.Sprintf("Proxied database protocol. Supported are: %v.", defaults.DatabaseProtocols)).StringVar(&dbConfigCreateFlags.StaticDatabaseProtocol)