From 4bc7cad5a14f014d8ff9ee731bc2f0aaa360d02d Mon Sep 17 00:00:00 2001 From: Marco Dinis Date: Thu, 13 Nov 2025 16:29:33 +0000 Subject: [PATCH 1/7] AWS EC2 Discovery: discover in all enabled regions --- api/types/matchers_aws.go | 21 ++- api/types/matchers_aws_test.go | 14 +- go.mod | 2 + go.sum | 2 + integrations/terraform-mwi/go.mod | 1 + integrations/terraform-mwi/go.sum | 2 + integrations/terraform/go.mod | 1 + integrations/terraform/go.sum | 2 + lib/cloud/awsconfig/awsconfig.go | 11 ++ lib/config/configuration.go | 4 + lib/config/configuration_test.go | 33 +++++ lib/srv/discovery/discovery.go | 15 ++ lib/srv/server/ec2_watcher.go | 223 +++++++++++++++++------------ lib/srv/server/ec2_watcher_test.go | 137 ++++++++++++++---- 14 files changed, 347 insertions(+), 121 deletions(-) diff --git a/api/types/matchers_aws.go b/api/types/matchers_aws.go index 591bbecb99fff..309baa80bf895 100644 --- a/api/types/matchers_aws.go +++ b/api/types/matchers_aws.go @@ -131,6 +131,11 @@ func isAlphanumericIncluding(s string, extraChars ...rune) bool { return true } +// IsRegionWildcard returns true if the matcher is configured to discover resources in all regions. +func (m *AWSMatcher) IsRegionWildcard() bool { + return len(m.Regions) == 1 && m.Regions[0] == Wildcard +} + // CheckAndSetDefaults that the matcher is correct and adds default values. func (m *AWSMatcher) CheckAndSetDefaults() error { for _, matcherType := range m.Types { @@ -145,10 +150,24 @@ func (m *AWSMatcher) CheckAndSetDefaults() error { } if len(m.Regions) == 0 { - return trace.BadParameter("discovery service requires at least one region") + m.Regions = []string{Wildcard} } for _, region := range m.Regions { + if region == Wildcard { + if len(m.Regions) > 1 { + return trace.BadParameter("when using %q as region, no other regions can be specified", Wildcard) + } + + // Only EC2 supports discovering regions. + // TODO(marco): add support for other resources types. + if len(m.Types) > 1 || m.Types[0] != AWSMatcherEC2 { + return trace.BadParameter("only EC2 resource discovery supports discovering all regions, " + + "enumerate all the regions or create a separate matcher for discovering EC2 resources") + } + break + } + if err := awsapiutils.IsValidRegion(region); err != nil { return trace.BadParameter("discovery service does not support region %q", region) } diff --git a/api/types/matchers_aws_test.go b/api/types/matchers_aws_test.go index ae5b153b5ecb3..e181a490e4d10 100644 --- a/api/types/matchers_aws_test.go +++ b/api/types/matchers_aws_test.go @@ -136,13 +136,21 @@ func TestAWSMatcherCheckAndSetDefaults(t *testing.T) { errCheck: isBadParameterErr, }, { - name: "wildcard is invalid for regions", + name: "wildcard is invalid for regions when using non-ec2 types", in: &AWSMatcher{ Types: []string{"ec2", "rds"}, Regions: []string{"*"}, }, errCheck: isBadParameterErr, }, + { + name: "wildcard is valid for the ec2 type", + in: &AWSMatcher{ + Types: []string{"ec2"}, + Regions: []string{"*"}, + }, + errCheck: require.NoError, + }, { name: "invalid type", in: &AWSMatcher{ @@ -179,12 +187,12 @@ func TestAWSMatcherCheckAndSetDefaults(t *testing.T) { errCheck: isBadParameterErr, }, { - name: "no region", + name: "no region is valid for ec2 type", in: &AWSMatcher{ Types: []string{"ec2"}, Regions: []string{}, }, - errCheck: isBadParameterErr, + errCheck: require.NoError, }, { name: "invalid assume role arn", diff --git a/go.mod b/go.mod index 85039c14ae792..7a25533156067 100644 --- a/go.mod +++ b/go.mod @@ -287,6 +287,8 @@ require ( software.sslmate.com/src/go-pkcs12 v0.6.0 ) +require github.com/aws/aws-sdk-go-v2/service/account v1.29.3 + require ( cel.dev/expr v0.25.0 // indirect cloud.google.com/go v0.121.6 // indirect diff --git a/go.sum b/go.sum index 2b962c5cee483..d1e9771f22d4e 100644 --- a/go.sum +++ b/go.sum @@ -823,6 +823,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEG github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k= +github.com/aws/aws-sdk-go-v2/service/account v1.29.3 h1:EqpyKDfuTQZFL6Cr1dutmyUx+Z9eDN2oFR1hoMXbmCs= +github.com/aws/aws-sdk-go-v2/service/account v1.29.3/go.mod h1:3LbGl+sLnaiyCVp2LJXj0gEoeV2Uw0QOsDpP1gDMBVA= github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.41.2 h1:zZ6TJ93crR09QVPxqoFJnlioltKejaPLxY1dCcBjU20= github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.41.2/go.mod h1:BDzrZs53Hsb5MyAICN2dmtFWaeLONzMaseXyF9Bagt0= github.com/aws/aws-sdk-go-v2/service/athena v1.55.10 h1:lhHg2H2XeRix8Zk2UKxsJXKk93066CAZCw0x5pMRvDw= diff --git a/integrations/terraform-mwi/go.mod b/integrations/terraform-mwi/go.mod index 1d51e95b3bbb6..b612eb8ccbc46 100644 --- a/integrations/terraform-mwi/go.mod +++ b/integrations/terraform-mwi/go.mod @@ -96,6 +96,7 @@ require ( github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/service/account v1.29.3 // indirect github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.41.2 // indirect github.com/aws/aws-sdk-go-v2/service/athena v1.55.10 // indirect github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.42.2 // indirect diff --git a/integrations/terraform-mwi/go.sum b/integrations/terraform-mwi/go.sum index 52857319856b3..6512f2bb3ecba 100644 --- a/integrations/terraform-mwi/go.sum +++ b/integrations/terraform-mwi/go.sum @@ -828,6 +828,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEG github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k= +github.com/aws/aws-sdk-go-v2/service/account v1.29.3 h1:EqpyKDfuTQZFL6Cr1dutmyUx+Z9eDN2oFR1hoMXbmCs= +github.com/aws/aws-sdk-go-v2/service/account v1.29.3/go.mod h1:3LbGl+sLnaiyCVp2LJXj0gEoeV2Uw0QOsDpP1gDMBVA= github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.41.2 h1:zZ6TJ93crR09QVPxqoFJnlioltKejaPLxY1dCcBjU20= github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.41.2/go.mod h1:BDzrZs53Hsb5MyAICN2dmtFWaeLONzMaseXyF9Bagt0= github.com/aws/aws-sdk-go-v2/service/athena v1.55.10 h1:lhHg2H2XeRix8Zk2UKxsJXKk93066CAZCw0x5pMRvDw= diff --git a/integrations/terraform/go.mod b/integrations/terraform/go.mod index 93842ad7f0842..ba26932770b01 100644 --- a/integrations/terraform/go.mod +++ b/integrations/terraform/go.mod @@ -100,6 +100,7 @@ require ( github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/service/account v1.29.3 // indirect github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.41.2 // indirect github.com/aws/aws-sdk-go-v2/service/athena v1.55.10 // indirect github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.42.2 // indirect diff --git a/integrations/terraform/go.sum b/integrations/terraform/go.sum index 06d13db1760d6..c2dc15f30c05c 100644 --- a/integrations/terraform/go.sum +++ b/integrations/terraform/go.sum @@ -848,6 +848,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEG github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k= +github.com/aws/aws-sdk-go-v2/service/account v1.29.3 h1:EqpyKDfuTQZFL6Cr1dutmyUx+Z9eDN2oFR1hoMXbmCs= +github.com/aws/aws-sdk-go-v2/service/account v1.29.3/go.mod h1:3LbGl+sLnaiyCVp2LJXj0gEoeV2Uw0QOsDpP1gDMBVA= github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.41.2 h1:zZ6TJ93crR09QVPxqoFJnlioltKejaPLxY1dCcBjU20= github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.41.2/go.mod h1:BDzrZs53Hsb5MyAICN2dmtFWaeLONzMaseXyF9Bagt0= github.com/aws/aws-sdk-go-v2/service/athena v1.55.10 h1:lhHg2H2XeRix8Zk2UKxsJXKk93066CAZCw0x5pMRvDw= diff --git a/lib/cloud/awsconfig/awsconfig.go b/lib/cloud/awsconfig/awsconfig.go index 6396f2be545e2..347918532559c 100644 --- a/lib/cloud/awsconfig/awsconfig.go +++ b/lib/cloud/awsconfig/awsconfig.go @@ -203,6 +203,17 @@ func (o *options) checkIntegrationCredentials() error { // when getting an AWS config. type OptionsFn func(*options) +// AssumedRoles extracts the assumed roles from the provided options. +// Only used in testing. +func AssumedRoles(opts ...OptionsFn) []AssumeRole { + var options options + for _, optFn := range opts { + optFn(&options) + } + + return options.assumeRoles +} + // WithAssumeRole configures options needed for assuming an AWS role. func WithAssumeRole(roleARN, externalID string) OptionsFn { return func(options *options) { diff --git a/lib/config/configuration.go b/lib/config/configuration.go index b28f40320242d..bea9938a44546 100644 --- a/lib/config/configuration.go +++ b/lib/config/configuration.go @@ -1575,6 +1575,10 @@ func applyDiscoveryConfig(fc *FileConfig, cfg *servicecfg.Config) error { } for _, region := range matcher.Regions { + if region == types.Wildcard { + continue + } + if !awsregion.IsKnownRegion(region) { const message = "AWS matcher uses unknown region" + "This is either a typo or a new AWS region that is unknown to the AWS SDK used to compile this binary. " diff --git a/lib/config/configuration_test.go b/lib/config/configuration_test.go index 4f318d341ba73..74333248e1acf 100644 --- a/lib/config/configuration_test.go +++ b/lib/config/configuration_test.go @@ -4845,6 +4845,23 @@ func TestDiscoveryConfig(t *testing.T) { desc: "AWS section is filled with invalid region", expectError: require.Error, expectEnabled: require.True, + mutate: func(cfg cfgMap) { + cfg["discovery_service"].(cfgMap)["enabled"] = "yes" + cfg["discovery_service"].(cfgMap)["aws"] = []cfgMap{ + { + "types": []string{"ec2"}, + "regions": []string{"invalid-region"}, + "tags": cfgMap{ + "discover_teleport": "yes", + }, + }, + } + }, + }, + { + desc: "for EC2 discovery, region can be set to wildcard", + expectError: require.NoError, + expectEnabled: require.True, mutate: func(cfg cfgMap) { cfg["discovery_service"].(cfgMap)["enabled"] = "yes" cfg["discovery_service"].(cfgMap)["aws"] = []cfgMap{ @@ -4857,6 +4874,22 @@ func TestDiscoveryConfig(t *testing.T) { }, } }, + expectedAWSMatchers: []types.AWSMatcher{{ + Types: []string{"ec2"}, + Regions: []string{"*"}, + Tags: map[string]apiutils.Strings{ + "discover_teleport": []string{"yes"}, + }, + Params: &types.InstallerParams{ + JoinMethod: types.JoinMethodIAM, + JoinToken: "aws-discovery-iam-token", + SSHDConfig: "/etc/ssh/sshd_config", + ScriptName: "default-installer", + InstallTeleport: true, + EnrollMode: types.InstallParamEnrollMode_INSTALL_PARAM_ENROLL_MODE_SCRIPT, + }, + SSM: &types.AWSSSM{DocumentName: "TeleportDiscoveryInstaller"}, + }}, }, { desc: "AWS section is filled with invalid join method", diff --git a/lib/srv/discovery/discovery.go b/lib/srv/discovery/discovery.go index 2441267e3e95f..3b8c44982b8a9 100644 --- a/lib/srv/discovery/discovery.go +++ b/lib/srv/discovery/discovery.go @@ -31,6 +31,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/account" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/ssm" @@ -130,6 +131,8 @@ type Config struct { // GetEC2Client gets an AWS EC2 client for the given region. GetEC2Client server.EC2ClientGetter + // GetAWSRegionsLister gets a client that is capable of listing AWS regions. + GetAWSRegionsLister server.RegionsListerGetter // GetSSMClient gets an AWS SSM client for the given region. GetSSMClient func(ctx context.Context, region string, opts ...awsconfig.OptionsFn) (server.SSMClient, error) // IntegrationOnlyCredentials discards any Matcher that don't have an Integration. @@ -280,6 +283,16 @@ kubernetes matchers are present.`) return ec2.NewFromConfig(cfg), nil } } + if c.GetAWSRegionsLister == nil { + c.GetAWSRegionsLister = func(ctx context.Context, opts ...awsconfig.OptionsFn) (account.ListRegionsAPIClient, error) { + region := "" // Account API is global, no region needed. + cfg, err := c.getAWSConfig(ctx, region, opts...) + if err != nil { + return nil, trace.Wrap(err) + } + return account.NewFromConfig(cfg), nil + } + } if c.AWSFetchersClients == nil { c.AWSFetchersClients = &awsFetchersClientsGetter{ Provider: awsconfig.ProviderFunc(c.getAWSConfig), @@ -599,6 +612,7 @@ func (s *Server) initAWSWatchers(matchers []types.AWSMatcher) error { s.staticServerAWSFetchers, err = server.MatchersToEC2InstanceFetchers(s.ctx, server.MatcherToEC2FetcherParams{ Matchers: ec2Matchers, EC2ClientGetter: s.GetEC2Client, + RegionsListerGetter: s.GetAWSRegionsLister, PublicProxyAddrGetter: s.publicProxyAddress, }) if err != nil { @@ -723,6 +737,7 @@ func (s *Server) awsServerFetchersFromMatchers(ctx context.Context, matchers []t fetchers, err := server.MatchersToEC2InstanceFetchers(ctx, server.MatcherToEC2FetcherParams{ Matchers: serverMatchers, EC2ClientGetter: s.GetEC2Client, + RegionsListerGetter: s.GetAWSRegionsLister, DiscoveryConfigName: discoveryConfigName, PublicProxyAddrGetter: s.publicProxyAddress, }) diff --git a/lib/srv/server/ec2_watcher.go b/lib/srv/server/ec2_watcher.go index ec0e62ad88da9..594fc431b718c 100644 --- a/lib/srv/server/ec2_watcher.go +++ b/lib/srv/server/ec2_watcher.go @@ -26,6 +26,8 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/account" + accounttypes "github.com/aws/aws-sdk-go-v2/service/account/types" "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/gravitational/trace" @@ -198,6 +200,9 @@ func NewEC2Watcher(ctx context.Context, fetchersFn func() []Fetcher, missedRotat // EC2ClientGetter gets an AWS EC2 client for the given region. type EC2ClientGetter func(ctx context.Context, region string, opts ...awsconfig.OptionsFn) (ec2.DescribeInstancesAPIClient, error) +// RegionsListerGetter gets a list of AWS regions. +type RegionsListerGetter func(ctx context.Context, opts ...awsconfig.OptionsFn) (account.ListRegionsAPIClient, error) + // MatcherToEC2FetcherParams contains parameters for converting AWS EC2 Matchers // into AWS EC2 Fetchers. type MatcherToEC2FetcherParams struct { @@ -205,6 +210,8 @@ type MatcherToEC2FetcherParams struct { Matchers []types.AWSMatcher // EC2ClientGetter gets an AWS EC2. EC2ClientGetter EC2ClientGetter + // RegionsListerGetter gets a client that is capable of listing AWS regions. + RegionsListerGetter RegionsListerGetter // DiscoveryConfigName is the name of the DiscoveryConfig that contains the matchers. // Empty if using static matchers (coming from the `teleport.yaml`). DiscoveryConfigName string @@ -216,67 +223,29 @@ type MatcherToEC2FetcherParams struct { // MatchersToEC2InstanceFetchers converts a list of AWS EC2 Matchers into a list of AWS EC2 Fetchers. func MatchersToEC2InstanceFetchers(ctx context.Context, matcherParams MatcherToEC2FetcherParams) ([]Fetcher, error) { - return matchersToEC2InstanceFetchers(matcherParams, func(ctx context.Context, region string, matcher *types.AWSMatcher, opts ...awsconfig.OptionsFn) (ec2.DescribeInstancesAPIClient, error) { - if matcher == nil { - return matcherParams.EC2ClientGetter(ctx, region, opts...) - } - - newOpts := make([]awsconfig.OptionsFn, 0, len(opts)+1) - copy(newOpts, opts) - - if matcher.AssumeRole != nil { - newOpts = append(newOpts, awsconfig.WithAssumeRole(matcher.AssumeRole.RoleARN, matcher.AssumeRole.ExternalID)) - } - - newOpts = append(newOpts, awsconfig.WithCredentialsMaybeIntegration(awsconfig.IntegrationMetadata{Name: matcher.Integration})) - - return matcherParams.EC2ClientGetter(ctx, region, newOpts...) - }) -} - -// matcherEC2ClientGetter is an EC2 client getter that builds an EC2 client for the given matcher. -// It includes the source of credentials (integration or ambient) and the assume role if any. -// Region is not considered part of the identity because each matcher can have multiple regions. -type matcherEC2ClientGetter func(ctx context.Context, region string, matcher *types.AWSMatcher, opts ...awsconfig.OptionsFn) (ec2.DescribeInstancesAPIClient, error) - -func (g matcherEC2ClientGetter) withMatcher(matcher *types.AWSMatcher) EC2ClientGetter { - return func(ctx context.Context, region string, opts ...awsconfig.OptionsFn) (ec2.DescribeInstancesAPIClient, error) { - return g(ctx, region, matcher, opts...) - } -} - -func matchersToEC2InstanceFetchers(matcherParams MatcherToEC2FetcherParams, getEC2Client matcherEC2ClientGetter) ([]Fetcher, error) { ret := []Fetcher{} for _, matcher := range matcherParams.Matchers { - for _, region := range matcher.Regions { - fetcher := newEC2InstanceFetcher(ec2FetcherConfig{ - ProxyPublicAddrGetter: matcherParams.PublicProxyAddrGetter, - Matcher: matcher, - Region: region, - Document: matcher.SSM.DocumentName, - EC2ClientGetter: getEC2Client.withMatcher(&matcher), - Labels: matcher.Tags, - Integration: matcher.Integration, - DiscoveryConfigName: matcherParams.DiscoveryConfigName, - }) - ret = append(ret, fetcher) - } + fetcher := newEC2InstanceFetcher(ec2FetcherConfig{ + Matcher: matcher, + ProxyPublicAddrGetter: matcherParams.PublicProxyAddrGetter, + EC2ClientGetter: matcherParams.EC2ClientGetter, + RegionsListerGetter: matcherParams.RegionsListerGetter, + DiscoveryConfigName: matcherParams.DiscoveryConfigName, + }) + ret = append(ret, fetcher) } return ret, nil } type ec2FetcherConfig struct { - Matcher types.AWSMatcher - Region string - Document string - EC2ClientGetter EC2ClientGetter - Labels types.Labels - Integration string - DiscoveryConfigName string + Matcher types.AWSMatcher // ProxyPublicAddrGetter returns the public proxy address to use for installation scripts. // This is only used if the matcher does not specify a ProxyAddress. // Example: proxy.example.com:3080 or proxy.example.com ProxyPublicAddrGetter func(ctx context.Context) (string, error) + EC2ClientGetter EC2ClientGetter + RegionsListerGetter RegionsListerGetter + DiscoveryConfigName string } type ec2InstanceFetcher struct { @@ -349,8 +318,8 @@ func newEC2InstanceFetcher(cfg ec2FetcherConfig) *ec2InstanceFetcher { Values: []string{string(ec2types.InstanceStateNameRunning)}, }} - if _, ok := cfg.Labels["*"]; !ok { - for key, val := range cfg.Labels { + if _, ok := cfg.Matcher.Tags["*"]; !ok { + for key, val := range cfg.Matcher.Tags { tagFilters = append(tagFilters, ec2types.Filter{ Name: aws.String("tag:" + key), Values: val, @@ -396,7 +365,7 @@ func ssmRunCommandParametersForCustomDocuments(cfg ec2FetcherConfig) map[string] } func ssmRunCommandParameters(ctx context.Context, cfg ec2FetcherConfig) (map[string]string, error) { - if cfg.Document == types.AWSSSMDocumentRunShellScript { + if cfg.Matcher.SSM.DocumentName == types.AWSSSMDocumentRunShellScript { // When using the pre-defined SSM Document AWS-RunShellScript, only the commands parameter is required. // It contains the full installation script that will be executed on the instance. script, err := installerScript(ctx, cfg.Matcher.Params, withProxyAddrGetter(cfg.ProxyPublicAddrGetter)) @@ -420,14 +389,8 @@ func (f *ec2InstanceFetcher) GetMatchingInstances(ctx context.Context, nodes []t return nil, trace.Wrap(err) } - insts := EC2Instances{ - Region: f.Region, - DocumentName: f.Document, - Parameters: ssmRunParams, - Rotation: rotation, - Integration: f.Integration, - DiscoveryConfigName: f.DiscoveryConfigName, - } + instancesByRegion := make(map[string]EC2Instances) + for _, node := range nodes { // Heartbeating and expiration keeps Teleport Agents up to date, no need to consider those nodes. // Agentless and EICE Nodes don't heartbeat, so they must be manually managed by the DiscoveryService. @@ -435,7 +398,7 @@ func (f *ec2InstanceFetcher) GetMatchingInstances(ctx context.Context, nodes []t continue } region, ok := node.GetLabel(types.AWSInstanceRegion) - if !ok || region != f.Region { + if !ok { continue } instID, ok := node.GetLabel(types.AWSInstanceIDLabel) @@ -450,41 +413,91 @@ func (f *ec2InstanceFetcher) GetMatchingInstances(ctx context.Context, nodes []t if !f.cachedInstances.exists(accountID, instID) { continue } - if insts.AccountID == "" { - insts.AccountID = accountID - } + if _, ok := instancesByRegion[region]; !ok { + instancesByRegion[region] = EC2Instances{ + Region: region, + DocumentName: f.Matcher.SSM.DocumentName, + Parameters: ssmRunParams, + Rotation: rotation, + Integration: f.Matcher.Integration, + DiscoveryConfigName: f.DiscoveryConfigName, + AccountID: accountID, + } + } + insts := instancesByRegion[region] insts.Instances = append(insts.Instances, EC2Instance{ InstanceID: instID, }) + + instancesByRegion[region] = insts } - if len(insts.Instances) == 0 { + if len(instancesByRegion) == 0 { return nil, trace.NotFound("no ec2 instances found") } - return chunkInstances(insts), nil + return chunkInstances(instancesByRegion), nil } -func chunkInstances(insts EC2Instances) []Instances { +func chunkInstances(instancesByRegion map[string]EC2Instances) []Instances { var instColl []Instances - for i := 0; i < len(insts.Instances); i += awsEC2APIChunkSize { - end := min(i+awsEC2APIChunkSize, len(insts.Instances)) - inst := EC2Instances{ - AccountID: insts.AccountID, - Region: insts.Region, - DocumentName: insts.DocumentName, - Parameters: insts.Parameters, - Instances: insts.Instances[i:end], - Rotation: insts.Rotation, - Integration: insts.Integration, - DiscoveryConfigName: insts.DiscoveryConfigName, + for _, insts := range instancesByRegion { + for i := 0; i < len(insts.Instances); i += awsEC2APIChunkSize { + end := min(i+awsEC2APIChunkSize, len(insts.Instances)) + inst := EC2Instances{ + AccountID: insts.AccountID, + Region: insts.Region, + DocumentName: insts.DocumentName, + Parameters: insts.Parameters, + Instances: insts.Instances[i:end], + Rotation: insts.Rotation, + Integration: insts.Integration, + DiscoveryConfigName: insts.DiscoveryConfigName, + } + instColl = append(instColl, Instances{EC2: &inst}) } - instColl = append(instColl, Instances{EC2: &inst}) } return instColl } +func (f *ec2InstanceFetcher) matcherRegions(ctx context.Context, awsOpts []awsconfig.OptionsFn) ([]string, error) { + if !f.Matcher.IsRegionWildcard() { + return f.Matcher.Regions, nil + } + + regionsListerClient, err := f.RegionsListerGetter(ctx, awsOpts...) + if err != nil { + return nil, trace.Wrap(err) + } + + paginator := account.NewListRegionsPaginator(regionsListerClient, &account.ListRegionsInput{ + RegionOptStatusContains: []accounttypes.RegionOptStatus{ + accounttypes.RegionOptStatusEnabled, + accounttypes.RegionOptStatusEnabledByDefault, + }, + }) + + var enabledRegions []string + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + convertedErr := libcloudaws.ConvertRequestFailureError(err) + if trace.IsAccessDenied(convertedErr) { + return nil, trace.BadParameter("Missing account:ListRegions permission in IAM Role, which is required to iterate over all regions. " + + "Add this permission to the IAM Role, or enumerate all the regions in the AWS matcher.") + } + return nil, convertedErr + } + + for _, region := range page.Regions { + enabledRegions = append(enabledRegions, aws.ToString(region.RegionName)) + } + } + + return enabledRegions, nil +} + // GetInstances fetches all EC2 instances matching configured filters. func (f *ec2InstanceFetcher) GetInstances(ctx context.Context, rotation bool) ([]Instances, error) { ssmRunParams, err := ssmRunCommandParameters(ctx, f.ec2FetcherConfig) @@ -492,13 +505,47 @@ func (f *ec2InstanceFetcher) GetInstances(ctx context.Context, rotation bool) ([ return nil, trace.Wrap(err) } - ec2Client, err := f.EC2ClientGetter(ctx, f.Region) + f.cachedInstances.clear() + var allInstances []Instances + + awsOpts := []awsconfig.OptionsFn{ + awsconfig.WithCredentialsMaybeIntegration(awsconfig.IntegrationMetadata{Name: f.Matcher.Integration}), + } + + if f.Matcher.AssumeRole != nil { + awsOpts = append(awsOpts, awsconfig.WithAssumeRole(f.Matcher.AssumeRole.RoleARN, f.Matcher.AssumeRole.ExternalID)) + } + + regions, err := f.matcherRegions(ctx, awsOpts) + if err != nil { + return nil, trace.Wrap(err) + } + + for _, region := range regions { + regionInstances, err := f.getInstancesInRegion(ctx, rotation, region, awsOpts, ssmRunParams) + if err != nil { + return nil, trace.Wrap(err) + } + + allInstances = append(allInstances, regionInstances...) + } + + if len(allInstances) == 0 { + return nil, trace.NotFound("no ec2 instances found") + } + + return allInstances, nil +} + +// getInstancesInRegion fetches all EC2 instances in a given region. +func (f *ec2InstanceFetcher) getInstancesInRegion(ctx context.Context, rotation bool, region string, awsOpts []awsconfig.OptionsFn, ssmRunParams map[string]string) ([]Instances, error) { + ec2Client, err := f.EC2ClientGetter(ctx, region, awsOpts...) if err != nil { return nil, trace.Wrap(err) } var instances []Instances - f.cachedInstances.clear() + paginator := ec2.NewDescribeInstancesPaginator(ec2Client, &ec2.DescribeInstancesInput{ Filters: f.Filters, }) @@ -515,12 +562,12 @@ func (f *ec2InstanceFetcher) GetInstances(ctx context.Context, rotation bool) ([ ownerID := aws.ToString(res.OwnerId) inst := EC2Instances{ AccountID: ownerID, - Region: f.Region, - DocumentName: f.Document, + Region: region, + DocumentName: f.Matcher.SSM.DocumentName, Instances: ToEC2Instances(res.Instances[i:end]), Parameters: ssmRunParams, Rotation: rotation, - Integration: f.Integration, + Integration: f.Matcher.Integration, AssumeRoleARN: f.Matcher.AssumeRole.RoleARN, ExternalID: f.Matcher.AssumeRole.ExternalID, DiscoveryConfigName: f.DiscoveryConfigName, @@ -534,10 +581,6 @@ func (f *ec2InstanceFetcher) GetInstances(ctx context.Context, rotation bool) ([ } } - if len(instances) == 0 { - return nil, trace.NotFound("no ec2 instances found") - } - return instances, nil } @@ -549,5 +592,5 @@ func (f *ec2InstanceFetcher) GetDiscoveryConfigName() string { // IntegrationName identifies the integration name whose credentials were used to fetch the resources. // Might be empty when the fetcher is using ambient credentials. func (f *ec2InstanceFetcher) IntegrationName() string { - return f.Integration + return f.Matcher.Integration } diff --git a/lib/srv/server/ec2_watcher_test.go b/lib/srv/server/ec2_watcher_test.go index 4c4921be1f2f4..fb56bd5815e4b 100644 --- a/lib/srv/server/ec2_watcher_test.go +++ b/lib/srv/server/ec2_watcher_test.go @@ -24,6 +24,8 @@ import ( "testing" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/account" + accounttypes "github.com/aws/aws-sdk-go-v2/service/account/types" "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/google/go-cmp/cmp" @@ -56,16 +58,17 @@ func (m *mockEC2Client) DescribeInstances(ctx context.Context, input *ec2.Descri return &output, nil } -func makeMockClients(m map[string]*ec2.DescribeInstancesOutput) matcherEC2ClientGetter { - return func(ctx context.Context, region string, matcher *types.AWSMatcher, opts ...awsconfig.OptionsFn) (ec2.DescribeInstancesAPIClient, error) { - var roleARN string - if matcher.AssumeRole != nil { - roleARN = matcher.AssumeRole.RoleARN - } - return &mockEC2Client{ - output: m[roleARN], - }, nil +type mockAWSAccountClient struct { + output *account.ListRegionsOutput + responseError error +} + +func (m *mockAWSAccountClient) ListRegions(ctx context.Context, input *account.ListRegionsInput, opts ...func(*account.Options)) (*account.ListRegionsOutput, error) { + if m.responseError != nil { + return nil, m.responseError } + + return m.output, nil } func instanceMatches(inst ec2types.Instance, filters []ec2types.Filter) bool { @@ -97,9 +100,11 @@ func TestNewEC2InstanceFetcherTags(t *testing.T) { { name: "with glob key", config: ec2FetcherConfig{ - Labels: types.Labels{ - "*": []string{}, - "hello": []string{"other"}, + Matcher: types.AWSMatcher{ + Tags: types.Labels{ + "*": []string{}, + "hello": []string{"other"}, + }, }, }, expectedFilters: []ec2types.Filter{ @@ -112,8 +117,10 @@ func TestNewEC2InstanceFetcherTags(t *testing.T) { { name: "with no glob key", config: ec2FetcherConfig{ - Labels: types.Labels{ - "hello": []string{"other"}, + Matcher: types.AWSMatcher{ + Tags: types.Labels{ + "hello": []string{"other"}, + }, }, }, expectedFilters: []ec2types.Filter{ @@ -174,6 +181,16 @@ func TestEC2Watcher(t *testing.T) { RoleARN: "alternate-role-arn", }, }, + { + Params: &types.InstallerParams{}, + Types: []string{"EC2"}, + Regions: []string{"*"}, + Tags: map[string]utils.Strings{"teleport": {"yes"}}, + SSM: &types.AWSSSM{}, + AssumeRole: &types.AssumeRole{ + RoleARN: "implicit-region", + }, + }, } present := ec2types.Instance{ @@ -223,7 +240,18 @@ func TestEC2Watcher(t *testing.T) { }, } - output := ec2.DescribeInstancesOutput{ + instanceImplicitRegion := ec2types.Instance{ + InstanceId: aws.String("instance-implicit-region"), + Tags: []ec2types.Tag{{ + Key: aws.String("teleport"), + Value: aws.String("yes"), + }}, + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, + } + + ec2DescribeInstancesOutNoAssumeRole := ec2.DescribeInstancesOutput{ Reservations: []ec2types.Reservation{{ Instances: []ec2types.Instance{ present, @@ -255,7 +283,7 @@ func TestEC2Watcher(t *testing.T) { }, }}, } - altAccountOutput := ec2.DescribeInstancesOutput{ + ec2DescribeInstancesOutAlternateAssumeRole := ec2.DescribeInstancesOutput{ Reservations: []ec2types.Reservation{{ Instances: []ec2types.Instance{ altAccountPresent, @@ -272,18 +300,51 @@ func TestEC2Watcher(t *testing.T) { }, }}, } - getClient := makeMockClients(map[string]*ec2.DescribeInstancesOutput{ - "": &output, - "alternate-role-arn": &altAccountOutput, - }) + ec2DescribeInstancesOutOnlyImplicitRegions := ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{{ + Instances: []ec2types.Instance{instanceImplicitRegion}, + }}, + } + + ec2ClientOutputsByRole := map[string]*ec2.DescribeInstancesOutput{ + "": &ec2DescribeInstancesOutNoAssumeRole, + "alternate-role-arn": &ec2DescribeInstancesOutAlternateAssumeRole, + "implicit-region": &ec2DescribeInstancesOutOnlyImplicitRegions, + } + + ec2ClientGetter := func(ctx context.Context, region string, opts ...awsconfig.OptionsFn) (ec2.DescribeInstancesAPIClient, error) { + assumedRoles := awsconfig.AssumedRoles(opts...) + var roleARN string + + for _, assumedRole := range assumedRoles { + roleARN = assumedRole.RoleARN + } + + return &mockEC2Client{ + output: ec2ClientOutputsByRole[roleARN], + }, nil + } + + regionsListerGetter := func(ctx context.Context, opts ...awsconfig.OptionsFn) (account.ListRegionsAPIClient, error) { + return &mockAWSAccountClient{ + output: &account.ListRegionsOutput{ + Regions: []accounttypes.Region{ + {RegionName: aws.String("eu-south-1")}, + {RegionName: aws.String("eu-south-2")}, + }, + }, + }, nil + } fetchersFn := func() []Fetcher { - fetchers, err := matchersToEC2InstanceFetchers(MatcherToEC2FetcherParams{ + fetchers, err := MatchersToEC2InstanceFetchers(t.Context(), MatcherToEC2FetcherParams{ Matchers: matchers, PublicProxyAddrGetter: func(ctx context.Context) (string, error) { return "proxy.example.com:3080", nil }, - }, getClient) + EC2ClientGetter: ec2ClientGetter, + RegionsListerGetter: regionsListerGetter, + }) require.NoError(t, err) return fetchers @@ -316,6 +377,18 @@ func TestEC2Watcher(t *testing.T) { Parameters: map[string]string{"token": "", "scriptName": "", "sshdConfigPath": ""}, AssumeRoleARN: "alternate-role-arn", }, + { + Region: "eu-south-1", + Instances: []EC2Instance{toEC2Instance(instanceImplicitRegion)}, + Parameters: map[string]string{"token": "", "scriptName": "", "sshdConfigPath": ""}, + AssumeRoleARN: "implicit-region", + }, + { + Region: "eu-south-2", + Instances: []EC2Instance{toEC2Instance(instanceImplicitRegion)}, + Parameters: map[string]string{"token": "", "scriptName": "", "sshdConfigPath": ""}, + AssumeRoleARN: "implicit-region", + }, } for _, instances := range expectedInstances { @@ -531,8 +604,10 @@ func TestSSMRunCommandParameters(t *testing.T) { JoinToken: "my-token", ScriptName: "default-installer", }, + SSM: &types.AWSSSM{ + DocumentName: "TeleportDiscoveryInstaller", + }, }, - Document: "TeleportDiscoveryInstaller", }, errCheck: require.NoError, expectedParams: map[string]string{ @@ -550,8 +625,10 @@ func TestSSMRunCommandParameters(t *testing.T) { ScriptName: "default-agentless-installer", SSHDConfig: "/etc/ssh/sshd_config", }, + SSM: &types.AWSSSM{ + DocumentName: "TeleportDiscoveryInstaller", + }, }, - Document: "TeleportDiscoveryInstaller", }, errCheck: require.NoError, expectedParams: map[string]string{ @@ -569,8 +646,10 @@ func TestSSMRunCommandParameters(t *testing.T) { JoinToken: "my-token", ScriptName: "default-installer", }, + SSM: &types.AWSSSM{ + DocumentName: "AWS-RunShellScript", + }, }, - Document: "AWS-RunShellScript", ProxyPublicAddrGetter: func(ctx context.Context) (string, error) { return "proxy.example.com", nil }, @@ -590,8 +669,10 @@ func TestSSMRunCommandParameters(t *testing.T) { ScriptName: "default-installer", Suffix: "cluster-green", }, + SSM: &types.AWSSSM{ + DocumentName: "AWS-RunShellScript", + }, }, - Document: "AWS-RunShellScript", ProxyPublicAddrGetter: func(ctx context.Context) (string, error) { return "proxy.example.com", nil }, @@ -611,8 +692,10 @@ func TestSSMRunCommandParameters(t *testing.T) { ScriptName: "default-installer", Suffix: "cluster-green", }, + SSM: &types.AWSSSM{ + DocumentName: "AWS-RunShellScript", + }, }, - Document: "AWS-RunShellScript", ProxyPublicAddrGetter: func(ctx context.Context) (string, error) { return "", trace.NotFound("proxy is not yet available") }, From 48937a55b03f37fdfad5c77928925921cf131be0 Mon Sep 17 00:00:00 2001 From: Marco Dinis Date: Fri, 14 Nov 2025 15:09:57 +0000 Subject: [PATCH 2/7] use wildcard --- go.mod | 3 +-- lib/srv/server/ec2_watcher.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7a25533156067..1e980a76e5e06 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.13 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.5 + github.com/aws/aws-sdk-go-v2/service/account v1.29.3 github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.41.2 github.com/aws/aws-sdk-go-v2/service/athena v1.55.10 github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.42.2 @@ -287,8 +288,6 @@ require ( software.sslmate.com/src/go-pkcs12 v0.6.0 ) -require github.com/aws/aws-sdk-go-v2/service/account v1.29.3 - require ( cel.dev/expr v0.25.0 // indirect cloud.google.com/go v0.121.6 // indirect diff --git a/lib/srv/server/ec2_watcher.go b/lib/srv/server/ec2_watcher.go index 594fc431b718c..c101ae2361501 100644 --- a/lib/srv/server/ec2_watcher.go +++ b/lib/srv/server/ec2_watcher.go @@ -318,7 +318,7 @@ func newEC2InstanceFetcher(cfg ec2FetcherConfig) *ec2InstanceFetcher { Values: []string{string(ec2types.InstanceStateNameRunning)}, }} - if _, ok := cfg.Matcher.Tags["*"]; !ok { + if _, ok := cfg.Matcher.Tags[types.Wildcard]; !ok { for key, val := range cfg.Matcher.Tags { tagFilters = append(tagFilters, ec2types.Filter{ Name: aws.String("tag:" + key), From 2fecd05efea3fbfbe09d1192f72fdf494c28de9d Mon Sep 17 00:00:00 2001 From: Marco Dinis Date: Wed, 26 Nov 2025 14:30:32 +0000 Subject: [PATCH 3/7] require at least one region --- api/types/matchers_aws.go | 2 +- api/types/matchers_aws_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/types/matchers_aws.go b/api/types/matchers_aws.go index 309baa80bf895..96eed5e7f1ada 100644 --- a/api/types/matchers_aws.go +++ b/api/types/matchers_aws.go @@ -150,7 +150,7 @@ func (m *AWSMatcher) CheckAndSetDefaults() error { } if len(m.Regions) == 0 { - m.Regions = []string{Wildcard} + return trace.BadParameter("discovery service requires at least one region, for EC2 you can also set the region to %q to iterate over all regions (requires account:ListRegions IAM permission)", Wildcard) } for _, region := range m.Regions { diff --git a/api/types/matchers_aws_test.go b/api/types/matchers_aws_test.go index e181a490e4d10..efe7d0a37c190 100644 --- a/api/types/matchers_aws_test.go +++ b/api/types/matchers_aws_test.go @@ -187,10 +187,10 @@ func TestAWSMatcherCheckAndSetDefaults(t *testing.T) { errCheck: isBadParameterErr, }, { - name: "no region is valid for ec2 type", + name: "wildcard region is valid for ec2 type", in: &AWSMatcher{ Types: []string{"ec2"}, - Regions: []string{}, + Regions: []string{"*"}, }, errCheck: require.NoError, }, From b5580ca5a2d003ff7f18cad6dec52d5973c52db0 Mon Sep 17 00:00:00 2001 From: Marco Dinis Date: Wed, 26 Nov 2025 16:24:17 +0000 Subject: [PATCH 4/7] downgrade discovery service for old clients --- .../discoveryconfigv1/service.go | 82 ++++++++++++++++- .../discoveryconfigv1/service_test.go | 92 +++++++++++++++++++ lib/auth/grpcserver.go | 8 ++ 3 files changed, 180 insertions(+), 2 deletions(-) diff --git a/lib/auth/discoveryconfig/discoveryconfigv1/service.go b/lib/auth/discoveryconfig/discoveryconfigv1/service.go index 97450b5e0246b..af3defc388b4f 100644 --- a/lib/auth/discoveryconfig/discoveryconfigv1/service.go +++ b/lib/auth/discoveryconfig/discoveryconfigv1/service.go @@ -20,21 +20,26 @@ package discoveryconfigv1 import ( "context" + "fmt" "log/slog" + "github.com/coreos/go-semver/semver" "github.com/gravitational/trace" "github.com/jonboulle/clockwork" "google.golang.org/protobuf/types/known/emptypb" "github.com/gravitational/teleport" discoveryconfigv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/discoveryconfig/v1" + "github.com/gravitational/teleport/api/metadata" "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/api/types/discoveryconfig" conv "github.com/gravitational/teleport/api/types/discoveryconfig/convert/v1" apievents "github.com/gravitational/teleport/api/types/events" + "github.com/gravitational/teleport/api/utils/aws" "github.com/gravitational/teleport/lib/authz" "github.com/gravitational/teleport/lib/events" "github.com/gravitational/teleport/lib/services" + "github.com/gravitational/teleport/lib/utils" ) // ServiceConfig holds configuration options for the DiscoveryConfig gRPC service. @@ -124,7 +129,11 @@ func (s *Service) ListDiscoveryConfigs(ctx context.Context, req *discoveryconfig dcs := make([]*discoveryconfigv1.DiscoveryConfig, len(results)) for i, r := range results { - dcs[i] = conv.ToProto(r) + downgraded, err := MaybeDowngradeDiscoveryConfig(ctx, r) + if err != nil { + return nil, trace.Wrap(err) + } + dcs[i] = conv.ToProto(downgraded) } return &discoveryconfigv1.ListDiscoveryConfigsResponse{ @@ -149,7 +158,12 @@ func (s *Service) GetDiscoveryConfig(ctx context.Context, req *discoveryconfigv1 return nil, trace.Wrap(err) } - return conv.ToProto(dc), nil + downgraded, err := MaybeDowngradeDiscoveryConfig(ctx, dc) + if err != nil { + return nil, trace.Wrap(err) + } + + return conv.ToProto(downgraded), nil } // CreateDiscoveryConfig creates a new DiscoveryConfig resource. @@ -377,3 +391,67 @@ func (s *Service) UpdateDiscoveryConfigStatus(ctx context.Context, req *discover return conv.ToProto(resp), nil } } + +// MaybeDowngradeDiscoveryConfig tests the client version passed through the gRPC metadata, +// and if necessary downgrades the Discovery Config resource for compatibility with the older client. +// The following rules are applied: +// - if version is lower than 18.4.2, the AWS wildcard region is replaced with "aws-global" sentinel region +// this ensures the client can still discover other resources without erroring out. +func MaybeDowngradeDiscoveryConfig(ctx context.Context, dc *discoveryconfig.DiscoveryConfig) (*discoveryconfig.DiscoveryConfig, error) { + clientVersionString, ok := metadata.ClientVersionFromContext(ctx) + if !ok { + // This client is not reporting its version via gRPC metadata, which means it's a client really old or a third-party client. + // For those, downgrading the resource will do more harm than good, so the resource is returned as is. + return dc, nil + } + + clientVersion, err := semver.NewVersion(clientVersionString) + if err != nil { + return nil, trace.BadParameter("unrecognized client version: %s is not a valid semver", clientVersionString) + } + + dc = maybeDowngradeDiscoveryConfigAWSWildcardRegion(dc, clientVersion) + return dc, nil +} + +var minSupportedDiscoveryConfigAWSWildcardRegionVersion = semver.Version{Major: 18, Minor: 4, Patch: 2} + +// For Auth Server v20.0.0, the expected minimum supported client version is v19.0.0, which supports the AWS wildcard region. +// This function should be deleted at that time. +// +// TODO(@marco): DELETE IN v20.0.0. +func maybeDowngradeDiscoveryConfigAWSWildcardRegion(dc *discoveryconfig.DiscoveryConfig, clientVersion *semver.Version) *discoveryconfig.DiscoveryConfig { + if supported, err := utils.MinVerWithoutPreRelease( + clientVersion.String(), + minSupportedDiscoveryConfigAWSWildcardRegionVersion.String()); supported || err != nil { + return dc + } + + var changed bool + + originalDiscoveryConfig := dc + + dc = dc.Clone() + awsMatchers := dc.Spec.AWS + awsMatchersWithoutRegionWildcard := make([]types.AWSMatcher, 0, len(awsMatchers)) + for _, awsMatcher := range awsMatchers { + if len(awsMatcher.Regions) == 1 && awsMatcher.Regions[0] == types.Wildcard { + awsMatcher.Regions = []string{aws.AWSGlobalRegion} + changed = true + } + awsMatchersWithoutRegionWildcard = append(awsMatchersWithoutRegionWildcard, awsMatcher) + } + + if !changed { + return originalDiscoveryConfig + } + + dc.Spec.AWS = awsMatchersWithoutRegionWildcard + reason := fmt.Sprintf(`Client version %q does not support discovering all regions. Either update the Discovery Service agent to at least %s or enumerate all the regions in %q discovery config.`, + clientVersion, minSupportedDiscoveryConfigAWSWildcardRegionVersion, dc.GetName()) + if dc.Metadata.Labels == nil { + dc.Metadata.Labels = make(map[string]string, 1) + } + dc.Metadata.Labels[types.TeleportDowngradedLabel] = reason + return dc +} diff --git a/lib/auth/discoveryconfig/discoveryconfigv1/service_test.go b/lib/auth/discoveryconfig/discoveryconfigv1/service_test.go index f09a1091613fb..d2bef2faade91 100644 --- a/lib/auth/discoveryconfig/discoveryconfigv1/service_test.go +++ b/lib/auth/discoveryconfig/discoveryconfigv1/service_test.go @@ -29,6 +29,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" discoveryconfigpb "github.com/gravitational/teleport/api/gen/proto/go/teleport/discoveryconfig/v1" + "github.com/gravitational/teleport/api/metadata" "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/api/types/discoveryconfig" convert "github.com/gravitational/teleport/api/types/discoveryconfig/convert/v1" @@ -559,3 +560,94 @@ func initSvc(t *testing.T, clusterName string) (context.Context, localClient, *S DiscoveryConfigService: localResourceService, }, resourceSvc } + +func TestDowngrade(t *testing.T) { + for _, tc := range []struct { + name string + clientVersion string + input *discoveryconfig.DiscoveryConfig + expected *discoveryconfig.DiscoveryConfig + }{ + { + name: "no downgrade for recent client", + clientVersion: "18.4.2", + input: func() *discoveryconfig.DiscoveryConfig { + dc, err := discoveryconfig.NewDiscoveryConfig( + header.Metadata{Name: "dc1"}, + discoveryconfig.Spec{ + DiscoveryGroup: "group1", + AWS: []types.AWSMatcher{ + { + Regions: []string{types.Wildcard}, + Types: []string{"ec2"}, + }, + }, + }, + ) + require.NoError(t, err) + return dc + }(), + expected: func() *discoveryconfig.DiscoveryConfig { + dc, err := discoveryconfig.NewDiscoveryConfig( + header.Metadata{Name: "dc1"}, + discoveryconfig.Spec{ + DiscoveryGroup: "group1", + AWS: []types.AWSMatcher{ + { + Regions: []string{types.Wildcard}, + Types: []string{"ec2"}, + }, + }, + }, + ) + require.NoError(t, err) + return dc + }(), + }, + { + name: "downgrade for old client", + clientVersion: "18.4.1", + input: func() *discoveryconfig.DiscoveryConfig { + dc, err := discoveryconfig.NewDiscoveryConfig( + header.Metadata{Name: "dc1"}, + discoveryconfig.Spec{ + DiscoveryGroup: "group1", + AWS: []types.AWSMatcher{ + { + Regions: []string{types.Wildcard}, + Types: []string{"ec2"}, + }, + }, + }, + ) + require.NoError(t, err) + return dc + }(), + expected: func() *discoveryconfig.DiscoveryConfig { + dc, err := discoveryconfig.NewDiscoveryConfig( + header.Metadata{Name: "dc1"}, + discoveryconfig.Spec{ + DiscoveryGroup: "group1", + AWS: []types.AWSMatcher{ + { + Regions: []string{types.Wildcard}, + Types: []string{"ec2"}, + }, + }, + }, + ) + require.NoError(t, err) + return dc + }(), + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := metadata.AddMetadataToContext(t.Context(), map[string]string{ + metadata.VersionKey: tc.clientVersion, + }) + downgraded, err := MaybeDowngradeDiscoveryConfig(ctx, tc.input) + require.NoError(t, err) + require.Equal(t, tc.expected, downgraded) + }) + } +} diff --git a/lib/auth/grpcserver.go b/lib/auth/grpcserver.go index 19d0bb2c754fd..ed2b0392ad723 100644 --- a/lib/auth/grpcserver.go +++ b/lib/auth/grpcserver.go @@ -90,6 +90,7 @@ import ( "github.com/gravitational/teleport/api/metadata" "github.com/gravitational/teleport/api/trail" "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/api/types/discoveryconfig" apievents "github.com/gravitational/teleport/api/types/events" "github.com/gravitational/teleport/api/types/installers" "github.com/gravitational/teleport/api/types/wrappers" @@ -585,6 +586,13 @@ func WatchEvents(watch *authpb.Watch, stream WatchEvent, componentName string, a } event.Resource = downgraded } + if dc, ok := event.Resource.(*discoveryconfig.DiscoveryConfig); ok { + downgraded, err := discoveryconfigv1.MaybeDowngradeDiscoveryConfig(stream.Context(), dc) + if err != nil { + return trace.Wrap(err) + } + event.Resource = downgraded + } out, err := client.EventToGRPC(event) if err != nil { return trace.Wrap(err) From 52185965fb2a6df92b60bbed3bbf8c3679c68b78 Mon Sep 17 00:00:00 2001 From: Marco Dinis Date: Fri, 28 Nov 2025 18:02:57 +0000 Subject: [PATCH 5/7] move region x type check to runtime --- api/types/matchers_aws.go | 7 ------- api/types/matchers_aws_test.go | 8 -------- lib/srv/discovery/fetchers/db/db.go | 10 ++++++++++ lib/srv/discovery/fetchers/eks.go | 4 ++++ lib/srv/server/ec2_watcher.go | 2 ++ 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/api/types/matchers_aws.go b/api/types/matchers_aws.go index 96eed5e7f1ada..34104d4ed06df 100644 --- a/api/types/matchers_aws.go +++ b/api/types/matchers_aws.go @@ -158,13 +158,6 @@ func (m *AWSMatcher) CheckAndSetDefaults() error { if len(m.Regions) > 1 { return trace.BadParameter("when using %q as region, no other regions can be specified", Wildcard) } - - // Only EC2 supports discovering regions. - // TODO(marco): add support for other resources types. - if len(m.Types) > 1 || m.Types[0] != AWSMatcherEC2 { - return trace.BadParameter("only EC2 resource discovery supports discovering all regions, " + - "enumerate all the regions or create a separate matcher for discovering EC2 resources") - } break } diff --git a/api/types/matchers_aws_test.go b/api/types/matchers_aws_test.go index efe7d0a37c190..338342e479017 100644 --- a/api/types/matchers_aws_test.go +++ b/api/types/matchers_aws_test.go @@ -135,14 +135,6 @@ func TestAWSMatcherCheckAndSetDefaults(t *testing.T) { }, errCheck: isBadParameterErr, }, - { - name: "wildcard is invalid for regions when using non-ec2 types", - in: &AWSMatcher{ - Types: []string{"ec2", "rds"}, - Regions: []string{"*"}, - }, - errCheck: isBadParameterErr, - }, { name: "wildcard is valid for the ec2 type", in: &AWSMatcher{ diff --git a/lib/srv/discovery/fetchers/db/db.go b/lib/srv/discovery/fetchers/db/db.go index 7e1fb19e060be..6c0b70d2db6d3 100644 --- a/lib/srv/discovery/fetchers/db/db.go +++ b/lib/srv/discovery/fetchers/db/db.go @@ -129,6 +129,9 @@ type AWSFetcherFactoryConfig struct { AWSConfigProvider awsconfig.Provider // AWSClients provides AWS SDK clients. AWSClients AWSClientProvider + + // Logger is the logger used by the factory and its fetchers. + Logger *slog.Logger } func (c *AWSFetcherFactoryConfig) checkAndSetDefaults() error { @@ -138,6 +141,9 @@ func (c *AWSFetcherFactoryConfig) checkAndSetDefaults() error { if c.AWSClients == nil { c.AWSClients = defaultAWSClients{} } + if c.Logger == nil { + c.Logger = slog.Default() + } return nil } @@ -175,6 +181,10 @@ func (f *AWSFetcherFactory) MakeFetchers(ctx context.Context, matchers []types.A for _, makeFetcher := range makeFetchers { for _, region := range matcher.Regions { + if region == types.Wildcard { + f.cfg.Logger.WarnContext(ctx, "AWS Database discovery does not support region discovery, remove the '*' from the regions field", "discovery_config", discoveryConfigName) + continue + } fetcher, err := makeFetcher(awsFetcherConfig{ Type: matcherType, AssumeRole: assumeRole, diff --git a/lib/srv/discovery/fetchers/eks.go b/lib/srv/discovery/fetchers/eks.go index 8f67d2d54468b..3b7497a274117 100644 --- a/lib/srv/discovery/fetchers/eks.go +++ b/lib/srv/discovery/fetchers/eks.go @@ -167,6 +167,10 @@ func MakeEKSFetchersFromAWSMatchers(logger *slog.Logger, clients AWSClientGetter for _, region := range matcher.Regions { switch t { case types.AWSMatcherEKS: + if region == types.Wildcard { + logger.WarnContext(context.Background(), "EKS discovery does not support region discovery, remove the '*' from the regions field", "discovery_config", discoveryConfigName) + continue + } fetcher, err := NewEKSFetcher( EKSFetcherConfig{ ClientGetter: clients, diff --git a/lib/srv/server/ec2_watcher.go b/lib/srv/server/ec2_watcher.go index c101ae2361501..40ad05ae186f0 100644 --- a/lib/srv/server/ec2_watcher.go +++ b/lib/srv/server/ec2_watcher.go @@ -440,6 +440,8 @@ func (f *ec2InstanceFetcher) GetMatchingInstances(ctx context.Context, nodes []t return chunkInstances(instancesByRegion), nil } +// chunkInstances splits instances into chunks of 50. +// This is required because SSM SendCommand API calls only accept up to 50 instance IDs at a time. func chunkInstances(instancesByRegion map[string]EC2Instances) []Instances { var instColl []Instances for _, insts := range instancesByRegion { From 43f35b51bbb627c4582354179d8bc0c2ffc9deaa Mon Sep 17 00:00:00 2001 From: Marco Dinis Date: Tue, 2 Dec 2025 12:12:25 +0000 Subject: [PATCH 6/7] add test back --- api/types/matchers_aws_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/types/matchers_aws_test.go b/api/types/matchers_aws_test.go index 338342e479017..b072850e389c0 100644 --- a/api/types/matchers_aws_test.go +++ b/api/types/matchers_aws_test.go @@ -186,6 +186,14 @@ func TestAWSMatcherCheckAndSetDefaults(t *testing.T) { }, errCheck: require.NoError, }, + { + name: "no region", + in: &AWSMatcher{ + Types: []string{"ec2"}, + Regions: []string{}, + }, + errCheck: isBadParameterErr, + }, { name: "invalid assume role arn", in: &AWSMatcher{ From 1e02293fd6fb5f54d0ffd3f3d87881439bb75bbf Mon Sep 17 00:00:00 2001 From: Marco Dinis Date: Tue, 2 Dec 2025 12:13:44 +0000 Subject: [PATCH 7/7] fix expected version --- lib/auth/discoveryconfig/discoveryconfigv1/service.go | 4 ++-- lib/auth/discoveryconfig/discoveryconfigv1/service_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/auth/discoveryconfig/discoveryconfigv1/service.go b/lib/auth/discoveryconfig/discoveryconfigv1/service.go index af3defc388b4f..6235642af8256 100644 --- a/lib/auth/discoveryconfig/discoveryconfigv1/service.go +++ b/lib/auth/discoveryconfig/discoveryconfigv1/service.go @@ -395,7 +395,7 @@ func (s *Service) UpdateDiscoveryConfigStatus(ctx context.Context, req *discover // MaybeDowngradeDiscoveryConfig tests the client version passed through the gRPC metadata, // and if necessary downgrades the Discovery Config resource for compatibility with the older client. // The following rules are applied: -// - if version is lower than 18.4.2, the AWS wildcard region is replaced with "aws-global" sentinel region +// - if version is lower than 18.5.0, the AWS wildcard region is replaced with "aws-global" sentinel region // this ensures the client can still discover other resources without erroring out. func MaybeDowngradeDiscoveryConfig(ctx context.Context, dc *discoveryconfig.DiscoveryConfig) (*discoveryconfig.DiscoveryConfig, error) { clientVersionString, ok := metadata.ClientVersionFromContext(ctx) @@ -414,7 +414,7 @@ func MaybeDowngradeDiscoveryConfig(ctx context.Context, dc *discoveryconfig.Disc return dc, nil } -var minSupportedDiscoveryConfigAWSWildcardRegionVersion = semver.Version{Major: 18, Minor: 4, Patch: 2} +var minSupportedDiscoveryConfigAWSWildcardRegionVersion = semver.Version{Major: 18, Minor: 5, Patch: 0} // For Auth Server v20.0.0, the expected minimum supported client version is v19.0.0, which supports the AWS wildcard region. // This function should be deleted at that time. diff --git a/lib/auth/discoveryconfig/discoveryconfigv1/service_test.go b/lib/auth/discoveryconfig/discoveryconfigv1/service_test.go index d2bef2faade91..b02b18af466a7 100644 --- a/lib/auth/discoveryconfig/discoveryconfigv1/service_test.go +++ b/lib/auth/discoveryconfig/discoveryconfigv1/service_test.go @@ -570,7 +570,7 @@ func TestDowngrade(t *testing.T) { }{ { name: "no downgrade for recent client", - clientVersion: "18.4.2", + clientVersion: "18.5.0", input: func() *discoveryconfig.DiscoveryConfig { dc, err := discoveryconfig.NewDiscoveryConfig( header.Metadata{Name: "dc1"}, @@ -606,7 +606,7 @@ func TestDowngrade(t *testing.T) { }, { name: "downgrade for old client", - clientVersion: "18.4.1", + clientVersion: "18.4.2", input: func() *discoveryconfig.DiscoveryConfig { dc, err := discoveryconfig.NewDiscoveryConfig( header.Metadata{Name: "dc1"},