Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
ef88209
Add support for ecs agent auto updates
bernardjkim Sep 7, 2023
d263297
fix unit test
bernardjkim Sep 15, 2023
241e070
Remove unused var
bernardjkim Sep 15, 2023
2ccebeb
Merge branch 'master' into bernard/ecs-auto-update
bernardjkim Sep 21, 2023
c8d0ddc
Addres feedback
bernardjkim Sep 21, 2023
ece42b3
Use list of available AWS database regions
bernardjkim Sep 21, 2023
b4690c2
Run update task on proxy instances
bernardjkim Sep 22, 2023
d5c57c9
Revert GenerateAWSOIDCToken
bernardjkim Sep 22, 2023
f028a24
Move const to start of file
bernardjkim Sep 24, 2023
d801171
Address feedback
bernardjkim Sep 26, 2023
a31d614
Create separate DeployServiceUpdater struct
bernardjkim Sep 28, 2023
aeee42d
Address feedback
bernardjkim Sep 29, 2023
406b0e1
debug
bernardjkim Sep 29, 2023
0f8dc75
Address feedback
bernardjkim Sep 29, 2023
2cc89a3
Make OwnershipTags explicitly required
bernardjkim Sep 29, 2023
fec081d
Add cluster alert
bernardjkim Oct 4, 2023
3978c82
Fix typo and update message
bernardjkim Oct 4, 2023
cf4c36e
Revert cluster alert
bernardjkim Oct 4, 2023
ffde7f2
Update err messages
bernardjkim Oct 5, 2023
33e8a5e
Merge branch 'master' into bernard/ecs-auto-update
bernardjkim Oct 9, 2023
23a020a
Merge branch 'master' into bernard/ecs-auto-update
bernardjkim Oct 9, 2023
a5a07c5
Merge branch 'master' into bernard/ecs-auto-update
bernardjkim Oct 9, 2023
7ede4f8
Merge branch 'master' into bernard/ecs-auto-update
bernardjkim Oct 9, 2023
5444138
Merge branch 'master' into bernard/ecs-auto-update
bernardjkim Oct 10, 2023
e381d26
Check minimum compatible server version
bernardjkim Oct 10, 2023
d27caf2
Update log msg
bernardjkim Oct 10, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions api/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -4022,6 +4022,22 @@ func (c *Client) ListIntegrations(ctx context.Context, pageSize int, nextKey str
return integrations, resp.GetNextKey(), nil
}

// ListAllIntegrations returns the list of all Integrations.
func (c *Client) ListAllIntegrations(ctx context.Context) ([]types.Integration, error) {
var result []types.Integration
var nextKey string
for {
integrations, nextKey, err := c.ListIntegrations(ctx, 0, nextKey)
if err != nil {
return nil, trace.Wrap(err)
}
result = append(result, integrations...)
if nextKey == "" {
return result, nil
}
}
}

// GetIntegration returns an Integration by its name.
func (c *Client) GetIntegration(ctx context.Context, name string) (types.Integration, error) {
ig, err := c.integrationsClient().GetIntegration(ctx, &integrationpb.GetIntegrationRequest{
Expand Down
29 changes: 29 additions & 0 deletions api/types/maintenance.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,10 @@ type ClusterMaintenanceConfig interface {
// SetAgentUpgradeWindow sets the agent upgrade window.
SetAgentUpgradeWindow(win AgentUpgradeWindow)

// WithinUpgradeWindow returns true if the time is within the configured
// upgrade window.
WithinUpgradeWindow(t time.Time) bool

CheckAndSetDefaults() error
}

Expand Down Expand Up @@ -229,3 +233,28 @@ func (m *ClusterMaintenanceConfigV1) GetAgentUpgradeWindow() (win AgentUpgradeWi
func (m *ClusterMaintenanceConfigV1) SetAgentUpgradeWindow(win AgentUpgradeWindow) {
m.Spec.AgentUpgrades = &win
}

// WithinUpgradeWindow returns true if the time is within the configured
// upgrade window.
func (m *ClusterMaintenanceConfigV1) WithinUpgradeWindow(t time.Time) bool {
upgradeWindow, ok := m.GetAgentUpgradeWindow()
if !ok {
return false
}

if len(upgradeWindow.Weekdays) == 0 {
if int(upgradeWindow.UTCStartHour) == t.Hour() {
return true
}
}

weekday := t.Weekday().String()
for _, upgradeWeekday := range upgradeWindow.Weekdays {
if weekday == upgradeWeekday {
if int(upgradeWindow.UTCStartHour) == t.Hour() {
return true
}
}
}
return false
}
57 changes: 57 additions & 0 deletions api/types/maintenance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,3 +214,60 @@ func TestWeekdayParser(t *testing.T) {
require.Equal(t, tt.expect, day)
}
}

func TestWithinUpgradeWindow(t *testing.T) {
t.Parallel()

tests := []struct {
desc string
upgradeWindow AgentUpgradeWindow
date string
withinWindow bool
}{
{
desc: "within upgrade window",
upgradeWindow: AgentUpgradeWindow{
UTCStartHour: 8,
},
date: "Mon, 02 Jan 2006 08:04:05 UTC",
withinWindow: true,
},
{
desc: "not within upgrade window",
upgradeWindow: AgentUpgradeWindow{
UTCStartHour: 8,
},
date: "Mon, 02 Jan 2006 09:04:05 UTC",
withinWindow: false,
},
{
desc: "within upgrade window weekday",
upgradeWindow: AgentUpgradeWindow{
UTCStartHour: 8,
Weekdays: []string{"Monday"},
},
date: "Mon, 02 Jan 2006 08:04:05 UTC",
withinWindow: true,
},
{
desc: "not within upgrade window weekday",
upgradeWindow: AgentUpgradeWindow{
UTCStartHour: 8,
Weekdays: []string{"Tuesday"},
},
date: "Mon, 02 Jan 2006 08:04:05 UTC",
withinWindow: false,
},
}

for _, tt := range tests {
t.Run(tt.desc, func(t *testing.T) {
cmc := NewClusterMaintenanceConfig()
cmc.SetAgentUpgradeWindow(tt.upgradeWindow)

date, err := time.Parse(time.RFC1123, tt.date)
require.NoError(t, err)
require.Equal(t, tt.withinWindow, cmc.WithinUpgradeWindow(date))
})
}
}
1 change: 1 addition & 0 deletions lib/authz/permissions.go
Original file line number Diff line number Diff line change
Expand Up @@ -618,6 +618,7 @@ func roleSpecForProxy(clusterName string) types.RoleSpecV6 {
types.NewRule(types.KindDatabaseService, services.RO()),
types.NewRule(types.KindSAMLIdPServiceProvider, services.RO()),
types.NewRule(types.KindUserGroup, services.RO()),
types.NewRule(types.KindClusterMaintenanceConfig, services.RO()),
types.NewRule(types.KindIntegration, append(services.RO(), types.VerbUse)),
// this rule allows cloud proxies to read
// plugins of `openai` type, since Assist uses the OpenAI API and runs in Proxy.
Expand Down
10 changes: 10 additions & 0 deletions lib/automaticupgrades/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ import (
const (
// automaticUpgradesEnvar defines the env var to lookup when deciding whether to enable AutomaticUpgrades feature.
automaticUpgradesEnvar = "TELEPORT_AUTOMATIC_UPGRADES"

// automaticUpgradesChannelEnvar defines a customer automatic upgrades version release channel.
automaticUpgradesChannelEnvar = "TELEPORT_AUTOMATIC_UPGRADES_CHANNEL"
Comment thread
bernardjkim marked this conversation as resolved.
)

// IsEnabled reads the TELEPORT_AUTOMATIC_UPGRADES and returns whether Automatic Upgrades are enabled or disabled.
Expand All @@ -46,3 +49,10 @@ func IsEnabled() bool {

return automaticUpgrades
}

// GetChannel returns the TELEPORT_AUTOMATIC_UPGRADES_CHANNEL value.
// Example of an acceptable value for TELEPORT_AUTOMATIC_UPGRADES_CHANNEL is:
// https://updates.releases.teleport.dev/v1/stable/cloud
func GetChannel() string {
return os.Getenv(automaticUpgradesChannelEnvar)
}
68 changes: 61 additions & 7 deletions lib/automaticupgrades/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,22 +34,54 @@ const (

// stableCloudVersionPath is the URL path that returns the current stable/cloud version.
stableCloudVersionPath = "/v1/stable/cloud/version"

// stableCloudCriticalPath is the URL path that returns the stable/cloud critical flag.
stableCloudCriticalPath = "/v1/stable/cloud/critical"
)

// Version returns the version that should be used for installing Teleport Services
// This is used when installing agents using scripts.
// Even when Teleport Auth/Proxy is using vX, the agents must always respect this version.
func Version(ctx context.Context, baseURL string) (string, error) {
if baseURL == "" {
baseURL = stableCloudVersionBaseURL
func Version(ctx context.Context, versionURL string) (string, error) {
versionURL, err := getVersionURL(versionURL)
if err != nil {
return "", trace.Wrap(err)
}

fullURL, err := url.JoinPath(baseURL, stableCloudVersionPath)
resp, err := sendRequest(ctx, versionURL)
if err != nil {
return "", trace.Wrap(err)
}

req, err := http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil)
return resp, nil
}

// Critical returns true if a critical upgrade is available.
func Critical(ctx context.Context, criticalURL string) (bool, error) {
criticalURL, err := getCriticalURL(criticalURL)
if err != nil {
return false, trace.Wrap(err)
}

critical, err := sendRequest(ctx, criticalURL)
if err != nil {
return false, trace.Wrap(err)
}

// Expects critical endpoint to return either the string "yes" or "no"
switch critical {
case "yes":
Comment thread
bernardjkim marked this conversation as resolved.
return true, nil
case "no":
return false, nil
default:
return false, trace.BadParameter("critical endpoint returned an unexpected value: %v", critical)
}
}

// sendRequest sends a GET request to the reqURL and returns the response value
func sendRequest(ctx context.Context, reqURL string) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
if err != nil {
return "", trace.Wrap(err)
}
Expand All @@ -69,7 +101,29 @@ func Version(ctx context.Context, baseURL string) (string, error) {
return "", trace.BadParameter("invalid status code %d, body: %s", resp.StatusCode, string(body))
}

versionString := strings.TrimSpace(string(body))
return strings.TrimSpace(string(body)), trace.Wrap(err)
}

// getVersionURL returns the versionURL or the default stable/cloud version url.
func getVersionURL(versionURL string) (string, error) {
if versionURL != "" {
return versionURL, nil
}
cloudStableVersionURL, err := url.JoinPath(stableCloudVersionBaseURL, stableCloudVersionPath)
if err != nil {
return "", trace.Wrap(err)
}
return cloudStableVersionURL, nil
}

return versionString, trace.Wrap(err)
// getCriticalURL returns the criticalURL or the default stable/cloud critical url.
func getCriticalURL(criticalURL string) (string, error) {
if criticalURL != "" {
return criticalURL, nil
}
cloudStableCriticalURL, err := url.JoinPath(stableCloudVersionBaseURL, stableCloudCriticalPath)
if err != nil {
return "", trace.Wrap(err)
}
return cloudStableCriticalURL, nil
}
121 changes: 118 additions & 3 deletions lib/automaticupgrades/version_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"net/http"
"net/http/httptest"
"net/url"
"testing"

"github.com/gravitational/trace"
Expand Down Expand Up @@ -68,19 +69,133 @@ func TestVersion(t *testing.T) {
} {
t.Run(tt.name, func(t *testing.T) {
httpTestServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/v1/stable/cloud/version")
assert.Equal(t, "/v1/stable/cloud/version", r.URL.Path)
w.WriteHeader(tt.mockStatusCode)
w.Write([]byte(tt.mockResponseString))
}))
defer httpTestServer.Close()

v, err := Version(ctx, httpTestServer.URL)
versionURL, err := url.JoinPath(httpTestServer.URL, "/v1/stable/cloud/version")
require.NoError(t, err)

v, err := Version(ctx, versionURL)
tt.errCheck(t, err)
if err != nil {
return
}

require.Equal(t, v, tt.expectedVersion)
require.Equal(t, tt.expectedVersion, v)
})
}
}

func TestCritical(t *testing.T) {
ctx := context.Background()

isBadParameterErr := func(tt require.TestingT, err error, i ...any) {
require.True(tt, trace.IsBadParameter(err), "expected bad parameter, got %v", err)
}

for _, tt := range []struct {
name string
mockStatusCode int
mockResponseString string
errCheck require.ErrorAssertionFunc
expectedCritical bool
}{
{
name: "critical available",
mockStatusCode: http.StatusOK,
mockResponseString: "yes\n",
errCheck: require.NoError,
expectedCritical: true,
},
{
name: "critical is not available",
mockStatusCode: http.StatusOK,
mockResponseString: "no\n",
errCheck: require.NoError,
expectedCritical: false,
},
{
name: "invalid status code (500)",
mockStatusCode: http.StatusInternalServerError,
errCheck: isBadParameterErr,
},
{
name: "invalid status code (403)",
mockStatusCode: http.StatusForbidden,
errCheck: isBadParameterErr,
},
} {
t.Run(tt.name, func(t *testing.T) {
httpTestServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/v1/stable/cloud/critical", r.URL.Path)
w.WriteHeader(tt.mockStatusCode)
w.Write([]byte(tt.mockResponseString))
}))
defer httpTestServer.Close()

criticalURL, err := url.JoinPath(httpTestServer.URL, "/v1/stable/cloud/critical")
require.NoError(t, err)

v, err := Critical(ctx, criticalURL)
tt.errCheck(t, err)
if err != nil {
return
}

require.Equal(t, tt.expectedCritical, v)
})
}
}

func TestGetVersionURL(t *testing.T) {
for _, tt := range []struct {
name string
versionURL string
expectedURL string
}{
{
name: "default stable/cloud version url",
versionURL: "",
expectedURL: "https://updates.releases.teleport.dev/v1/stable/cloud/version",
},
{
name: "custom version url",
versionURL: "https://custom.dev/version",
expectedURL: "https://custom.dev/version",
},
} {
t.Run(tt.name, func(t *testing.T) {
v, err := getVersionURL(tt.versionURL)
require.NoError(t, err)
require.Equal(t, tt.expectedURL, v)
})
}
}

func TestGetCriticalURL(t *testing.T) {
for _, tt := range []struct {
name string
criticalURL string
expectedURL string
}{
{
name: "default stable/cloud critical url",
criticalURL: "",
expectedURL: "https://updates.releases.teleport.dev/v1/stable/cloud/critical",
},
{
name: "custom critical url",
criticalURL: "https://custom.dev/critical",
expectedURL: "https://custom.dev/critical",
},
} {
t.Run(tt.name, func(t *testing.T) {
v, err := getCriticalURL(tt.criticalURL)
require.NoError(t, err)
require.Equal(t, tt.expectedURL, v)
})
}
}
2 changes: 1 addition & 1 deletion lib/cloud/aws/policy_statements.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func StatementForECSManageService() *Statement {
Actions: []string{
"ecs:DescribeClusters", "ecs:CreateCluster", "ecs:PutClusterCapacityProviders",
"ecs:DescribeServices", "ecs:CreateService", "ecs:UpdateService",
"ecs:RegisterTaskDefinition",
"ecs:RegisterTaskDefinition", "ecs:DescribeTaskDefinition", "ecs:DeregisterTaskDefinition",
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have to think of some way to let the users know that their current set of permissions might not be enough, and that they must re-run the script.

Maybe a warning in the logs?
Cluster Alert might also work

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we doing anything here?

For new deployments, it should be good (we add all the permissions).
For current deployments, it will fail.
Do we have any kind of alerting?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We will log the following message if permissions are lacking, but I don't know how effective that will be. https://github.com/gravitational/teleport/pull/31982/files#diff-b2b691cbb3f300d6396f5b1ff26ac2569a944859ac8041067d8c97b23037f9c3R321-R325

I'm not familiar with Cluster Alerts, do you think this would be a better approach?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The log messages are stored internally (as in Teleport Cloud infra).
They will not be visible to the customer.

Maybe we can get away with looking for those logs ourselves and letting the affected customers know.
But honestly, that feels hacky, error-prone and not scalable.

With a ClusterAlert the customer would know.
I'm not very familiar with those as well, but I believe it would work better (even if we do this, let's keep the warning as well).
@r0mant What do you think?

As for method to fix:

Re-run deploy service configuration script to update permissions.

We don't have an easy way for the user to re-run the script.
It is only generated at that step when enrolling an RDS database.

Can we generate the script URL?
It should look something like this:

curl 'https://<tenant>.teleport.sh/webapi/scripts/integrations/configure/deployservice-iam.sh?integrationName=<integration-name>&awsRegion=<aws-region>&taskRole=<taskRole>&role=<integrationRole>'

I think all the variables are present in the current context.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've added a cluster alert here fec081d

Do you know of a way to get the task role in the current context? The only way I'm aware of is extracting the task role from the task definition. But that would require that the instance already has permissions to describe the task definition.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I don't think we can.
I thought it was under Service, but it is actually under Task Definition 😭

In that case, we can only ask them to fill it with the same role that was used previously.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay the alert might look something like the following:

Open Amazon CloudShell and copy/paste the following command to reconfigure integration. Replace TASK_ROLE with deploy service task role name. bash -c "$(curl 'https://bernard-dev.cloud.gravitational.io/webapi/scripts/integrations/configure/deployservice-iam.sh?awsRegion=us-west-2&integrationName=database-access&role=bernard-dev-database-access&taskRole=TASK_ROLE')"

It is pretty verbose tho. We might want to consider adding a docs page later and linking that instead.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's leave cluster alerts out of scope for now and revisit at a later time.


// EC2 DescribeSecurityGroups is required so that the user can list the SG and then pick which ones they want to apply to the ECS Service.
"ec2:DescribeSecurityGroups",
Expand Down
Loading