Feature/dynamic host port assignment

.github/workflows/gitsecrets.yml

@@ -8,7 +8,7 @@ jobs:
     name: Git Secrets Scan
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           path: src/github.com/aws/amazon-ecs-agent
       - name: Git Secrets Scan Script

.github/workflows/linux.yml

@@ -8,7 +8,7 @@ jobs:
     name: Linux unit tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: true
           path: src/github.com/aws/amazon-ecs-agent

.github/workflows/static.yml

@@ -8,7 +8,7 @@ jobs:
     name: Static Analysis
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           path: src/github.com/aws/amazon-ecs-agent
       - name: get GO_VERSION
@@ -24,11 +24,11 @@ jobs:
             echo "invalid GO version"
             exit 1
           fi
-          echo "::set-output name=GO_VERSION::$go_version"
-      - uses: actions/setup-go@v2
+          echo "GO_VERSION=$go_version" >> $GITHUB_OUTPUT
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ steps.get-go-version.outputs.GO_VERSION }}
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           path: src/github.com/aws/amazon-ecs-agent
       - name: run static checks
@@ -44,7 +44,7 @@ jobs:
     name: Static Analysis Init
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           path: src/github.com/aws/amazon-ecs-agent
       - name: get GO_VERSION
@@ -60,11 +60,11 @@ jobs:
             echo "invalid GO version"
             exit 1
           fi
-          echo "::set-output name=GO_VERSION::$go_version"
-      - uses: actions/setup-go@v2
+          echo "GO_VERSION=$go_version" >> $GITHUB_OUTPUT
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ steps.get-go-version.outputs.GO_VERSION }}
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           path: src/github.com/aws/amazon-ecs-agent
       - name: run static checks
@@ -80,7 +80,7 @@ jobs:
     name: Cross platform build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           path: src/github.com/aws/amazon-ecs-agent
       - name: get GO_VERSION
@@ -96,11 +96,11 @@ jobs:
             echo "invalid GO version"
             exit 1
           fi
-          echo "::set-output name=GO_VERSION::$go_version"
-      - uses: actions/setup-go@v2
+          echo "GO_VERSION=$go_version" >> $GITHUB_OUTPUT
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ steps.get-go-version.outputs.GO_VERSION }}
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: true
           path: src/github.com/aws/amazon-ecs-agent

.github/workflows/windows.yml

@@ -8,7 +8,7 @@ jobs:
     name: Windows unit tests
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           path: src/github.com/aws/amazon-ecs-agent
       - name: get GO_VERSION
@@ -23,11 +23,11 @@ jobs:
             echo "invalid GO version"
             exit 1
           }
-          Write-Output "::set-output name=GO_VERSION_WINDOWS::$go_version_win"
-      - uses: actions/setup-go@v2
+          Write-Output "GO_VERSION_WINDOWS=$go_version_win" >> $GITHUB_OUTPUT
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ steps.get-go-version.outputs.GO_VERSION_WINDOWS }}
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: true
           path: src/github.com/aws/amazon-ecs-agent

CHANGELOG.md

@@ -1,4 +1,14 @@
 # Changelog
+
+## 1.69.0
+* Enhancement - Use T.TempDir to create temporary test directory [#3159](https://github.com/aws/amazon-ecs-agent/pull/3159) and [#3560](https://github.com/aws/amazon-ecs-agent/pull/3560)
+* Enhancement - remove set-output GitHub action command [#3487](https://github.com/aws/amazon-ecs-agent/pull/3487)
+* Enhancement - periodically disconnect from ACS [#3586](https://github.com/aws/amazon-ecs-agent/pull/3586)
+* Bug - Fixed a bug that incorrectly advertised the gMSA and fsx capability [#3540](https://github.com/aws/amazon-ecs-agent/pull/3540)
+* Bug - Remove fallback to Docker for host port ranges assignment [#3569](https://github.com/aws/amazon-ecs-agent/pull/3569)
+* Bug - fix ecs-init log message [#3577](https://github.com/aws/amazon-ecs-agent/pull/3577)
+* Bug - Update CNI plugin versions, IMDS access works properly over IPv6 [#3581](https://github.com/aws/amazon-ecs-agent/pull/3581)
+
 ## 1.68.2
 * Enhancement: Skip sending STSC events for internal tasks [#3541](https://github.com/aws/amazon-ecs-agent/pull/3541) and [#3559](https://github.com/aws/amazon-ecs-agent/pull/3559)
 * Enhancement: Update go version in module file, update most vendored build dependencies to latest library [#3534](https://github.com/aws/amazon-ecs-agent/pull/3534) and [#3551](https://github.com/aws/amazon-ecs-agent/pull/3551)

VERSION

@@ -1 +1 @@
-1.68.2
+1.69.0

agent/acs/handler/acs_handler.go

@@ -21,6 +21,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	acsclient "github.com/aws/amazon-ecs-agent/agent/acs/client"
@@ -39,6 +40,7 @@ import (
 	"github.com/aws/amazon-ecs-agent/agent/utils/ttime"
 	"github.com/aws/amazon-ecs-agent/agent/version"
 	"github.com/aws/amazon-ecs-agent/agent/wsclient"
+
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/cihub/seelog"
 )
@@ -54,6 +56,10 @@ const (
 
 	inactiveInstanceReconnectDelay = 1 * time.Hour
 
+	// connectionTime is the maximum time after which agent closes its connection to ACS
+	connectionTime   = 15 * time.Minute
+	connectionJitter = 30 * time.Minute
+
 	connectionBackoffMin        = 250 * time.Millisecond
 	connectionBackoffMax        = 2 * time.Minute
 	connectionBackoffJitter     = 0.2
@@ -70,6 +76,10 @@ const (
 	// 1: default protocol version
 	// 2: ACS will proactively close the connection when heartbeat acks are missing
 	acsProtocolVersion = 2
+	// numOfHandlersSendingAcks is the number of handlers that send acks back to ACS and that are not saved across
+	// sessions. We use this to send pending acks, before agent initiates a disconnect to ACS.
+	// they are: refreshCredentialsHandler, taskManifestHandler, payloadHandler and heartbeatHandler
+	numOfHandlersSendingAcks = 4
 )
 
 // Session defines an interface for handler's long-lived connection with ACS.
@@ -100,14 +110,16 @@ type session struct {
 	doctor                          *doctor.Doctor
 	_heartbeatTimeout               time.Duration
 	_heartbeatJitter                time.Duration
+	connectionTime                  time.Duration
+	connectionJitter                time.Duration
 	_inactiveInstanceReconnectDelay time.Duration
 }
 
 // sessionResources defines the resource creator interface for starting
 // a session with ACS. This interface is intended to define methods
 // that create resources used to establish the connection to ACS
 // It is confined to just the createACSClient() method for now. It can be
-// extended to include the acsWsURL() and newDisconnectionTimer() methods
+// extended to include the acsWsURL() and newHeartbeatTimer() methods
 // when needed
 // The goal is to make it easier to test and inject dependencies
 type sessionResources interface {
@@ -182,6 +194,8 @@ func NewSession(
 		doctor:                          doctor,
 		_heartbeatTimeout:               heartbeatTimeout,
 		_heartbeatJitter:                heartbeatJitter,
+		connectionTime:                  connectionTime,
+		connectionJitter:                connectionJitter,
 		_inactiveInstanceReconnectDelay: inactiveInstanceReconnectDelay,
 	}
 }
@@ -224,7 +238,7 @@ func (acsSession *session) Start() error {
 				}
 			}
 			if shouldReconnectWithoutBackoff(acsError) {
-				// If ACS closed the connection, there's no need to backoff,
+				// If ACS or agent closed the connection, there's no need to backoff,
 				// reconnect immediately
 				seelog.Infof("ACS Websocket connection closed for a valid reason: %v", acsError)
 				acsSession.backoff.Reset()
@@ -360,11 +374,17 @@ func (acsSession *session) startACSSession(client wsclient.ClientServer) error {
 	}
 
 	seelog.Info("Connected to ACS endpoint")
-	// Start inactivity timer for closing the connection
-	timer := newDisconnectionTimer(client, acsSession.heartbeatTimeout(), acsSession.heartbeatJitter())
-	// Any message from the server resets the disconnect timeout
-	client.SetAnyRequestHandler(anyMessageHandler(timer, client))
-	defer timer.Stop()
+	// Start a connection timer; agent will send pending acks and close its ACS websocket connection
+	// after this timer expires
+	connectionTimer := newConnectionTimer(client, acsSession.connectionTime, acsSession.connectionJitter,
+		&refreshCredsHandler, &taskManifestHandler, &payloadHandler, &heartbeatHandler)
+	defer connectionTimer.Stop()
+
+	// Start a heartbeat timer for closing the connection
+	heartbeatTimer := newHeartbeatTimer(client, acsSession.heartbeatTimeout(), acsSession.heartbeatJitter())
+	// Any message from the server resets the heartbeat timer
+	client.SetAnyRequestHandler(anyMessageHandler(heartbeatTimer, client))
+	defer heartbeatTimer.Stop()
 
 	acsSession.resources.connectedToACS()
 
@@ -393,7 +413,7 @@ func (acsSession *session) startACSSession(client wsclient.ClientServer) error {
 		case err := <-serveErr:
 			// Stop receiving and sending messages from and to ACS when
 			// client.Serve returns an error. This can happen when the
-			// the connection is closed by ACS or the agent
+			// connection is closed by ACS or the agent
 			if err == nil || err == io.EOF {
 				seelog.Info("ACS Websocket connection closed for a valid reason")
 			} else {
@@ -478,9 +498,9 @@ func acsWsURL(endpoint, cluster, containerInstanceArn string, taskEngine engine.
 	return acsURL + "?" + query.Encode()
 }
 
-// newDisconnectionTimer creates a new time object, with a callback to
+// newHeartbeatTimer creates a new time object, with a callback to
 // disconnect from ACS on inactivity
-func newDisconnectionTimer(client wsclient.ClientServer, timeout time.Duration, jitter time.Duration) ttime.Timer {
+func newHeartbeatTimer(client wsclient.ClientServer, timeout time.Duration, jitter time.Duration) ttime.Timer {
 	timer := time.AfterFunc(retry.AddJitter(timeout, jitter), func() {
 		seelog.Warn("ACS Connection hasn't had any activity for too long; closing connection")
 		if err := client.Close(); err != nil {
@@ -492,6 +512,64 @@ func newDisconnectionTimer(client wsclient.ClientServer, timeout time.Duration,
 	return timer
 }
 
+// newConnectionTimer creates a new timer, after which agent sends any pending acks to ACS and closes
+// its websocket connection
+func newConnectionTimer(
+	client wsclient.ClientServer,
+	connectionTime time.Duration,
+	connectionJitter time.Duration,
+	refreshCredsHandler *refreshCredentialsHandler,
+	taskManifestHandler *taskManifestHandler,
+	payloadHandler *payloadRequestHandler,
+	heartbeatHandler *heartbeatHandler,
+) ttime.Timer {
+	expiresAt := retry.AddJitter(connectionTime, connectionJitter)
+	timer := time.AfterFunc(expiresAt, func() {
+		seelog.Debugf("Sending pending acks to ACS before closing the connection")
+
+		wg := sync.WaitGroup{}
+		wg.Add(numOfHandlersSendingAcks)
+
+		// send pending creds refresh acks to ACS
+		go func() {
+			refreshCredsHandler.sendPendingAcks()
+			wg.Done()
+		}()
+
+		// send pending task manifest acks and task stop verification acks to ACS
+		go func() {
+			taskManifestHandler.sendPendingTaskManifestMessageAck()
+			taskManifestHandler.handlePendingTaskStopVerificationAck()
+			wg.Done()
+		}()
+
+		// send pending payload acks to ACS
+		go func() {
+			payloadHandler.sendPendingAcks()
+			wg.Done()
+		}()
+
+		// send pending heartbeat acks to ACS
+		go func() {
+			heartbeatHandler.sendPendingHeartbeatAck()
+			wg.Done()
+		}()
+
+		// wait for acks from all the handlers above to be sent to ACS before closing the websocket connection.
+		// the methods used to read pending acks are non-blocking, so it is safe to wait here.
+		wg.Wait()
+
+		seelog.Infof("Closing ACS websocket connection after %v minutes", expiresAt.Minutes())
+		// WriteCloseMessage() writes a close message using websocket control messages
+		// Ref: https://pkg.go.dev/github.com/gorilla/websocket#hdr-Control_Messages
+		err := client.WriteCloseMessage()
+		if err != nil {
+			seelog.Warnf("Error writing close message: %v", err)
+		}
+	})
+	return timer
+}
+
 // anyMessageHandler handles any server message. Any server message means the
 // connection is active and thus the heartbeat disconnect should not occur
 func anyMessageHandler(timer ttime.Timer, client wsclient.ClientServer) func(interface{}) {