-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Blue green cutover and improvements in logging (#7)
* Changing how script waits for deployment to end (wip) * Wait for deployment to be created * Update Deploy.sh Due the known issue on Codedeploy, CodeDeploy will fail the deployment if the ECS service is unhealthy/unstable for 5mins for replacement taskset during the wait status, this 5mins is a non-configurable value as today. For the reason above we wait for 10 minutes before consider the deployment in ready status as successful * Create stop.sh Adding script stop.sh, to be triggered if customer doesn't approval the new version to be deployed, after accessing and testing the application through test listener * Update Dockerfile Adding script stop.sh * Cleanup and working on status to fail early if user cancels or deployment fails, instead of waiting for 10 minutes * Refactoring script to wait for deployment when cutover is enabled Co-authored-by: elliot-dnx <[email protected]>
- Loading branch information
Showing
8 changed files
with
173 additions
and
82 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| version: '3.4' | ||
|
|
||
| services: | ||
| app: | ||
| build: . | ||
| volumes: | ||
| - .:/work | ||
| environment: | ||
| - AWS_ACCESS_KEY_ID | ||
| - AWS_ACCOUNT_ID | ||
| - AWS_DEFAULT_REGION | ||
| - AWS_ROLE | ||
| - AWS_SECRET_ACCESS_KEY | ||
| - AWS_SECURITY_TOKEN | ||
| - AWS_SESSION_EXPIRATION | ||
| - AWS_SESSION_TOKEN | ||
| entrypoint: "" | ||
| command: /bin/bash |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,17 +1,6 @@ | ||
| { | ||
| "version": 1, | ||
| "Resources": [ | ||
| { | ||
| "TargetService": { | ||
| "Type": "AWS::ECS::Service", | ||
| "Properties": { | ||
| "TaskDefinition": "$TASK_ARN", | ||
| "LoadBalancerInfo": { | ||
| "ContainerName": "$APP_NAME", | ||
| "ContainerPort": $CONTAINER_PORT | ||
| } | ||
| } | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| "revisionType": "AppSpecContent", | ||
| "appSpecContent": { | ||
| "content": "{\"version\":1,\"Resources\":[{\"TargetService\":{\"Type\":\"AWS::ECS::Service\",\"Properties\":{\"TaskDefinition\":\"$TASK_ARN\",\"LoadBalancerInfo\":{\"ContainerName\":\"$APP_NAME\",\"ContainerPort\":$CONTAINER_PORT}}}}]}" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,31 +1,20 @@ | ||
| #!/bin/bash -e | ||
|
|
||
| ERROR=0 | ||
| if [[ -z "$AWS_DEFAULT_REGION" ]]; then echo "---> ERROR: Missing variable AWS_DEFAULT_REGION"; ERROR=1; fi | ||
| if [[ -z "$APP_NAME" ]]; then echo "---> ERROR: Missing variable APP_NAME"; ERROR=1; fi | ||
| if [[ -z "$CLUSTER_NAME" ]]; then echo "---> ERROR: Missing variable CLUSTER_NAME"; ERROR=1; fi | ||
| if [[ -z "$CONTAINER_PORT" ]]; then echo "---> ERROR: Missing variable CONTAINER_PORT"; ERROR=1; fi | ||
| if [[ -z "$IMAGE_NAME" ]]; then echo "---> ERROR: Missing variable IMAGE_NAME"; ERROR=1; fi | ||
| if [[ "$ERROR" == "1" ]]; then exit 1; fi | ||
|
|
||
| # Fetch deployment ID pending cutover to the green(new) enviroment | ||
| DEPLOYMENT_ID=$(aws deploy list-deployments --application-name=$CLUSTER_NAME-$APP_NAME --deployment-group=$CLUSTER_NAME-$APP_NAME --max-items=1 --query="deployments[0]" --output=text | head -n 1) | ||
|
|
||
| DEPLOYMENT_PID=$! | ||
|
|
||
| #echo "---> For More Deployment info: https://$AWS_DEFAULT_REGION.console.aws.amazon.com/codesuite/codedeploy/deployments/$DEPLOYMENT_ID" | ||
|
|
||
| #echo "---> Waiting for Deployment ..." | ||
|
|
||
| aws deploy continue-deployment --deployment-id $DEPLOYMENT_ID --deployment-wait-type "READY_WAIT" | ||
|
|
||
| wait $DEPLOYMENT_PID | ||
| RET=$? | ||
|
|
||
| if [ $RET -eq 0 ]; then | ||
| echo "---> Deployment completed!" | ||
| echo "---> Cutover engaged!" | ||
| else | ||
| echo "---> ERROR: Deployment FAILED!" | ||
| echo "---> ERROR: Cutover FAILED!" | ||
| fi | ||
|
|
||
| exit $RET |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| #!/bin/bash -e | ||
|
|
||
| if [[ -z "$AWS_DEFAULT_REGION" ]]; then echo "---> ERROR: Missing variable AWS_DEFAULT_REGION"; ERROR=1; fi | ||
| if [[ -z "$APP_NAME" ]]; then echo "---> ERROR: Missing variable APP_NAME"; ERROR=1; fi | ||
| if [[ -z "$CLUSTER_NAME" ]]; then echo "---> ERROR: Missing variable CLUSTER_NAME"; ERROR=1; fi | ||
|
|
||
| # Fetch deployment ID pending cutover to the green(new) enviroment | ||
| DEPLOYMENT_ID=$(aws deploy list-deployments --application-name=$CLUSTER_NAME-$APP_NAME --deployment-group=$CLUSTER_NAME-$APP_NAME --max-items=1 --query="deployments[0]" --output=text | head -n 1) | ||
|
|
||
| aws deploy stop-deployment --deployment-id $DEPLOYMENT_ID | ||
|
|
||
| RET=$? | ||
|
|
||
| if [ $RET -eq 0 ]; then | ||
| echo "---> Deployment stopped!" | ||
| else | ||
| echo "---> ERROR: Deployment stopped FAILED!" | ||
| fi | ||
|
|
||
| exit $RET |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,49 +1,56 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| import boto3, json, time, os, datetime | ||
| import boto3, json, time, os, datetime, sys | ||
|
|
||
| aws_ecs = boto3.client('ecs') | ||
| logs = boto3.client('logs') | ||
|
|
||
| cluster_name=os.environ['CLUSTER_NAME'] | ||
| app_name=os.environ['APP_NAME'] | ||
| task_arn=os.environ['TASK_ID'] | ||
| task_number=task_arn.split(":task/",1)[1] #get the task number id | ||
| task_arn=sys.argv[1] | ||
|
|
||
| task_id=task_arn.split(":task/",1)[1] #get the task number id | ||
| last_event = None | ||
| log_group_name='/ecs/'+cluster_name+'/'+app_name | ||
|
|
||
| extra_args = { | ||
| 'logGroupName': log_group_name, | ||
| 'logStreamName': app_name+'/'+app_name+'/'+task_id, | ||
| 'startFromHead': True | ||
| } | ||
|
|
||
| while True: | ||
| try: | ||
| response = aws_ecs.describe_tasks( | ||
| cluster=cluster_name, | ||
| tasks=[task_arn]) | ||
|
|
||
| logs = boto3.client('logs') | ||
| task_status = response['tasks'][0]['lastStatus'] | ||
| print('Task status', task_status) | ||
| logGroupName='/ecs/'+cluster_name+'/'+app_name | ||
| print('Searching logs for ', logGroupName) | ||
| time.sleep(5) | ||
|
|
||
| logStreams = logs.describe_log_streams( | ||
| logGroupName=logGroupName, | ||
| logStreamNamePrefix=app_name+'/'+app_name+'/'+task_number, | ||
| limit=1, | ||
| descending=True) | ||
|
|
||
| for stream in logStreams['logStreams']: | ||
| streamName=stream['logStreamName'] | ||
| print('log Streams', streamName) | ||
| logStreamEvents = logs.get_log_events( | ||
| logGroupName=logGroupName, | ||
| logStreamName=streamName, | ||
| startFromHead=True) | ||
| for log in logStreamEvents['events']: | ||
| print(log['message']) | ||
| if task_status == 'STOPPED': | ||
| break | ||
| time.sleep(5) | ||
|
|
||
| log_stream_events = logs.get_log_events(**extra_args) | ||
|
|
||
| for event in log_stream_events['events']: | ||
| print("%s" % (event['message'])) | ||
|
|
||
| if 'nextToken' in extra_args and log_stream_events['nextForwardToken'] == extra_args['nextToken']: | ||
| if task_status == "STOPPED": | ||
| print("======== TASK STOPPED ========") | ||
| print("Task ID: %s" % task_id) | ||
| print("Task ARN: %s" % task_arn) | ||
| print("Service Name: %s" % app_name) | ||
| print("Cluster Name: %s" % cluster_name) | ||
| print("Started at: %s" % response['tasks'][0]['startedAt']) | ||
| print("Stopped at: %s" % response['tasks'][0]['stoppedAt']) | ||
| print("Stopped Reason: %s" % response['tasks'][0]['stoppedReason']) | ||
| if 'stopCode' in response['tasks'][0]: | ||
| print("Stop Code: %s" % response['tasks'][0]['stopCode']) | ||
| print("") | ||
| break | ||
| time.sleep(1) | ||
| else: | ||
| extra_args['nextToken'] = log_stream_events['nextForwardToken'] | ||
|
|
||
| except Exception as e: | ||
| print("error: " + str(e)) | ||
| print("Error: " + str(e)) | ||
| break | ||
|
|
||
|
|