Merge pull request #350 from aws/1.5.x

1.5.x

bin/codedeploy-agent

@@ -2,7 +2,7 @@
 
 $:.unshift File.join(File.dirname(File.expand_path('..', __FILE__)), 'lib')
 
-ruby_versions = ["2.7", "2.6", "2.5", "2.4", "2.3", "2.2", "2.1", "2.0"]
+ruby_versions = ["3.0", "2.7", "2.6", "2.5", "2.4", "2.3", "2.2", "2.1", "2.0"]
 actual_ruby_version = RUBY_VERSION.split('.').map{|s|s.to_i}
 left_bound = '2.0.0'.split('.').map{|s|s.to_i}
 ruby_bin = nil

bin/install

@@ -213,10 +213,10 @@ EOF
   end
 
   def supported_ruby_versions
-    ['2.7', '2.6', '2.5', '2.4', '2.3', '2.2', '2.1', '2.0']
+    ['3.0', '2.7', '2.6', '2.5', '2.4', '2.3', '2.2', '2.1', '2.0']
   end
 
-  # check ruby version, only version 2.x works
+  # check ruby version, only version 2.x 3.x works
   def check_ruby_version_and_symlink
     @log.info("Starting Ruby version check.")
     actual_ruby_version = RUBY_VERSION.split('.').map{|s|s.to_i}[0,2]
@@ -241,9 +241,9 @@ EOF
   end
 
   def unsupported_ruby_version_error
-    @log.error("Current running Ruby version for "+ENV['USER']+" is "+RUBY_VERSION+", but Ruby version 2.x needs to be installed.")
+    @log.error("Current running Ruby version for "+ENV['USER']+" is "+RUBY_VERSION+", but Ruby version 2.x, 3.x needs to be installed.")
     @log.error('If you already have the proper Ruby version installed, please either create a symlink to /usr/bin/ruby2.x,')
-    @log.error( "or run this install script with right interpreter. Otherwise please install Ruby 2.x for "+ENV['USER']+" user.")
+    @log.error( "or run this install script with right interpreter. Otherwise please install Ruby 2.x, 3.x for "+ENV['USER']+" user.")
     @log.error('You can get more information by running the script with --help option.')
   end
 
@@ -295,15 +295,14 @@ EOF
     end
     @type = ARGV.shift.downcase;
   end
-
   def force_ruby2x(ruby_interpreter_path)
     # change interpreter when symlink /usr/bin/ruby2.x exists, but running with non-supported ruby version
     actual_ruby_version = RUBY_VERSION.split('.').map{|s|s.to_i}
     left_bound = '2.0.0'.split('.').map{|s|s.to_i}
-    right_bound = '2.7.0'.split('.').map{|s|s.to_i}
+    right_bound = '3.0.0'.split('.').map{|s|s.to_i}
     if (actual_ruby_version <=> left_bound) < 0
       if(!@reexeced)
-        @log.info("The current Ruby version is not 2.x! Restarting the installer with #{ruby_interpreter_path}")
+        @log.info("The current Ruby version is not 2.x or 3.0.x! Restarting the installer with #{ruby_interpreter_path}")
         exec("#{ruby_interpreter_path}", __FILE__, '--re-execed' , *@args)
       else
         unsupported_ruby_version_error

codedeploy_agent.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = 'aws_codedeploy_agent'
-  spec.version       = '1.4.1'
+  spec.version       = '1.5.0'
   spec.summary       = 'Packages AWS CodeDeploy agent libraries'
   spec.description   = 'AWS CodeDeploy agent is responsible for doing the actual work of deploying software on an individual EC2 instance'
   spec.author        = 'Amazon Web Services'
@@ -17,7 +17,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency('rubyzip', '~> 1.3.0')
   spec.add_dependency('logging', '~> 1.8')
   spec.add_dependency('aws-sdk-core', '~> 3')
-  spec.add_dependency('aws-sdk-code-generator', '~> 0.2.2.pre')
   spec.add_dependency('aws-sdk-s3', '~> 1')
   spec.add_dependency('simple_pid', '~> 0.2.1')
   spec.add_dependency('docopt', '~> 0.5.0')

lib/instance_agent/plugins/codedeploy/command_acknowledgement_request_builder.rb

@@ -0,0 +1,47 @@
+module InstanceAgent; module Plugins; module CodeDeployPlugin
+class CommandAcknowledgementRequestBuilder
+  @@MIN_ACK_TIMEOUT = 60
+  @@MAX_ACK_TIMEOUT = 4200
+
+  def initialize(logger)
+    @logger = logger
+  end
+
+  def build(diagnostics, host_command_identifier, timeout)
+    result = build_default(diagnostics, host_command_identifier)
+    if timeout && timeout > 0
+      result[:host_command_max_duration_in_seconds] = correct_timeout(timeout)
+    end
+
+    result
+  end
+
+  private
+
+  def build_default(diagnostics, host_command_identifier)
+    {
+      :diagnostics => diagnostics,
+      :host_command_identifier => host_command_identifier
+    }
+  end
+
+  def correct_timeout(timeout)
+    result = timeout
+    if timeout < @@MIN_ACK_TIMEOUT
+      log(:info, "Command timeout of #{timeout} is below minimum value of #{@@MIN_ACK_TIMEOUT} " +
+        "seconds. Sending #{@@MIN_ACK_TIMEOUT} to the service instead.")
+      result = @@MIN_ACK_TIMEOUT
+    elsif timeout > @@MAX_ACK_TIMEOUT
+      log(:warn, "Command timeout of #{timeout} exceeds maximum accepted value #{@@MAX_ACK_TIMEOUT} " +
+        "seconds. Sending #{@@MAX_ACK_TIMEOUT} to the service instead. Commands may time out.")
+      result = @@MAX_ACK_TIMEOUT
+    end
+
+    result
+  end
+
+  def log(severity, message)
+    raise ArgumentError, "Unknown severity #{severity.inspect}" unless InstanceAgent::Log::SEVERITIES.include?(severity.to_s)
+    @logger.send(severity.to_sym, "#{self.class.to_s}: #{message}")
+  end
+end end end end

lib/instance_agent/plugins/codedeploy/command_executor.rb

@@ -79,6 +79,28 @@ def is_command_noop?(command_name, deployment_spec)
           return true
         end
 
+        def total_timeout_for_all_lifecycle_events(command_name, deployment_spec)
+          parsed_spec = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentSpecification.parse(deployment_spec)
+          timeout_sums = ((@hook_mapping || {command_name => []})[command_name] || []).map do |lifecycle_event|
+            create_hook_executor(lifecycle_event, parsed_spec).total_timeout_for_all_scripts
+          end
+
+          total_timeout = nil
+          if timeout_sums.empty?
+            log(:info, "Command #{command_name} has no script timeouts specified in appspec.")
+          # If any lifecycle events' scripts don't specify a timeout, don't set a value.
+          # The default will be the maximum at the server.
+          elsif timeout_sums.include?(nil)
+            log(:info, "Command #{command_name} has at least one script that does not specify a timeout. " +
+              "No timeout override will be sent.")
+          else
+            total_timeout = timeout_sums.reduce(0) {|running_sum, item| running_sum + item}
+            log(:info, "Command #{command_name} has total script timeout #{total_timeout} in appspec.")
+          end
+
+          total_timeout
+        end
+
         def execute_command(command, deployment_specification)
           method_name = command_method(command.command_name)
           log(:debug, "Command #{command.command_name} maps to method #{method_name}")

lib/instance_agent/plugins/codedeploy/command_poller.rb

@@ -67,6 +67,29 @@ def validate
           end
         end
 
+        # Called during initialization of the child process
+        def recover_from_crash?
+          begin
+            if DeploymentCommandTracker.check_deployment_event_inprogress?() then
+              log(:warn, "Deployment tracking file found: #{DeploymentCommandTracker.deployment_dir_path()}. The agent likely restarted while running a customer-supplied script. Failing the lifecycle event.")
+              host_command_identifier = DeploymentCommandTracker.most_recent_host_command_identifier()
+
+              log(:info, "Calling PutHostCommandComplete: 'Failed' #{host_command_identifier}")
+              @deploy_control_client.put_host_command_complete(
+                :command_status => "Failed",
+                :diagnostics => {:format => "JSON", :payload => gather_diagnostics_from_failure_after_restart("Failing in-progress lifecycle event after an agent restart.")},
+                :host_command_identifier => host_command_identifier)
+
+              DeploymentCommandTracker.clean_ongoing_deployment_dir()
+              return true
+            end
+            # We want to catch-all exceptions so that the child process always can startup succesfully.
+          rescue Exception => e
+            log(:error, "Exception thrown during restart recovery: #{e}")
+            return nil
+          end
+        end
+
         def perform
           return unless command = next_command
 
@@ -109,7 +132,7 @@ def process_command(command, spec)
           log(:debug, "Calling #{@plugin.to_s}.execute_command")
           begin
             deployment_id = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentSpecification.parse(spec).deployment_id
-            DeploymentCommandTracker.create_ongoing_deployment_tracking_file(deployment_id)
+            DeploymentCommandTracker.create_ongoing_deployment_tracking_file(deployment_id, command.host_command_identifier)
             #Successful commands will complete without raising an exception
             @plugin.execute_command(command, spec)
 
@@ -232,6 +255,16 @@ def gather_diagnostics_from_error(error)
           gather_diagnostics_from_script_error(script_error)
         end
 
+        private
+        def gather_diagnostics_from_failure_after_restart(msg = "")
+          begin
+            raise ScriptError.new(ScriptError::FAILED_AFTER_RESTART_CODE, "", ScriptLog.new), "Failed: #{msg}"
+          rescue ScriptError => e
+            script_error = e
+          end
+          gather_diagnostics_from_script_error(script_error)
+        end
+
         private
         def gather_diagnostics(msg = "")
           begin

lib/instance_agent/plugins/codedeploy/deployment_command_tracker.rb

@@ -13,21 +13,21 @@ class FileDoesntExistException < Exception; end
         class DeploymentCommandTracker 
             DEPLOYMENT_EVENT_FILE_STALE_TIMELIMIT_SECONDS = 86400 # 24 hour limit in secounds
 
-            def self.create_ongoing_deployment_tracking_file(deployment_id)
+            def self.create_ongoing_deployment_tracking_file(deployment_id, host_command_identifier)
               FileUtils.mkdir_p(deployment_dir_path())
-              FileUtils.touch(deployment_event_tracking_file_path(deployment_id));    
+              File.write(deployment_event_tracking_file_path(deployment_id), host_command_identifier)
             end
 
             def self.delete_deployment_tracking_file_if_stale?(deployment_id, timeout)
-              if(Time.now - File.ctime(deployment_event_tracking_file_path(deployment_id)) > timeout)
+              if(Time.now - File.mtime(deployment_event_tracking_file_path(deployment_id)) > timeout)
                 delete_deployment_command_tracking_file(deployment_id)
                 return true;
               end
               return false;
             end
 
             def self.check_deployment_event_inprogress?
-              if(File.exists?deployment_dir_path())
+              if(File.exist?(deployment_dir_path()))
                 return directories_and_files_inside(deployment_dir_path()).any?{|deployment_id| check_if_lifecycle_event_is_stale?(deployment_id)}
               else
                 return false
@@ -36,7 +36,7 @@ def self.check_deployment_event_inprogress?
 
             def self.delete_deployment_command_tracking_file(deployment_id)
               ongoing_deployment_event_file_path = deployment_event_tracking_file_path(deployment_id)
-                if File.exists?ongoing_deployment_event_file_path
+              if File.exist?(ongoing_deployment_event_file_path)
                     File.delete(ongoing_deployment_event_file_path);
                 else
                     InstanceAgent::Log.warn("the tracking file does not exist")
@@ -46,8 +46,18 @@ def self.delete_deployment_command_tracking_file(deployment_id)
             def self.directories_and_files_inside(directory)
               Dir.entries(directory) - %w(.. .)
             end
-
-            private
+
+            def self.most_recent_host_command_identifier
+              # check_deployment_event_inprogress handles deleting stale files for us.
+              if check_deployment_event_inprogress? then
+                most_recent_id = directories_and_files_inside(deployment_dir_path()).max_by{ |filename| File.mtime(deployment_event_tracking_file_path(filename)) }
+                most_recent_file = deployment_event_tracking_file_path(most_recent_id)
+                return File.read(most_recent_file)
+              else
+                return nil
+              end
+            end
+
             def self.deployment_dir_path
               File.join(InstanceAgent::Config.config[:root_dir], InstanceAgent::Config.config[:ongoing_deployment_tracking])
             end
@@ -57,8 +67,12 @@ def self.check_if_lifecycle_event_is_stale?(deployment_id)
             end
 
             def self.deployment_event_tracking_file_path(deployment_id)
-              ongoing_deployment_file_path = File.join(deployment_dir_path(), deployment_id)
-            end  
+              return File.join(deployment_dir_path(), deployment_id)
+            end
+
+            def self.clean_ongoing_deployment_dir
+              FileUtils.rm_r(InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.deployment_dir_path()) rescue Errno::ENOENT
+            end
           end
         end
     end          

lib/instance_agent/plugins/codedeploy/hook_executor.rb

@@ -48,6 +48,8 @@ class ScriptError < StandardError
         SCRIPT_FAILED_CODE = 4
         UNKNOWN_ERROR_CODE = 5
         OUTPUTS_LEFT_OPEN_CODE = 6
+        FAILED_AFTER_RESTART_CODE = 7
+
         def initialize(error_code, script_name, log)
           @error_code = error_code
           @script_name = script_name
@@ -113,6 +115,12 @@ def is_noop?
           return @app_spec.nil? || @app_spec.hooks[@lifecycle_event].nil? || @app_spec.hooks[@lifecycle_event].empty?
         end
 
+        def total_timeout_for_all_scripts
+          return nil if is_noop?
+          timeouts = @app_spec.hooks[@lifecycle_event].map {|script| script.timeout}
+          timeouts.reduce(0) {|running_sum, item| running_sum + item}
+        end
+
         def execute
           return if @app_spec.nil?
           if (hooks = @app_spec.hooks[@lifecycle_event]) &&

lib/instance_agent/runner/child.rb

@@ -8,6 +8,8 @@ class Child < ProcessManager::Daemon::Child
 
       attr_accessor :runner
 
+      @prepare_run_done = false
+
       def load_plugins(plugins)
         ProcessManager::Log.debug("Registering Plugins: #{plugins.inspect}.")
         plugins.each do |plugin|
@@ -31,24 +33,30 @@ def prepare_run
         with_error_handling do
           @runner = @plugins[index].runner
           ProcessManager.set_program_name(description)
+          @runner.recover_from_crash?()
         end
+
+        @prepare_run_done = true
       end
 
       def run
         with_error_handling do
           runner.run
         end
       end
-      
+
       # Stops the master after recieving the kill signal
-      # is overriden from ProcessManager::Daemon::Child 
+      # is overriden from ProcessManager::Daemon::Child
       def stop
-        @runner.graceful_shutdown
+        if @prepare_run_done
+          @runner.graceful_shutdown
+        end
+
         ProcessManager::Log.info('agent exiting now')
         super
       end
 
-      # Catches the trap signals and does a default or custom action 
+      # Catches the trap signals and does a default or custom action
       # is overriden from ProcessManager::Daemon::Child
       def trap_signals
         [:INT, :QUIT, :TERM].each do |sig|

lib/winagent.rb

@@ -41,6 +41,7 @@ def service_main
         begin
           @polling_mutex.synchronize do
             @runner ||= InstanceAgent::Plugins::CodeDeployPlugin::CommandPoller.runner
+            @runner.recover_from_crash?
             @runner.run
           end
         rescue SystemExit