diff --git a/README.md b/README.md index 23fc8ad61..95042cd9f 100644 --- a/README.md +++ b/README.md @@ -677,9 +677,11 @@ That'll post a line like follows to a preconfigured chatbot in Basecamp: [My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de ``` -### Custom healthcheck +### Healthcheck -MRSK defaults to checking the health of your application again `/up` on port 3000 up to 7 times. You can tailor the behaviour with the `healthcheck` setting: +MRSK uses Docker healtchecks to check the health of your application during deployment. Traefik uses this same healthcheck status to determine when a container is ready to receive traffic. + +The healthcheck defaults to testing the HTTP response to the path `/up` on port 3000, up to 7 times. You can tailor this behaviour with the `healthcheck` setting: ```yaml healthcheck: @@ -690,7 +692,29 @@ healthcheck: This will ensure your application is configured with a traefik label for the healthcheck against `/healthz` and that the pre-deploy healthcheck that MRSK performs is done against the same path on port 4000. -The healthcheck also allows for an optional `max_attempts` setting, which will attempt the healthcheck up to the specified number of times before failing the deploy. This is useful for applications that take a while to start up. The default is 7. +You can also specify a custom healthcheck command, which is useful for non-HTTP services: + +```yaml +healthcheck: + cmd: /bin/check_health +``` + +The top-level healthcheck configuration applies to all services that use +Traefik, by default. You can also specialize the configuration at the role +level: + +```yaml +servers: + job: + hosts: ... + cmd: bin/jobs + healthcheck: + cmd: bin/check +``` + +The healthcheck allows for an optional `max_attempts` setting, which will attempt the healthcheck up to the specified number of times before failing the deploy. This is useful for applications that take a while to start up. The default is 7. + +Note that the HTTP health checks assume that the `curl` command is avilable inside the container. If that's not the case, use the healthcheck's `cmd` option to specify an alternative check that the container supports. ## Commands diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index 882c80c8d..6e4fd496c 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -6,8 +6,6 @@ def boot using_version(version_or_latest) do |version| say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta - cli = self - on(MRSK.hosts) do execute *MRSK.auditor.record("Tagging #{MRSK.config.absolute_image} as the latest image"), verbosity: :debug execute *MRSK.app.tag_current_as_latest @@ -17,19 +15,24 @@ def boot roles = MRSK.roles_on(host) roles.each do |role| - execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug + app = MRSK.app(role: role) + auditor = MRSK.auditor(role: role) + + execute *auditor.record("Booted app version #{version}"), verbosity: :debug - if capture_with_info(*MRSK.app(role: role).container_id_for_version(version), raise_on_non_zero_exit: false).present? + if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present? tmp_version = "#{version}_#{SecureRandom.hex(8)}" info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}" - execute *MRSK.auditor(role: role).record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug - execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version) + execute *auditor.record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug + execute *app.rename_container(version: version, new_version: tmp_version) end - old_version = capture_with_info(*MRSK.app(role: role).current_running_version, raise_on_non_zero_exit: false).strip - execute *MRSK.app(role: role).run - sleep MRSK.config.readiness_delay - execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present? + old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip + execute *app.run + + Mrsk::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) } + + execute *app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present? end end end diff --git a/lib/mrsk/cli/healthcheck.rb b/lib/mrsk/cli/healthcheck.rb index 5e9f42dca..81677d629 100644 --- a/lib/mrsk/cli/healthcheck.rb +++ b/lib/mrsk/cli/healthcheck.rb @@ -1,7 +1,4 @@ class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base - - class HealthcheckError < StandardError; end - default_command :perform desc "perform", "Health check current app version" @@ -9,38 +6,10 @@ def perform on(MRSK.primary_host) do begin execute *MRSK.healthcheck.run - - target = "Health check against #{MRSK.config.healthcheck["path"]}" - attempt = 1 - max_attempts = MRSK.config.healthcheck["max_attempts"] - - begin - status = capture_with_info(*MRSK.healthcheck.curl) - - if status == "200" - info "#{target} succeeded with 200 OK!" - else - raise HealthcheckError, "#{target} failed with status #{status}" - end - rescue SSHKit::Command::Failed - if attempt <= max_attempts - info "#{target} failed to respond, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..." - sleep attempt - attempt += 1 - - retry - else - raise - end - end - rescue SSHKit::Command::Failed, HealthcheckError => e + Mrsk::Utils::HealthcheckPoller.wait_for_healthy { capture_with_info(*MRSK.healthcheck.status) } + rescue Mrsk::Utils::HealthcheckPoller::HealthcheckError => e error capture_with_info(*MRSK.healthcheck.logs) - - if e.message =~ /curl/ - raise SSHKit::Command::Failed, "#{target} failed to return 200 OK!" - else - raise - end + raise ensure execute *MRSK.healthcheck.stop, raise_on_non_zero_exit: false execute *MRSK.healthcheck.remove, raise_on_non_zero_exit: false diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb index 08e6795df..2d1cc132f 100644 --- a/lib/mrsk/commands/app.rb +++ b/lib/mrsk/commands/app.rb @@ -15,6 +15,7 @@ def run "--name", container_name, "-e", "MRSK_CONTAINER_NAME=\"#{container_name}\"", *role.env_args, + *role.health_check_args, *config.logging_args, *config.volume_args, *role.label_args, @@ -27,6 +28,10 @@ def start docker :start, container_name end + def status(version:) + pipe container_id_for_version(version), xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT)) + end + def stop(version: nil) pipe \ version ? container_id_for_version(version) : current_running_container_id, diff --git a/lib/mrsk/commands/base.rb b/lib/mrsk/commands/base.rb index 25ca00752..49911a049 100644 --- a/lib/mrsk/commands/base.rb +++ b/lib/mrsk/commands/base.rb @@ -2,6 +2,8 @@ module Mrsk::Commands class Base delegate :sensitive, :argumentize, to: Mrsk::Utils + DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'" + attr_accessor :config def initialize(config) diff --git a/lib/mrsk/commands/healthcheck.rb b/lib/mrsk/commands/healthcheck.rb index 1d467033b..14932f3ac 100644 --- a/lib/mrsk/commands/healthcheck.rb +++ b/lib/mrsk/commands/healthcheck.rb @@ -11,14 +11,15 @@ def run "--label", "service=#{container_name}", "-e", "MRSK_CONTAINER_NAME=\"#{container_name}\"", *web.env_args, + *web.health_check_args, *config.volume_args, *web.option_args, config.absolute_image, web.cmd end - def curl - [ :curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--max-time", "2", health_url ] + def status + pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT)) end def logs diff --git a/lib/mrsk/configuration/role.rb b/lib/mrsk/configuration/role.rb index 6bca4fb82..881b331e7 100644 --- a/lib/mrsk/configuration/role.rb +++ b/lib/mrsk/configuration/role.rb @@ -35,6 +35,21 @@ def env_args argumentize_env_with_secrets env end + def health_check_args + if health_check_cmd.present? + optionize({ "health-cmd" => health_check_cmd, "health-interval" => "1s" }) + else + [] + end + end + + def health_check_cmd + options = specializations["healthcheck"] || {} + options = config.healthcheck.merge(options) if running_traefik? + + options["cmd"] || http_health_check(port: options["port"], path: options["path"]) + end + def cmd specializations["cmd"] end @@ -75,8 +90,6 @@ def traefik_labels if running_traefik? { "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)", - "traefik.http.services.#{traefik_service}.loadbalancer.healthcheck.path" => config.healthcheck["path"], - "traefik.http.services.#{traefik_service}.loadbalancer.healthcheck.interval" => "1s", "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5", "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms", "traefik.http.routers.#{traefik_service}.middlewares" => "#{traefik_service}-retry@docker" @@ -125,4 +138,8 @@ def merged_env_with_secrets new_env["clear"] = (clear_app_env + clear_role_env).uniq end end + + def http_health_check(port:, path:) + "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present? + end end diff --git a/lib/mrsk/utils/healthcheck_poller.rb b/lib/mrsk/utils/healthcheck_poller.rb new file mode 100644 index 000000000..d7b8be65d --- /dev/null +++ b/lib/mrsk/utils/healthcheck_poller.rb @@ -0,0 +1,39 @@ +class Mrsk::Utils::HealthcheckPoller + TRAEFIK_HEALTHY_DELAY = 1 + + class HealthcheckError < StandardError; end + + class << self + def wait_for_healthy(pause_after_ready: false, &block) + attempt = 1 + max_attempts = MRSK.config.healthcheck["max_attempts"] + + begin + case status = block.call + when "healthy" + sleep TRAEFIK_HEALTHY_DELAY if pause_after_ready + when "running" # No health check configured + sleep MRSK.config.readiness_delay if pause_after_ready + else + raise HealthcheckError, "container not ready (#{status})" + end + rescue HealthcheckError => e + if attempt <= max_attempts + info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..." + sleep attempt + attempt += 1 + retry + else + raise + end + end + + info "Container is healthy!" + end + + private + def info(message) + SSHKit.config.output.info(message) + end + end +end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 78b166d4c..54292d65a 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -2,8 +2,11 @@ class CliAppTest < CliTestCase test "boot" do - # Stub current version fetch - SSHKit::Backend::Abstract.any_instance.stubs(:capture).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("running") # health check run_command("boot").tap do |output| assert_match "docker tag dhh/app:latest dhh/app:latest", output @@ -19,6 +22,10 @@ class CliAppTest < CliTestCase .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false) .returns("12345678") # running version + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("running") # health check + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false) .returns("123") # old version diff --git a/test/cli/healthcheck_test.rb b/test/cli/healthcheck_test.rb index a3441a7ed..a3577aa4e 100644 --- a/test/cli/healthcheck_test.rb +++ b/test/cli/healthcheck_test.rb @@ -5,62 +5,58 @@ class CliHealthcheckTest < CliTestCase # Prevent expected failures from outputting to terminal Thread.report_on_exception = false - SSHKit::Backend::Abstract.any_instance.stubs(:sleep) # No sleeping when retrying + Mrsk::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying + SSHKit::Backend::Abstract.any_instance.stubs(:execute) .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false) SSHKit::Backend::Abstract.any_instance.stubs(:execute) - .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "MRSK_CONTAINER_NAME=\"healthcheck-app\"", "dhh/app:999") + .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "MRSK_CONTAINER_NAME=\"healthcheck-app\"", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999") SSHKit::Backend::Abstract.any_instance.stubs(:execute) .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false) # Fail twice to test retry logic SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) - .with(:curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--max-time", "2", "http://localhost:3999/up") - .raises(SSHKit::Command::Failed) + .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("starting") .then - .raises(SSHKit::Command::Failed) + .returns("unhealthy") .then - .returns("200") + .returns("healthy") run_command("perform").tap do |output| - assert_match "Health check against /up failed to respond, retrying in 1s (attempt 1/7)...", output - assert_match "Health check against /up failed to respond, retrying in 2s (attempt 2/7)...", output - assert_match "Health check against /up succeeded with 200 OK!", output + assert_match "container not ready (starting), retrying in 1s (attempt 1/7)...", output + assert_match "container not ready (unhealthy), retrying in 2s (attempt 2/7)...", output + assert_match "Container is healthy!", output end end - test "perform failing because of curl" do + test "perform failing to become healthy" do # Prevent expected failures from outputting to terminal Thread.report_on_exception = false - SSHKit::Backend::Abstract.any_instance.stubs(:execute) # No need to execute anything here - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) - .with(:curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--max-time", "2", "http://localhost:3999/up") - .returns("curl: command not found") - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :logs, "--tail", 50, "2>&1") - - exception = assert_raises SSHKit::Runner::ExecuteError do - run_command("perform") - end - assert_match "Health check against /up failed to return 200 OK!", exception.message - end + Mrsk::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying - test "perform failing for unknown reason" do - # Prevent expected failures from outputting to terminal - Thread.report_on_exception = false + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false) + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "MRSK_CONTAINER_NAME=\"healthcheck-app\"", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999") + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false) - SSHKit::Backend::Abstract.any_instance.stubs(:execute) # No need to execute anything here + # Continually report unhealthy SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) - .with(:curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--max-time", "2", "http://localhost:3999/up") - .returns("500") + .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("unhealthy") + + # Capture logs when failing SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :logs, "--tail", 50, "2>&1") + .returns("some log output") exception = assert_raises do run_command("perform") end - assert_match "Health check against /up failed with status 500", exception.message + assert_match "container not ready (unhealthy)", exception.message end private diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 87afe3f1c..f74b21c96 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -13,7 +13,7 @@ class CommandsAppTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app-web.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app-web.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -21,7 +21,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:volumes] = ["/local/path:/container/path" ] assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app-web.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app-web.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -29,7 +29,23 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:healthcheck] = { "path" => "/healthz" } assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app-web.loadbalancer.healthcheck.path=\"/healthz\" --label traefik.http.services.app-web.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + new_command.run.join(" ") + end + + test "run with custom healthcheck command" do + @config[:healthcheck] = { "cmd" => "/bin/up" } + + assert_equal \ + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + new_command.run.join(" ") + end + + test "run with role-specific healthcheck options" do + @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } } + + assert_equal \ + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -44,7 +60,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app-web.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app-web.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end diff --git a/test/commands/healthcheck_test.rb b/test/commands/healthcheck_test.rb index cc44e5029..68387487d 100644 --- a/test/commands/healthcheck_test.rb +++ b/test/commands/healthcheck_test.rb @@ -10,7 +10,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" dhh/app:123", + "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123", new_command.run.join(" ") end @@ -18,7 +18,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase @config[:healthcheck] = { "port" => 3001 } assert_equal \ - "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" dhh/app:123", + "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3001/up || exit 1\" --health-interval \"1s\" dhh/app:123", new_command.run.join(" ") end @@ -26,29 +26,29 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase @destination = "staging" assert_equal \ - "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e MRSK_CONTAINER_NAME=\"healthcheck-app-staging\" dhh/app:123", + "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e MRSK_CONTAINER_NAME=\"healthcheck-app-staging\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123", new_command.run.join(" ") end - test "run with custom options" do - @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } } + test "run with custom healthcheck" do + @config[:healthcheck] = { "cmd" => "/bin/up" } + assert_equal \ - "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" --mount \"somewhere\" dhh/app:123", + "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"/bin/up\" --health-interval \"1s\" dhh/app:123", new_command.run.join(" ") end - test "curl" do + test "run with custom options" do + @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } } assert_equal \ - "curl --silent --output /dev/null --write-out '%{http_code}' --max-time 2 http://localhost:3999/up", - new_command.curl.join(" ") + "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123", + new_command.run.join(" ") end - test "curl with custom path" do - @config[:healthcheck] = { "path" => "/healthz" } - + test "status" do assert_equal \ - "curl --silent --output /dev/null --write-out '%{http_code}' --max-time 2 http://localhost:3999/healthz", - new_command.curl.join(" ") + "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'", + new_command.status.join(" ") end test "stop" do diff --git a/test/configuration/role_test.rb b/test/configuration/role_test.rb index 20ea39efa..f3814066a 100644 --- a/test/configuration/role_test.rb +++ b/test/configuration/role_test.rb @@ -42,7 +42,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase end test "special label args for web" do - assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.services.app-web.loadbalancer.healthcheck.path=\"/up\"", "--label", "traefik.http.services.app-web.loadbalancer.healthcheck.interval=\"1s\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args + assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args end test "custom labels" do @@ -57,8 +57,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase end test "overwriting default traefik label" do - @deploy[:labels] = { "traefik.http.routers.app.rule" => "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"" } - assert_equal "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"", @config.role(:web).labels["traefik.http.routers.app.rule"] + @deploy[:labels] = { "traefik.http.routers.app-web.rule" => "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"" } + assert_equal "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"", @config.role(:web).labels["traefik.http.routers.app-web.rule"] end test "default traefik label on non-web role" do @@ -66,15 +66,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] } }) - assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.services.app-beta.loadbalancer.healthcheck.path=\"/up\"", "--label", "traefik.http.services.app-beta.loadbalancer.healthcheck.interval=\"1s\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args - end - - test "default traefik label for non-web role with destination" do - config = Mrsk::Configuration.new(@deploy_with_roles.tap { |c| - c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] } - }, destination: "staging") - - assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "destination=\"staging\"", "--label", "traefik.http.routers.app-beta-staging.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.services.app-beta-staging.loadbalancer.healthcheck.path=\"/up\"", "--label", "traefik.http.services.app-beta-staging.loadbalancer.healthcheck.interval=\"1s\"", "--label", "traefik.http.middlewares.app-beta-staging-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-staging-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta-staging.middlewares=\"app-beta-staging-retry@docker\"" ], config.role(:beta).label_args + assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args end test "env overwritten by role" do