From 4e751a2c16a389725189b612235dab2d311eb8b4 Mon Sep 17 00:00:00 2001 From: JonJagger Date: Mon, 4 Nov 2024 12:23:44 +0000 Subject: [PATCH 1/2] Simplify generating and gathering test coverage metrics --- .github/workflows/main.yml | 6 +- .rubocop.yml | 101 ++++++++++++------ Dockerfile | 2 +- Makefile | 2 +- bin/check_coverage.sh | 36 ++++--- bin/lib.sh | 15 ++- bin/run_tests.sh | 27 +++-- source/client/Dockerfile | 2 +- test/client/lib/check_test_metrics.rb | 62 +++++++++++ test/client/lib/coverage.rb | 12 ++- test/client/lib/id58_test_base.rb | 22 ++-- ...ov_json.rb => simplecov_formatter_json.rb} | 2 +- test/client/lib/slim_json_reporter.rb | 20 ++++ test/client/metrics.rb | 35 ------ test/server/lib/check_test_metrics.rb | 62 +++++++++++ test/server/lib/coverage.rb | 12 ++- test/server/lib/id58_test_base.rb | 23 ++-- ...ov_json.rb => simplecov_formatter_json.rb} | 2 +- test/server/lib/slim_json_reporter.rb | 20 ++++ test/server/metrics.rb | 35 ------ 20 files changed, 319 insertions(+), 179 deletions(-) create mode 100644 test/client/lib/check_test_metrics.rb rename test/client/lib/{simplecov_json.rb => simplecov_formatter_json.rb} (97%) create mode 100644 test/client/lib/slim_json_reporter.rb delete mode 100644 test/client/metrics.rb create mode 100644 test/server/lib/check_test_metrics.rb rename test/server/lib/{simplecov_json.rb => simplecov_formatter_json.rb} (97%) create mode 100644 test/server/lib/slim_json_reporter.rb delete mode 100644 test/server/metrics.rb diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 193facce..8739ea90 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -281,7 +281,7 @@ jobs: kosli attest generic \ --description="unit-test branch-coverage and metrics" \ --name=differ.unit-test-coverage \ - --user-data=./reports/server/coverage.json + --user-data=./reports/server/coverage_metrics.json integration-tests: @@ -304,7 +304,7 @@ jobs: - name: Run integration tests run: - make test_client + make image_client test_client - name: Get integration test coverage id: coverage @@ -335,7 +335,7 @@ jobs: kosli attest generic \ --description="integration-test branch-coverage and metrics" \ --name=differ.integration-test-coverage \ - --user-data=./reports/client/coverage.json + --user-data=./reports/client/coverage_metrics.json snyk-container-scan: diff --git a/.rubocop.yml b/.rubocop.yml index db76eac2..108a3c93 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,35 +1,46 @@ AllCops: NewCops: enable -Metrics/BlockLength: +Layout/EmptyLineAfterMagicComment: Exclude: - - 'test/**/*.rb' - - source/server/app_base.rb + - test/*/lib/id58_test_base.rb -Metrics/MethodLength: +Layout/ExtraSpacing: Exclude: - - 'test/**/*.rb' - - source/client/html_demo.rb - - source/server/git_diff_parser.rb - - source/server/external/sheller.rb - - source/server/differ.rb - - source/client/http_json_hash/service_error.rb + - test/*/lib/check_test_metrics.rb -Style/Documentation: +Layout/LineLength: Exclude: - - '*/**/*.rb' + - test/*/lib/simplecov_formatter_json.rb -Metrics/CyclomaticComplexity: +Layout/SpaceInsideArrayLiteralBrackets: + Exclude: + - test/*/lib/check_test_metrics.rb + +Layout/SpaceInsideReferenceBrackets: + Exclude: + - test/*/lib/check_test_metrics.rb + +Lint/RescueException: Exclude: + - source/client/client.rb + +Lint/SymbolConversion: + Exclude: + - source/server/app_base.rb - test/*/lib/id58_test_base.rb - - source/server/git_diff_parser.rb Metrics/AbcSize: Exclude: - - test/*/lib/simplecov_json.rb - test/*/lib/id58_test_base.rb - source/server/git_diff_parser.rb + - test/*/lib/simplecov_formatter_json.rb + - test/*/lib/check_test_metrics.rb +Metrics/BlockLength: + Exclude: + - test/**/*.rb + - source/server/app_base.rb Metrics/ClassLength: Exclude: @@ -40,41 +51,61 @@ Metrics/ClassLength: - test/server/diff_summary_test.rb - source/server/git_diff_parser.rb -Style/FormatStringToken: +Metrics/CyclomaticComplexity: Exclude: - test/*/lib/id58_test_base.rb + - source/server/git_diff_parser.rb -Layout/LineLength: +Metrics/MethodLength: + Exclude: + - test/**/*.rb + - source/client/html_demo.rb + - source/server/git_diff_parser.rb + - source/server/external/sheller.rb + - source/server/differ.rb + - source/client/http_json_hash/service_error.rb + +Metrics/ParameterLists: + Exclude: + - test/server/saver.rb + - source/client/saver.rb + +Naming/VariableNumber: Exclude: - - test/*/lib/simplecov_json.rb + - source/server/external/gitter.rb + +Security/Eval: + Exclude: + - test/*/lib/check_test_metrics.rb Style/ClassVars: Exclude: - test/*/lib/id58_test_base.rb -Naming/VariableNumber: +Style/ClassAndModuleChildren: Exclude: - - source/server/external/gitter.rb + - test/*/lib/slim_json_reporter.rb -Metrics/ParameterLists: +Style/Documentation: Exclude: - - test/server/saver.rb - - source/client/saver.rb + - '*/**/*.rb' -Style/OpenStructUse: +Style/DocumentDynamicEvalDefinition: Exclude: - - test/server/prober_test.rb - - test/server/http_adapter_stub.rb - - test/client/http_response_unpacker_test.rb + - test/*/lib/check_test_metrics.rb -Lint/RescueException: +Style/EvalWithLocation: Exclude: - - source/client/client.rb + - test/*/lib/check_test_metrics.rb -Lint/SymbolConversion: +Style/FormatString: + Exclude: + - test/*/lib/check_test_metrics.rb + +Style/FormatStringToken: Exclude: - - source/server/app_base.rb - test/*/lib/id58_test_base.rb + - test/*/lib/check_test_metrics.rb Style/FetchEnvVar: Exclude: @@ -88,5 +119,13 @@ Style/HashTransformKeys: Exclude: - source/server/app_base.rb +Style/OpenStructUse: + Exclude: + - test/server/prober_test.rb + - test/server/http_adapter_stub.rb + - test/client/http_response_unpacker_test.rb +Style/TrailingCommaInArrayLiteral: + Exclude: + - test/*/lib/check_test_metrics.rb diff --git a/Dockerfile b/Dockerfile index 261ada14..f73c70b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM cyberdojo/sinatra-base:11ddc45 +FROM cyberdojo/sinatra-base:026c095 LABEL maintainer=jon@jaggersoft.com diff --git a/Makefile b/Makefile index b1e36f02..ca8305af 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ all_client: test_client coverage_client image_client: ${PWD}/bin/build_image.sh client -test_client: image_client +test_client: ${PWD}/bin/run_tests.sh client coverage_client: diff --git a/bin/check_coverage.sh b/bin/check_coverage.sh index 578777a6..eb15e06a 100755 --- a/bin/check_coverage.sh +++ b/bin/check_coverage.sh @@ -40,28 +40,34 @@ check_args() check_coverage() { - # Process test-run results and coverage data against metrics.rb values and - # - print output showing individual metrics and their pass/fail status - # - return zero if all metrics pass, otherwise non-zero - # Does not create any new files. - check_args "$@" + export $(echo_versioner_env_vars) + + local -r TYPE="${1}" # {server|client} + local -r TEST_LOG=test.log + local -r HOST_TEST_DIR="${ROOT_DIR}/test/${TYPE}" + local -r HOST_REPORTS_DIR="${ROOT_DIR}/reports/${TYPE}" # where report json files have been written to + local -r CONTAINER_TMP_DIR=/tmp - local -r TYPE="${1}" # client | server + exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/${TEST_LOG}" + exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/test_metrics.json" + exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/coverage_metrics.json" + exit_non_zero_unless_file_exists "${HOST_TEST_DIR}/lib/check_test_metrics.rb" set +e docker run \ + --read-only \ + --rm \ + --entrypoint="" \ + --env COVERAGE_ROOT="${CONTAINER_TMP_DIR}" \ --env COVERAGE_CODE_TAB_NAME=app \ --env COVERAGE_TEST_TAB_NAME=test \ - --rm \ - --volume "${ROOT_DIR}/reports/${TYPE}":/reports/:ro \ - --volume "${ROOT_DIR}/test/${TYPE}"/metrics.rb:/app/metrics.rb:ro \ - cyberdojo/check-test-results:latest \ - sh -c \ - "ruby /app/check_test_results.rb \ - /reports/test.log \ - /reports/index.html \ - /reports/coverage.json" + --volume ${HOST_REPORTS_DIR}/test_metrics.json:${CONTAINER_TMP_DIR}/test_metrics.json:ro \ + --volume ${HOST_REPORTS_DIR}/coverage_metrics.json:${CONTAINER_TMP_DIR}/coverage_metrics.json:ro \ + --volume ${HOST_TEST_DIR}/lib/check_test_metrics.rb:${CONTAINER_TMP_DIR}/check_test_metrics.rb:ro \ + "${CYBER_DOJO_DIFFER_IMAGE}:${CYBER_DOJO_DIFFER_TAG}" \ + sh -c "ruby ${CONTAINER_TMP_DIR}/check_test_metrics.rb" \ + | tee -a "${HOST_REPORTS_DIR}/${TEST_LOG}" local -r STATUS=${PIPESTATUS[0]} set -e diff --git a/bin/lib.sh b/bin/lib.sh index 8bd908dd..d93dcd90 100644 --- a/bin/lib.sh +++ b/bin/lib.sh @@ -25,15 +25,22 @@ stderr() >&2 echo "ERROR: ${message}" } +exit_non_zero_unless_file_exists() +{ + local -r filename="${1}" + if [ ! -f "${filename}" ]; then + stderr "${filename} does not exist" + exit 42 + fi +} + copy_in_saver_test_data() { - local -r SRC_PATH=${ROOT_DIR}/test/server/data/cyber-dojo local -r SAVER_CID=$(docker ps --filter status=running --format '{{.Names}}' | grep "saver") + local -r SRC_PATH=${ROOT_DIR}/test/server/data/cyber-dojo local -r DEST_PATH=/cyber-dojo # You cannot docker cp to a tmpfs, so tar-piping instead... - pushd "${SRC_PATH}" || exit 99 - tar -c . | docker exec -i "${SAVER_CID}" tar x -C ${DEST_PATH} - popd || exit 99 + tar --no-xattrs -c -C "${SRC_PATH}" - . | docker exec -i "${SAVER_CID}" tar x -C ${DEST_PATH} } containers_down() diff --git a/bin/run_tests.sh b/bin/run_tests.sh index 3db4924b..ccefce78 100755 --- a/bin/run_tests.sh +++ b/bin/run_tests.sh @@ -72,15 +72,14 @@ run_tests() copy_in_saver_test_data local -r TYPE="${1}" # {server|client} + local -r TEST_LOG=test.log + local -r CONTAINER_COVERAGE_DIR="/tmp/reports" + local -r HOST_REPORTS_DIR="${ROOT_DIR}/reports/${TYPE}" echo '==================================' echo "Running ${TYPE} tests" echo '==================================' - local -r COVERAGE_CODE_TAB_NAME=app - local -r COVERAGE_TEST_TAB_NAME=test - local -r TEST_LOG=test.log - # CONTAINER_NAME is running with read-only:true and a non-root user (in docker-compose.yml). I want to # keep those settings in the docker-exec call below since that is how I want the microservice to run. # The [docker exec run.sh] is creating coverage files which I process on the host after it completes. @@ -89,27 +88,27 @@ run_tests() # on both my M2 laptop, and in the CI workflow. So I am writing the coverage files to /tmp and # tar-piping them out. - local -r CONTAINER_COVERAGE_DIR="/tmp/reports" - set +e docker exec \ - --env COVERAGE_CODE_TAB_NAME=${COVERAGE_CODE_TAB_NAME} \ - --env COVERAGE_TEST_TAB_NAME=${COVERAGE_TEST_TAB_NAME} \ + --env COVERAGE_CODE_TAB_NAME=app \ + --env COVERAGE_TEST_TAB_NAME=test \ --user "${USER}" \ "${CONTAINER_NAME}" \ sh -c "/differ/test/lib/run.sh ${CONTAINER_COVERAGE_DIR} ${TEST_LOG} ${*:2}" local -r STATUS=$? set -e - local -r HOST_REPORTS_DIR="${ROOT_DIR}/reports/${TYPE}" - rm -rf "${HOST_REPORTS_DIR}" &> /dev/null || true mkdir -p "${HOST_REPORTS_DIR}" &> /dev/null || true - docker exec \ - "${CONTAINER_NAME}" \ - tar Ccf "${CONTAINER_COVERAGE_DIR}" - . \ - | tar Cxf "${HOST_REPORTS_DIR}" - + docker exec --user "${USER}" "${CONTAINER_NAME}" tar Ccf "${CONTAINER_COVERAGE_DIR}" - . \ + | tar Cxf "${HOST_REPORTS_DIR}" - + + # Check we generated the expected files. + exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/${TEST_LOG}" + exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/index.html" + exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/test_metrics.json" + exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/coverage_metrics.json" echo "${TYPE} test branch-coverage report is at:" echo "${HOST_REPORTS_DIR}/index.html" diff --git a/source/client/Dockerfile b/source/client/Dockerfile index b42fe2a9..b762408b 100644 --- a/source/client/Dockerfile +++ b/source/client/Dockerfile @@ -1,4 +1,4 @@ -FROM cyberdojo/sinatra-base:11ddc45 +FROM cyberdojo/sinatra-base:026c095 LABEL maintainer=jon@jaggersoft.com WORKDIR /differ diff --git a/test/client/lib/check_test_metrics.rb b/test/client/lib/check_test_metrics.rb new file mode 100644 index 00000000..10edf1a0 --- /dev/null +++ b/test/client/lib/check_test_metrics.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Uses data from two json files: +# - reports/client/test_metrics.json generated in slim_json_reporter.rb by minitest. See id58_test_base.rb +# - reports/client/coverage_metrics.json generated in simplecov_formatter_json.rb by simplecov. See coverage.rb + +require 'json' + +def coloured(arg) + red = 31 + green = 32 + colourize(arg ? green : red, arg) +end + +def colourize(code, word) + "\e[#{code}m #{word} \e[0m" +end + +def table_data + cov_root = ENV.fetch('COVERAGE_ROOT') + stats = JSON.parse(File.read("#{cov_root}/test_metrics.json")) + + cov_json = JSON.parse(File.read("#{cov_root}/coverage_metrics.json")) + test_cov = cov_json['groups'][ENV.fetch('COVERAGE_TEST_TAB_NAME')] + code_cov = cov_json['groups'][ENV.fetch('COVERAGE_CODE_TAB_NAME')] + + [ + [ nil ], + [ 'test.count', stats['test_count'], '>=', 34 ], + [ 'test.duration', stats['total_time'], '<=', 50 ], + [ nil ], + [ 'test.failures', stats['failure_count'], '<=', 0 ], + [ 'test.errors', stats['error_count' ], '<=', 0 ], + [ 'test.skips', stats['skip_count' ], '<=', 0 ], + [ nil ], + [ 'test.lines.total', test_cov['lines' ]['total' ], '<=', 234 ], + [ 'test.lines.missed', test_cov['lines' ]['missed'], '<=', 0 ], + [ 'test.branches.total', test_cov['branches']['total' ], '<=', 0 ], + [ 'test.branches.missed', test_cov['branches']['missed'], '<=', 0 ], + [ nil ], + [ 'code.lines.total', code_cov['lines' ]['total' ], '<=', 101 ], + [ 'code.lines.missed', code_cov['lines' ]['missed'], '<=', 0 ], + [ 'code.branches.total', code_cov['branches']['total' ], '<=', 8 ], + [ 'code.branches.missed', code_cov['branches']['missed'], '<=', 0 ], + ] +end + +results = [] +table_data.each do |name, value, op, limit| + if name.nil? + puts + next + end + # puts "name=#{name}, value=#{value}, op=#{op}, limit=#{limit}" # debug + result = eval("#{value} #{op} #{limit}") + puts '%s | %s %s %s | %s' % [ + name.rjust(25), value.to_s.rjust(5), " #{op}", limit.to_s.rjust(5), coloured(result) + ] + results << result +end +puts +exit results.all? diff --git a/test/client/lib/coverage.rb b/test/client/lib/coverage.rb index 57e82ea3..a7fffa9c 100644 --- a/test/client/lib/coverage.rb +++ b/test/client/lib/coverage.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'simplecov' -require_relative 'simplecov_json' +require_relative 'simplecov_formatter_json' SimpleCov.start do enable_coverage :branch @@ -9,13 +9,15 @@ add_filter('test/lib/id58_test_base.rb') coverage_dir(ENV.fetch('COVERAGE_ROOT', nil)) # add_group('debug') { |src| puts src.filename; false } - test_tab = ENV.fetch('COVERAGE_TEST_TAB_NAME', nil) - code_tab = ENV.fetch('COVERAGE_CODE_TAB_NAME', nil) + test_tab = ENV.fetch('COVERAGE_TEST_TAB_NAME') + code_tab = ENV.fetch('COVERAGE_CODE_TAB_NAME') test_re = %r{^/differ/test} add_group(test_tab) { |src| src.filename =~ test_re } add_group(code_tab) { |src| src.filename !~ test_re } end -formatters = [SimpleCov::Formatter::HTMLFormatter, - SimpleCov::Formatter::JSONFormatter] +formatters = [ + SimpleCov::Formatter::HTMLFormatter, + SimpleCov::Formatter::JSONFormatter +] SimpleCov.formatters = SimpleCov::Formatter::MultiFormatter.new(formatters) diff --git a/test/client/lib/id58_test_base.rb b/test/client/lib/id58_test_base.rb index c076775d..ca6a6434 100644 --- a/test/client/lib/id58_test_base.rb +++ b/test/client/lib/id58_test_base.rb @@ -2,9 +2,15 @@ require 'English' require 'minitest/autorun' -require 'minitest/ci' +require 'minitest/reporters' +require_relative 'slim_json_reporter' -Minitest::Ci.report_dir = "#{ENV.fetch('COVERAGE_ROOT')}/junit" +reporters = [ + Minitest::Reporters::DefaultReporter.new, + Minitest::Reporters::SlimJsonReporter.new, + Minitest::Reporters::JUnitReporter.new("#{ENV.fetch('COVERAGE_ROOT')}/junit") +] +Minitest::Reporters.use!(reporters) def require_app(required) require_relative "../../#{required}" @@ -21,8 +27,6 @@ def initialize(arg) @@seen_ids = [] @@timings = {} - # - - - - - - - - - - - - - - - - - - - - - - - def self.test(id58_suffix, *lines, &test_block) src = test_block.source_location src_file = File.basename(src[0]) @@ -50,12 +54,10 @@ def self.test(id58_suffix, *lines, &test_block) define_method("test_\n#{name}".to_sym, &execute_around) end - # - - - - - - - - - - - - - - - - - - - - - - - Minitest.after_run do slow = @@timings.select { |_name, secs| secs > 0.000 } sorted = slow.sort_by { |_name, secs| -secs }.to_h - size = [sorted.size, 10].min + size = [sorted.size, 5].min puts puts 'Slowest tests are...' unless sorted.empty? sorted.each_with_index do |(name, secs), index| @@ -65,8 +67,6 @@ def self.test(id58_suffix, *lines, &test_block) puts end - # - - - - - - - - - - - - - - - - - - - - - - - ID58_ALPHABET = %w[ 0 1 2 3 4 5 6 7 8 9 A B C D E F G H J K L M N P Q R S T U V W X Y Z @@ -102,14 +102,10 @@ def self.checked_id58(id58_suffix, lines) id58 end - # - - - - - - - - - - - - - - - - - - - - - - - def id58_setup; end def id58_teardown; end - # - - - - - - - - - - - - - - - - - - - - - - - def id58 @_id58 end diff --git a/test/client/lib/simplecov_json.rb b/test/client/lib/simplecov_formatter_json.rb similarity index 97% rename from test/client/lib/simplecov_json.rb rename to test/client/lib/simplecov_formatter_json.rb index 6b349b64..3a521cea 100644 --- a/test/client/lib/simplecov_json.rb +++ b/test/client/lib/simplecov_formatter_json.rb @@ -41,7 +41,7 @@ def output_filepath end def output_filename - 'coverage.json' + 'coverage_metrics.json' end def output_message(result) diff --git a/test/client/lib/slim_json_reporter.rb b/test/client/lib/slim_json_reporter.rb new file mode 100644 index 00000000..110426f9 --- /dev/null +++ b/test/client/lib/slim_json_reporter.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'json' +require 'minitest/reporters' + +class Minitest::Reporters::SlimJsonReporter < Minitest::Reporters::BaseReporter + def report + super + filename = "#{ENV.fetch('COVERAGE_ROOT')}/test_metrics.json" + metrics = { + total_time: formar('%.2f', total_time), + assertion_count: assertions, + test_count: count, + failure_count: failures, + error_count: errors, + skip_count: skips + } + File.write(filename, JSON.pretty_generate(metrics)) + end +end diff --git a/test/client/metrics.rb b/test/client/metrics.rb deleted file mode 100644 index 3499b616..00000000 --- a/test/client/metrics.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -# max values used by cyberdojo/check-test-results image -# which is called from sh/test_in_containers.sh - -MAX = { - failures: 0, - errors: 0, - warnings: 0, - skips: 0, - - duration: 50, - - app: { - lines: { - total: 101, - missed: 0 - }, - branches: { - total: 8, - missed: 0 - } - }, - - test: { - lines: { - total: 226, - missed: 0 - }, - branches: { - total: 0, - missed: 0 - } - } -}.freeze diff --git a/test/server/lib/check_test_metrics.rb b/test/server/lib/check_test_metrics.rb new file mode 100644 index 00000000..bbdcff32 --- /dev/null +++ b/test/server/lib/check_test_metrics.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Uses data from two json files: +# - reports/server/test_metrics.json generated in slim_json_reporter.rb by minitest. See id58_test_base.rb +# - reports/server/coverage_metrics.json generated in simplecov_formatter_json.rb by simplecov. See coverage.rb + +require 'json' + +def coloured(arg) + red = 31 + green = 32 + colourize(arg ? green : red, arg) +end + +def colourize(code, word) + "\e[#{code}m #{word} \e[0m" +end + +def table_data + cov_root = ENV.fetch('COVERAGE_ROOT') + stats = JSON.parse(File.read("#{cov_root}/test_metrics.json")) + + cov_json = JSON.parse(File.read("#{cov_root}/coverage_metrics.json")) + test_cov = cov_json['groups'][ENV.fetch('COVERAGE_TEST_TAB_NAME')] + code_cov = cov_json['groups'][ENV.fetch('COVERAGE_CODE_TAB_NAME')] + + [ + [ nil ], + [ 'test.count', stats['test_count'], '>=', 73 ], + [ 'test.duration', stats['total_time'], '<=', 50 ], + [ nil ], + [ 'test.failures', stats['failure_count'], '<=', 0 ], + [ 'test.errors', stats['error_count' ], '<=', 0 ], + [ 'test.skips', stats['skip_count' ], '<=', 0 ], + [ nil ], + [ 'test.lines.total', test_cov['lines' ]['total' ], '<=', 525 ], + [ 'test.lines.missed', test_cov['lines' ]['missed'], '<=', 0 ], + [ 'test.branches.total', test_cov['branches']['total' ], '<=', 0 ], + [ 'test.branches.missed', test_cov['branches']['missed'], '<=', 0 ], + [ nil ], + [ 'code.lines.total', code_cov['lines' ]['total' ], '<=', 352 ], + [ 'code.lines.missed', code_cov['lines' ]['missed'], '<=', 0 ], + [ 'code.branches.total', code_cov['branches']['total' ], '<=', 60 ], + [ 'code.branches.missed', code_cov['branches']['missed'], '<=', 1 ], + ] +end + +results = [] +table_data.each do |name, value, op, limit| + if name.nil? + puts + next + end + # puts "name=#{name}, value=#{value}, op=#{op}, limit=#{limit}" # debug + result = eval("#{value} #{op} #{limit}") + puts '%s | %s %s %s | %s' % [ + name.rjust(25), value.to_s.rjust(5), " #{op}", limit.to_s.rjust(5), coloured(result) + ] + results << result +end +puts +exit results.all? diff --git a/test/server/lib/coverage.rb b/test/server/lib/coverage.rb index 57e82ea3..a7fffa9c 100644 --- a/test/server/lib/coverage.rb +++ b/test/server/lib/coverage.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'simplecov' -require_relative 'simplecov_json' +require_relative 'simplecov_formatter_json' SimpleCov.start do enable_coverage :branch @@ -9,13 +9,15 @@ add_filter('test/lib/id58_test_base.rb') coverage_dir(ENV.fetch('COVERAGE_ROOT', nil)) # add_group('debug') { |src| puts src.filename; false } - test_tab = ENV.fetch('COVERAGE_TEST_TAB_NAME', nil) - code_tab = ENV.fetch('COVERAGE_CODE_TAB_NAME', nil) + test_tab = ENV.fetch('COVERAGE_TEST_TAB_NAME') + code_tab = ENV.fetch('COVERAGE_CODE_TAB_NAME') test_re = %r{^/differ/test} add_group(test_tab) { |src| src.filename =~ test_re } add_group(code_tab) { |src| src.filename !~ test_re } end -formatters = [SimpleCov::Formatter::HTMLFormatter, - SimpleCov::Formatter::JSONFormatter] +formatters = [ + SimpleCov::Formatter::HTMLFormatter, + SimpleCov::Formatter::JSONFormatter +] SimpleCov.formatters = SimpleCov::Formatter::MultiFormatter.new(formatters) diff --git a/test/server/lib/id58_test_base.rb b/test/server/lib/id58_test_base.rb index c076775d..d4628db9 100644 --- a/test/server/lib/id58_test_base.rb +++ b/test/server/lib/id58_test_base.rb @@ -1,10 +1,15 @@ # frozen_string_literal: true - require 'English' require 'minitest/autorun' -require 'minitest/ci' +require 'minitest/reporters' +require_relative 'slim_json_reporter' -Minitest::Ci.report_dir = "#{ENV.fetch('COVERAGE_ROOT')}/junit" +reporters = [ + Minitest::Reporters::DefaultReporter.new, + Minitest::Reporters::SlimJsonReporter.new, + Minitest::Reporters::JUnitReporter.new("#{ENV.fetch('COVERAGE_ROOT')}/junit") +] +Minitest::Reporters.use!(reporters) def require_app(required) require_relative "../../#{required}" @@ -21,8 +26,6 @@ def initialize(arg) @@seen_ids = [] @@timings = {} - # - - - - - - - - - - - - - - - - - - - - - - - def self.test(id58_suffix, *lines, &test_block) src = test_block.source_location src_file = File.basename(src[0]) @@ -50,12 +53,10 @@ def self.test(id58_suffix, *lines, &test_block) define_method("test_\n#{name}".to_sym, &execute_around) end - # - - - - - - - - - - - - - - - - - - - - - - - Minitest.after_run do slow = @@timings.select { |_name, secs| secs > 0.000 } sorted = slow.sort_by { |_name, secs| -secs }.to_h - size = [sorted.size, 10].min + size = [sorted.size, 5].min puts puts 'Slowest tests are...' unless sorted.empty? sorted.each_with_index do |(name, secs), index| @@ -65,8 +66,6 @@ def self.test(id58_suffix, *lines, &test_block) puts end - # - - - - - - - - - - - - - - - - - - - - - - - ID58_ALPHABET = %w[ 0 1 2 3 4 5 6 7 8 9 A B C D E F G H J K L M N P Q R S T U V W X Y Z @@ -102,14 +101,10 @@ def self.checked_id58(id58_suffix, lines) id58 end - # - - - - - - - - - - - - - - - - - - - - - - - def id58_setup; end def id58_teardown; end - # - - - - - - - - - - - - - - - - - - - - - - - def id58 @_id58 end diff --git a/test/server/lib/simplecov_json.rb b/test/server/lib/simplecov_formatter_json.rb similarity index 97% rename from test/server/lib/simplecov_json.rb rename to test/server/lib/simplecov_formatter_json.rb index 6b349b64..3a521cea 100644 --- a/test/server/lib/simplecov_json.rb +++ b/test/server/lib/simplecov_formatter_json.rb @@ -41,7 +41,7 @@ def output_filepath end def output_filename - 'coverage.json' + 'coverage_metrics.json' end def output_message(result) diff --git a/test/server/lib/slim_json_reporter.rb b/test/server/lib/slim_json_reporter.rb new file mode 100644 index 00000000..dc5c9ada --- /dev/null +++ b/test/server/lib/slim_json_reporter.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'json' +require 'minitest/reporters' + +class Minitest::Reporters::SlimJsonReporter < Minitest::Reporters::BaseReporter + def report + super + filename = "#{ENV.fetch('COVERAGE_ROOT')}/test_metrics.json" + metrics = { + total_time: format('%.2f', total_time), + assertion_count: assertions, + test_count: count, + failure_count: failures, + error_count: errors, + skip_count: skips + } + File.write(filename, JSON.pretty_generate(metrics)) + end +end diff --git a/test/server/metrics.rb b/test/server/metrics.rb deleted file mode 100644 index d1348ff7..00000000 --- a/test/server/metrics.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -# max values used by cyberdojo/check-test-results image -# which is called from sh/coverage_in_container.sh - -MAX = { - failures: 0, - errors: 0, - warnings: 0, - skips: 0, - - duration: 50, - - app: { - lines: { - total: 352, - missed: 0 - }, - branches: { - total: 60, - missed: 1 - } - }, - - test: { - lines: { - total: 517, - missed: 0 - }, - branches: { - total: 0, - missed: 0 - } - } -}.freeze From 433119bc8845671257c57c6b8e4ff07b563138d8 Mon Sep 17 00:00:00 2001 From: JonJagger Date: Mon, 4 Nov 2024 12:27:05 +0000 Subject: [PATCH 2/2] Fix typo in method name --- test/client/lib/slim_json_reporter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/client/lib/slim_json_reporter.rb b/test/client/lib/slim_json_reporter.rb index 110426f9..dc5c9ada 100644 --- a/test/client/lib/slim_json_reporter.rb +++ b/test/client/lib/slim_json_reporter.rb @@ -8,7 +8,7 @@ def report super filename = "#{ENV.fetch('COVERAGE_ROOT')}/test_metrics.json" metrics = { - total_time: formar('%.2f', total_time), + total_time: format('%.2f', total_time), assertion_count: assertions, test_count: count, failure_count: failures,