Macro code coverage reporting

.github/workflows/ci.yml

@@ -64,7 +64,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: extractions/setup-just@v3
       - name: Install kcov
-        if: matrix.crystal == 'latest'
+        if: ${{ matrix.crystal == vars.COVERAGE_CHANNEL }}
         run: |
           sudo apt-get update &&
           sudo apt-get install binutils-dev libssl-dev libcurl4-openssl-dev libelf-dev libstdc++-12-dev zlib1g-dev libdw-dev libiberty-dev
@@ -91,16 +91,16 @@ jobs:
         run: just test-compiled
         shell: bash
       - uses: codecov/codecov-action@v5
-        if: matrix.crystal == 'latest' && github.event_name != 'schedule' # Only want to upload coverage report once in the matrix
+        if: ${{ matrix.crystal == vars.COVERAGE_CHANNEL && github.event_name != 'schedule' }} # Only want to upload coverage report once in the matrix
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true
           directory: coverage
-          files: '**/cov.xml' # There is no `unreachable.codecov.json` file when running _only_ compiled specs
+          files: '**/cov.xml,**/macro_coverage.*.codecov.json' # There is no `unreachable.codecov.json` file when running _only_ compiled specs
           flags: compiled
           verbose: true
       - uses: codecov/test-results-action@v1
-        if: matrix.crystal == 'latest' && github.event_name != 'schedule' # Only want to upload coverage report once in the matrix
+        if: ${{ matrix.crystal == vars.COVERAGE_CHANNEL && github.event_name != 'schedule' }} # Only want to upload coverage report once in the matrix
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true
@@ -129,7 +129,7 @@ jobs:
           fetch-depth: 0
       - uses: extractions/setup-just@v3
       - name: Install kcov
-        if: matrix.os == 'ubuntu-latest' && matrix.crystal == 'latest'
+        if: ${{ matrix.os == 'ubuntu-latest' && matrix.crystal == vars.COVERAGE_CHANNEL }}
         run: |
           sudo apt-get update &&
           sudo apt-get install binutils-dev libssl-dev libcurl4-openssl-dev libelf-dev libstdc++-12-dev zlib1g-dev libdw-dev libiberty-dev
@@ -160,7 +160,7 @@ jobs:
         run: just test-unit
         shell: bash
       - uses: codecov/codecov-action@v5
-        if: matrix.os == 'ubuntu-latest' && matrix.crystal == 'latest' && github.event_name != 'schedule' # Only want to upload coverage report once in the matrix
+        if: ${{ matrix.os == 'ubuntu-latest' && matrix.crystal == vars.COVERAGE_CHANNEL && github.event_name != 'schedule' }} # Only want to upload coverage report once in the matrix
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true
@@ -169,7 +169,7 @@ jobs:
           flags: unit
           verbose: true
       - uses: codecov/test-results-action@v1
-        if: matrix.os == 'ubuntu-latest' && matrix.crystal == 'latest' && github.event_name != 'schedule' # Only want to upload coverage report once in the matrix
+        if: ${{ matrix.os == 'ubuntu-latest' && matrix.crystal == vars.COVERAGE_CHANNEL && github.event_name != 'schedule' }} # Only want to upload coverage report once in the matrix
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true

scripts/test.sh

@@ -6,21 +6,41 @@ function runSpecs() (
   $CRYSTAL spec "${DEFAULT_BUILD_OPTIONS[@]}" "${DEFAULT_OPTIONS[@]}" "src/components/$1/spec"
 )
 
-# Coverage generation logic based on https://hannes.kaeufler.net/posts/measuring-code-coverage-in-crystal-with-kcov
+# Runtime coverage generation logic based on https://hannes.kaeufler.net/posts/measuring-code-coverage-in-crystal-with-kcov.
+# Additionally generates a coverage report for unreachable code.
+#
+# Compiled time code generates a macro code coverage report for the entire component, and each compiled sub-process spec.
 #
 # $1 component name
 function runSpecsWithCoverage() (
   set -e
   rm -rf "coverage/$1"
   mkdir -p coverage/bin "coverage/$1"
+
+  # Build spec binary that covers entire `spec/` directory to run coverage against.
   echo "require \"../../src/components/$1/spec/**\"" > "./coverage/bin/$1.cr" && \
   $CRYSTAL build "${DEFAULT_BUILD_OPTIONS[@]}" "./coverage/bin/$1.cr" -o "./coverage/bin/$1" && \
-  kcov $(if $IS_CI != "true"; then echo "--cobertura-only"; fi) --clean --include-path="./src/components/$1" "./coverage/$1" "./coverage/bin/$1" --junit_output="./coverage/$1/junit.xml" "${DEFAULT_OPTIONS[@]}"
+  ATHENA_SPEC_COVERAGE_OUTPUT_DIR="$(realpath ./coverage/$1/)" \
+    kcov $(if $IS_CI != "true"; then echo "--cobertura-only"; fi) \
+      --clean \
+      --include-path="./src/components/$1"\
+      "./coverage/$1"\
+      "./coverage/bin/$1"\
+      --junit_output="./coverage/$1/junit.xml"\
+      "${DEFAULT_OPTIONS[@]}"
+
+  if [ "$TYPE" != "unit" ]
+  then
+    # Generate macro coverage report.
+    # The report itself is sent to STDOUT while other output is sent to STDERR.
+    # We can ignore STDERR since those failures would be captured as part of running the specs themselves.
+    $CRYSTAL tool macro_code_coverage --no-color "./coverage/bin/$1.cr" > "./coverage/$1/macro_coverage.root.codecov.json"
+  fi
 
-  # Scope this to only non-compiled tests as that's all it's relevant for.
-  if [ $TYPE != "compiled" ]
+  # Only runtime code can be unreachable.
+  if [ "$TYPE" != "compiled" ]
   then
-    $CRYSTAL tool unreachable --format=codecov "./coverage/bin/$1.cr" > "./coverage/$1/unreachable.codecov.json"
+    $CRYSTAL tool unreachable --no-color --format=codecov "./coverage/bin/$1.cr" > "./coverage/$1/unreachable.codecov.json"
   fi
 )
 
@@ -39,39 +59,39 @@ IS_CI=${CI:="false"}
 COMPONENT=${1-all}
 TYPE=${2-all}
 
-if [ $TYPE == "unit" ]
+if [ "$TYPE" == "unit" ]
 then
   DEFAULT_OPTIONS+=("--tag=~compiled")
-elif [ $TYPE == "compiled" ]
+elif [ "$TYPE" == "compiled" ]
 then
   DEFAULT_OPTIONS+=("--tag=compiled")
-elif [ $TYPE != "all" ]
+elif [ "$TYPE" != "all" ]
 then
   echo "Second argument must be 'unit', 'compiled', or 'all' got '${2}'."
   exit 1
 fi
 
 EXIT_CODE=0
 
-if [ $COMPONENT != "all" ]
+if [ "$COMPONENT" != "all" ]
 then
-  if [ $HAS_KCOV = "true" ]
+  if [ "$HAS_KCOV" = "true" ]
   then
-    runSpecsWithCoverage $COMPONENT
+    runSpecsWithCoverage "$COMPONENT"
   else
-    runSpecs $COMPONENT
+    runSpecs "$COMPONENT"
   fi
   exit $?
 fi
 
 for component in $(find src/components/ -maxdepth 2 -type f -name shard.yml | xargs -I{} dirname {} | xargs -I{} basename {} | sort); do
   echo "::group::$component"
 
-  if [ $HAS_KCOV = "true" ]
+  if [ "$HAS_KCOV" = "true" ]
   then
-    runSpecsWithCoverage $component
+    runSpecsWithCoverage "$component"
   else
-    runSpecs $component
+    runSpecs "$component"
   fi
 
   if [ $? -eq 1 ]; then

shard.yml

@@ -1,6 +1,9 @@
 name: athena-ecosystem
 version: 0.0.0
 
+# Min Crystal version required to run the test suite/develop on Athena.
+crystal: ~> 1.17
+
 license: MIT
 
 repository: https://github.com/athena-framework/athena
@@ -51,4 +54,4 @@ development_dependencies:
     version: ~> 1.6.3
   athena-spec:
     github: athena-framework/spec
-    version: ~> 0.3.2
+    version: ~> 0.4.0

src/components/console/spec/compiler_spec.cr

@@ -4,7 +4,7 @@ describe Athena::Console do
   describe "compiler errors", tags: "compiled" do
     describe "when a command configured via annotation doesn't have a name" do
       it "non hidden no aliases" do
-        ASPEC::Methods.assert_error "Console command 'NoNameCommand' has an 'ACONA::AsCommand' annotation but is missing the commands's name. It was not provided as the first positional argument nor via the 'name' field.", <<-CR
+        ASPEC::Methods.assert_compile_time_error "Console command 'NoNameCommand' has an 'ACONA::AsCommand' annotation but is missing the commands's name. It was not provided as the first positional argument nor via the 'name' field.", <<-CR
           require "./spec_helper.cr"
 
           @[ACONA::AsCommand]
@@ -19,7 +19,7 @@ describe Athena::Console do
       end
 
       it "hidden" do
-        ASPEC::Methods.assert_error "Console command 'NoNameCommand' has an 'ACONA::AsCommand' annotation but is missing the commands's name. It was not provided as the first positional argument nor via the 'name' field.", <<-CR
+        ASPEC::Methods.assert_compile_time_error "Console command 'NoNameCommand' has an 'ACONA::AsCommand' annotation but is missing the commands's name. It was not provided as the first positional argument nor via the 'name' field.", <<-CR
           require "./spec_helper.cr"
 
           @[ACONA::AsCommand(hidden: true)]

src/components/console/spec/helper/helper_set_spec.cr

@@ -0,0 +1,13 @@
+require "../spec_helper"
+
+describe ACON::Helper::HelperSet do
+  describe "compiler errors", tags: "compiled" do
+    it "when the provided helper type is not an `ACON::Helper::Interface`" do
+      ASPEC::Methods.assert_compile_time_error "Helper class type 'String' is not an 'ACON::Helper::Interface'.", <<-CR
+        require "../spec_helper.cr"
+
+        ACON::Helper::HelperSet.new[String]?
+      CR
+    end
+  end
+end

src/components/console/spec/question/question_spec.cr

@@ -22,6 +22,15 @@ struct QuestionTest < ASPEC::TestCase
     @question = ACON::Question(String?).new "Test Question", nil
   end
 
+  @[Tags("compiled")]
+  def test_nil_generic_arg : Nil
+    ASPEC::Methods.assert_compile_time_error "An ACON::Question generic argument cannot be 'Nil'. Use 'String?' instead.", <<-CR
+      require "../spec_helper.cr"
+
+      ACON::Question(Nil).new "Nil Question", nil
+    CR
+  end
+
   def test_default : Nil
     @question.default.should be_nil
     default = ACON::Question(String).new("Test Question", "FOO").default

src/components/console/src/helper/helper_set.cr

@@ -31,7 +31,11 @@ class Athena::Console::Helper::HelperSet
 
   # Returns the helper of the provided *helper_class*, or `nil` if it is not defined.
   def []?(helper_class : T.class) : T? forall T
-    {% T.raise "Helper class type '#{T}' is not an 'ACON::Helper::Interface'." unless T <= ACON::Helper::Interface %}
+    {%
+      unless T <= ACON::Helper::Interface
+        T.raise "Helper class type '#{T}' is not an 'ACON::Helper::Interface'."
+      end
+    %}
 
     @helpers[helper_class]?.as? T
   end

src/components/console/src/question/base.cr

@@ -34,7 +34,11 @@ module Athena::Console::Question::Base(T)
   property? trimmable : Bool = true
 
   def initialize(@question : String, @default : T)
-    {% T.raise "An ACON::Question generic argument cannot be 'Nil'. Use 'String?' instead." if T == Nil %}
+    {%
+      if T == Nil
+        T.raise "An ACON::Question generic argument cannot be 'Nil'. Use 'String?' instead."
+      end
+    %}
   end
 
   # :nodoc:

src/components/dependency_injection/spec/athena-dependency_injection_spec.cr

@@ -6,6 +6,16 @@ class ValueStore
 end
 
 describe Athena::DependencyInjection do
+  describe "compiler errors", tags: "compiled" do
+    it "errors when passing a non-module to add_compiler_pass" do
+      ASPEC::Methods.assert_compile_time_error "Pass type must be a module.", <<-CR
+        require "./spec_helper.cr"
+
+        ADI.add_compiler_pass String
+      CR
+    end
+  end
+
   describe ".container" do
     it "returns a container" do
       ADI.container.should be_a ADI::ServiceContainer

src/components/dependency_injection/spec/compiler_passes/auto_wire_spec.cr

@@ -1,7 +1,7 @@
 require "../spec_helper"
 
-private def assert_error(message : String, code : String, *, line : Int32 = __LINE__) : Nil
-  ASPEC::Methods.assert_error message, <<-CR, line: line
+private def assert_compile_time_error(message : String, code : String, *, line : Int32 = __LINE__) : Nil
+  ASPEC::Methods.assert_compile_time_error message, <<-CR, line: line
     require "../spec_helper.cr"
     #{code}
   CR
@@ -36,7 +36,7 @@ record SameInstanceClient, a : SameInstancePrimary, b : SameInstanceAliasInterfa
 describe ADI::ServiceContainer do
   describe "compiler errors", tags: "compiled" do
     it "does not resolve an un-aliased interface when there is only 1 implementation" do
-      assert_error "Failed to resolve value for parameter 'a : SomeInterface' of service 'bar' (Bar).", <<-CR
+      assert_compile_time_error "Failed to resolve value for parameter 'a : SomeInterface' of service 'bar' (Bar).", <<-CR
         module SomeInterface; end
 
         @[ADI::Register]

src/components/dependency_injection/spec/compiler_passes/define_getters_spec.cr

@@ -1,7 +1,7 @@
 require "../spec_helper"
 
-private def assert_error(message : String, code : String, *, line : Int32 = __LINE__) : Nil
-  ASPEC::Methods.assert_error message, <<-CR, line: line
+private def assert_compile_time_error(message : String, code : String, *, line : Int32 = __LINE__) : Nil
+  ASPEC::Methods.assert_compile_time_error message, <<-CR, line: line
     require "../spec_helper.cr"
     #{code}
   CR
@@ -53,7 +53,7 @@ describe ADI::ServiceContainer::DefineGetters, tags: "compiled" do
   describe "compiler errors" do
     describe "aliases" do
       it "does not expose named getter for non-public string aliases" do
-        assert_error "undefined method 'bar' for Athena::DependencyInjection::ServiceContainer", <<-'CR'
+        assert_compile_time_error "undefined method 'bar' for Athena::DependencyInjection::ServiceContainer", <<-'CR'
           module SomeInterface; end
 
           @[ADI::Register]
@@ -67,7 +67,7 @@ describe ADI::ServiceContainer::DefineGetters, tags: "compiled" do
       end
 
       it "does not expose typed getter for non-public typed aliases" do
-        assert_error "undefined method 'get' for Athena::DependencyInjection::ServiceContainer", <<-'CR'
+        assert_compile_time_error "undefined method 'get' for Athena::DependencyInjection::ServiceContainer", <<-'CR'
           module SomeInterface; end
 
           @[ADI::Register]

src/components/dependency_injection/spec/compiler_passes/merge_configs_spec.cr

@@ -2,7 +2,7 @@ require "../spec_helper"
 
 describe ADI::ServiceContainer::MergeConfigs do
   it "deep merges consecutive `ADI.configure` call", tags: "compiled" do
-    ASPEC::Methods.assert_success <<-CR
+    ASPEC::Methods.assert_compiles <<-'CR'
       require "../spec_helper"
 
       module Schema
@@ -53,9 +53,14 @@ describe ADI::ServiceContainer::MergeConfigs do
 
       macro finished
         macro finished
-          it { \\{{ADI::CONFIG["test"]["default_locale"]}}.should eq "en" }
-          it { \\{{ADI::CONFIG["test"]["cors"]["defaults"]["allow_credentials"]}}.should be_true }
-          it { \\{{ADI::CONFIG["test"]["cors"]["defaults"]["allow_credentials"]}}.should eq ["*"] }
+          \{%
+            config = ADI::CONFIG["test"]
+
+            raise "#{config}" unless config["default_locale"] == "en"
+            raise "#{config}" unless config["cors"]["defaults"]["allow_credentials"] == true
+            raise "#{config}" unless config["cors"]["defaults"]["allow_origin"].size == 1
+            raise "#{config}" unless config["cors"]["defaults"]["allow_origin"][0] == "*"
+          %}
         end
       end
     CR