diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 32c5b7b3b..cf488ee41 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -18,13 +18,13 @@ jobs:
         include:
           - elixir: 1.12.x
             otp: 22.x
-            tests_may_fail: true
+            tests_may_fail: false
           - elixir: 1.12.x
             otp: 23.x
-            tests_may_fail: true
+            tests_may_fail: false
           - elixir: 1.12.x
             otp: 24.x
-            tests_may_fail: true
+            tests_may_fail: false
           - elixir: 1.13.x
             otp: 22.x
             tests_may_fail: false
@@ -80,6 +80,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - elixir: 1.12.x
+            otp: 22.x
+          - elixir: 1.12.x
+            otp: 23.x
+          - elixir: 1.12.x
+            otp: 24.x
           - elixir: 1.13.x
             otp: 22.x
           - elixir: 1.13.x
diff --git a/apps/language_server/test/providers/references_test.exs b/apps/language_server/test/providers/references_test.exs
index 398d6a131..bf778faa0 100644
--- a/apps/language_server/test/providers/references_test.exs
+++ b/apps/language_server/test/providers/references_test.exs
@@ -217,9 +217,13 @@ defmodule ElixirLS.LanguageServer.Providers.ReferencesTest do
 
     list = References.references(parser_context, uri, line, char, true, File.cwd!())
 
-    assert length(list) == 2
-    assert Enum.any?(list, &(&1["uri"] |> String.ends_with?("references_erlang.ex")))
-    assert Enum.any?(list, &(&1["uri"] |> String.ends_with?("references_referenced.ex")))
+    if Version.match?(System.version(), ">= 1.13.0-dev") do
+      assert length(list) == 2
+      assert Enum.any?(list, &(&1["uri"] |> String.ends_with?("references_erlang.ex")))
+      assert Enum.any?(list, &(&1["uri"] |> String.ends_with?("references_referenced.ex")))
+    else
+      assert length(list) == 3
+    end
   end
 
   test "finds remote references to erlang module" do
@@ -239,9 +243,13 @@ defmodule ElixirLS.LanguageServer.Providers.ReferencesTest do
 
     list = References.references(parser_context, uri, line, char, true, File.cwd!())
 
-    assert length(list) == 2
-    assert Enum.any?(list, &(&1["uri"] |> String.ends_with?("references_erlang.ex")))
-    assert Enum.any?(list, &(&1["uri"] |> String.ends_with?("references_referenced.ex")))
+    if Version.match?(System.version(), ">= 1.13.0-dev") do
+      assert length(list) == 2
+      assert Enum.any?(list, &(&1["uri"] |> String.ends_with?("references_erlang.ex")))
+      assert Enum.any?(list, &(&1["uri"] |> String.ends_with?("references_referenced.ex")))
+    else
+      assert length(list) == 3
+    end
   end
 
   test "finds alias references" do