add support for 1.17 tokenizer

fix tests on 1.17
elixir-lsp · May 26, 2024 · a41777a · a41777a
1 parent 5f2defd
commit a41777a
Show file tree

Hide file tree

Showing 6 changed files with 29 additions and 15 deletions.
diff --git a/lib/elixir_sense/core/metadata_builder.ex b/lib/elixir_sense/core/metadata_builder.ex
@@ -1813,7 +1813,7 @@ defmodule ElixirSense.Core.MetadataBuilder do
   end
 
   # stepped range struct
-  def get_binding_type(_state, {:"..//", _, [_, _, _]}) do
+  def get_binding_type(_state, {:..//, _, [_, _, _]}) do
     {:struct, [], {:atom, Range}}
   end
 

diff --git a/lib/elixir_sense/core/normalized/code/fragment.ex b/lib/elixir_sense/core/normalized/code/fragment.ex
@@ -17,7 +17,7 @@ defmodule ElixirSense.Core.Normalized.Code.Fragment do
     end
   rescue
     e ->
-      if Version.match?(System.version(), ">= 1.16.0-dev") do
+      if Version.match?(System.version(), ">= 1.17.0-dev") do
         Logger.error(
           "Code.Fragment.cursor_context raised #{Exception.format(:error, e, __STACKTRACE__)}. Please report that to elixir project."
         )
@@ -44,7 +44,7 @@ defmodule ElixirSense.Core.Normalized.Code.Fragment do
     end
   rescue
     e ->
-      if Version.match?(System.version(), ">= 1.16.0-dev") do
+      if Version.match?(System.version(), ">= 1.17.0-dev") do
         Logger.error(
           "Code.Fragment.surround_context raised #{Exception.format(:error, e, __STACKTRACE__)}. Please report that to elixir project."
         )
@@ -71,7 +71,7 @@ defmodule ElixirSense.Core.Normalized.Code.Fragment do
     end
   rescue
     e ->
-      if Version.match?(System.version(), ">= 1.16.0-dev") do
+      if Version.match?(System.version(), ">= 1.17.0-dev") do
         try do
           Logger.error(
             "Code.Fragment.container_cursor_to_quoted raised #{Exception.format(:error, e, __STACKTRACE__)}. Please report that to elixir project."

diff --git a/lib/elixir_sense/core/normalized/tokenizer.ex b/lib/elixir_sense/core/normalized/tokenizer.ex
@@ -24,15 +24,20 @@ defmodule ElixirSense.Core.Normalized.Tokenizer do
       end
 
     case result do
+      # < 1.17
       {:ok, _line, _column, _warning, tokens} ->
         Enum.reverse(tokens)
 
+      # >= 1.17
+      {:ok, _line, _column, _warning, tokens, _terminators} ->
+        tokens
+
       {:error, _, _, _, sofar} ->
         sofar
     end
   rescue
     e ->
-      if Version.match?(System.version(), ">= 1.16.0-dev") do
+      if Version.match?(System.version(), ">= 1.17.0-dev") do
         Logger.error(
           ":elixir_tokenizer.tokenize raised #{Exception.format(:error, e, __STACKTRACE__)}. Please report that to elixir project."
         )

diff --git a/lib/elixir_sense/core/normalized/typespec.ex b/lib/elixir_sense/core/normalized/typespec.ex
@@ -13,7 +13,7 @@ defmodule ElixirSense.Core.Normalized.Typespec do
       # workaround for crash
       # Keyword.fetch({:error, :beam_lib, {:not_a_beam_file, ""}}, :module)
       # fixed in elixir 1.16.0
-      if Version.match?(System.version(), ">= 1.16.0-dev") do
+      if Version.match?(System.version(), ">= 1.17.0-dev") do
         Logger.error(
           "Code.Typespec.fetch_specs raised #{Exception.format(:error, e, __STACKTRACE__)}. Please report that to elixir project."
         )
@@ -33,7 +33,7 @@ defmodule ElixirSense.Core.Normalized.Typespec do
       # workaround for crash
       # Keyword.fetch({:error, :beam_lib, {:not_a_beam_file, ""}}, :module)
       # fixed in elixir 1.16.0
-      if Version.match?(System.version(), ">= 1.16.0-dev") do
+      if Version.match?(System.version(), ">= 1.17.0-dev") do
         Logger.error(
           "Code.Typespec.fetch_types raised #{Exception.format(:error, e, __STACKTRACE__)}. Please report that to elixir project."
         )
@@ -53,7 +53,7 @@ defmodule ElixirSense.Core.Normalized.Typespec do
       # workaround for crash
       # Keyword.fetch({:error, :beam_lib, {:not_a_beam_file, ""}}, :module)
       # fixed in elixir 1.16.0
-      if Version.match?(System.version(), ">= 1.16.0-dev") do
+      if Version.match?(System.version(), ">= 1.17.0-dev") do
         Logger.error(
           "Code.Typespec.fetch_callbacks raised #{Exception.format(:error, e, __STACKTRACE__)}. Please report that to elixir project."
         )

diff --git a/lib/elixir_sense/core/type_info.ex b/lib/elixir_sense/core/type_info.ex
@@ -230,7 +230,7 @@ defmodule ElixirSense.Core.TypeInfo do
         |> to_string()
       rescue
         e ->
-          if Version.match?(System.version(), ">= 1.16.0-dev") do
+          if Version.match?(System.version(), ">= 1.17.0-dev") do
             Logger.error(
               "Macro.to_string(#{inspect(sanitized)}) returned invalid code. Please report that to elixir project."
             )

diff --git a/test/elixir_sense/core/source_test.exs b/test/elixir_sense/core/source_test.exs
@@ -224,12 +224,21 @@ defmodule ElixirSense.Core.SourceTest do
     end
 
     test "inside fn body" do
-      assert %{
-               candidate: {Enum, :map},
-               elixir_prefix: false,
-               npar: 1,
-               pos: {{1, 11}, {1, nil}}
-             } = which_func("var = Enum.map([1,2], fn(i) -> i*")
+      if Version.match?(System.version(), ">= 1.17.0-rc.0") do
+        assert %{
+                 candidate: {nil, :*},
+                 elixir_prefix: false,
+                 npar: 1,
+                 pos: {{1, 33}, {1, nil}}
+               } = which_func("var = Enum.map([1,2], fn(i) -> i*")
+      else
+        assert %{
+                 candidate: {Enum, :map},
+                 elixir_prefix: false,
+                 npar: 1,
+                 pos: {{1, 11}, {1, nil}}
+               } = which_func("var = Enum.map([1,2], fn(i) -> i*")
+      end
     end
 
     test "inside a list" do