draft of warning on confusable identifiers

lib/elixir/src/elixir_tokenizer.erl

@@ -133,13 +133,14 @@ tokenize(String, Line, Opts) ->
   tokenize(String, Line, 1, Opts).
 
 tokenize([], Line, Column, #elixir_tokenizer{cursor_completion=Cursor} = Scope, Tokens) when Cursor /= false ->
-  #elixir_tokenizer{terminators=Terminators, warnings=Warnings} = Scope,
+  #elixir_tokenizer{file=File, identifier_tokenizer=IdentifierTokenizer, terminators=Terminators, warnings=Warnings} = Scope,
 
   {CursorColumn, CursorTerminators, CursorTokens} =
     add_cursor(Line, Column, Cursor, Terminators, Tokens),
 
   AccTokens = cursor_complete(Line, CursorColumn, CursorTerminators, CursorTokens),
-  {ok, Line, Column, Warnings, AccTokens};
+  UnicodeWarnings = maybe_unicode_warnings(IdentifierTokenizer, Tokens, File),
+  {ok, Line, Column, Warnings ++ UnicodeWarnings, AccTokens};
 
 tokenize([], EndLine, Column, #elixir_tokenizer{terminators=[{Start, StartLine, _} | _]} = Scope, Tokens) ->
   End = terminator(Start),
@@ -148,8 +149,10 @@ tokenize([], EndLine, Column, #elixir_tokenizer{terminators=[{Start, StartLine,
   Formatted = io_lib:format(Message, [End, Start, StartLine]),
   error({EndLine, Column, [Formatted, Hint], []}, [], Scope, Tokens);
 
-tokenize([], Line, Column, #elixir_tokenizer{warnings=Warnings}, Tokens) ->
-  {ok, Line, Column, Warnings, lists:reverse(Tokens)};
+tokenize([], Line, Column, #elixir_tokenizer{file=File, identifier_tokenizer=IdentifierTokenizer, warnings=Warnings}, TokensReversed) ->
+  Tokens = lists:reverse(TokensReversed),
+  UnicodeWarnings = maybe_unicode_warnings(IdentifierTokenizer, Tokens, File),
+  {ok, Line, Column, Warnings ++ UnicodeWarnings, Tokens};
 
 % VC merge conflict
 
@@ -1642,4 +1645,10 @@ prune_tokens([], [], Terminators) ->
 
 drop_including([{Token, _} | Tokens], Token) -> Tokens;
 drop_including([_ | Tokens], Token) -> drop_including(Tokens, Token);
-drop_including([], _Token) -> [].
+drop_including([], _Token) -> [].
+
+maybe_unicode_warnings(IdentifierTokenizer, Tokens, File) ->
+  case erlang:function_exported(IdentifierTokenizer, unicode_lint_warnings, 1) of
+    true -> IdentifierTokenizer:unicode_lint_warnings(Tokens, File);
+    false -> []
+  end.

lib/elixir/test/elixir/kernel/string_tokenizer_test.exs

@@ -69,4 +69,22 @@ defmodule Kernel.StringTokenizerTest do
     assert {:error, _} = Code.string_to_quoted("Ola?")
     assert {:error, _} = Code.string_to_quoted("Ola!")
   end
+
+  test "tokenizes confusables with warnings" do
+    assert {:ok, _, _, warnings, _} = :elixir_tokenizer.tokenize('а=1; a=1', 1, 1, file: "f")
+
+    msg = "confusable: 'a' on L1 looks like 'а' up on L1" |> String.to_charlist()
+    assert [{_linecol, _file, ^msg}] = warnings
+  end
+
+  test "unicode tr39 'skeleton' calculation" do
+    import String.UnicodeSecurity, only: [skeleton: 1]
+
+    # cases similar to those from unicode-security .rs
+    assert skeleton("") === ""
+    assert skeleton("ｓ") === "s"
+    assert skeleton("ｓｓｓ") === "sss"
+    assert skeleton("ﶛ") === "نمى"
+    assert skeleton("ﶛﶛ") === "نمىنمى"
+  end
 end