Experimental project structure

This commit represents a new structure for the experimental project, and
a path forward. With these changes the project now has:

  * A build option enabling the experimental server
  * Per-message routing for the experimental server. If enabled, it can
    either share messages with the existing server or take them over.
    Presently, the find references and formatting providers are
    implemented and "exclusive", meaning that they're handled solely by
    the experimental server
  * A consistent interface for building providers.
  * A consistent way to convert lsp messages into data structures and
    back again. This conversion is handled automatically for providers.
  * A genserver-like interface for providers to implement
  * Data structures representing LSP messages that are simple to define
    and build.
  * Fast and efficient conversion between utf8 and utf16.
  * A separation between what a provider does and how it responds to
    messages. This allows the work that underpins providers to be tested
    independently from the language server.

apps/language_server/.formatter.exs

@@ -6,12 +6,14 @@ impossible_to_format = [
 proto_dsl = [
   defenum: 1,
   defnotification: 2,
-  defrequest: 2,
+  defnotification: 3,
+  defrequest: 3,
   defresponse: 1,
   deftype: 1
 ]
 
 [
+  import_deps: [:patch],
   export: [
     locals_without_parens: proto_dsl
   ],

apps/language_server/lib/language_server.ex

@@ -9,17 +9,19 @@ defmodule ElixirLS.LanguageServer do
 
   @impl Application
   def start(_type, _args) do
-    children = [
-      Experimental.SourceFile.Store,
-      {ElixirLS.LanguageServer.Server, ElixirLS.LanguageServer.Server},
-      Experimental.Server,
-      {ElixirLS.LanguageServer.PacketRouter, [LanguageServer.Server, Experimental.Server]},
-      {ElixirLS.LanguageServer.JsonRpc,
-       name: ElixirLS.LanguageServer.JsonRpc, language_server: LanguageServer.PacketRouter},
-      {ElixirLS.LanguageServer.Providers.WorkspaceSymbols, []},
-      {ElixirLS.LanguageServer.Tracer, []},
-      {ElixirLS.LanguageServer.ExUnitTestTracer, []}
-    ]
+    Experimental.LanguageServer.persist_enabled_state()
+
+    children =
+      [
+        maybe_experimental_supervisor(),
+        {ElixirLS.LanguageServer.Server, ElixirLS.LanguageServer.Server},
+        maybe_packet_router(),
+        jsonrpc(),
+        {ElixirLS.LanguageServer.Providers.WorkspaceSymbols, []},
+        {ElixirLS.LanguageServer.Tracer, []},
+        {ElixirLS.LanguageServer.ExUnitTestTracer, []}
+      ]
+      |> Enum.reject(&is_nil/1)
 
     opts = [strategy: :one_for_one, name: LanguageServer.Supervisor, max_restarts: 0]
     Supervisor.start_link(children, opts)
@@ -38,4 +40,25 @@ defmodule ElixirLS.LanguageServer do
 
     :ok
   end
+
+  defp maybe_experimental_supervisor do
+    if Experimental.LanguageServer.enabled?() do
+      Experimental.Supervisor
+    end
+  end
+
+  defp maybe_packet_router do
+    if Experimental.LanguageServer.enabled?() do
+      {ElixirLS.LanguageServer.PacketRouter, [LanguageServer.Server, Experimental.Server]}
+    end
+  end
+
+  defp jsonrpc do
+    if Experimental.LanguageServer.enabled?() do
+      {ElixirLS.LanguageServer.JsonRpc,
+       name: ElixirLS.LanguageServer.JsonRpc, language_server: LanguageServer.PacketRouter}
+    else
+      {ElixirLS.LanguageServer.JsonRpc, name: ElixirLS.LanguageServer.JsonRpc}
+    end
+  end
 end

apps/language_server/lib/language_server/experimental/code_unit.ex

@@ -0,0 +1,159 @@
+defmodule ElixirLS.LanguageServer.Experimental.CodeUnit do
+  @moduledoc """
+  Code unit and offset conversions
+
+  The LSP protocol speaks in positions, which defines where something happens in a document.
+  Positions have a start and an end, which are defined as code unit _offsets_ from the beginning
+  of a line. this module helps to convert between utf8, which most of the world speaks
+  natively, and utf16, which has been forced upon us by microsoft.
+
+  Converting between offsets and code units is 0(n), and allocations only happen if a
+  multi-byte character is detected, at which point, only that character is allocated.
+  This exploits the fact that most source code consists of ascii characters, with at best,
+  sporadic multi-byte characters in it. Thus, the vast majority of documents will not require
+  any allocations at all.
+  """
+  @type utf8_code_unit :: non_neg_integer()
+  @type utf16_code_unit :: non_neg_integer()
+  @type utf8_offset :: non_neg_integer()
+  @type utf16_offset :: non_neg_integer()
+
+  @type error :: {:error, :misaligned} | {:error, :out_of_bounds}
+
+  # public
+
+  @doc """
+  Converts a utf8 character offset into a utf16 character offset. This implementation
+  clamps the maximum size of an offset so that any initial character position can be
+  passed in and the offset returned will reflect the end of the line.
+  """
+  @spec utf16_offset(String.t(), utf8_offset()) :: utf16_offset()
+  def utf16_offset(binary, character_position) do
+    do_utf16_offset(binary, character_position, 0)
+  end
+
+  @doc """
+  Converts a utf16 character offset into a utf8 character offset. This implementation
+  clamps the maximum size of an offset so that any initial character position can be
+  passed in and the offset returned will reflect the end of the line.
+  """
+  @spec utf8_offset(String.t(), utf16_offset()) :: utf8_offset()
+  def utf8_offset(binary, character_position) do
+    do_utf8_offset(binary, character_position, 0)
+  end
+
+  @spec to_utf8(String.t(), utf16_code_unit()) :: {:ok, utf8_code_unit()} | error
+  def to_utf8(binary, utf16_unit) do
+    do_to_utf8(binary, utf16_unit + 1, 0)
+  end
+
+  @spec to_utf16(String.t(), utf8_code_unit()) :: {:ok, utf16_code_unit()} | error
+  def to_utf16(binary, utf16_unit) do
+    do_to_utf16(binary, utf16_unit + 1, 0)
+  end
+
+  # Private
+
+  # UTF-16
+
+  defp do_utf16_offset(_, 0, offset) do
+    offset
+  end
+
+  defp do_utf16_offset(<<>>, _, offset) do
+    # this clause pegs the offset at the end of the string
+    # no matter the character index
+    offset
+  end
+
+  defp do_utf16_offset(<<c, rest::binary>>, remaining, offset) when c < 128 do
+    do_utf16_offset(rest, remaining - 1, offset + 1)
+  end
+
+  defp do_utf16_offset(<<c::utf8, rest::binary>>, remaining, offset) do
+    s = <<c::utf8>>
+    increment = utf16_size(s)
+    do_utf16_offset(rest, remaining - 1, offset + increment)
+  end
+
+  defp do_to_utf16(_, 0, utf16_unit) do
+    {:ok, utf16_unit - 1}
+  end
+
+  defp do_to_utf16(_, utf8_unit, _) when utf8_unit < 0 do
+    {:error, :misaligned}
+  end
+
+  defp do_to_utf16(<<>>, _remaining, _utf16_unit) do
+    {:error, :out_of_bounds}
+  end
+
+  defp do_to_utf16(<<c, rest::binary>>, utf8_unit, utf16_unit) when c < 128 do
+    do_to_utf16(rest, utf8_unit - 1, utf16_unit + 1)
+  end
+
+  defp do_to_utf16(<<c::utf8, rest::binary>>, utf8_unit, utf16_unit) do
+    utf8_string = <<c::utf8>>
+    increment = utf16_size(utf8_string)
+    decrement = byte_size(utf8_string)
+
+    do_to_utf16(rest, utf8_unit - decrement, utf16_unit + increment)
+  end
+
+  defp utf16_size(binary) when is_binary(binary) do
+    binary
+    |> :unicode.characters_to_binary(:utf8, :utf16)
+    |> byte_size()
+    |> div(2)
+  end
+
+  # UTF-8
+
+  defp do_utf8_offset(_, 0, offset) do
+    offset
+  end
+
+  defp do_utf8_offset(<<>>, _, offset) do
+    # this clause pegs the offset at the end of the string
+    # no matter the character index
+    offset
+  end
+
+  defp do_utf8_offset(<<c, rest::binary>>, remaining, offset) when c < 128 do
+    do_utf8_offset(rest, remaining - 1, offset + 1)
+  end
+
+  defp do_utf8_offset(<<c::utf8, rest::binary>>, remaining, offset) do
+    s = <<c::utf8>>
+    increment = utf8_size(s)
+    decrement = utf16_size(s)
+    do_utf8_offset(rest, remaining - decrement, offset + increment)
+  end
+
+  defp do_to_utf8(_, 0, utf8_unit) do
+    {:ok, utf8_unit - 1}
+  end
+
+  defp do_to_utf8(_, utf_16_units, _) when utf_16_units < 0 do
+    {:error, :misaligned}
+  end
+
+  defp do_to_utf8(<<>>, _remaining, _utf8_unit) do
+    {:error, :out_of_bounds}
+  end
+
+  defp do_to_utf8(<<c, rest::binary>>, utf16_unit, utf8_unit) when c < 128 do
+    do_to_utf8(rest, utf16_unit - 1, utf8_unit + 1)
+  end
+
+  defp do_to_utf8(<<c::utf8, rest::binary>>, utf16_unit, utf8_unit) do
+    utf8_code_units = byte_size(<<c::utf8>>)
+    utf16_code_units = utf16_size(<<c::utf8>>)
+
+    do_to_utf8(rest, utf16_unit - utf16_code_units, utf8_unit + utf8_code_units)
+  end
+
+  defp utf8_size(binary) when is_binary(binary) do
+    byte_size(binary)
+  end
+end

apps/language_server/lib/language_server/experimental/format.ex

@@ -0,0 +1,158 @@
+defmodule ElixirLS.LanguageServer.Experimental.Format do
+  alias ElixirLS.LanguageServer.Experimental.Format.Diff
+  alias ElixirLS.LanguageServer.Experimental.SourceFile
+  alias ElixirLS.LanguageServer.Experimental.SourceFile.Conversions
+  alias ElixirLS.LanguageServer.Experimental.Protocol.Types.TextEdit
+
+  require Logger
+  @type formatter_function :: (String.t() -> any) | nil
+
+  @spec text_edits(SourceFile.t(), String.t() | nil) :: {:ok, [TextEdit.t()]} | {:error, any}
+  def text_edits(%SourceFile{} = document, project_path_or_uri) do
+    with {:ok, unformatted, formatted} <- do_format(document, project_path_or_uri) do
+      edits = Diff.diff(unformatted, formatted)
+      {:ok, edits}
+    end
+  end
+
+  @spec format(SourceFile.t(), String.t() | nil) :: {:ok, String.t()} | {:error, any}
+  def format(%SourceFile{} = document, project_path_or_uri) do
+    with {:ok, _, formatted_code} <- do_format(document, project_path_or_uri) do
+      {:ok, formatted_code}
+    end
+  end
+
+  defp do_format(%SourceFile{} = document, project_path_or_uri)
+       when is_binary(project_path_or_uri) do
+    project_path = Conversions.ensure_path(project_path_or_uri)
+
+    with :ok <- check_current_directory(document, project_path),
+         {:ok, formatter, options} <- formatter_for(document.path),
+         :ok <-
+           check_inputs_apply(document, project_path, Keyword.get(options, :inputs)) do
+      document
+      |> SourceFile.to_string()
+      |> formatter.()
+    end
+  end
+
+  defp do_format(%SourceFile{} = document, _) do
+    formatter = build_formatter([])
+
+    document
+    |> SourceFile.to_string()
+    |> formatter.()
+  end
+
+  @spec formatter_for(String.t()) :: {:ok, formatter_function, keyword()} | :error
+  defp formatter_for(uri_or_path) do
+    path = Conversions.ensure_path(uri_or_path)
+
+    try do
+      true = Code.ensure_loaded?(Mix.Tasks.Format)
+
+      if function_exported?(Mix.Tasks.Format, :formatter_for_file, 1) do
+        {formatter_function, options} = Mix.Tasks.Format.formatter_for_file(path)
+
+        wrapped_formatter_function = wrap_with_try_catch(formatter_function)
+
+        {:ok, wrapped_formatter_function, options}
+      else
+        options = Mix.Tasks.Format.formatter_opts_for_file(path)
+        formatter = build_formatter(options)
+        {:ok, formatter, Mix.Tasks.Format.formatter_opts_for_file(path)}
+      end
+    rescue
+      e ->
+        message = Exception.message(e)
+
+        Logger.warn(
+          "Unable to get formatter options for #{path}: #{inspect(e.__struct__)} #{message}"
+        )
+
+        {:error, :no_formatter_available}
+    end
+  end
+
+  defp build_formatter(opts) do
+    fn code ->
+      formatted_iodata = Code.format_string!(code, opts)
+      IO.iodata_to_binary([formatted_iodata, ?\n])
+    end
+    |> wrap_with_try_catch()
+  end
+
+  defp wrap_with_try_catch(formatter_fn) do
+    fn code ->
+      try do
+        {:ok, code, formatter_fn.(code)}
+      rescue
+        e ->
+          {:error, e}
+      end
+    end
+  end
+
+  defp check_current_directory(%SourceFile{} = document, project_path) do
+    cwd = File.cwd!()
+
+    if subdirectory?(document.path, parent: project_path) or
+         subdirectory?(document.path, parent: cwd) do
+      :ok
+    else
+      message =
+        "Cannot format file from current directory " <>
+          "(Currently in #{Path.relative_to(cwd, project_path)})"
+
+      {:error, message}
+    end
+  end
+
+  defp check_inputs_apply(%SourceFile{} = document, project_path, inputs)
+       when is_list(inputs) do
+    formatter_dir = dominating_formatter_exs_dir(document, project_path)
+
+    inputs_apply? =
+      Enum.any?(inputs, fn input_glob ->
+        glob = Path.join(formatter_dir, input_glob)
+        PathGlobVendored.match?(document.path, glob, match_dot: true)
+      end)
+
+    if inputs_apply? do
+      :ok
+    else
+      {:error, :input_mismatch}
+    end
+  end
+
+  defp check_inputs_apply(_, _, _), do: :ok
+
+  defp subdirectory?(child, parent: parent) do
+    normalized_parent = Path.absname(parent)
+    String.starts_with?(child, normalized_parent)
+  end
+
+  # Finds the directory with the .formatter.exs that's the nearest parent to the
+  # source file, or the project dir if none was found.
+  defp dominating_formatter_exs_dir(%SourceFile{} = document, project_path) do
+    document.path
+    |> Path.dirname()
+    |> dominating_formatter_exs_dir(project_path)
+  end
+
+  defp dominating_formatter_exs_dir(project_dir, project_dir) do
+    project_dir
+  end
+
+  defp dominating_formatter_exs_dir(current_dir, project_path) do
+    formatter_exs_name = Path.join(current_dir, ".formatter.exs")
+
+    if File.exists?(formatter_exs_name) do
+      current_dir
+    else
+      current_dir
+      |> Path.dirname()
+      |> dominating_formatter_exs_dir(project_path)
+    end
+  end
+end