Skip indexing of generated files (#1255)

* Add ability to skip generated files (by tag in their header)
* Allow generated files to be indexed on-demand
* Better indexing report on completion

apps/els_core/src/els_config.erl

@@ -131,6 +131,7 @@ do_initialize(RootUri, Capabilities, InitOptions, {ConfigPath, Config}) ->
   ElvisConfigPath = maps:get("elvis_config_path", Config, undefined),
   BSPEnabled = maps:get("bsp_enabled", Config, auto),
   IncrementalSync = maps:get("incremental_sync", Config, true),
+  Indexing = maps:get("indexing", Config, #{}),
   CompilerTelemetryEnabled
     = maps:get("compiler_telemetry_enabled", Config, false),
   EDocCustomTags = maps:get("edoc_custom_tags", Config, []),
@@ -159,6 +160,8 @@ do_initialize(RootUri, Capabilities, InitOptions, {ConfigPath, Config}) ->
   ok = set(compiler_telemetry_enabled, CompilerTelemetryEnabled),
   ok = set(edoc_custom_tags, EDocCustomTags),
   ok = set(incremental_sync, IncrementalSync),
+  ok = set(indexing, maps:merge( els_config_indexing:default_config()
+                               , Indexing)),
   %% Calculated from the above
   ok = set(apps_paths     , project_paths(RootPath, AppsDirs, false)),
   ok = set(deps_paths     , project_paths(RootPath, DepsDirs, false)),

apps/els_core/src/els_config_indexing.erl

@@ -0,0 +1,44 @@
+-module(els_config_indexing).
+
+-include("els_core.hrl").
+
+-export([ default_config/0 ]).
+
+%% Getters
+-export([ get_skip_generated_files/0
+        , get_generated_files_tag/0
+        ]).
+
+-type config() :: #{ string() => string() }.
+
+-spec default_config() -> config().
+default_config() ->
+  #{ "skip_generated_files" => default_skip_generated_files()
+   , "generated_files_tag" => default_generated_files_tag()
+   }.
+
+-spec get_skip_generated_files() -> boolean().
+get_skip_generated_files() ->
+  Value = maps:get("skip_generated_files",
+                   els_config:get(indexing),
+                   default_skip_generated_files()),
+  normalize_boolean(Value).
+
+-spec get_generated_files_tag() -> string().
+get_generated_files_tag() ->
+  maps:get("generated_files_tag",
+           els_config:get(indexing),
+           default_generated_files_tag()).
+
+-spec default_skip_generated_files() -> string().
+default_skip_generated_files() ->
+  "false".
+
+-spec default_generated_files_tag() -> string().
+default_generated_files_tag() ->
+  "@generated".
+
+-spec normalize_boolean(boolean() | string()) -> boolean().
+normalize_boolean("true") -> true;
+normalize_boolean("false") -> false;
+normalize_boolean(Bool) when is_boolean(Bool) -> Bool.

apps/els_lsp/src/els_indexing.erl

@@ -22,11 +22,6 @@
 %%==============================================================================
 -type mode()  :: 'deep' | 'shallow'.
 
-%%==============================================================================
-%% Macros
-%%==============================================================================
--define(SERVER, ?MODULE).
-
 %%==============================================================================
 %% Exported functions
 %%==============================================================================
@@ -50,9 +45,33 @@ find_and_index_file(FileName) ->
 
 -spec index_file(binary()) -> {ok, uri()}.
 index_file(Path) ->
-  try_index_file(Path, 'deep'),
+  GeneratedFilesTag = els_config_indexing:get_generated_files_tag(),
+  try_index_file(Path, 'deep', false, GeneratedFilesTag),
   {ok, els_uri:uri(Path)}.
 
+-spec index_if_not_generated(uri(), binary(), mode(), boolean(), string()) ->
+        ok | skipped.
+index_if_not_generated(Uri, Text, Mode, false, _GeneratedFilesTag) ->
+  index(Uri, Text, Mode);
+index_if_not_generated(Uri, Text, Mode, true, GeneratedFilesTag) ->
+  case is_generated_file(Text, GeneratedFilesTag) of
+    true ->
+      ?LOG_DEBUG("Skip indexing for generated file ~p", [Uri]),
+      skipped;
+    false ->
+      ok = index(Uri, Text, Mode)
+  end.
+
+-spec is_generated_file(binary(), string()) -> boolean().
+is_generated_file(Text, Tag) ->
+  [Line|_] = string:split(Text, "\n", leading),
+  case re:run(Line, Tag) of
+    {match, _} ->
+      true;
+    nomatch ->
+      false
+  end.
+
 -spec index(uri(), binary(), mode()) -> ok.
 index(Uri, Text, Mode) ->
   MD5 = erlang:md5(Text),
@@ -132,10 +151,23 @@ start() ->
 
 -spec start(binary(), [{string(), 'deep' | 'shallow'}]) -> ok.
 start(Group, Entries) ->
-  Task = fun({Dir, Mode}, _) -> index_dir(Dir, Mode) end,
+  SkipGeneratedFiles = els_config_indexing:get_skip_generated_files(),
+  GeneratedFilesTag = els_config_indexing:get_generated_files_tag(),
+  Task = fun({Dir, Mode}, {Succeeded0, Skipped0, Failed0}) ->
+             {Su, Sk, Fa} = index_dir(Dir, Mode,
+                                      SkipGeneratedFiles, GeneratedFilesTag),
+             {Succeeded0 + Su, Skipped0 + Sk, Failed0 + Fa}
+         end,
   Config = #{ task => Task
             , entries => Entries
             , title => <<"Indexing ", Group/binary>>
+            , initial_state => {0, 0, 0}
+            , on_complete =>
+                fun({Succeeded, Skipped, Failed}) ->
+                    ?LOG_INFO("Completed indexing for ~s "
+                              "(succeeded: ~p, skipped: ~p, failed: ~p)",
+                              [Group, Succeeded, Skipped, Failed])
+                end
             },
   {ok, _Pid} = els_background_job:new(Config),
   ok.
@@ -144,14 +176,17 @@ start(Group, Entries) ->
 %% Internal functions
 %%==============================================================================
 
+
 %% @doc Try indexing a file.
--spec try_index_file(binary(), mode()) -> ok | {error, any()}.
-try_index_file(FullName, Mode) ->
+-spec try_index_file(binary(), mode(), boolean(), string()) ->
+        ok | skipped | {error, any()}.
+try_index_file(FullName, Mode, SkipGeneratedFiles, GeneratedFilesTag) ->
   Uri = els_uri:uri(FullName),
   try
     ?LOG_DEBUG("Indexing file. [filename=~s, uri=~s]", [FullName, Uri]),
     {ok, Text} = file:read_file(FullName),
-    ok         = index(Uri, Text, Mode)
+    index_if_not_generated(Uri, Text, Mode,
+                           SkipGeneratedFiles, GeneratedFilesTag)
   catch Type:Reason:St ->
       ?LOG_ERROR("Error indexing file "
                  "[filename=~s, uri=~s] "
@@ -179,32 +214,42 @@ register_reference(Uri, #{kind := Kind, id := Id, range := Range})
                           , #{id => Id, uri => Uri, range => Range}
                           ).
 
--spec index_dir(string(), mode()) -> {non_neg_integer(), non_neg_integer()}.
+-spec index_dir(string(), mode()) ->
+        {non_neg_integer(), non_neg_integer(), non_neg_integer()}.
 index_dir(Dir, Mode) ->
+  SkipGeneratedFiles = els_config_indexing:get_skip_generated_files(),
+  GeneratedFilesTag = els_config_indexing:get_generated_files_tag(),
+  index_dir(Dir, Mode, SkipGeneratedFiles, GeneratedFilesTag).
+
+-spec index_dir(string(), mode(), boolean(), string()) ->
+        {non_neg_integer(), non_neg_integer(), non_neg_integer()}.
+index_dir(Dir, Mode, SkipGeneratedFiles, GeneratedFilesTag) ->
   ?LOG_DEBUG("Indexing directory. [dir=~s] [mode=~s]", [Dir, Mode]),
-  F = fun(FileName, {Succeeded, Failed}) ->
-          case try_index_file(els_utils:to_binary(FileName), Mode) of
-            ok              -> {Succeeded + 1, Failed};
-            {error, _Error} -> {Succeeded, Failed + 1}
+  F = fun(FileName, {Succeeded, Skipped, Failed}) ->
+          case try_index_file(els_utils:to_binary(FileName), Mode,
+                              SkipGeneratedFiles, GeneratedFilesTag) of
+            ok -> {Succeeded + 1, Skipped, Failed};
+            skipped -> {Succeeded, Skipped + 1, Failed};
+            {error, _Error} -> {Succeeded, Skipped, Failed + 1}
           end
       end,
   Filter = fun(Path) ->
                Ext = filename:extension(Path),
                lists:member(Ext, [".erl", ".hrl", ".escript"])
            end,
 
-  {Time, {Succeeded, Failed}} = timer:tc( els_utils
-                                        , fold_files
-                                        , [ F
-                                          , Filter
-                                          , Dir
-                                          , {0, 0}
-                                          ]
-                                        ),
+  {Time, {Succeeded, Skipped, Failed}} = timer:tc( els_utils
+                                                 , fold_files
+                                                 , [ F
+                                                   , Filter
+                                                   , Dir
+                                                   , {0, 0, 0}
+                                                   ]
+                                                 ),
   ?LOG_DEBUG("Finished indexing directory. [dir=~s] [mode=~s] [time=~p] "
-             "[succeeded=~p] "
-             "[failed=~p]", [Dir, Mode, Time/1000/1000, Succeeded, Failed]),
-  {Succeeded, Failed}.
+             "[succeeded=~p] [skipped=~p] [failed=~p]",
+             [Dir, Mode, Time/1000/1000, Succeeded, Skipped, Failed]),
+  {Succeeded, Skipped, Failed}.
 
 -spec entries_apps() -> [{string(), 'deep' | 'shallow'}].
 entries_apps() ->

apps/els_lsp/test/els_indexer_SUITE.erl

@@ -13,13 +13,17 @@
         , index_erl_file/1
         , index_hrl_file/1
         , index_unkown_extension/1
+        , do_not_skip_generated_file_by_tag_by_default/1
+        , skip_generated_file_by_tag/1
+        , skip_generated_file_by_custom_tag/1
         ]).
 
 %%==============================================================================
 %% Includes
 %%==============================================================================
 -include_lib("common_test/include/ct.hrl").
 -include_lib("stdlib/include/assert.hrl").
+-include_lib("els_core/include/els_core.hrl").
 
 %%==============================================================================
 %% Types
@@ -42,10 +46,29 @@ end_per_suite(Config) ->
   els_test_utils:end_per_suite(Config).
 
 -spec init_per_testcase(atom(), config()) -> config().
+init_per_testcase(TestCase, Config) when
+    TestCase =:= skip_generated_file_by_tag ->
+  meck:new(els_config_indexing, [passthrough, no_link]),
+  meck:expect(els_config_indexing, get_skip_generated_files, fun() -> true end),
+  els_test_utils:init_per_testcase(TestCase, Config);
+init_per_testcase(TestCase, Config) when
+    TestCase =:= skip_generated_file_by_custom_tag ->
+  meck:new(els_config_indexing, [passthrough, no_link]),
+  meck:expect(els_config_indexing,
+              get_skip_generated_files,
+              fun() -> true end),
+  meck:expect(els_config_indexing,
+              get_generated_files_tag,
+              fun() -> "@customgeneratedtag" end),
+  els_test_utils:init_per_testcase(TestCase, Config);
 init_per_testcase(TestCase, Config) ->
   els_test_utils:init_per_testcase(TestCase, Config).
 
 -spec end_per_testcase(atom(), config()) -> ok.
+end_per_testcase(TestCase, Config) when
+    TestCase =:= skip_generated_file_by_tag ->
+  meck:unload(els_config_indexing),
+  els_test_utils:end_per_testcase(TestCase, Config);
 end_per_testcase(TestCase, Config) ->
   els_test_utils:end_per_testcase(TestCase, Config).
 
@@ -88,3 +111,54 @@ index_unkown_extension(Config) ->
   {ok, Uri} = els_indexing:index_file(Path),
   {ok, [#{kind := other}]} = els_dt_document:lookup(Uri),
   ok.
+
+-spec do_not_skip_generated_file_by_tag_by_default(config()) -> ok.
+do_not_skip_generated_file_by_tag_by_default(Config) ->
+  DataDir = data_dir(Config),
+  GeneratedByTagUri = uri(DataDir, "generated_file_by_tag.erl"),
+  GeneratedByCustomTagUri = uri(DataDir, "generated_file_by_custom_tag.erl"),
+  ?assertEqual({4, 0, 0}, els_indexing:index_dir(DataDir, 'deep')),
+  {ok, [#{ id := generated_file_by_tag
+         , kind := module
+         }
+       ]} = els_dt_document:lookup(GeneratedByTagUri),
+  {ok, [#{ id := generated_file_by_custom_tag
+         , kind := module
+         }
+       ]} = els_dt_document:lookup(GeneratedByCustomTagUri),
+  ok.
+
+-spec skip_generated_file_by_tag(config()) -> ok.
+skip_generated_file_by_tag(Config) ->
+  DataDir = data_dir(Config),
+  GeneratedByTagUri = uri(DataDir, "generated_file_by_tag.erl"),
+  GeneratedByCustomTagUri = uri(DataDir, "generated_file_by_custom_tag.erl"),
+  ?assertEqual({3, 1, 0}, els_indexing:index_dir(DataDir, 'deep')),
+  {ok, []} = els_dt_document:lookup(GeneratedByTagUri),
+  {ok, [#{ id := generated_file_by_custom_tag
+         , kind := module
+         }
+       ]} = els_dt_document:lookup(GeneratedByCustomTagUri),
+  ok.
+
+-spec skip_generated_file_by_custom_tag(config()) -> ok.
+skip_generated_file_by_custom_tag(Config) ->
+  DataDir = data_dir(Config),
+  GeneratedByTagUri = uri(DataDir, "generated_file_by_tag.erl"),
+  GeneratedByCustomTagUri = uri(DataDir, "generated_file_by_custom_tag.erl"),
+  ?assertEqual({3, 1, 0}, els_indexing:index_dir(DataDir, 'deep')),
+  {ok, [#{ id := generated_file_by_tag
+         , kind := module
+         }
+       ]} = els_dt_document:lookup(GeneratedByTagUri),
+  {ok, []} = els_dt_document:lookup(GeneratedByCustomTagUri),
+  ok.
+
+-spec data_dir(proplists:proplist()) -> binary().
+data_dir(Config) ->
+  ?config(data_dir, Config).
+
+-spec uri(binary(), string()) -> uri().
+uri(DataDir, FileName) ->
+  Path = els_utils:to_binary(filename:join(DataDir, FileName)),
+  els_uri:uri(Path).

apps/els_lsp/test/els_indexer_SUITE_data/generated_file_by_custom_tag.erl

@@ -0,0 +1,7 @@
+%% This file is @customgeneratedtag
+-module(generated_file_by_custom_tag).
+
+-export([main/0]).
+
+main() ->
+  ok.

apps/els_lsp/test/els_indexer_SUITE_data/generated_file_by_tag.erl

@@ -0,0 +1,7 @@
+%% This file is @generated
+-module(generated_file_by_tag).
+
+-export([main/0]).
+
+main() ->
+  ok.