diff --git a/apps/language_server/lib/language_server/server.ex b/apps/language_server/lib/language_server/server.ex
index e5e9d27c7..c9bf9261f 100644
--- a/apps/language_server/lib/language_server/server.ex
+++ b/apps/language_server/lib/language_server/server.ex
@@ -407,7 +407,11 @@ defmodule ElixirLS.LanguageServer.Server do
       Enum.any?(changes, fn %{"uri" => uri = "file:" <> _, "type" => type} ->
         path = SourceFile.path_from_uri(uri)
 
-        Path.extname(path) in (additional_watched_extensions ++ @default_watched_extensions) and
+        relative_path = Path.relative_to(path, state.project_dir)
+        first_path_segment = relative_path |> Path.split() |> hd
+
+        first_path_segment not in [".elixir_ls", "_build"] and
+          Path.extname(path) in (additional_watched_extensions ++ @default_watched_extensions) and
           (type in [1, 3] or not Map.has_key?(state.source_files, uri) or
              state.source_files[uri].dirty?)
       end)