haste name_reducers and paths

Summary:
We want to be able to figure out the id of a haste module by using it's filename instead of having to read the file to extract the `providesModule` annotations. The plan is to completely kill the usage of `providesModule` at Facebook (internally and open source projects). I'm adding this as a configurable opt-in option (and preserving support for `providesModule`) for the sake of (a) non-fb consumers of our open source tools that rely on `providesModule` and (b) seamlessly rolling this out throughout Facebook projects (the plan is to add support for using the filename to all tools and then make the switch and remove the `providesModule` annotations over the next 3-4 weeks).

`name_reducers` are basically a series of replacements that are applied in order to reduce a filename to a module id. These rules only apply to file paths that match the whitelist and do not match the blacklist. I had to use this instead of just using the basename (minus .js.flow) because react native has a bunch of platform specific extensions (`.ios`/`.android`/`.web`) that shouldn't be part of the module id.

I've hardly written any OCaml in my life, so please do pardon (and point out) atrocities that I may have committed.

Reviewed By: avikchaudhuri

Differential Revision: D4488705

fbshipit-source-id: f16bb1a034fb86bf6e6461f9e5f1cf2564f57d1f

Author: voideanvalue

src/commands/lsCommand.ml

@@ -216,6 +216,18 @@ let make_options ~root ~strip_root ~ignore_flag ~include_flag =
     opt_ignore_non_literal_requires = false;
     opt_max_header_tokens = FlowConfig.(
       flowconfig.options.Opts.max_header_tokens
+    );
+    opt_haste_name_reducers = FlowConfig.(
+      flowconfig.options.Opts.haste_name_reducers
+    );
+    opt_haste_paths_blacklist = FlowConfig.(
+      flowconfig.options.Opts.haste_paths_blacklist
+    );
+    opt_haste_paths_whitelist = FlowConfig.(
+      flowconfig.options.Opts.haste_paths_whitelist
+    );
+    opt_haste_use_name_reducers = FlowConfig.(
+      flowconfig.options.Opts.haste_use_name_reducers
     )
   }
 

src/commands/serverCommands.ml

@@ -358,6 +358,18 @@ module OptionParser(Config : CONFIG) = struct
       );
       opt_max_header_tokens = FlowConfig.(
         flowconfig.options.Opts.max_header_tokens
+      );
+      opt_haste_name_reducers = FlowConfig.(
+        flowconfig.options.Opts.haste_name_reducers
+      );
+      opt_haste_paths_blacklist = FlowConfig.(
+        flowconfig.options.Opts.haste_paths_blacklist
+      );
+      opt_haste_paths_whitelist = FlowConfig.(
+        flowconfig.options.Opts.haste_paths_whitelist
+      );
+      opt_haste_use_name_reducers = FlowConfig.(
+        flowconfig.options.Opts.haste_use_name_reducers
       )
     } in
     Main.start options

src/common/flowConfig.ml

@@ -51,6 +51,10 @@ module Opts = struct
     esproposal_decorators: Options.esproposal_feature_mode;
     esproposal_export_star_as: Options.esproposal_feature_mode;
     facebook_fbt: string option;
+    haste_name_reducers: (Str.regexp * string) list;
+    haste_paths_blacklist: string list;
+    haste_paths_whitelist: string list;
+    haste_use_name_reducers: bool;
     ignore_non_literal_requires: bool;
     moduleSystem: moduleSystem;
     module_name_mappers: (Str.regexp * string) list;
@@ -149,6 +153,10 @@ module Opts = struct
     esproposal_decorators = Options.ESPROPOSAL_WARN;
     esproposal_export_star_as = Options.ESPROPOSAL_WARN;
     facebook_fbt = None;
+    haste_name_reducers = [(Str.regexp "^\\(.*/\\)?\\([a-zA-Z0-9$_.-]+\\)\\.js\\(\\.flow\\)?$", "\\2")];
+    haste_paths_blacklist = ["\\(.*\\)?/node_modules/.*"];
+    haste_paths_whitelist = ["<PROJECT_ROOT>/.*"];
+    haste_use_name_reducers = false;
     ignore_non_literal_requires = false;
     moduleSystem = Node;
     module_name_mappers = [];
@@ -286,6 +294,18 @@ module Opts = struct
 
   let optparse_filepath str =
     Path.make str
+
+  let optparse_mapping str =
+    let regexp_str = "^'\\([^']*\\)'[ \t]*->[ \t]*'\\([^']*\\)'$" in
+    let regexp = Str.regexp regexp_str in
+    (if not (Str.string_match regexp str 0) then
+      raise (UserError (
+        "Expected a mapping of form: " ^
+        "'single-quoted-string' -> 'single-quoted-string'"
+      ))
+    );
+    (Str.matched_group 1 str, Str.matched_group 2 str)
+
 end
 
 type config = {
@@ -448,6 +468,53 @@ let parse_options config lines =
       });
     }
 
+    |> define_opt "module.system.haste.name_reducers" {
+      _initializer = INIT_FN (fun opts -> {
+        opts with haste_name_reducers = [];
+      });
+      flags = [ALLOW_DUPLICATE];
+      optparser = (fun str ->
+        let (pattern, template) = optparse_mapping str in
+        (Str.regexp pattern, template)
+      );
+      setter = (fun opts v -> {
+        opts with haste_name_reducers = v::(opts.haste_name_reducers);
+      });
+    }
+
+    |> define_opt "module.system.haste.paths.blacklist" {
+      _initializer = INIT_FN (fun opts -> {
+        opts with haste_paths_blacklist = [];
+      });
+      flags = [ALLOW_DUPLICATE];
+      optparser = optparse_string;
+      setter = (fun opts v -> {
+        opts with haste_paths_blacklist = v::(opts.haste_paths_blacklist);
+      });
+    }
+
+    |> define_opt "module.system.haste.paths.whitelist" {
+      _initializer = INIT_FN (fun opts -> {
+        opts with haste_paths_whitelist = [];
+      });
+      flags = [ALLOW_DUPLICATE];
+      optparser = optparse_string;
+      setter = (fun opts v -> {
+        opts with haste_paths_whitelist = v::(opts.haste_paths_whitelist);
+      });
+    }
+
+    |> define_opt "module.system.haste.use_name_reducers" {
+      _initializer = INIT_FN (fun opts -> {
+        opts with haste_use_name_reducers = false;
+      });
+      flags = [];
+      optparser = optparse_boolean;
+      setter = (fun opts v ->
+        {opts with haste_use_name_reducers = v;}
+      );
+    }
+
     |> define_opt "log.file" {
       _initializer = USE_DEFAULT;
       flags = [];
@@ -499,18 +566,7 @@ let parse_options config lines =
       _initializer = USE_DEFAULT;
       flags = [ALLOW_DUPLICATE];
       optparser = (fun str ->
-        let regexp_str = "^'\\([^']*\\)'[ \t]*->[ \t]*'\\([^']*\\)'$" in
-        let regexp = Str.regexp regexp_str in
-        (if not (Str.string_match regexp str 0) then
-          raise (Opts.UserError (
-            "Expected a mapping of form: " ^
-            "'single-quoted-string' -> 'single-quoted-string'"
-          ))
-        );
-
-        let pattern = Str.matched_group 1 str in
-        let template = Str.matched_group 2 str in
-
+        let (pattern, template) = optparse_mapping str in
         (Str.regexp pattern, template)
       );
       setter = (fun opts v ->
@@ -523,18 +579,7 @@ let parse_options config lines =
       _initializer = USE_DEFAULT;
       flags = [ALLOW_DUPLICATE];
       optparser = (fun str ->
-        let regexp_str = "^'\\([^']*\\)'[ \t]*->[ \t]*'\\([^']*\\)'$" in
-        let regexp = Str.regexp regexp_str in
-        (if not (Str.string_match regexp str 0) then
-          raise (Opts.UserError (
-            "Expected a mapping of form: " ^
-            "'single-quoted-string' -> 'single-quoted-string'"
-          ))
-        );
-
-        let file_ext = Str.matched_group 1 str in
-        let template = Str.matched_group 2 str in
-
+        let (file_ext, template) = optparse_mapping str in
         (Str.regexp ("^\\(.*\\)\\." ^ (Str.quote file_ext) ^ "$"), template)
       );
       setter = (fun opts v ->

src/common/flowConfig.mli

@@ -20,6 +20,10 @@ module Opts : sig
     esproposal_decorators: Options.esproposal_feature_mode;
     esproposal_export_star_as: Options.esproposal_feature_mode;
     facebook_fbt: string option;
+    haste_name_reducers: (Str.regexp * string) list;
+    haste_paths_blacklist: string list;
+    haste_paths_whitelist: string list;
+    haste_use_name_reducers: bool;
     ignore_non_literal_requires: bool;
     moduleSystem: moduleSystem;
     module_name_mappers: (Str.regexp * string) list;

src/common/options.ml

@@ -37,6 +37,10 @@ type t = {
   opt_esproposal_decorators: esproposal_feature_mode;
   opt_esproposal_export_star_as: esproposal_feature_mode;
   opt_facebook_fbt: string option;
+  opt_haste_name_reducers: (Str.regexp * string) list;
+  opt_haste_paths_blacklist: string list;
+  opt_haste_paths_whitelist: string list;
+  opt_haste_use_name_reducers: bool;
   opt_ignore_non_literal_requires: bool;
   opt_ignores: (string * Str.regexp) list;
   opt_includes: Path_matcher.t;
@@ -94,6 +98,10 @@ let esproposal_class_instance_fields opts =
   opts.opt_esproposal_class_instance_fields
 let esproposal_decorators opts = opts.opt_esproposal_decorators
 let esproposal_export_star_as opts = opts.opt_esproposal_export_star_as
+let haste_name_reducers opts = opts.opt_haste_name_reducers
+let haste_paths_blacklist opts = opts.opt_haste_paths_blacklist
+let haste_paths_whitelist opts = opts.opt_haste_paths_whitelist
+let haste_use_name_reducers opts = opts.opt_haste_use_name_reducers
 let ignores opts = opts.opt_ignores
 let includes opts = opts.opt_includes
 let is_check_mode opts = opts.opt_check_mode

src/services/inference/module_js.ml

@@ -261,7 +261,7 @@ let package_incompatible package ast =
    model both Haste and Node, but should be further generalized. *)
 module type MODULE_SYSTEM = sig
   (* Given a file and docblock info, make the name of the module it exports. *)
-  val exported_module: filename -> Docblock.t -> Modulename.t
+  val exported_module: Options.t -> filename -> Docblock.t -> Modulename.t
 
   (* Given a file and a reference in it to an imported module, make the name of
      the module it refers to. If given an optional reference to an accumulator,
@@ -354,7 +354,7 @@ let lazy_seq: 'a option Lazy.t list -> 'a option =
 (*******************************)
 
 module Node = struct
-  let exported_module file _ =
+  let exported_module _ file _ =
     if Loc.check_suffix file Files.flow_ext
     then Modulename.Filename (Loc.chop_suffix file Files.flow_ext)
     else Modulename.Filename file
@@ -520,27 +520,61 @@ module Haste: MODULE_SYSTEM = struct
     | Loc.ResourceFile file ->
         Str.string_match mock_path file 0
 
-  let rec exported_module file info =
+  let expand_project_root_token options str =
+    let root = Path.to_string (Options.root options)
+      |> Sys_utils.normalize_filename_dir_sep in
+    str
+      |> Str.split_delim FlowConfig.project_root_token
+      |> String.concat root
+      |> Str.regexp
+
+  let is_haste_file options file =
+    let matched_haste_paths_whitelist file = List.exists
+      (fun r -> Str.string_match (expand_project_root_token options r) (string_of_filename file) 0)
+      (Options.haste_paths_whitelist options) in
+    let matched_haste_paths_blacklist file = List.exists
+      (fun r -> Str.string_match (expand_project_root_token options r) (string_of_filename file) 0)
+      (Options.haste_paths_blacklist options) in
+    (matched_haste_paths_whitelist file) && not (matched_haste_paths_blacklist file)
+
+  let haste_name options file =
+    let reduce_name name (regexp, template) =
+      Str.global_replace regexp template name
+    in
+    List.fold_left
+      reduce_name
+      (string_of_filename file)
+      (Options.haste_name_reducers options)
+
+  let rec exported_module options file info =
     if is_mock file
     then Modulename.String (short_module_name_of file)
+    else if Options.haste_use_name_reducers options
+    then
+      if is_haste_file options file
+      then Modulename.String (haste_name options file)
+      else exported_non_haste_module options file
     else match Docblock.providesModule info with
       | Some m -> Modulename.String m
       | None ->
           (* If foo.js.flow doesn't have a @providesModule, then look at foo.js
            * and use its @providesModule instead *)
-          if Loc.check_suffix file Files.flow_ext
-          then
-            let file_without_flow_ext = Loc.chop_suffix file Files.flow_ext in
-            if Parsing_service_js.has_ast file_without_flow_ext
-            then
-              (** TODO [perf]: mark as expensive and investigate! **)
-              let _, info =
-                Parsing_service_js.get_ast_and_info_unsafe file_without_flow_ext in
-              exported_module file_without_flow_ext info
-            else
-              Modulename.Filename (file_without_flow_ext)
-          else
-            Modulename.Filename file
+          exported_non_haste_module options file
+
+  and exported_non_haste_module options file =
+    if Loc.check_suffix file Files.flow_ext
+    then
+      let file_without_flow_ext = Loc.chop_suffix file Files.flow_ext in
+      if Parsing_service_js.has_ast file_without_flow_ext
+      then
+        (** TODO [perf]: mark as expensive and investigate! **)
+        let _, info =
+          Parsing_service_js.get_ast_and_info_unsafe file_without_flow_ext in
+        exported_module options file_without_flow_ext info
+      else
+        Modulename.Filename (file_without_flow_ext)
+    else
+      Modulename.Filename file
 
   let expanded_name r =
     match Str.split_delim (Str.regexp_string "/") r with
@@ -624,7 +658,7 @@ let get_module_system opts =
 
 let exported_module ~options file info =
   let module M = (val (get_module_system options)) in
-  M.exported_module file info
+  M.exported_module options file info
 
 let imported_module ~options cx loc ?path_acc r =
   let module M = (val (get_module_system options)) in

tests/declaration_files_incremental_haste_name_reducers/.flowconfig

@@ -0,0 +1,6 @@
+[options]
+module.system=haste
+module.system.haste.use_name_reducers=true
+module.system.haste.name_reducers='^.*/\([a-zA-Z0-9$_.-]+\)\.js\(\.flow\)?$' -> '\1'
+module.system.haste.paths.whitelist=<PROJECT_ROOT>/.*
+module.system.haste.paths.blacklist=.*/node_modules/.*

tests/declaration_files_incremental_haste_name_reducers/.testconfig

@@ -0,0 +1 @@
+shell: test.sh

tests/declaration_files_incremental_haste_name_reducers/A.js

@@ -0,0 +1,4 @@
+/* @flow */
+
+class AImplementation {}
+export function foo(): AImplementation { return new AImplementation(); }

tests/declaration_files_incremental_haste_name_reducers/A.js.flow.ignored

@@ -0,0 +1,2 @@
+declare class ADefinition {}
+declare export function foo(): ADefinition;