a few hacks that improve the build time (BinaryAnalysisPlatform#1028)

* a few hacks that improve the build time

Building the whole BAP repository from scratch is now more than twice
as fast and takes about 8 minutes or even less with faster CPUs and
SSD drives.

What is more important, rebuilding BAP now takes less than a minute
even if something was changed deep in the BAP's core, like
disassembler or even regular or bap-types.

Below is the list of tricks that made this possible.

1) Disable -short-paths. This option was responsible for nearly x2
slowdown. Since nowadays we mostly do typechecking with merlin having
short paths doesn't really matter anymore. So screw them. Short paths
are also disable in bapbuild.

2) Disable cross-module optimizations. When cross-module optimizations
are enabled (by default) every time a module changes all dependent
modules are recompiled. This behavior sort of defeats the purpose of
separate compilation and leads to 20 minutes of recompilation when
some module in disassembler is changed. Now it won't happen and when
any module is changed no other modules will be recompiled. If the
interface is changed only the modules that import this interface will
be recompiled. Theoretically, it may lead to less optimal builds. We
didn't notice any slowdown yet, but we may consider putting this
particular trick under a compilation flag and disable during releases.

3) Hotfix ocamlbuild linking rule. There is a bug in ocamlbuild that
prevents caching from working properly. The rule for linking library
mentions .so as its product, but it never builds it (and can't (and
shouldn't)). As a result the caching system notices a missing object
and triggers relinking of all libraries. Ideally, it should be fixed
on the upstream (I will PR to them) but we don't want to wait for
upsteam, especially since the fix is simple - we just override the
rule by our own, that didn't mention any .so files.

4) Disables compression when plugins are packed. Plugins are zip files
underneath the hood and when we pack and unpack them it takes time to
compress them. It goes much faster (especially for custom llvm builds
with debugging symbols) when the compression is disabled. This trick
is also a candidate to be made optional.

5) Packs a plugin in one step. Instead of doing 4 packings/repackings
we now do it only once.

6) Unlimits the number of jobs tried by ocamlbuild. This may break on
Travis, but let's try. We all know that ocamlbuild is not very good in
parallelizing tasks, but it tries its best. I didn't see more than 12
jobs at once, so it should be allright.

* fixes an invalid depenency in the cmxa -> cmxs rule.

It should depend on '.a' also, as `cmxa` is a collection of `cmx` so
it doesn't change when cross-module optimizations are disabled, or
when the set of imported modules is unchanged.

* implements build caching for plugins

Now all built plugins are cached inside the _build folder, instead of
being rebuilt every time in a temporary folder. And md5sum of the
transitive closure of plugin dependencies is used as the cache
validator. If any dependency is changed, then the plugin is relinked
and repacked.

Additionally, the script builds only those plugins that were migrated
into the build tree, so when we're installing via opam it doesn't try
to rebuild already installed plugins.

This is an extperimental patch, I'm concerned with its
portability. We use md5sum utility, which is not present on MacOS X.
However, there is md5 there, with a slightly different interface,
which we're trying to use instead. Probably, we should rewrite this
in OCaml instead, for better portability and performance.

Finally, it introduces new system dependency, md5[sum], so our docker
builds may fail.

* adds the development mode to configure

Adds a new variable called `development` to the configure script that
is disabled by default. When enabled (with the `--enable-development`
option) it will enable certain options and optimizations that are in
general not suitable for the released version. Right now this option,
when enabled, disables cross-module optimizations. We might later
attach more build time optimizations and tweaks to that option.

The cross-module optimizations, when disabled will significantly
improve the rebuilding time (at no to neglible preformance
degradation). Turning them off won't affect a lot, the initial
build time so there is no need to disable them during the initial
installation.

Author: ivg

Makefile

@@ -15,17 +15,23 @@ doc:
 all:
 	$(SETUP) -all $(BAPALLFLAGS)
 
-install:
-	$(SETUP) -install $(BAPINSTALLFLAGS)
+.PHONY: plugins
+install-plugins:
 	sh tools/build_plugins.sh
 	if [ -f ./postinstall.native ]; then ./postinstall.native; fi
+	if [ -f ./postinstall.byte ]; then ./postinstall.byte; fi
+
+install-libs:
+	$(SETUP) -install $(BAPINSTALLFLAGS)
+
+reinstall-libs:
+	$(SETUP) -reinstall $(BAPINSTALLFLAGS)
 
 uninstall:
 	$(SETUP) -uninstall $(BAPUNINSTALLFLAGS)
 
-reinstall:
-	make uninstall
-	make install
+install: install-libs install-plugins
+reinstall: reinstall-libs install-plugins
 
 clean:
 	$(SETUP) -clean $(BAPCLEANFLAGS)

lib/bap_build/bap_build.ml

@@ -19,7 +19,6 @@ module Plugin_rules = struct
   let default_tags = [
     "thread";
     "debug";
-    "short_paths";
     "custom";
     "pp(ppx-jane -dump-ast -inline-test-drop)"
   ] @ List.map default_predicates ~f:(sprintf "predicate(%s)")

lib/bap_bundle/bap_bundle.ml

@@ -126,7 +126,7 @@ module Std = struct
           | Some man -> man
           | None -> Manifest.create "noname" in
         let mdata = Manifest.to_string man in
-        Zip.add_entry mdata zip Nameof.manifest;
+        Zip.add_entry ~level:0 mdata zip Nameof.manifest;
         Zip.close_out zip
     end
 
@@ -194,9 +194,9 @@ module Std = struct
       let zout = Zip.open_out bundle.path in
       Hashtbl.iteri files ~f:(fun ~key:name ~data ->
           match data with
-          | `Data s -> Zip.add_entry s zout name
-          | `Copy f -> Zip.copy_file_to_entry f zout name
-          | `Move f -> Zip.copy_file_to_entry f zout name;
+          | `Data s -> Zip.add_entry ~level:0 s zout name
+          | `Copy f -> Zip.copy_file_to_entry ~level:0 f zout name
+          | `Move f -> Zip.copy_file_to_entry ~level:0 f zout name;
             Sys.remove f);
       Zip.close_out zout
 

myocamlbuild.ml.in

@@ -71,14 +71,44 @@ let predicate_used_libs = function
           (S [A "-predicates"; A ("used_"^lib)]))
   | _ -> ()
 
+let disable_crossmodule_optimization = function
+  | After_rules ->
+    if bool_of_string @@ expand "$development"
+    then flag ["ocaml"; "compile"] (A "-opaque")
+  | _ -> ()
+
 
+let link_shared_native ~cmxa ~dst =
+  Cmd (S [
+      !Options.ocamlopt;
+      A "-shared";
+      A "-linkall";
+      A "-I";
+      A (Filename.dirname cmxa);
+      P cmxa; A "-o"; Px dst])
+
+let register_cmxs_of_cmxa_rule () =
+  rule "bap: cmxa -> cmxs"
+    ~prods:["%.cmxs"]
+    ~deps:["%.cmxa"; "%.a"]
+    (fun env _ ->
+       link_shared_native
+         ~cmxa:(env "%.cmxa")
+         ~dst:(env "%.cmxs"))
+
+let override_broken_cmxs_rule = function
+  | Before_rules ->
+    register_cmxs_of_cmxa_rule ()
+  | _ -> ()
 
 let () =
   mark_tags ();
   Ocamlbuild_plugin.dispatch (fun stage ->
       Rules.dispatch stage;
       dispatch_default stage;
+      override_broken_cmxs_rule stage;
       pr_6184_hack stage;
       pass_pp_to_link stage;
       predicate_used_libs stage;
+      disable_crossmodule_optimization stage;
       predicate_used_packages stage)

oasis/common

@@ -11,9 +11,7 @@ Plugins:     META (0.4)
 AlphaFeatures: ocamlbuild_more_args, compiled_setup_ml
 BuildTools: ocamlbuild
 XOCamlbuildExtraArgs:
-     -j 4
-     -use-ocamlfind
-     -classic-display
+     -j 0
      -plugin-tags "'package(findlib)'"
 
 PreConfCommand: $rm setup.data
@@ -22,3 +20,7 @@ PostDistcleanCommand: $rm  _tags myocamlbuild.ml setup.ml setup.data
 Flag everything
   Description: Build every feature by default
   Default: false
+
+Flag development
+  Description: Enable development mode
+  Default: false

oasis/common.tags.in

@@ -1,6 +1,4 @@
 # OASIS_START
 # OASIS_STOP
-true: short_paths
-true: bin_annot
 true: debug
 true: warn(+a-4-6-7-9-27-29-32..42-44-45-48-50-60)

tools/bap_config.ab

@@ -1,2 +1,3 @@
 PLUGINS_DIR="$plugindir"
 DATA_DIR="$datadir/bap"
+OS="$system"

tools/build_plugins.sh

@@ -4,54 +4,61 @@ set -ue
 
 . $(dirname $0)/bap_config
 
+md5sum=md5sum
+
+if [ $OS = "macosx" ]; then
+    md5sum=md5
+fi
+
+built_plugins=_build/built_plugins
+[ -d $built_plugins ] || mkdir $built_plugins
+
+compute_digest() {
+    plugin_lib=$1
+    output=$2
+    $md5sum `ocamlfind query -predicates byte -a-format -r $plugin_lib` > $output
+}
 
 build_plugin() {
     plugin=bap_plugin_$1
-    TMPDIR=`mktemp -d`
-    cd $TMPDIR
-    touch $plugin.ml
-    bapbuild -package bap-plugin-$1 $plugin.plugin
-    DESC=`ocamlfind query -format "%D" bap-plugin-$1`
-    CONS=`ocamlfind query -format "%(constraints)" bap-plugin-$1`
-    TAGS=`ocamlfind query -format "%(tags)" bap-plugin-$1`
-    if [ ! -z "$CONS" ]; then
-        bapbundle update -cons "$CONS" $plugin.plugin
-    fi
-    if [ ! -z "$TAGS" ]; then
-        bapbundle update -tags "$TAGS" $plugin.plugin
-    fi
-    bapbundle update -desc "$DESC" $plugin.plugin
-    bapbundle update -name $1 $plugin.plugin
+    plugin_lib=bap-plugin-$1
+    BUILDIR=$built_plugins/$1
 
-    mv $plugin.plugin $1.plugin
-    bapbundle install $1.plugin
-    cd -
-    rm -rf $TMPDIR
-}
+    [ -d $BUILDIR ] || mkdir $BUILDIR
+    cd $BUILDIR
+
+    [ -f digest ] || touch digest
 
-cd plugins
+    compute_digest $plugin_lib new_digest
 
-for plugin in `ls`; do
-    if ocamlfind query bap-plugin-$plugin 2>/dev/null
+    if cmp -s digest new_digest
     then
-        plugin_lib=`ocamlfind query bap-plugin-$plugin`
-        if [ -f $PLUGINS_DIR/$plugin.plugin ]
-        then
-            existing_plugin=$PLUGINS_DIR/$plugin.plugin
-            if [ $plugin_lib  -nt $existing_plugin ]
-            then
-                build_plugin $plugin &
-            else
-                echo "Not rebuilding $plugin"
-            fi
+        echo "$1: cache hit"
+    else
+        echo "$1: cache miss"
+        touch $plugin.ml
+        bapbuild -clean
+        bapbuild -package bap-plugin-$1 $plugin.plugin
+        DESC=`ocamlfind query -format "%D" bap-plugin-$1`
+        CONS=`ocamlfind query -format "%(constraints)" bap-plugin-$1`
+        TAGS=`ocamlfind query -format "%(tags)" bap-plugin-$1`
+        if [ -z $CONS ]; then
+            bapbundle update -name $1 -desc "$DESC" -tags "core,$TAGS" $plugin.plugin
         else
-            echo "Building $plugin as it wasn't built yet"
-            build_plugin $plugin &
+            bapbundle update -name $1 -desc "$DESC" -cons "$CONS" -tags "core,$TAGS" $plugin.plugin
         fi
-    else
-        echo "Not building $plugin as it wasn't selected"
+        mv $plugin.plugin $1.plugin
     fi
-done
+    bapbundle install $1.plugin
+    mv new_digest digest
+}
 
-cd ..
+
+for plugin in `ls _build/plugins`; do
+    (build_plugin $plugin)&
+    echo `pwd`
+done
 wait
+
+
+echo "Finished updating plugins"