auto merge of #11787 : alexcrichton/rust/refactor, r=brson

It was decided a long, long time ago that libextra should not exist, but rather its modules should be split out into smaller independent libraries maintained outside of the compiler itself. The theory was to use `rustpkg` to manage dependencies in order to move everything out of the compiler, but maintain an ease of usability.

Sadly, the work on `rustpkg` isn't making progress as quickly as expected, but the need for dissolving libextra is becoming more and more pressing. Because of this, we've thought that a good interim solution would be to simply package more libraries with the rust distribution itself. Instead of dissolving libextra into libraries outside of the mozilla/rust repo, we can dissolve libraries into the mozilla/rust repo for now.

Work on this has been excruciatingly painful in the past because the makefiles are completely opaque to all but a few. Adding a new library involved adding about 100 lines spread out across 8 files (incredibly error prone). The first commit of this pull request targets this pain point. It does not rewrite the build system, but rather refactors large portions of it. Afterwards, adding a new library is as simple as modifying 2 lines (easy, right?). The build system automatically keeps track of dependencies between crates (rust *and* native), promotes binaries between stages, tracks dependencies of installed tools, etc, etc.

With this newfound buildsystem power, I chose the `extra::flate` module as the first candidate for removal from libextra. While a small module, this module is relative complex in that is has a C dependency and the compiler requires it (messing with the dependency graph a bit). Albeit I modified more than 2 lines of makefiles to accomodate libflate (the native dependency required 2 extra lines of modifications), but the removal process was easy to do and straightforward.

---

Testing-wise, I've cross-compiled, run tests, built some docs, installed, uninstalled, etc. I'm still working out a few kinks, and I'm sure that there's gonna be built system issues after this, but it should be working well for basic use!

cc #8784

Makefile.in

@@ -222,72 +222,27 @@ GENERATED :=
 
 
 ######################################################################
-# Crates
+# Cleaning out old crates
 ######################################################################
 
-define DEF_LIBS
-
-CFG_RUNTIME_$(1) :=$(call CFG_STATIC_LIB_NAME_$(1),rustrt)
-CFG_RUSTLLVM_$(1) :=$(call CFG_STATIC_LIB_NAME_$(1),rustllvm)
-CFG_STDLIB_$(1) :=$(call CFG_LIB_NAME_$(1),std)
-CFG_EXTRALIB_$(1) :=$(call CFG_LIB_NAME_$(1),extra)
-CFG_LIBRUSTC_$(1) :=$(call CFG_LIB_NAME_$(1),rustc)
-CFG_LIBSYNTAX_$(1) :=$(call CFG_LIB_NAME_$(1),syntax)
-CFG_LIBRUSTPKG_$(1) :=$(call CFG_LIB_NAME_$(1),rustpkg)
-CFG_LIBRUSTDOC_$(1) :=$(call CFG_LIB_NAME_$(1),rustdoc)
-CFG_LIBRUSTUV_$(1) :=$(call CFG_LIB_NAME_$(1),rustuv)
-CFG_LIBGREEN_$(1) :=$(call CFG_LIB_NAME_$(1),green)
-CFG_LIBNATIVE_$(1) :=$(call CFG_LIB_NAME_$(1),native)
-
-EXTRALIB_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),extra)
-STDLIB_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),std)
-LIBRUSTC_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),rustc)
-LIBSYNTAX_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),syntax)
-LIBRUSTPKG_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),rustpkg)
-LIBRUSTDOC_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),rustdoc)
-LIBRUSTUV_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),rustuv)
-LIBGREEN_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),green)
-LIBNATIVE_GLOB_$(1) :=$(call CFG_LIB_GLOB_$(1),native)
-EXTRALIB_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),extra)
-STDLIB_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),std)
-LIBRUSTC_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),rustc)
-LIBSYNTAX_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),syntax)
-LIBRUSTPKG_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),rustpkg)
-LIBRUSTDOC_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),rustdoc)
-LIBRUSTUV_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),rustuv)
-LIBGREEN_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),green)
-LIBNATIVE_DSYM_GLOB_$(1) :=$(call CFG_LIB_DSYM_GLOB_$(1),native)
-
-EXTRALIB_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,extra)
-STDLIB_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,std)
-LIBRUSTUV_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,rustuv)
-LIBSYNTAX_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,syntax)
-LIBRUSTC_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,rustc)
-LIBNATIVE_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,native)
-LIBGREEN_RGLOB_$(1) :=$(call CFG_RLIB_GLOB,green)
-
-endef
-
 # $(1) is the path for directory to match against
 # $(2) is the glob to use in the match
-# $(3) is filename (usually the target being created) to filter out from match
-#      (i.e. filename is not out-of-date artifact from prior Rust version/build)
 #
 # Note that a common bug is to accidentally construct the glob denoted
 # by $(2) with a space character prefix, which invalidates the
 # construction $(1)$(2).
-define CHECK_FOR_OLD_GLOB_MATCHES_EXCEPT
-  $(Q)MATCHES="$(filter-out %$(3),$(wildcard $(1)/$(2)))"; if [ -n "$$MATCHES" ] ; then echo "warning: there are previous" \'$(2)\' "libraries:" $$MATCHES; fi
+define CHECK_FOR_OLD_GLOB_MATCHES
+  $(Q)MATCHES="$(wildcard $(1))"; if [ -n "$$MATCHES" ] ; then echo "warning: there are previous" \'$(notdir $(2))\' "libraries:" $$MATCHES; fi
 endef
 
 # Same interface as above, but deletes rather than just listing the files.
 ifdef VERBOSE
-define REMOVE_ALL_OLD_GLOB_MATCHES_EXCEPT
-  $(Q)MATCHES="$(filter-out %$(3),$(wildcard $(1)/$(2)))"; if [ -n "$$MATCHES" ] ; then echo "warning: removing previous" \'$(2)\' "libraries:" $$MATCHES; rm $$MATCHES ; fi
+define REMOVE_ALL_OLD_GLOB_MATCHES
+  $(Q)MATCHES="$(wildcard $(1))"; if [ -n "$$MATCHES" ] ; then echo "warning: removing previous" \'$(notdir $(1))\' "libraries:" $$MATCHES; rm $$MATCHES ; fi
 endef
 else
-define REMOVE_ALL_OLD_GLOB_MATCHES_EXCEPT
-  $(Q)MATCHES="$(filter-out %$(3),$(wildcard $(1)/$(2)))"; if [ -n "$$MATCHES" ] ; then rm $$MATCHES ; fi
+define REMOVE_ALL_OLD_GLOB_MATCHES
+  $(Q)MATCHES="$(wildcard $(1))"; if [ -n "$$MATCHES" ] ; then rm $$MATCHES ; fi
 endef
 endif
 
@@ -298,72 +253,15 @@ endif
 # soon. (This is in contrast to the macros above, which are meant to
 # be run at the outset of a command list in a rule.)
 ifdef VERBOSE
-define LIST_ALL_OLD_GLOB_MATCHES_EXCEPT
-  @echo "info: now are following matches for" '$(2)' "libraries:"
-  @( cd $(1) && ( ls $(2) 2>/dev/null || true ) | grep -v $(3) || true )
+define LIST_ALL_OLD_GLOB_MATCHES
+  @echo "info: now are following matches for" '$(notdir $(1))' "libraries:"
+  @( ls $(1) 2>/dev/null || true )
 endef
 else
-define LIST_ALL_OLD_GLOB_MATCHES_EXCEPT
+define LIST_ALL_OLD_GLOB_MATCHES
 endef
 endif
 
-$(foreach target,$(CFG_TARGET),\
-  $(eval $(call DEF_LIBS,$(target))))
-
-######################################################################
-# Standard library variables
-######################################################################
-
-STDLIB_CRATE := $(S)src/libstd/lib.rs
-STDLIB_INPUTS := $(wildcard $(addprefix $(S)src/libstd/,        \
-                                           *.rs */*.rs */*/*rs */*/*/*rs))
-
-######################################################################
-# Extra library variables
-######################################################################
-
-EXTRALIB_CRATE := $(S)src/libextra/lib.rs
-EXTRALIB_INPUTS := $(wildcard $(addprefix $(S)src/libextra/,          \
-                                          *.rs */*.rs))
-
-######################################################################
-# Rust UV library variables
-######################################################################
-
-LIBRUSTUV_CRATE := $(S)src/librustuv/lib.rs
-LIBRUSTUV_INPUTS := $(wildcard $(addprefix $(S)src/librustuv/,          \
-                                          *.rs */*.rs))
-
-######################################################################
-# Green threading library variables
-######################################################################
-
-LIBGREEN_CRATE := $(S)src/libgreen/lib.rs
-LIBGREEN_INPUTS := $(wildcard $(addprefix $(S)src/libgreen/,          \
-                                          *.rs */*.rs))
-
-######################################################################
-# Native threading library variables
-######################################################################
-
-LIBNATIVE_CRATE := $(S)src/libnative/lib.rs
-LIBNATIVE_INPUTS := $(wildcard $(addprefix $(S)src/libnative/,          \
-                                          *.rs */*.rs))
-
-######################################################################
-# rustc crate variables
-######################################################################
-
-COMPILER_CRATE := $(S)src/librustc/lib.rs
-COMPILER_INPUTS := $(wildcard $(addprefix $(S)src/librustc/,      \
-                           *.rs */*.rs */*/*.rs */*/*/*.rs))
-
-LIBSYNTAX_CRATE := $(S)src/libsyntax/lib.rs
-LIBSYNTAX_INPUTS := $(wildcard $(addprefix $(S)src/libsyntax/, \
-                           *.rs */*.rs */*/*.rs */*/*/*.rs))
-
-DRIVER_CRATE := $(S)src/driver/driver.rs
-
 ######################################################################
 # LLVM macros
 ######################################################################
@@ -424,14 +322,12 @@ export CFG_RUSTLIBDIR
 export CFG_LIBDIR_RELATIVE
 export CFG_DISABLE_INJECT_STD_VERSION
 
-######################################################################
-# Subprograms
-######################################################################
-
 ######################################################################
 # Per-stage targets and runner
 ######################################################################
 
+include $(CFG_SRC_DIR)mk/crates.mk
+
 define SREQ
 # $(1) is the stage number
 # $(2) is the target triple
@@ -447,84 +343,37 @@ TROOT$(1)_T_$(2)_H_$(3) = $$(HLIB$(1)_H_$(3))/$$(CFG_RUSTLIBDIR)/$(2)
 TBIN$(1)_T_$(2)_H_$(3) = $$(TROOT$(1)_T_$(2)_H_$(3))/bin
 TLIB$(1)_T_$(2)_H_$(3) = $$(TROOT$(1)_T_$(2)_H_$(3))/lib
 
-# The name of the standard and extra libraries used by rustc
-HSTDLIB_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_STDLIB_$(3))
-TSTDLIB_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_STDLIB_$(2))
-
-HEXTRALIB_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_EXTRALIB_$(3))
-TEXTRALIB_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_EXTRALIB_$(2))
-
-HLIBRUSTC_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_LIBRUSTC_$(3))
-TLIBRUSTC_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTC_$(2))
-
-HLIBRUSTUV_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_LIBRUSTUV_$(3))
-TLIBRUSTUV_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTUV_$(2))
-
-HLIBGREEN_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_LIBGREEN_$(3))
-TLIBGREEN_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBGREEN_$(2))
-
-HLIBNATIVE_DEFAULT$(1)_H_$(3) = \
-  $$(HLIB$(1)_H_$(3))/$(CFG_LIBNATIVE_$(3))
-TLIBNATIVE_DEFAULT$(1)_T_$(2)_H_$(3) = \
-  $$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBNATIVE_$(2))
-
 # Preqrequisites for using the stageN compiler
 ifeq ($(1),0)
 HSREQ$(1)_H_$(3) = $$(HBIN$(1)_H_$(3))/rustc$$(X_$(3))
 else
 HSREQ$(1)_H_$(3) = \
 	$$(HBIN$(1)_H_$(3))/rustc$$(X_$(3)) \
-	$$(HSTDLIB_DEFAULT$(1)_H_$(3)) \
-	$$(HEXTRALIB_DEFAULT$(1)_H_$(3)) \
-	$$(HLIBSYNTAX_DEFAULT$(1)_H_$(3)) \
-	$$(HLIBRUSTC_DEFAULT$(1)_H_$(3)) \
-	$$(HLIBRUSTUV_DEFAULT$(1)_H_$(3)) \
-	$$(HLIBGREEN_DEFAULT$(1)_H_$(3)) \
-	$$(HLIBNATIVE_DEFAULT$(1)_H_$(3)) \
+	$$(HLIB$(1)_H_$(3))/stamp.rustc \
+	$$(foreach dep,$$(RUST_DEPS_rustc),$$(HLIB$(1)_H_$(3))/stamp.$$(dep)) \
 	$$(MKFILE_DEPS)
 endif
 
 # Prerequisites for using the stageN compiler to build target artifacts
 TSREQ$(1)_T_$(2)_H_$(3) = \
 	$$(HSREQ$(1)_H_$(3)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_RUNTIME_$(2)) \
 	$$(TLIB$(1)_T_$(2)_H_$(3))/libmorestack.a
 
-# Prerequisites for a working stageN compiler and libraries, for a specific target
+# Prerequisites for a working stageN compiler and libraries, for a specific
+# target
 SREQ$(1)_T_$(2)_H_$(3) = \
 	$$(TSREQ$(1)_T_$(2)_H_$(3)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_STDLIB_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_EXTRALIB_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTUV_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBGREEN_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBNATIVE_$(2))
+	$$(foreach dep,$$(TARGET_CRATES),\
+	    $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(dep))
 
-# Prerequisites for a working stageN compiler and libraries, for a specific target
+# Prerequisites for a working stageN compiler and complete set of target
+# libraries
 CSREQ$(1)_T_$(2)_H_$(3) = \
 	$$(TSREQ$(1)_T_$(2)_H_$(3)) \
 	$$(HBIN$(1)_H_$(3))/rustpkg$$(X_$(3)) \
 	$$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
-	$$(HLIB$(1)_H_$(3))/$(CFG_LIBRUSTPKG_$(3)) \
-	$$(HLIB$(1)_H_$(3))/$(CFG_LIBRUSTDOC_$(3)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_STDLIB_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_EXTRALIB_$(2))  \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBSYNTAX_$(2))  \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTC_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTPKG_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTDOC_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBRUSTUV_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBGREEN_$(2)) \
-	$$(TLIB$(1)_T_$(2)_H_$(3))/$(CFG_LIBNATIVE_$(2))
+	$$(foreach dep,$$(CRATES),$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(dep)) \
+	$$(foreach dep,$$(HOST_CRATES),$$(HLIB$(1)_H_$(3))/stamp.$$(dep))
 
 ifeq ($(1),0)
 # Don't run the the stage0 compiler under valgrind - that ship has sailed
@@ -562,15 +411,15 @@ endif
 STAGE$(1)_T_$(2)_H_$(3) := 						\
 	$$(Q)$$(RPATH_VAR$(1)_T_$(2)_H_$(3))                            \
 		$$(call CFG_RUN_TARG_$(3),$(1),				\
-		$$(CFG_VALGRIND_COMPILE$(1)) 			\
+		$$(CFG_VALGRIND_COMPILE$(1))				\
 		$$(HBIN$(1)_H_$(3))/rustc$$(X_$(3))			\
 		--cfg $$(CFGFLAG$(1)_T_$(2)_H_$(3))			\
 		$$(CFG_RUSTC_FLAGS) $$(EXTRAFLAGS_STAGE$(1)) --target=$(2)) \
                 $$(RUSTC_FLAGS_$(2))
 
-PERF_STAGE$(1)_T_$(2)_H_$(3) := 				\
+PERF_STAGE$(1)_T_$(2)_H_$(3) :=						\
 	$$(Q)$$(call CFG_RUN_TARG_$(3),$(1),				\
-		$$(CFG_PERF_TOOL) 						\
+		$$(CFG_PERF_TOOL) 					\
 		$$(HBIN$(1)_H_$(3))/rustc$$(X_$(3))			\
 		--cfg $$(CFGFLAG$(1)_T_$(2)_H_$(3))			\
 		$$(CFG_RUSTC_FLAGS) $$(EXTRAFLAGS_STAGE$(1)) --target=$(2)) \
@@ -594,13 +443,12 @@ define DEF_RUSTC_STAGE_TARGET
 # $(2) == stage
 
 rustc-stage$(2)-H-$(1):							\
-	$$(foreach target,$$(CFG_TARGET),	\
-		$$(SREQ$(2)_T_$$(target)_H_$(1)))
+	$$(foreach target,$$(CFG_TARGET),$$(SREQ$(2)_T_$$(target)_H_$(1)))
 
 endef
 
-$(foreach host,$(CFG_HOST),							\
- $(eval $(foreach stage,1 2 3,									\
+$(foreach host,$(CFG_HOST),						\
+ $(eval $(foreach stage,1 2 3,						\
   $(eval $(call DEF_RUSTC_STAGE_TARGET,$(host),$(stage))))))
 
 rustc-stage1: rustc-stage1-H-$(CFG_BUILD)
@@ -701,7 +549,6 @@ include $(CFG_SRC_DIR)mk/target.mk
 include $(CFG_SRC_DIR)mk/host.mk
 include $(CFG_SRC_DIR)mk/stage0.mk
 include $(CFG_SRC_DIR)mk/rustllvm.mk
-include $(CFG_SRC_DIR)mk/tools.mk
 include $(CFG_SRC_DIR)mk/docs.mk
 include $(CFG_SRC_DIR)mk/llvm.mk
 

doc/index.md

@@ -37,6 +37,8 @@ li {list-style-type: none; }
 * [The Rust parser, `libsyntax`](syntax/index.html)
 * [The Rust compiler, `librustc`](rustc/index.html)
 
+* [The `flate` compression library](flate/index.html)
+
 # Tooling
 
 * [The `rustdoc` manual](rustdoc.html)
@@ -47,11 +49,11 @@ li {list-style-type: none; }
 * [Language FAQ](complement-lang-faq.html)
 * [Project FAQ](complement-project-faq.html)
 * [Usage FAQ](complement-usage-faq.html)
-* [Code cheatsheet](complement-cheatsheet.html) - "How do I do X?"  
+* [Code cheatsheet](complement-cheatsheet.html) - "How do I do X?"
 * [How to submit a bug report](complement-bugreport.html)
 
 # External resources
 
-* The Rust [IRC channel](http://chat.mibbit.com/?server=irc.mozilla.org&channel=%23rust) - `#rust` on irc.mozilla.org  
+* The Rust [IRC channel](http://chat.mibbit.com/?server=irc.mozilla.org&channel=%23rust) - `#rust` on irc.mozilla.org
 * The Rust community on [Reddit](http://reddit.com/r/rust)
 * The Rust [wiki](https://github.com/mozilla/rust/wiki)