Attempt to cache git modules

.travis.yml

@@ -133,13 +133,14 @@ before_script:
 script:
   - >
       if [ "$ALLOW_PR" = "" ] && [ "$TRAVIS_BRANCH" != "auto" ]; then
-          echo skipping, not a full build;
-      elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
-          travis_retry stamp sh -c 'git submodule deinit -f . && git submodule update --init' &&
-          stamp src/ci/run.sh;
+          echo skipping, not a full build
       else
-          travis_retry stamp sh -c 'git submodule deinit -f . && git submodule update --init' &&
-          stamp src/ci/docker/run.sh $IMAGE;
+          stamp src/ci/init_repo.sh . "$HOME/rustsrc" &&
+          if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+              stamp src/ci/run.sh;
+          else
+              stamp src/ci/docker/run.sh $IMAGE;
+          fi
       fi
 
 after_success:
@@ -157,20 +158,29 @@ after_failure:
   - cat /tmp/sccache.log
 
 # Save tagged docker images we created and load them if they're available
+# Travis saves caches whether the build failed or not, nuke rustsrc if
+# the failure was while updating it (as it may be in an bad state)
+# https://github.com/travis-ci/travis-ci/issues/4472
 before_cache:
   - docker history -q rust-ci |
     grep -v missing |
     xargs docker save |
     gzip > $HOME/docker/rust-ci.tar.gz
+  - if [ ! -f $HOME/rustsrc/cache_valid1 ]; then
+        echo "WARNING rustsrc cache was invalid when saving";
+        rm -rf $HOME/rustsrc && mkdir $HOME/rustsrc;
+    fi
 before_install:
   - zcat $HOME/docker/rust-ci.tar.gz | docker load || true
+  - mkdir -p $HOME/rustsrc
 
 notifications:
   email: false
 
 cache:
   directories:
     - $HOME/docker
+    - $HOME/rustsrc
 
 before_deploy:
   - mkdir -p deploy/$TRAVIS_COMMIT

appveyor.yml

@@ -141,7 +141,8 @@ install:
   - set SCCACHE_ERROR_LOG=%CD%/sccache.log
 
 test_script:
-  - appveyor-retry sh -c 'git submodule deinit -f . && git submodule update --init'
+  - mkdir C:\cache\rustsrc
+  - sh src/ci/init_repo.sh . /c/cache/rustsrc
   - set SRC=.
   - set NO_CCACHE=1
   - sh src/ci/run.sh
@@ -150,6 +151,7 @@ on_failure:
   - cat %CD%/sccache.log
 
 cache:
+  - C:\cache\rustsrc
   - "build/i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
   - "build/x86_64-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
   - "i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"

src/ci/docker/run.sh

@@ -57,6 +57,7 @@ exec docker \
   --env DEPLOY_ALT=$DEPLOY_ALT \
   --env LOCAL_USER_ID=`id -u` \
   --volume "$HOME/.cargo:/cargo" \
+  --volume "$HOME/rustsrc:$HOME/rustsrc" \
   --privileged \
   --rm \
   rust-ci \

src/ci/init_repo.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+set -o xtrace
+
+ci_dir=$(cd $(dirname $0) && pwd)
+. "$ci_dir/shared.sh"
+
+REPO_DIR="$1"
+CACHE_DIR="$2"
+
+cache_src_dir="$CACHE_DIR/src"
+# If the layout of the cache directory changes, bump the number here
+# (and anywhere else this file is referenced) so the cache is wiped
+cache_valid_file="$CACHE_DIR/cache_valid1"
+
+if [ ! -d "$REPO_DIR" -o ! -d "$REPO_DIR/.git" ]; then
+    echo "Error: $REPO_DIR does not exist or is not a git repo"
+    exit 1
+fi
+cd $REPO_DIR
+if [ ! -d "$CACHE_DIR" ]; then
+    echo "Error: $CACHE_DIR does not exist or is not an absolute path"
+    exit 1
+fi
+
+# Wipe the cache if it's not valid, or mark it as invalid while we update it
+if [ ! -f "$cache_valid_file" ]; then
+    rm -rf "$CACHE_DIR" && mkdir "$CACHE_DIR"
+else
+    rm "$cache_valid_file"
+fi
+
+# Update the cache (a pristine copy of the rust source master)
+if [ ! -d "$cache_src_dir/.git" ]; then
+    retry sh -c "rm -rf $cache_src_dir && mkdir -p $cache_src_dir && \
+        git clone https://github.com/rust-lang/rust.git $cache_src_dir"
+fi
+retry sh -c "cd $cache_src_dir && git reset --hard && git pull"
+retry sh -c "cd $cache_src_dir && \
+    git submodule deinit -f . && git submodule sync && git submodule update --init"
+
+# Cache was updated without errors, mark it as valid
+touch "$cache_valid_file"
+
+# Update the submodules of the repo we're in, using the pristine repo as
+# a cache for any object files
+# No, `git submodule foreach` won't work:
+# http://stackoverflow.com/questions/12641469/list-submodules-in-a-git-repository
+modules="$(git config --file .gitmodules --get-regexp '\.path$' | cut -d' ' -f2)"
+for module in $modules; do
+    if [ ! -d "$cache_src_dir/$module" ]; then
+        echo "WARNING: $module not found in pristine repo"
+        retry sh -c "git submodule deinit -f $module && git submodule update --init $module"
+        continue
+    fi
+    retry sh -c "git submodule deinit -f $module && \
+        git submodule update --init --reference $cache_src_dir/$module $module"
+done

src/ci/shared.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/false
 # Copyright 2016 The Rust Project Developers. See the COPYRIGHT
 # file at the top-level directory of this distribution and at
 # http://rust-lang.org/COPYRIGHT.
@@ -9,13 +9,16 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
+# This file is intended to be sourced with `. shared.sh` or
+# `source shared.sh`, hence the invalid shebang and not being
+# marked as an executable file in git.
+
 # See http://unix.stackexchange.com/questions/82598
 function retry {
+  echo "Attempting with retry:" "$@"
   local n=1
   local max=5
-  local delay=15
   while true; do
-    echo "Attempting:" "$@"
     "$@" && break || {
       if [[ $n -lt $max ]]; then
         ((n++))