ARROW-2516: [CI] Filter changes in AppVeyor builds [skip travis]

.travis.yml

@@ -33,7 +33,7 @@ before_install:
         sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
         sudo apt-get update -qq
       fi
-  - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+  - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
 
 
 matrix:

appveyor.yml

@@ -37,46 +37,34 @@ matrix:
 environment:
   global:
     USE_CLCACHE: true
+    PYTHON: "3.6"
+    ARCH: "64"
 
   matrix:
     # NOTE: clcache seems to work best with Ninja and worst with msbuild
     # (as generated by cmake)
     - JOB: "Build"
       GENERATOR: Ninja
-      PYTHON: "3.6"
-      ARCH: "64"
       CONFIGURATION: "Release"
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       BOOST_ROOT: C:\Libraries\boost_1_64_0
     - JOB: "Build_Debug"
       GENERATOR: Ninja
-      PYTHON: "3.6"
-      ARCH: "64"
       CONFIGURATION: "Debug"
     - JOB: "Build"
       GENERATOR: Ninja
-      PYTHON: "3.6"
-      ARCH: "64"
       CONFIGURATION: "Release"
     - JOB: "Static_Crt_Build"
       GENERATOR: Ninja
-      PYTHON: "3.6"
-      ARCH: "64"
     - JOB: "Toolchain"
       GENERATOR: Visual Studio 14 2015 Win64
-      PYTHON: "3.6"
-      ARCH: "64"
       CONFIGURATION: "Release"
     - JOB: "Cmake_Script_Tests"
       GENERATOR: NMake Makefiles
-      PYTHON: "3.6"
-      ARCH: "64"
       CONFIGURATION: "Release"
       BUILD_SCRIPT: "CMake_Build_Script"
     - JOB: "Build"
       GENERATOR: NMake Makefiles
-      PYTHON: "3.6"
-      ARCH: "64"
       CONFIGURATION: "Release"
     - JOB: "Rust_Stable"
       RUST_VERSION: stable
@@ -89,6 +77,13 @@ environment:
   APPVEYOR_SAVE_CACHE_ON_ERROR: true
 
 install:
+  - python ci\detect-changes.py > generated_changes.bat
+  # Populate ARROW_CI_* variables
+  - call generated_changes.bat
+  # Decide to exit if current job doesn't exercise affected topics
+  - call ci\appveyor-filter-changes.bat
+
+before_build:
   - call ci\appveyor-install.bat
 
 build_script:

ci/appveyor-filter-changes.bat

@@ -0,0 +1,32 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements.  See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership.  The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License.  You may obtain a copy of the License at
+@rem
+@rem   http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied.  See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+if "%JOB%" == "Rust_Stable" (
+    if "%ARROW_CI_RUST_AFFECTED%" == "0" (
+        echo ===
+        echo === No Rust changes, exiting job
+        echo ===
+        appveyor exit
+    )
+) else (
+    if "%ARROW_CI_PYTHON_AFFECTED%" == "0" (
+        echo ===
+        echo === No C++ or Python changes, exiting job
+        echo ===
+        appveyor exit
+    )
+)

ci/appveyor-install.bat

@@ -25,5 +25,5 @@ if "%JOB%" == "Rust_Stable" (
     cargo -V
 ) else (
     set "PATH=C:\Miniconda36-x64;C:\Miniconda36-x64\Scripts;C:\Miniconda36-x64\Library\bin;%PATH%"
-    call ci\appveyor-setup.bat
+    call ci\appveyor-cpp-setup.bat
 )

ci/travis_detect_changes.py → ci/detect-changes.py

@@ -95,6 +95,21 @@ def list_travis_affected_files():
         return list_affected_files(commit_range)
 
 
+def list_appveyor_affected_files():
+    """
+    Return a list of files affected in the current AppVeyor build.
+    This only works for PR builds.
+    """
+    # Re-fetch PR base branch (e.g. origin/master), pointing FETCH_HEAD to it
+    run_cmd(["git", "fetch", "-q", "origin",
+             "+refs/heads/{0}".format(os.environ['APPVEYOR_REPO_BRANCH'])])
+    # Compute base changeset between FETCH_HEAD (PR base) and HEAD (PR head)
+    merge_base = run_cmd(["git", "merge-base",
+                          "HEAD", "FETCH_HEAD"]).decode().strip()
+    # Compute changes files between base changeset and HEAD
+    return list_affected_files("{0}..HEAD".format(merge_base))
+
+
 def get_affected_topics(affected_files):
     """
     Return a dict of topics affected by the given files.
@@ -120,10 +135,9 @@ def get_affected_topics(affected_files):
                 affected[k] = True
             break
         elif p in ('cpp', 'format'):
-            # All languages are potentially affected
-            for k in LANGUAGE_TOPICS:
+            # Test C++ and bindings to the C++ library
+            for k in ('cpp', 'python', 'c_glib', 'integration'):
                 affected[k] = True
-            affected['integration'] = True
         elif p in ('java', 'js'):
             affected[p] = True
             affected['integration'] = True
@@ -146,6 +160,14 @@ def get_unix_shell_eval(env):
                       for k, v in env.items()))
 
 
+def get_windows_shell_eval(env):
+    """
+    Return a shell-evalable string to setup some environment variables.
+    """
+    return "\n".join(('set "{0}={1}"'.format(k, v)
+                      for k, v in env.items()))
+
+
 def run_from_travis():
     if (os.environ['TRAVIS_REPO_SLUG'] == 'apache/arrow' and
         os.environ['TRAVIS_BRANCH'] == 'master' and
@@ -172,12 +194,36 @@ def run_from_travis():
     return get_unix_shell_eval(make_env_for_topics(affected))
 
 
+def run_from_appveyor():
+    if not os.environ.get('APPVEYOR_PULL_REQUEST_HEAD_COMMIT'):
+        # Not a PR build, test everything
+        affected = dict.fromkeys(ALL_TOPICS, True)
+    else:
+        affected_files = list_appveyor_affected_files()
+        perr("Affected files:", affected_files)
+        affected = get_affected_topics(affected_files)
+        assert set(affected) <= set(ALL_TOPICS), affected
+
+    perr("Affected topics:")
+    perr(pprint.pformat(affected))
+    return get_windows_shell_eval(make_env_for_topics(affected))
+
+
 if __name__ == "__main__":
     # This script should have its output evaluated by a shell,
-    # e.g. "eval `python ci/travis_detect_changes.py`"
-    try:
-        print(run_from_travis())
-    except:
-        # Make sure the enclosing eval will return an error
-        print("exit 1")
-        raise
+    # e.g. "eval `python ci/detect-changes.py`"
+    if os.environ.get('TRAVIS'):
+        try:
+            print(run_from_travis())
+        except:
+            # Make sure the enclosing eval will return an error
+            print("exit 1")
+            raise
+    elif os.environ.get('APPVEYOR'):
+        try:
+            print(run_from_appveyor())
+        except:
+            print("exit 1")
+            raise
+    else:
+        sys.exit("Script must be run under Travis-CI or AppVeyor")