Merge pull request #715 from s1113950/collectionsSupport

Ansible 2.10 + Collections support

.ci/ansible_install.py

@@ -11,7 +11,8 @@
         'pip install '
             '-r tests/requirements.txt '
             '-r tests/ansible/requirements.txt',
-        'pip install -q ansible=={0}'.format(ci_lib.ANSIBLE_VERSION)
+        # encoding is required for installing ansible 2.10 with pip2, otherwise we get a UnicodeDecode error
+        'LC_CTYPE=en_US.UTF-8 LANG=en_US.UTF-8 pip install -q ansible=={0}'.format(ci_lib.ANSIBLE_VERSION)
     ]
 ]
 

.ci/ansible_tests.py

@@ -37,9 +37,6 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('job_setup'):
-    # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
-    run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION)
-
     os.chdir(TESTS_DIR)
     os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7))
 
@@ -75,7 +72,7 @@ def pause_if_interactive():
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
     try:
-        run('./run_ansible_playbook.py %s -i "%s" %s',
+        run('./run_ansible_playbook.py %s -i "%s" -vvv %s',
             playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
     except:
         pause_if_interactive()

.ci/azure-pipelines-steps.yml

@@ -8,23 +8,7 @@ steps:
 - script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py"
   displayName: "Run prep_azure.py"
 
-# The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
-# broken symlinks, incorrect permissions and missing codecs. So we use the
-# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
-# stuff into. The virtualenv can probably be removed again, but this was a
-# hard-fought battle and for now I am tired of this crap.
 - script: |
-    # need wheel before building virtualenv because of bdist_wheel and setuptools deps
-    # Mac's System Integrity Protection prevents symlinking /usr/bin
-    # and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html
-    # the || will activate when running python3 tests
-    # TODO: get python3 tests passing
-    (sudo ln -fs /usr/bin/python$(python.version) /usr/bin/python &&
-      /usr/bin/python -m pip install -U pip wheel setuptools &&
-      /usr/bin/python -m pip install -U virtualenv &&
-      /usr/bin/python -m virtualenv /tmp/venv -p /usr/bin/python$(python.version)) ||
-    (sudo /usr/bin/python$(python.version) -m pip install -U pip wheel setuptools &&
-      /usr/bin/python$(python.version) -m venv /tmp/venv)
     echo "##vso[task.prependpath]/tmp/venv/bin"
 
   displayName: activate venv

.ci/azure-pipelines.yml

@@ -6,19 +6,30 @@
 jobs:
 
 - job: Mac
+  # vanilla Ansible is really slow
+  timeoutInMinutes: 120
   steps:
   - template: azure-pipelines-steps.yml
   pool:
-    vmImage: macOS-10.14
+    vmImage: macOS-10.15
   strategy:
     matrix:
       Mito27_27:
-        python.version: '2.7.18'
+        python.version: '2.7'
         MODE: mitogen
-      Ans288_27:
-        python.version: '2.7.18'
+        VER: 2.10.0
+      # TODO: test python3, python3 tests are broken
+      Ans210_27:
+        python.version: '2.7'
+        MODE: localhost_ansible
+        VER: 2.10.0
+
+      # NOTE: this hangs when ran in Ubuntu 18.04
+      Vanilla_210_27:
+        python.version: '2.7'
         MODE: localhost_ansible
-        VER: 2.8.8
+        VER: 2.10.0
+        STRATEGY: linear
 
 
 - job: Linux
@@ -35,6 +46,7 @@ jobs:
         python.version: '2.7'
         MODE: mitogen
         DISTRO: debian
+        VER: 2.10.0
 
       #MitoPy27CentOS6_26:
         #python.version: '2.7'
@@ -45,12 +57,13 @@ jobs:
         python.version: '3.6'
         MODE: mitogen
         DISTRO: centos6
+        VER: 2.10.0
 
       Mito37Debian_27:
         python.version: '3.7'
         MODE: mitogen
         DISTRO: debian
-        VER: 2.9.6
+        VER: 2.10.0
 
       #Py26CentOS7:
         #python.version: '2.7'
@@ -94,17 +107,12 @@ jobs:
         #DISTROS: debian
         #STRATEGY: linear
 
-      Ansible_280_27:
+      Ansible_210_27:
         python.version: '2.7'
         MODE: ansible
-        VER: 2.8.0
+        VER: 2.10.0
 
-      Ansible_280_35:
+      Ansible_210_35:
         python.version: '3.5'
         MODE: ansible
-        VER: 2.8.0
-
-      Ansible_296_37:
-        python.version: '3.7'
-        MODE: ansible
-        VER: 2.9.6
+        VER: 2.10.0

.ci/ci_lib.py

@@ -49,6 +49,10 @@ def have_apt():
     proc = subprocess.Popen('apt --help >/dev/null 2>/dev/null', shell=True)
     return proc.wait() == 0
 
+def have_brew():
+    proc = subprocess.Popen('brew help >/dev/null 2>/dev/null', shell=True)
+    return proc.wait() == 0
+
 
 def have_docker():
     proc = subprocess.Popen('docker info >/dev/null 2>/dev/null', shell=True)

.ci/debops_common_install.py

@@ -10,9 +10,11 @@
         # Must be installed separately, as PyNACL indirect requirement causes
         # newer version to be installed if done in a single pip run.
         'pip install "pycparser<2.19"',
-        'pip install -qqqU debops==0.7.2 ansible==%s' % ci_lib.ANSIBLE_VERSION,
+        'pip install -qqq debops[ansible]==2.1.2 ansible==%s' % ci_lib.ANSIBLE_VERSION,
     ],
     [
         'docker pull %s' % (ci_lib.image_for_distro('debian'),),
     ],
 ])
+
+ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2')

.ci/debops_common_tests.py

@@ -26,12 +26,14 @@
     ci_lib.run('debops-init %s', project_dir)
     os.chdir(project_dir)
 
+    ansible_strategy_plugin = "{}/ansible_mitogen/plugins/strategy".format(ci_lib.GIT_ROOT)
+
     with open('.debops.cfg', 'w') as fp:
         fp.write(
             "[ansible defaults]\n"
-            "strategy_plugins = %s/ansible_mitogen/plugins/strategy\n"
+            "strategy_plugins = {}\n"
             "strategy = mitogen_linear\n"
-            % (ci_lib.GIT_ROOT,)
+            .format(ansible_strategy_plugin)
         )
 
     with open(vars_path, 'w') as fp:

.ci/localhost_ansible_install.py

@@ -7,7 +7,8 @@
         # Must be installed separately, as PyNACL indirect requirement causes
         # newer version to be installed if done in a single pip run.
         # Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml
-        'pip install "pycparser<2.19" "idna<2.7"',
+        # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
+        'pip install "pycparser<2.19" "idna<2.7" virtualenv',
         'pip install '
             '-r tests/requirements.txt '
             '-r tests/ansible/requirements.txt',

.ci/localhost_ansible_tests.py

@@ -20,12 +20,15 @@
 
 
 with ci_lib.Fold('job_setup'):
-    # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
-    run("pip install -q virtualenv ansible==%s", ci_lib.ANSIBLE_VERSION)
-
     os.chmod(KEY_PATH, int('0600', 8))
+    # NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n", 
+    # there's a checksum error with "brew install http://git.io/sshpass.rb" though, so installing manually
     if not ci_lib.exists_in_path('sshpass'):
-        run("brew install http://git.io/sshpass.rb")
+        os.system("curl -O -L  https://sourceforge.net/projects/sshpass/files/sshpass/1.05/sshpass-1.05.tar.gz && \
+                tar xvf sshpass-1.05.tar.gz && \
+                cd sshpass-1.05 && \
+                ./configure && \
+                sudo make install")
 
 
 with ci_lib.Fold('machine_prep'):

.ci/prep_azure.py

@@ -30,8 +30,20 @@
         ]
     ]
 
+# setup venv, need all python commands in 1 list to be subprocessed at the same time
+venv_steps = []
+
+need_to_fix_psycopg2 = False
+
+is_python3 = os.environ['PYTHONVERSION'].startswith('3')
+
+# @dw: The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
+# broken symlinks, incorrect permissions and missing codecs. So we use the
+# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
+# stuff into. The virtualenv can probably be removed again, but this was a
+# hard-fought battle and for now I am tired of this crap.
 if ci_lib.have_apt():
-    batches.append([
+    venv_steps.extend([
         'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
         'sudo add-apt-repository ppa:deadsnakes/ppa',
         'sudo apt-get update',
@@ -40,8 +52,39 @@
             'python{pv}-dev '
             'libsasl2-dev '
             'libldap2-dev '
-            .format(pv=os.environ['PYTHONVERSION'])
+            .format(pv=os.environ['PYTHONVERSION']),
+        'sudo ln -fs /usr/bin/python{pv} /usr/local/bin/python{pv}'
+        .format(pv=os.environ['PYTHONVERSION'])
+    ])
+    if is_python3:
+        venv_steps.append('sudo apt-get -y install python{pv}-venv'.format(pv=os.environ['PYTHONVERSION']))
+# TODO: somehow `Mito36CentOS6_26` has both brew and apt installed https://dev.azure.com/dw-mitogen/Mitogen/_build/results?buildId=1031&view=logs&j=7bdbcdc6-3d3e-568d-ccf8-9ddca1a9623a&t=73d379b6-4eea-540f-c97e-046a2f620483
+elif is_python3 and ci_lib.have_brew():
+    # Mac's System Integrity Protection prevents symlinking /usr/bin
+    # and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html
+    # so we'll use /usr/local/bin/python for everything
+    # /usr/local/bin/python2.7 already exists!
+    need_to_fix_psycopg2 = True
+    venv_steps.append(
+        'brew install python@{pv} postgresql'
+        .format(pv=os.environ['PYTHONVERSION'])
+    )
+
+# need wheel before building virtualenv because of bdist_wheel and setuptools deps
+venv_steps.append('/usr/local/bin/python{pv} -m pip install -U pip wheel setuptools'.format(pv=os.environ['PYTHONVERSION']))
+
+if os.environ['PYTHONVERSION'].startswith('2'):
+    venv_steps.extend([
+        '/usr/local/bin/python{pv} -m pip install -U virtualenv'.format(pv=os.environ['PYTHONVERSION']),
+        '/usr/local/bin/python{pv} -m virtualenv /tmp/venv -p /usr/local/bin/python{pv}'.format(pv=os.environ['PYTHONVERSION'])
     ])
+else:
+    venv_steps.append('/usr/local/bin/python{pv} -m venv /tmp/venv'.format(pv=os.environ['PYTHONVERSION']))
+# fixes https://stackoverflow.com/questions/59595649/can-not-install-psycopg2-on-macos-catalina https://github.com/Azure/azure-cli/issues/12854#issuecomment-619213863
+if need_to_fix_psycopg2:
+    venv_steps.append('/tmp/venv/bin/pip3 install psycopg2==2.8.5 psycopg2-binary')
+
+batches.append(venv_steps)
 
 
 if ci_lib.have_docker():

.ci/travis.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# workaround from https://stackoverflow.com/a/26082445 to handle Travis 4MB log limit
+set -e
+
+export PING_SLEEP=30s
+export WORKDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+export BUILD_OUTPUT=$WORKDIR/build.out
+
+touch $BUILD_OUTPUT
+
+dump_output() {
+    echo Tailing the last 1000 lines of output:
+    tail -1000 $BUILD_OUTPUT  
+}
+error_handler() {
+    echo ERROR: An error was encountered with the build.
+    dump_output
+    kill $PING_LOOP_PID
+    exit 1
+}
+# If an error occurs, run our error handler to output a tail of the build
+trap 'error_handler' ERR
+
+# Set up a repeating loop to send some output to Travis.
+
+bash -c "while true; do echo \$(date) - building ...; sleep $PING_SLEEP; done" &
+PING_LOOP_PID=$!
+
+.ci/${MODE}_tests.py >> $BUILD_OUTPUT 2>&1
+
+# The build finished without returning an error so dump a tail of the output
+dump_output
+
+# nicely terminate the ping output loop
+kill $PING_LOOP_PID