run our backups script in a virtualenv

playbooks/roles/backups/defaults/main.yml

@@ -25,5 +25,7 @@ BACKUPS_GS_SERVICE_KEY_FILE: '/root/backup-service-account.p12'
 BACKUPS_GS_SERVICE_CLIENT_ID: ''
 BACKUPS_GS_SERVICE_KEY_FILE_PASSWORD: 'notasecret'
 
+BACKUPS_VIRTUALENV: /opt/backups/ve
+
 # Prometheus Pushgateway
 PUSHGATEWAY: 'https://pushgateway.infra.appsembler.com'

playbooks/roles/backups/tasks/main.yml

@@ -12,7 +12,11 @@
   tags: ['backups']
 
 - name: Install Python packages required for backups on GCP
-  pip: name={{ item.name }} version={{ item.version }}
+  pip:
+    name: "{{ item.name }}"
+    version: "{{ item.version }}"
+    virtualenv: "{{ BACKUPS_VIRTUALENV }}"
+    state: present
   with_items:
     - {name: gcs-oauth2-boto-plugin, version: 1.9}
     - {name: oauth2client, version: 1.5.2}
@@ -22,12 +26,19 @@
   tags: ['backups']
 
 - name: Install filechunkio for multipart S3 uploads
-  pip: name=filechunkio state=present
+  pip:
+    name: filechunkio
+    state: present
+    virtualenv: "{{ BACKUPS_VIRTUALENV }}"
   when: BACKUPS_PROVIDER == 's3'
   tags: ['backups']
 
 - name: Install raven package for Sentry alerts
-  pip: name=raven version="6.9.0"
+  pip:
+    name: raven
+    version: "6.9.0"
+    state: present
+    virtualenv: "{{ BACKUPS_VIRTUALENV }}"
   tags: ['backups']
 
 - name: Copy boto configuration
@@ -58,12 +69,12 @@
   tags: ['backups']
 
 - name: Create cron job for Mongo backups
-  cron: name="Backup Mongo" cron_file=edx_backup_mongo user=root job="/usr/local/bin/edx_backup mongodb" special_time=daily
+  cron: name="Backup Mongo" cron_file=edx_backup_mongo user=root job="{{ BACKUPS_VIRTUALENV }}/bin/python /usr/local/bin/edx_backup mongodb" special_time=daily
   when: BACKUPS_MONGO
   tags: ['backups']
 
 - name: Create cron job for Mongo backups monitor
-  cron: name="Backup Mongo" cron_file=edx_backup_monitor_mongo user=root job="/usr/local/bin/edx_backups_monitor mongodb" special_time=daily
+  cron: name="Backup Mongo" cron_file=edx_backup_monitor_mongo user=root job="{{ BACKUPS_VIRTUALENV }}/bin/python /usr/local/bin/edx_backups_monitor mongodb" special_time=daily
   when: BACKUPS_MONGO
   tags: ['backups']
 
@@ -104,7 +115,7 @@
   tags: ['backups']
 
 - name: Create cron job for MySQL backups
-  cron: name="Backup MySQL" cron_file=edx_backup_mysql user=root job="/usr/local/bin/edx_backup mysql" special_time=daily
+  cron: name="Backup MySQL" cron_file=edx_backup_mysql user=root job="{{ BACKUPS_VIRTUALENV }}/bin/python /usr/local/bin/edx_backup mysql" special_time=daily
   when: BACKUPS_MYSQL
   tags: ['backups']
 
@@ -128,7 +139,7 @@
 - name: Run a backup for mysql/mongodb to make sure it works
   become: yes
   become_user: root
-  shell: /usr/local/bin/edx_backup {{ item.service }}
+  shell: "{{ BACKUPS_VIRTUALENV }}/bin/python /usr/local/bin/edx_backup {{ item.service }}"
   environment:
     BACKUP_DIR: '{{ BACKUPS_DIR }}'
     BACKUP_BUCKET: '{{ BACKUPS_BUCKET }}'
@@ -147,18 +158,3 @@
     - service: mysql
       enabled: "{{ BACKUPS_MYSQL }}"
   tags: ['backups']
-
-# one of the system-wide pip installs that runs at some point previous
-# to install libraries upgrades urllib3 to a version that breaks
-# ansible on subsequent deploys.
-# See: https://appsembler.atlassian.net/wiki/spaces/ED/pages/390004931/Failed+to+install+repo+key
-# the *correct* fix for this would be to have our backups script and
-# its dependencies installed into a virtualenv so it doesn't conflict
-# with system-wide libraries. That's a large change though
-# so for now, as a workaround, we just downgrade urllib3 back to a known safe version
-- name: pin urllib3 back down to a version that doesn't break ansible
-  pip: name={{ item.name }} version={{ item.version }}
-  with_items:
-    - {name: urllib3, version: 1.13.1}
-  when: BACKUPS_PROVIDER == 'gs'
-  tags: ['backups']