Merge pull request #1229 from YakDriver/0.20.0

Release watchmaker version 0.20.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.19.0
+current_version = 0.20.0
 commit = False
 tag = False
 tag_name = {new_version}

.gitmodules

@@ -46,3 +46,6 @@
 [submodule "src/watchmaker/static/salt/formulas/vault-auth-formula"]
 	path = src/watchmaker/static/salt/formulas/vault-auth-formula
 	url = https://github.com/plus3it/vault-auth-formula
+[submodule "src/vendor/pypa/get-pip"]
+	path = src/vendor/pypa/get-pip
+	url = https://github.com/pypa/get-pip.git

CHANGELOG.md

@@ -1,5 +1,15 @@
 ## Changelog
 
+### 0.20.0
+
+**Commit Delta**: [Change from 0.19.0 release](https://github.com/plus3it/watchmaker/compare/0.19.0...0.20.0)
+
+**Released**: 2020.05.06
+
+**Summary**:
+
+*   Adds capability to install Python packages using Pip in Salt's Python interpreter
+
 ### 0.19.0
 
 **Commit Delta**: [Change from 0.18.2 release](https://github.com/plus3it/watchmaker/compare/0.18.2...0.19.0)

ci/build.sh

@@ -14,4 +14,4 @@ pip install -r requirements/build.txt
 pip install --editable .
 
 # creates standalone
-gravitybee --src-dir src --sha file --with-latest --extra-data static --extra-pkgs boto3 --extra-modules boto3
+gravitybee --src-dir src --sha file --with-latest --extra-data static --extra-data ../vendor/pypa/get-pip/2.6 --extra-pkgs boto3 --extra-modules boto3

docs/configuration.md

@@ -107,6 +107,28 @@ Parameters supported by the Salt Worker:
     ou_path: "OU=Super Cool App,DC=example,DC=com"
     ```
 
+-   `pip_install` (_list_): The Python package(s) that formulas require.
+
+    ```yaml
+    pip_install:
+        - hvac
+        - numpy
+    ```
+
+-   `pip_args` (_list_): Options to be passed to pip when installing package(s). Options with values should be passed with the option and value as separate list items.
+
+    Linux example where emoji is a value that corresponds to the `--progress-bar` option:
+
+    ```yaml
+    linux:
+      - salt:
+          pip_args:
+            - --ignore-installed
+            - --progress-bar=emoji
+    ```
+
+-   `pip_index` (_string_): Base URL of Python Package Index.
+
 -   `salt_states` (_string, comma-separated_): User-defined salt states to
     apply.
 
@@ -196,6 +218,7 @@ all:
       computer_name: null
       environment: null
       ou_path: null
+      pip_install: null
       salt_content: null
       salt_states: Highstate
       user_formulas:

requirements/basics.txt

@@ -2,7 +2,7 @@ setuptools==36.8.0;python_version<="2.6"
 setuptools==43.0.0;python_version<="3.4" and python_version>="2.7"
 setuptools==46.1.3;python_version>"3.4"
 virtualenv==15.2.0;python_version<="2.6"
-virtualenv==20.0.18;python_version>="2.7"
+virtualenv==20.0.20;python_version>="2.7"
 wheel==0.29.0;python_version<="2.6"
 wheel==0.33.6;python_version<="3.4" and python_version>="2.7"
 wheel==0.34.2;python_version>"3.4"

requirements/build.txt

@@ -1 +1 @@
-gravitybee==0.1.40
+gravitybee==0.1.41

requirements/check.txt

@@ -1,6 +1,6 @@
 -r docs-check.txt
 
-check-manifest==0.41
+check-manifest==0.42
 flake8==3.7.9
 flake8-bugbear==20.1.4
 flake8-builtins==1.5.2
@@ -12,5 +12,5 @@ isort==4.3.21
 m2r==0.2.1
 pep8-naming==0.10.0
 pydocstyle==5.0.2
-pylint==2.4.4
+pylint==2.5.2
 readme-renderer==26.0

requirements/tox.txt

@@ -2,4 +2,4 @@
 
 tox==2.9.1;python_version<="2.6"
 tox==3.14.0;python_version<="3.4" and python_version>="2.7"
-tox==3.14.6;python_version>"3.4"
+tox==3.15.0;python_version>"3.4"

setup.cfg

@@ -3,7 +3,7 @@
 [metadata]
 name = watchmaker
 description = Applied Configuration Management
-version = 0.19.0
+version = 0.20.0
 long_description = file: README.md, CHANGELOG.md
 long_description_content_type = text/markdown
 author = Plus3IT Maintainers of Watchmaker
@@ -73,6 +73,7 @@ exclude =
     dist,
     htmlcov,
     */static/salt/formulas/*
+    */vendor/pypa/*
 ignore = FI15,FI16,FI17,FI18,FI5,D107,W503,W504
 
 [tool:pytest]

src/watchmaker/workers/salt.py

@@ -95,6 +95,18 @@ class SaltBase(WorkerBase, PlatformManagerBase):
             E.g. ``"OU=SuperCoolApp,DC=example,DC=com"``
             (*Default*: ``''``)
 
+        pip_install: (:obj:`list`)
+            Python packages to be installed prior to applying the high state.
+            (*Default*: ``[]``)
+
+        pip_args: (:obj:`list`)
+            Options to pass to pip when installing packages.
+            (*Default*: ``[]``)
+
+        pip_index: (:obj:`str`)
+            URL used for an index by pip.
+            (*Default*: ``https://pypi.org/simple``)
+
     """
 
     def __init__(self, *args, **kwargs):
@@ -112,6 +124,10 @@ def __init__(self, *args, **kwargs):
         self.ou_path = kwargs.pop('ou_path', None) or ''
         self.admin_groups = kwargs.pop('admin_groups', None) or ''
         self.admin_users = kwargs.pop('admin_users', None) or ''
+        self.pip_install = kwargs.pop('pip_install', None) or []
+        self.pip_args = kwargs.pop('pip_args', None) or []
+        self.pip_index = kwargs.pop('pip_index', None) or \
+            'https://pypi.org/simple'
         self.salt_states = kwargs.pop('salt_states', None) or ''
         self.exclude_states = kwargs.pop('exclude_states', None) or ''
 
@@ -340,6 +356,10 @@ def _set_grain(self, grain, value):
         ]
         self.run_salt(cmd)
 
+    def _get_grain(self, grain):
+        grain_full = self.run_salt(['grains.get', grain])
+        return grain_full['stdout'].decode().split('\n')[1].strip()
+
     def _get_failed_states(self, state_ret):
         failed_states = {}
         try:
@@ -360,6 +380,72 @@ def _get_failed_states(self, state_ret):
             failed_states = state_ret
         return failed_states
 
+    def _install_pip(self, py_exec):
+        get_pip = os.path.join(
+            os.path.abspath(
+                os.path.join(os.path.dirname(__file__), '..', '..')),
+            'vendor',
+            'pypa',
+            'get-pip',
+            '2.6',
+            'get-pip.py')
+        self.log.info(
+            'Attempting pip install using get-pip (%s)...', get_pip)
+        cmd = [
+            py_exec,
+            get_pip,
+            "--index-url",
+            self.pip_index
+        ]
+        self.call_process(cmd)
+
+    def _upgrade_pip(self, py_exec):
+        self.log.debug('Attempting to upgrade pip...')
+        cmd = [
+            py_exec,
+            "-m",
+            "pip",
+            "install",
+            "--upgrade",
+            "pip",
+            "--index-url",
+            self.pip_index
+        ]
+        self.call_process(cmd, raise_error=False)
+
+        ver = self.call_process([py_exec, '-m', 'pip', '--version'])
+        self.log.debug('Pip version: %s', ver['stdout'])
+
+    def _install_pip_packages(self):
+        py_exec = self._get_grain('pythonexecutable')
+        self.log.debug('Salt Python interpreter: `%s`', py_exec)
+
+        try:
+            ver = self.call_process([
+                py_exec,
+                '-m',
+                'pip',
+                '--version'])
+            self.log.debug('Pip version: %s', ver['stdout'])
+        except WatchmakerException:
+            self.log.debug('Pip not installed for Salt interpreter!')
+            self._install_pip(py_exec)
+
+        self._upgrade_pip(py_exec)
+
+        self.log.info(
+            'Pip installing module(s): `%s`', " ".join(self.pip_install))
+        pip_cmd = [
+            py_exec,
+            '-m',
+            'pip',
+            'install'
+        ]
+        pip_cmd.extend(self.pip_install)
+        pip_cmd.extend(['--index-url', self.pip_index])
+        pip_cmd.extend(self.pip_args)
+        self.call_process(pip_cmd)
+
     def run_salt(self, command, **kwargs):
         """
         Execute salt command.
@@ -717,6 +803,8 @@ def install(self):
         if os.path.exists(self.salt_call):
             salt_running, salt_enabled = self.service_status(salt_svc)
         self._install_package()
+        if self.pip_install:
+            self._install_pip_packages()
         salt_stopped = self.service_stop(salt_svc)
         self._build_salt_formula(self.salt_srv)
         if salt_enabled:
@@ -849,6 +937,8 @@ def install(self):
         if os.path.exists(self.salt_call):
             salt_running, salt_enabled = self.service_status(salt_svc)
         self._install_package()
+        if self.pip_install:
+            self._install_pip_packages()
         salt_stopped = self.service_stop(salt_svc)
         self._build_salt_formula(self.salt_srv)
         if salt_enabled: