diff --git a/.bumpversion.cfg b/.bumpversion.cfg index aa3614bd2..b454edbcb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.19.0 +current_version = 0.20.0 commit = False tag = False tag_name = {new_version} diff --git a/.gitmodules b/.gitmodules index ca7b5e321..839ebe96f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -46,3 +46,6 @@ [submodule "src/watchmaker/static/salt/formulas/vault-auth-formula"] path = src/watchmaker/static/salt/formulas/vault-auth-formula url = https://github.com/plus3it/vault-auth-formula +[submodule "src/vendor/pypa/get-pip"] + path = src/vendor/pypa/get-pip + url = https://github.com/pypa/get-pip.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 38cd17fda..a72c05589 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ ## Changelog +### 0.20.0 + +**Commit Delta**: [Change from 0.19.0 release](https://github.com/plus3it/watchmaker/compare/0.19.0...0.20.0) + +**Released**: 2020.05.06 + +**Summary**: + +* Adds capability to install Python packages using Pip in Salt's Python interpreter + ### 0.19.0 **Commit Delta**: [Change from 0.18.2 release](https://github.com/plus3it/watchmaker/compare/0.18.2...0.19.0) diff --git a/ci/build.sh b/ci/build.sh index 40874a8a2..e55bf4413 100644 --- a/ci/build.sh +++ b/ci/build.sh @@ -14,4 +14,4 @@ pip install -r requirements/build.txt pip install --editable . # creates standalone -gravitybee --src-dir src --sha file --with-latest --extra-data static --extra-pkgs boto3 --extra-modules boto3 +gravitybee --src-dir src --sha file --with-latest --extra-data static --extra-data ../vendor/pypa/get-pip/2.6 --extra-pkgs boto3 --extra-modules boto3 diff --git a/docs/configuration.md b/docs/configuration.md index f0a918c11..c465d817f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -107,6 +107,28 @@ Parameters supported by the Salt Worker: ou_path: "OU=Super Cool App,DC=example,DC=com" ``` +- `pip_install` (_list_): The Python package(s) that formulas require. + + ```yaml + pip_install: + - hvac + - numpy + ``` + +- `pip_args` (_list_): Options to be passed to pip when installing package(s). Options with values should be passed with the option and value as separate list items. + + Linux example where emoji is a value that corresponds to the `--progress-bar` option: + + ```yaml + linux: + - salt: + pip_args: + - --ignore-installed + - --progress-bar=emoji + ``` + +- `pip_index` (_string_): Base URL of Python Package Index. + - `salt_states` (_string, comma-separated_): User-defined salt states to apply. @@ -196,6 +218,7 @@ all: computer_name: null environment: null ou_path: null + pip_install: null salt_content: null salt_states: Highstate user_formulas: diff --git a/requirements/basics.txt b/requirements/basics.txt index 75152985b..c08c3e03d 100644 --- a/requirements/basics.txt +++ b/requirements/basics.txt @@ -2,7 +2,7 @@ setuptools==36.8.0;python_version<="2.6" setuptools==43.0.0;python_version<="3.4" and python_version>="2.7" setuptools==46.1.3;python_version>"3.4" virtualenv==15.2.0;python_version<="2.6" -virtualenv==20.0.18;python_version>="2.7" +virtualenv==20.0.20;python_version>="2.7" wheel==0.29.0;python_version<="2.6" wheel==0.33.6;python_version<="3.4" and python_version>="2.7" wheel==0.34.2;python_version>"3.4" diff --git a/requirements/build.txt b/requirements/build.txt index ac051c994..f890e8d59 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -1 +1 @@ -gravitybee==0.1.40 +gravitybee==0.1.41 diff --git a/requirements/check.txt b/requirements/check.txt index cedc93901..e27d3f3b0 100644 --- a/requirements/check.txt +++ b/requirements/check.txt @@ -1,6 +1,6 @@ -r docs-check.txt -check-manifest==0.41 +check-manifest==0.42 flake8==3.7.9 flake8-bugbear==20.1.4 flake8-builtins==1.5.2 @@ -12,5 +12,5 @@ isort==4.3.21 m2r==0.2.1 pep8-naming==0.10.0 pydocstyle==5.0.2 -pylint==2.4.4 +pylint==2.5.2 readme-renderer==26.0 diff --git a/requirements/tox.txt b/requirements/tox.txt index be458cd4a..7ca3e3dfc 100644 --- a/requirements/tox.txt +++ b/requirements/tox.txt @@ -2,4 +2,4 @@ tox==2.9.1;python_version<="2.6" tox==3.14.0;python_version<="3.4" and python_version>="2.7" -tox==3.14.6;python_version>"3.4" +tox==3.15.0;python_version>"3.4" diff --git a/setup.cfg b/setup.cfg index f528cd8fa..84301273b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ [metadata] name = watchmaker description = Applied Configuration Management -version = 0.19.0 +version = 0.20.0 long_description = file: README.md, CHANGELOG.md long_description_content_type = text/markdown author = Plus3IT Maintainers of Watchmaker @@ -73,6 +73,7 @@ exclude = dist, htmlcov, */static/salt/formulas/* + */vendor/pypa/* ignore = FI15,FI16,FI17,FI18,FI5,D107,W503,W504 [tool:pytest] diff --git a/src/vendor/pypa/get-pip b/src/vendor/pypa/get-pip new file mode 160000 index 000000000..1fe530e9e --- /dev/null +++ b/src/vendor/pypa/get-pip @@ -0,0 +1 @@ +Subproject commit 1fe530e9e3d800be94e04f6428460fc4fb94f5a9 diff --git a/src/watchmaker/workers/salt.py b/src/watchmaker/workers/salt.py index b0cdfad76..673e714a3 100644 --- a/src/watchmaker/workers/salt.py +++ b/src/watchmaker/workers/salt.py @@ -95,6 +95,18 @@ class SaltBase(WorkerBase, PlatformManagerBase): E.g. ``"OU=SuperCoolApp,DC=example,DC=com"`` (*Default*: ``''``) + pip_install: (:obj:`list`) + Python packages to be installed prior to applying the high state. + (*Default*: ``[]``) + + pip_args: (:obj:`list`) + Options to pass to pip when installing packages. + (*Default*: ``[]``) + + pip_index: (:obj:`str`) + URL used for an index by pip. + (*Default*: ``https://pypi.org/simple``) + """ def __init__(self, *args, **kwargs): @@ -112,6 +124,10 @@ def __init__(self, *args, **kwargs): self.ou_path = kwargs.pop('ou_path', None) or '' self.admin_groups = kwargs.pop('admin_groups', None) or '' self.admin_users = kwargs.pop('admin_users', None) or '' + self.pip_install = kwargs.pop('pip_install', None) or [] + self.pip_args = kwargs.pop('pip_args', None) or [] + self.pip_index = kwargs.pop('pip_index', None) or \ + 'https://pypi.org/simple' self.salt_states = kwargs.pop('salt_states', None) or '' self.exclude_states = kwargs.pop('exclude_states', None) or '' @@ -340,6 +356,10 @@ def _set_grain(self, grain, value): ] self.run_salt(cmd) + def _get_grain(self, grain): + grain_full = self.run_salt(['grains.get', grain]) + return grain_full['stdout'].decode().split('\n')[1].strip() + def _get_failed_states(self, state_ret): failed_states = {} try: @@ -360,6 +380,72 @@ def _get_failed_states(self, state_ret): failed_states = state_ret return failed_states + def _install_pip(self, py_exec): + get_pip = os.path.join( + os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..')), + 'vendor', + 'pypa', + 'get-pip', + '2.6', + 'get-pip.py') + self.log.info( + 'Attempting pip install using get-pip (%s)...', get_pip) + cmd = [ + py_exec, + get_pip, + "--index-url", + self.pip_index + ] + self.call_process(cmd) + + def _upgrade_pip(self, py_exec): + self.log.debug('Attempting to upgrade pip...') + cmd = [ + py_exec, + "-m", + "pip", + "install", + "--upgrade", + "pip", + "--index-url", + self.pip_index + ] + self.call_process(cmd, raise_error=False) + + ver = self.call_process([py_exec, '-m', 'pip', '--version']) + self.log.debug('Pip version: %s', ver['stdout']) + + def _install_pip_packages(self): + py_exec = self._get_grain('pythonexecutable') + self.log.debug('Salt Python interpreter: `%s`', py_exec) + + try: + ver = self.call_process([ + py_exec, + '-m', + 'pip', + '--version']) + self.log.debug('Pip version: %s', ver['stdout']) + except WatchmakerException: + self.log.debug('Pip not installed for Salt interpreter!') + self._install_pip(py_exec) + + self._upgrade_pip(py_exec) + + self.log.info( + 'Pip installing module(s): `%s`', " ".join(self.pip_install)) + pip_cmd = [ + py_exec, + '-m', + 'pip', + 'install' + ] + pip_cmd.extend(self.pip_install) + pip_cmd.extend(['--index-url', self.pip_index]) + pip_cmd.extend(self.pip_args) + self.call_process(pip_cmd) + def run_salt(self, command, **kwargs): """ Execute salt command. @@ -717,6 +803,8 @@ def install(self): if os.path.exists(self.salt_call): salt_running, salt_enabled = self.service_status(salt_svc) self._install_package() + if self.pip_install: + self._install_pip_packages() salt_stopped = self.service_stop(salt_svc) self._build_salt_formula(self.salt_srv) if salt_enabled: @@ -849,6 +937,8 @@ def install(self): if os.path.exists(self.salt_call): salt_running, salt_enabled = self.service_status(salt_svc) self._install_package() + if self.pip_install: + self._install_pip_packages() salt_stopped = self.service_stop(salt_svc) self._build_salt_formula(self.salt_srv) if salt_enabled: