diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 131cbdc3b..9a8f10f3f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.15.2 +current_version = 0.16.0 commit = False tag = False tag_name = {new_version} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2fd2e6ad4..b195d05e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,9 @@ image: ${IMAGE} before_script: - - sudo yum -y install python34 python34-pip python34-libs python34-setuptools - - sudo python3 -m pip install --upgrade pip + - sudo yum -y install python36 python36-libs python36-setuptools + - sudo python3.6 -m ensurepip --upgrade + - test -f /usr/bin/pip3 || sudo ln -sf /usr/local/bin/pip3 /usr/bin/pip3 - sudo pip3 install --upgrade -r requirements/docs.txt - sudo pip3 install -e . diff --git a/.gitmodules b/.gitmodules index 3c46a1ce1..da242f1fd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -43,3 +43,6 @@ [submodule "src/watchmaker/static/salt/formulas/amazon-inspector-formula"] path = src/watchmaker/static/salt/formulas/amazon-inspector-formula url = https://github.com/plus3it/amazon-inspector-formula.git +[submodule "src/watchmaker/static/salt/content"] + path = src/watchmaker/static/salt/content + url = https://github.com/plus3it/watchmaker-salt-content.git diff --git a/.travis.yml b/.travis.yml index d2ba3d3ea..4743747d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,6 +54,9 @@ notifications: on_failure: always jobs: include: + - stage: test + python: 3.7 + dist: xenial - stage: test python: 3.6 env: diff --git a/CHANGELOG.md b/CHANGELOG.md index cd8f14c11..23e51b6d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ ## Changelog +### 0.16.0 + +**Commit Delta**: [Change from 0.15.2 release](https://github.com/plus3it/watchmaker/compare/0.15.2...0.16.0) + +**Released**: 2019.05.10 + +**Summary**: + +* Adds salt content locally as a submodule to better support Watchmaker standalone packages +* dotnet4-formula + - Updates formula to support the use of Python3 versions of Salt +* join-domain-formula + - Adds additional enhancements and logic to better handle the domin-join process in Linux + ### 0.15.2 **Commit Delta**: [Change from 0.15.1 release](https://github.com/plus3it/watchmaker/compare/0.15.1...0.15.2) diff --git a/appveyor.yml b/appveyor.yml index 9381ce258..49b6d4a75 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,6 +14,41 @@ init: install: - git submodule update --init --recursive test_script: + # ----- python 3.7, x64 + # python envs + - set PYTHON_HOME=C:\Python37-x64 + - set PYTHON_VERSION=3.7 + - set PYTHON_ARCH=64 + + # install + - '%PYTHON_HOME%\Scripts\pip install --upgrade tox setuptools virtualenv wheel' + + # versions + - '%PYTHON_HOME%\Scripts\virtualenv --version' + - '%PYTHON_HOME%\Scripts\tox --version' + - '%PYTHON_HOME%\Scripts\pip --version' + + # test + - set TOXENV=3.7-codecov + - '%WITH_ENV% %PYTHON_HOME%\Scripts\tox -v' + + # ----- python 3.7, x32 + # python envs + - set PYTHON_HOME=C:\Python37 + - set PYTHON_ARCH=32 + + # install + - '%PYTHON_HOME%\Scripts\pip install --upgrade tox setuptools virtualenv wheel' + + # versions + - '%PYTHON_HOME%\Scripts\virtualenv --version' + - '%PYTHON_HOME%\Scripts\tox --version' + - '%PYTHON_HOME%\Scripts\pip --version' + + # test + - set TOXENV=3.7-codecov + - '%WITH_ENV% %PYTHON_HOME%\Scripts\tox -v' + # ----- python 3.6, x64 # python envs - set PYTHON_HOME=C:\Python36-x64 diff --git a/requirements/build.txt b/requirements/build.txt index 887c465a9..ba4f57321 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -1 +1 @@ -gravitybee==0.1.26 +gravitybee==0.1.27 diff --git a/requirements/check.txt b/requirements/check.txt index 9b4d85afb..87c087aa4 100644 --- a/requirements/check.txt +++ b/requirements/check.txt @@ -1,4 +1,4 @@ -check-manifest==0.37 +check-manifest==0.38 flake8==3.7.7 flake8-bugbear==19.3.0 flake8-builtins==1.4.1 @@ -6,10 +6,10 @@ flake8-docstrings==1.3.0 flake8-isort==2.7.0 flake8-future-import==0.4.5 flake8-print==3.1.0 -isort==4.3.17 +isort==4.3.18 m2r==0.2.1 pep8-naming==0.8.2 pydocstyle==3.0.0 -pygments==2.3.1 +pygments==2.4.0 pylint==2.3.1 readme-renderer==24.0 diff --git a/requirements/deploy.txt b/requirements/deploy.txt index d0e244531..541c38ec9 100644 --- a/requirements/deploy.txt +++ b/requirements/deploy.txt @@ -1 +1 @@ -satsuki==0.1.13 +satsuki==0.1.15 diff --git a/requirements/docs.txt b/requirements/docs.txt index c784a0da5..150676973 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,4 +1,4 @@ m2r==0.2.1 -setuptools==41.0.0 +setuptools==41.0.1 sphinx==2.0.1 sphinx-rtd-theme==0.4.3 diff --git a/requirements/pip.txt b/requirements/pip.txt index b79fbf1bd..b9b0f5d95 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1 +1 @@ -pip==19.0.3 +pip==19.1.1 diff --git a/requirements/test.txt b/requirements/test.txt index 0fe3af8a0..dfa17d73c 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,10 +1,11 @@ -mock==2.0.0 +mock==2.0.0;python_version=="2.6" # pyup: ==2.0.0 +mock==3.0.5;python_version>="2.7" pytest==3.2.5;python_version=="2.6" or python_version=="3.3" # pyup: ==3.2.5 -pytest==4.4.0;python_version=="2.7" or python_version>="3.4" +pytest==4.4.2;python_version=="2.7" or python_version>="3.4" pytest-travis-fold==1.3.0 pytest-catchlog==1.2.2;python_version=="2.6" pytest-cov==2.5.1;python_version=="2.6" # pyup: ==2.5.1 -pytest-cov==2.6.1;python_version>="2.7" +pytest-cov==2.7.1;python_version>="2.7" pytest-mock==1.6.3;python_version=="2.6" # pyup: ==1.6.3 -pytest-mock==1.10.3;python_version>="2.7" +pytest-mock==1.10.4;python_version>="2.7" wheel==0.29.0;python_version<="2.6" # pyup: ==0.29.0 diff --git a/setup.cfg b/setup.cfg index b05bf8616..9a9766963 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ [metadata] name = watchmaker description = Applied Configuration Management -version = 0.15.2 +version = 0.16.0 author = Plus3IT Maintainers of Watchmaker author_email = projects@plus3it.com url = https://github.com/plus3it/watchmaker @@ -31,7 +31,8 @@ classifiers = [options] install_requires = - backoff + backoff;python_version>="2.7" + backoff<1.7;python_version<"2.7" click;python_version>="2.7" click<7;python_version<"2.7" defusedxml;platform_system=="Windows" @@ -39,7 +40,7 @@ install_requires = six pypiwin32;platform_system=="Windows" PyYAML;python_version>="2.7" - PyYAML<4;python_version<"2.7" + PyYAML<4;python_version<"2.7" wheel<=0.29.0;python_version<"2.7" packages = find: include_package_data = True diff --git a/src/watchmaker/static/config.yaml b/src/watchmaker/static/config.yaml index 02e09a4b3..0df10e13e 100644 --- a/src/watchmaker/static/config.yaml +++ b/src/watchmaker/static/config.yaml @@ -5,7 +5,7 @@ all: computer_name: None environment: None ou_path: None - salt_content: https://s3.amazonaws.com/watchmaker/salt-content.zip + salt_content: None salt_states: Highstate user_formulas: # To add extra formulas, specify them as a map of diff --git a/src/watchmaker/static/salt/content b/src/watchmaker/static/salt/content new file mode 160000 index 000000000..e70b56cd0 --- /dev/null +++ b/src/watchmaker/static/salt/content @@ -0,0 +1 @@ +Subproject commit e70b56cd0600ad74f12ef75fe64e8188c2b9d848 diff --git a/src/watchmaker/static/salt/formulas/dotnet4-formula b/src/watchmaker/static/salt/formulas/dotnet4-formula index 5c47fb02e..00b01d24a 160000 --- a/src/watchmaker/static/salt/formulas/dotnet4-formula +++ b/src/watchmaker/static/salt/formulas/dotnet4-formula @@ -1 +1 @@ -Subproject commit 5c47fb02eff2e0f0efffcb35d62b43b05343c850 +Subproject commit 00b01d24ad0ea2d649c580953090f99237e66588 diff --git a/src/watchmaker/static/salt/formulas/join-domain-formula b/src/watchmaker/static/salt/formulas/join-domain-formula index 1fab4840f..a9acd1233 160000 --- a/src/watchmaker/static/salt/formulas/join-domain-formula +++ b/src/watchmaker/static/salt/formulas/join-domain-formula @@ -1 +1 @@ -Subproject commit 1fab4840f9ea75786283666f9c11402c21e22fe3 +Subproject commit a9acd1233a96f1904bffb67d504191a7c1252a05 diff --git a/src/watchmaker/utils/__init__.py b/src/watchmaker/utils/__init__.py index c0e2cfdc5..e072b9409 100644 --- a/src/watchmaker/utils/__init__.py +++ b/src/watchmaker/utils/__init__.py @@ -4,6 +4,7 @@ unicode_literals, with_statement) import os +import shutil import ssl import backoff @@ -58,3 +59,26 @@ def urlopen_retry(uri): pass return urllib.request.urlopen(uri, **kwargs) + + +def copytree(src, dst, force=False, **kwargs): + r""" + Copy OS directory trees from source to destination. + + Args: + src: (:obj:`str`) + Source directory tree to be copied. + (*Default*: None) + + dst: (:obj:`str`) + Destination where directory tree is to be copied. + (*Default*: None) + + force: (:obj:`bool`) + Whether to delete destination prior to copy. + (*Default*: ``False``) + """ + if force and os.path.exists(dst): + shutil.rmtree(dst) + + shutil.copytree(src, dst, **kwargs) diff --git a/src/watchmaker/workers/salt.py b/src/watchmaker/workers/salt.py index 5dd83a6da..a80fbdb1d 100644 --- a/src/watchmaker/workers/salt.py +++ b/src/watchmaker/workers/salt.py @@ -169,7 +169,6 @@ def _prepare_for_install(self): ] for salt_dir in [ - self.salt_base_env, self.salt_formula_root, self.salt_conf_path ]: @@ -177,7 +176,7 @@ def _prepare_for_install(self): os.makedirs(salt_dir) except OSError: if not os.path.isdir(salt_dir): - msg = ('Unable create directory - {0}'.format(salt_dir)) + msg = ('Unable to create directory - {0}'.format(salt_dir)) self.log.error(msg) raise SystemError(msg) @@ -194,11 +193,11 @@ def _get_formulas_conf(self): formulas_path = os.sep.join((static.__path__[0], 'salt', 'formulas')) for formula in os.listdir(formulas_path): formula_path = os.path.join(self.salt_formula_root, '', formula) - if os.path.exists(formula_path): - shutil.rmtree(formula_path) - shutil.copytree( + watchmaker.utils.copytree( os.sep.join((formulas_path, formula)), - formula_path) + formula_path, + force=True + ) # Obtain & extract any Salt formulas specified in user_formulas. for formula_name, formula_url in self.user_formulas.items(): @@ -241,7 +240,7 @@ def _get_formulas_conf(self): ] def _build_salt_formula(self, extract_dir): - if self.salt_content: + if self.salt_content and self.salt_content != 'None': salt_content_filename = watchmaker.utils.basename_from_uri( self.salt_content ) @@ -255,6 +254,23 @@ def _build_salt_formula(self, extract_dir): to_directory=extract_dir ) + bundled_content = os.sep.join( + (static.__path__[0], 'salt', 'content') + ) + for subdir in next(os.walk(bundled_content))[1]: + if ( + not subdir.startswith('.') and + not os.path.exists(os.sep.join((extract_dir, subdir))) + ): + watchmaker.utils.copytree( + os.sep.join((bundled_content, subdir)), + os.sep.join((extract_dir, subdir)) + ) + self.log.info( + 'Using bundled content from %s', + os.sep.join((bundled_content, subdir)) + ) + with codecs.open( os.path.join(self.salt_conf_path, 'minion'), 'r+', diff --git a/tests/test_saltworker.py b/tests/test_saltworker.py index ce023ce46..20164d662 100644 --- a/tests/test_saltworker.py +++ b/tests/test_saltworker.py @@ -1,12 +1,20 @@ # -*- coding: utf-8 -*- +# pylint: disable=redefined-outer-name,protected-access """Salt worker main test module.""" from __future__ import (absolute_import, division, print_function, unicode_literals, with_statement) +import os + import pytest from watchmaker.exceptions import InvalidValue -from watchmaker.workers.salt import SaltBase +from watchmaker.workers.salt import SaltBase, SaltWindows + +try: + from unittest.mock import MagicMock, patch +except ImportError: + from mock import MagicMock, patch @pytest.fixture @@ -53,3 +61,203 @@ def test_valid_environment(saltworker_client): saltworker_client.ent_env = 'dev' saltworker_client.valid_envs = [None, 'dev', 'test', 'prod'] assert saltworker_client.before_install() is None + + +@patch.dict(os.environ, {'systemdrive': 'C:'}) +def test_windows_missing_prepdir(): + """Ensure that error raised when missing prep directory.""" + system_params = {} + salt_config = {} + + system_params['logdir'] = "logdir" + system_params['workingdir'] = "workingdir" + + with pytest.raises(KeyError, match='prepdir'): + SaltWindows(system_params, **salt_config) + + +@patch.dict(os.environ, {'systemdrive': 'C:'}) +def test_windows_missing_logdir(): + """Ensure that error raised when missing log directory.""" + system_params = {} + salt_config = {} + + system_params['prepdir'] = "prepdir" + system_params['workingdir'] = "workingdir" + + with pytest.raises(KeyError, match='logdir'): + SaltWindows(system_params, **salt_config) + + +@patch.dict(os.environ, {'systemdrive': 'C:'}) +def test_windows_missing_workingdir(): + """Ensure that error raised when missing working directory.""" + system_params = {} + salt_config = {} + + system_params['logdir'] = "logdir" + system_params['prepdir'] = "prepdir" + + with pytest.raises(KeyError, match='workingdir'): + SaltWindows(system_params, **salt_config) + + +@patch.dict(os.environ, {'systemdrive': 'C:'}) +def test_windows_defaults(): + """Ensure that default values are populated as expected.""" + system_params = {} + salt_config = {} + + system_params['prepdir'] = "8cbda638-6b60-5628-870b-40fdf8add9f8" + system_params['logdir'] = "21fa57e2-9302-5934-978d-4ae40d5a2a55" + system_params['workingdir'] = "c990ee27-ff12-5d7e-9957-1d27d114c0ff" + + salt_config['installer_url'] = "5de41ea1-902c-5e7c-ae86-89587057c6b3" + salt_config['ash_role'] = "b116b3e1-ee3f-5a83-9dd1-3d01d0d5e343" + + win_salt = SaltWindows(system_params, **salt_config) + + # assertions =================== + assert win_salt.installer_url == salt_config['installer_url'] + assert win_salt.ash_role == salt_config['ash_role'] + assert win_salt.salt_root == os.sep.join(("C:", 'Salt')) + assert win_salt.salt_call == os.sep.join(("C:", 'Salt', 'salt-call.bat')) + assert win_salt.salt_wam_root == os.sep.join(( + system_params['prepdir'], + 'Salt', + )) + assert win_salt.salt_conf_path == os.sep.join(( + system_params['prepdir'], + 'Salt', + 'conf', + )) + assert win_salt.salt_srv == os.sep.join(( + system_params['prepdir'], + 'Salt', + 'srv', + )) + assert win_salt.salt_win_repo == os.sep.join(( + system_params['prepdir'], + 'Salt', + 'srv', + 'winrepo', + )) + assert win_salt.salt_log_dir == system_params['logdir'] + assert win_salt.salt_working_dir == system_params['workingdir'] + assert win_salt.salt_working_dir_prefix == 'Salt-' + + assert win_salt.salt_base_env == os.sep.join(( + win_salt.salt_srv, + 'states' + )) + assert win_salt.salt_formula_root == os.sep.join(( + win_salt.salt_srv, + 'formulas' + )) + assert win_salt.salt_pillar_root == os.sep.join(( + win_salt.salt_srv, + 'pillar' + )) + assert win_salt.salt_conf['file_client'] == 'local' + assert win_salt.salt_conf['hash_type'] == 'sha512' + assert win_salt.salt_conf['pillar_roots'] == {'base': [str(os.sep.join(( + win_salt.salt_srv, + 'pillar' + )))]} + assert win_salt.salt_conf['pillar_merge_lists'] + assert win_salt.salt_conf['conf_dir'] == os.sep.join(( + system_params['prepdir'], + 'Salt', + 'conf', + )) + assert win_salt.salt_conf['winrepo_source_dir'] == 'salt://winrepo' + assert win_salt.salt_conf['winrepo_dir'] == os.sep.join(( + system_params['prepdir'], + 'Salt', + 'srv', + 'winrepo', + 'winrepo', + )) + + +@patch.dict(os.environ, {'systemdrive': 'C:'}) +def test_windows_install(): + """Ensure that install runs as expected.""" + system_params = {} + salt_config = {} + system_params['prepdir'] = "0dcd877d-56cb-50c2-954a-80d1084b2216" + system_params['logdir'] = "647c2a49-baf9-511b-a17a-d6ebf0edd91c" + system_params['workingdir'] = "3d6ab2ef-09ad-59f1-a365-ee5f22c95c79" + + salt_config['installer_url'] = "20c913cf-d825-533e-8649-4ab2fed5d9c1" + salt_config['ash_role'] = "f1d27775-9a3d-5e87-ab42-a79ac329ae4b" + + saltworker_win = SaltWindows(system_params, **salt_config) + + saltworker_win._prepare_for_install = MagicMock(return_value=None) + saltworker_win._install_package = MagicMock(return_value=None) + saltworker_win.service_stop = MagicMock(return_value=None) + saltworker_win._build_salt_formula = MagicMock(return_value=None) + saltworker_win.service_disable = MagicMock(return_value=True) + saltworker_win._set_grain = MagicMock(return_value=None) + saltworker_win.process_grains = MagicMock(return_value=None) + saltworker_win.run_salt = MagicMock(return_value=None) + saltworker_win.working_dir = system_params['workingdir'] + saltworker_win.cleanup = MagicMock(return_value=None) + + saltworker_win.install() + + # assertions =================== + assert saltworker_win._prepare_for_install.call_count == 1 + assert saltworker_win._install_package.call_count == 1 + saltworker_win.service_stop.assert_called_with('salt-minion') + saltworker_win._build_salt_formula.assert_called_with( + saltworker_win.salt_srv + ) + saltworker_win.service_disable.assert_called_with('salt-minion') + saltworker_win._set_grain.assert_called_with( + 'ash-windows', {'lookup': {'role': salt_config['ash_role']}} + ) + assert saltworker_win.process_grains.call_count == 1 + saltworker_win.run_salt.assert_called_with('pkg.refresh_db') + assert saltworker_win.cleanup.call_count == 1 + + +@patch.dict(os.environ, {'systemdrive': 'C:'}) +@patch('codecs.open', autospec=True) +@patch('os.makedirs', autospec=True) +@patch('yaml.safe_dump', autospec=True) +def test_windows_prep_install(mock_safe, mock_makedirs, mock_codec): + """Ensure that prep portion of install runs as expected.""" + system_params = {} + salt_config = {} + system_params['prepdir'] = "ac2bf0c3-7985-569f-bfbd-3a8d8a13ce7d" + system_params['logdir'] = "5b0976f8-fcbc-50af-9459-8060589e70d9" + system_params['workingdir'] = "860f630a-f85d-5ed2-bd7a-bbdb48a53b2b" + + salt_config['installer_url'] = "5f0c8635-4a10-5802-8145-732052a0b44b" + salt_config['ash_role'] = "fddc3dc3-3684-5924-bf55-bb1dbc4e4c08" + + saltworker_win = SaltWindows(system_params, **salt_config) + + saltworker_win.create_working_dir = MagicMock( + return_value=system_params['workingdir'] + ) + saltworker_win._prepare_for_install() + + # assertions =================== + saltworker_win.create_working_dir.assert_called_with( + system_params['workingdir'], + 'Salt-' + ) + mock_makedirs.assert_called_with(saltworker_win.salt_conf_path) + mock_codec.assert_called_with( + os.path.join(saltworker_win.salt_conf_path, 'minion'), + 'w', + encoding="utf-8" + ) + mock_safe.assert_called_with( + saltworker_win.salt_conf, + mock_codec.return_value.__enter__(), + default_flow_style=False + ) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 000000000..d6aa1bba7 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# pylint: disable=redefined-outer-name,protected-access +"""Salt worker main test module.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals, with_statement) + +import watchmaker.utils + +try: + from unittest.mock import patch +except ImportError: + from mock import patch + + +@patch('os.path.exists', autospec=True) +@patch('shutil.rmtree', autospec=True) +@patch('shutil.copytree', autospec=True) +def test_copytree_no_force(mock_copy, mock_rm, mock_exists): + """Test that copytree results in correct calls without force option.""" + random_src = 'aba51e65-afd2-5020-8117-195f75e64258' + random_dst = 'f74d03de-7c1d-596f-83f3-73748f2e238f' + + watchmaker.utils.copytree(random_src, random_dst) + mock_copy.assert_called_with(random_src, random_dst) + assert mock_rm.call_count == 0 + assert mock_exists.call_count == 0 + + watchmaker.utils.copytree(random_src, random_dst, force=False) + mock_copy.assert_called_with(random_src, random_dst) + assert mock_rm.call_count == 0 + assert mock_exists.call_count == 0 + + +@patch('os.path.exists', autospec=True) +@patch('shutil.rmtree', autospec=True) +@patch('shutil.copytree', autospec=True) +def test_copytree_force(mock_copy, mock_rm, mock_exists): + """Test that copytree results in correct calls with force option.""" + random_src = '44b6df59-db6f-57cb-a570-ccd55d782561' + random_dst = '72fe7962-a7af-5f2f-899b-54798bc5e79f' + + watchmaker.utils.copytree(random_src, random_dst, force=True) + mock_copy.assert_called_with(random_src, random_dst) + mock_rm.assert_called_with(random_dst) + mock_exists.assert_called_with(random_dst) diff --git a/tox.ini b/tox.ini index 34f113197..2cb67c89f 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ [tox] envlist = check, - {2.6,2.7,3.4,3.5,3.6,pypy}-codecov, + {2.6,2.7,3.4,3.5,3.6,3.7,pypy}-codecov, report, docs