diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 86f17edaa..aa3614bd2 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.18.2 +current_version = 0.19.0 commit = False tag = False tag_name = {new_version} diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ff4d3d42..38cd17fda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ ## Changelog +### 0.19.0 + +**Commit Delta**: [Change from 0.18.2 release](https://github.com/plus3it/watchmaker/compare/0.18.2...0.19.0) + +**Released**: 2020.05.01 + +**Summary**: + +* Updates Watchmaker file permissions and makes them more restrictive +* Adds new SaltWorker optional argument `--salt-content-path` that allows specifying glob pattern for + salt files located within salt archive file + ### 0.18.2 **Commit Delta**: [Change from 0.18.1 release](https://github.com/plus3it/watchmaker/compare/0.18.1...0.18.2) diff --git a/docs/configuration.md b/docs/configuration.md index 5f4861368..f0a918c11 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -134,6 +134,11 @@ Parameters supported by the Salt Worker: - `salt_content` (_string_): URL to the Salt content file that contains further configuration specific to the salt install. +- `salt_content_path` (_string_): The path within the Salt content file + specified using `salt_content` where salt files are located. + Can be used to provide the path within the archive file where + the Salt configuration files are located. + - `install_method` (_string_): (Linux-only) The method used to install Salt. Currently supports: `yum`, `git` diff --git a/requirements/basics.txt b/requirements/basics.txt index daf889957..75152985b 100644 --- a/requirements/basics.txt +++ b/requirements/basics.txt @@ -2,7 +2,7 @@ setuptools==36.8.0;python_version<="2.6" setuptools==43.0.0;python_version<="3.4" and python_version>="2.7" setuptools==46.1.3;python_version>"3.4" virtualenv==15.2.0;python_version<="2.6" -virtualenv==20.0.15;python_version>="2.7" +virtualenv==20.0.18;python_version>="2.7" wheel==0.29.0;python_version<="2.6" wheel==0.33.6;python_version<="3.4" and python_version>="2.7" wheel==0.34.2;python_version>"3.4" diff --git a/requirements/build.txt b/requirements/build.txt index ea413ba94..ac051c994 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -1 +1 @@ -gravitybee==0.1.36 +gravitybee==0.1.40 diff --git a/requirements/check.txt b/requirements/check.txt index 0471d8fc9..cedc93901 100644 --- a/requirements/check.txt +++ b/requirements/check.txt @@ -5,7 +5,7 @@ flake8==3.7.9 flake8-bugbear==20.1.4 flake8-builtins==1.5.2 flake8-docstrings==1.5.0 -flake8-isort==2.9.1 +flake8-isort==3.0.0 flake8-future-import==0.4.6 flake8-print==3.1.4 isort==4.3.21 @@ -13,4 +13,4 @@ m2r==0.2.1 pep8-naming==0.10.0 pydocstyle==5.0.2 pylint==2.4.4 -readme-renderer==25.0 +readme-renderer==26.0 diff --git a/requirements/docs.txt b/requirements/docs.txt index c34d39f39..b4e586316 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -2,5 +2,5 @@ recommonmark==0.6.0 setuptools==46.1.3 -sphinx==2.4.4 +sphinx==3.0.3 sphinx-rtd-theme==0.4.3 diff --git a/requirements/pip.txt b/requirements/pip.txt index 982dbd95a..dd4d9ec2f 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,3 +1,3 @@ pip==9.0.3;python_version<="2.6" pip==19.1;python_version<="3.4" and python_version>="2.7" -pip==20.0.2;python_version>"3.4" +pip==20.1;python_version>"3.4" diff --git a/requirements/test.txt b/requirements/test.txt index 981999dc0..398dcc43d 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -9,5 +9,6 @@ pytest-catchlog==1.2.2;python_version<"2.7" pytest-cov==2.5.1;python_version<"2.7" # pyup: ==2.5.1 pytest-cov==2.8.1;python_version>="2.7" pytest-mock==1.6.3;python_version<"2.7" # pyup: ==1.6.3 -pytest-mock==2.0.0;python_version>="2.7" +pytest-mock==2.0.0;python_version<"3.5" and python_version>="2.7" # pyup: ==2.0.0 +pytest-mock==3.1.0;python_version>="3.5" wheel==0.29.0;python_version<="2.6" # pyup: ==0.29.0 diff --git a/setup.cfg b/setup.cfg index 49ffd0484..f528cd8fa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ [metadata] name = watchmaker description = Applied Configuration Management -version = 0.18.2 +version = 0.19.0 long_description = file: README.md, CHANGELOG.md long_description_content_type = text/markdown author = Plus3IT Maintainers of Watchmaker @@ -41,12 +41,13 @@ install_requires = defusedxml;platform_system=="Windows" futures;python_version<"3" six - pypiwin32;platform_system=="Windows" + pywin32;platform_system=="Windows" PyYAML;python_version>="3.5" PyYAML<=5.2;python_version<"3.5" PyYAML<4;python_version<"2.7" wheel<=0.29.0;python_version<"2.7" compatibleversion>=0.1.2 + oschmod>=0.1.3 packages = find: include_package_data = True diff --git a/src/watchmaker/__init__.py b/src/watchmaker/__init__.py index b626db33d..1ced10f70 100644 --- a/src/watchmaker/__init__.py +++ b/src/watchmaker/__init__.py @@ -10,6 +10,7 @@ import platform import subprocess +import oschmod import pkg_resources import setuptools import yaml @@ -407,6 +408,7 @@ def _set_system_params(self): self.system_drive = '/' self.workers_manager = LinuxWorkersManager self.system_params = self._get_linux_system_params() + os.umask(0o077) elif 'windows' in self.system: self.system_drive = os.environ['SYSTEMDRIVE'] self.workers_manager = WindowsWorkersManager @@ -431,6 +433,7 @@ def install(self): # Create watchmaker directories try: os.makedirs(self.system_params['workingdir']) + oschmod.set_mode(self.system_params['prepdir'], 0o700) except OSError: if not os.path.exists(self.system_params['workingdir']): msg = ( diff --git a/src/watchmaker/logger/__init__.py b/src/watchmaker/logger/__init__.py index 10303355f..cd807b930 100644 --- a/src/watchmaker/logger/__init__.py +++ b/src/watchmaker/logger/__init__.py @@ -14,6 +14,8 @@ import subprocess import xml.etree.ElementTree +import oschmod + IS_WINDOWS = platform.system() == 'Windows' MESSAGE_TYPES = ('Information', 'Warning', 'Error') @@ -88,6 +90,7 @@ def make_log_dir(log_dir): """ if not os.path.exists(log_dir): os.makedirs(log_dir) + oschmod.set_mode(log_dir, 0o700) def log_system_details(log): @@ -141,6 +144,7 @@ def prepare_logging(log_dir, log_level): make_log_dir(log_dir) log_filename = os.sep.join((log_dir, 'watchmaker.log')) hdlr = logging.FileHandler(log_filename) + oschmod.set_mode(log_filename, 0o600) hdlr.setLevel(level) hdlr.setFormatter(logging.Formatter(logformat)) logging.getLogger().addHandler(hdlr) diff --git a/src/watchmaker/static/salt/formulas/ash-windows-formula b/src/watchmaker/static/salt/formulas/ash-windows-formula index 3c18ca7ea..dc16b7641 160000 --- a/src/watchmaker/static/salt/formulas/ash-windows-formula +++ b/src/watchmaker/static/salt/formulas/ash-windows-formula @@ -1 +1 @@ -Subproject commit 3c18ca7ea5d6e6f5551edd3e97ba0f295802c4b9 +Subproject commit dc16b7641f2383e2aaf058ed388a7e5f792e6eb9 diff --git a/src/watchmaker/static/salt/formulas/vault-auth-formula b/src/watchmaker/static/salt/formulas/vault-auth-formula index 3b27e1fcf..b7ee109e2 160000 --- a/src/watchmaker/static/salt/formulas/vault-auth-formula +++ b/src/watchmaker/static/salt/formulas/vault-auth-formula @@ -1 +1 @@ -Subproject commit 3b27e1fcfabe0540dd6c2837a7da28ed2be90c2d +Subproject commit b7ee109e215e9f23fef9c1bfb1efbc659bc14b59 diff --git a/src/watchmaker/utils/__init__.py b/src/watchmaker/utils/__init__.py index 6717f9d5c..ac4a97311 100644 --- a/src/watchmaker/utils/__init__.py +++ b/src/watchmaker/utils/__init__.py @@ -119,3 +119,20 @@ def clean_none(value): return None return value + + +def copy_subdirectories(src_dir, dest_dir, log=None): + """Copy subdirectories within given src dir into dest dir.""" + for subdir in next(os.walk(src_dir))[1]: + if ( + not subdir.startswith('.') and + not os.path.exists(os.sep.join((dest_dir, subdir))) + ): + copytree( + os.sep.join((src_dir, subdir)), + os.sep.join((dest_dir, subdir)) + ) + if log: + log.info('Copied from %s to %s', + os.sep.join((src_dir, subdir)), + os.sep.join((dest_dir, subdir))) diff --git a/src/watchmaker/workers/salt.py b/src/watchmaker/workers/salt.py index 42721c7e0..b0cdfad76 100644 --- a/src/watchmaker/workers/salt.py +++ b/src/watchmaker/workers/salt.py @@ -5,6 +5,7 @@ import ast import codecs +import glob import json import os import shutil @@ -40,6 +41,16 @@ class SaltBase(WorkerBase, PlatformManagerBase): - *Linux*: ``/srv/watchmaker/salt`` - *Windows*: ``C:\Watchmaker\Salt\srv`` + salt_content_path: (:obj:`str`) + Used in conjunction with the "salt_content" arg. + Glob pattern for the location of salt content files inside the + provided salt_content archive. To be used when salt content files + are located within a sub-path of the archive, rather than at its + top-level. Multiple paths matching the given pattern will result + in error. + E.g. ``salt_content_path='*/'`` + (*Default*: ``''``) + salt_states: (:obj:`str`) Comma-separated string of salt states to execute. Accepts two special keywords (case-insensitive). @@ -97,6 +108,7 @@ def __init__(self, *args, **kwargs): self.valid_envs = kwargs.pop('valid_environments', []) or [] self.salt_debug_log = kwargs.pop('salt_debug_log', None) or '' self.salt_content = kwargs.pop('salt_content', None) or '' + self.salt_content_path = kwargs.pop('salt_content_path', None) or '' self.ou_path = kwargs.pop('ou_path', None) or '' self.admin_groups = kwargs.pop('admin_groups', None) or '' self.admin_users = kwargs.pop('admin_users', None) or '' @@ -109,6 +121,8 @@ def __init__(self, *args, **kwargs): self.salt_debug_log, self.log) self.salt_content = watchmaker.utils.config_none_deprecate( self.salt_content, self.log) + self.salt_content_path = watchmaker.utils.config_none_deprecate( + self.salt_content_path, self.log) self.ou_path = watchmaker.utils.config_none_deprecate( self.ou_path, self.log) self.admin_groups = watchmaker.utils.config_none_deprecate( @@ -262,27 +276,51 @@ def _build_salt_formula(self, extract_dir): salt_content_filename )) self.retrieve_file(self.salt_content, salt_content_file) - self.extract_contents( - filepath=salt_content_file, - to_directory=extract_dir - ) + if not self.salt_content_path: + self.extract_contents( + filepath=salt_content_file, + to_directory=extract_dir + ) + else: + self.log.debug( + 'Using salt content path: %s', + self.salt_content_path + ) + temp_extract_dir = os.sep.join(( + self.working_dir, 'salt-archive')) + self.extract_contents( + filepath=salt_content_file, + to_directory=temp_extract_dir + ) + salt_content_src = os.sep.join(( + temp_extract_dir, self.salt_content_path)) + salt_content_glob = glob.glob(salt_content_src) + self.log.debug('salt_content_glob: %s', salt_content_glob) + if len(salt_content_glob) > 1: + msg = 'Found multiple paths matching' \ + ' \'{0}\' in {1}'.format( + self.salt_content_path, + self.salt_content) + self.log.critical(msg) + raise WatchmakerException(msg) + try: + salt_files_dir = salt_content_glob[0] + except IndexError: + msg = 'Salt content glob path \'{0}\' not' \ + ' found in {1}'.format( + self.salt_content_path, + self.salt_content) + self.log.critical(msg) + raise WatchmakerException(msg) + + watchmaker.utils.copy_subdirectories( + salt_files_dir, extract_dir, self.log) bundled_content = os.sep.join( (static.__path__[0], 'salt', 'content') ) - for subdir in next(os.walk(bundled_content))[1]: - if ( - not subdir.startswith('.') and - not os.path.exists(os.sep.join((extract_dir, subdir))) - ): - watchmaker.utils.copytree( - os.sep.join((bundled_content, subdir)), - os.sep.join((extract_dir, subdir)) - ) - self.log.info( - 'Using bundled content from %s', - os.sep.join((bundled_content, subdir)) - ) + watchmaker.utils.copy_subdirectories( + bundled_content, extract_dir, self.log) with codecs.open( os.path.join(self.salt_conf_path, 'minion'), diff --git a/tests/test_logger.py b/tests/test_logger.py index d4f4a0d02..88901b96d 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -12,6 +12,7 @@ import sys import xml.etree.ElementTree +import oschmod import pytest from watchmaker import logger @@ -65,21 +66,24 @@ def test_logger_handler(): """ Tests prepare_logging() use case. - Tests that prepare_logging() will set logger level appropriately - and attach a FileHandler if a directory is passed. + Tests that prepare_logging() will set logger level appropriately, attach a + FileHandler if a directory is passed, and set log file to the correct mode. """ - logger.prepare_logging('./logfiles', 'debug') + logger.prepare_logging('logfiles', 'debug') this_logger = logging.getLogger() if logger.HAS_PYWIN32: log_hdlr = this_logger.handlers.pop() - assert type(log_hdlr) == logging.handlers.NTEventLogHandler + assert isinstance(log_hdlr, logging.handlers.NTEventLogHandler) assert log_hdlr.level == logging.DEBUG log_hdlr = this_logger.handlers.pop() - assert type(log_hdlr) == logging.FileHandler + assert isinstance(log_hdlr, logging.FileHandler) assert log_hdlr.level == logging.DEBUG + assert oschmod.get_mode( + os.path.join('logfiles', 'watchmaker.log')) == 0o600 + def test_log_if_no_log_directory_given(caplog): """ diff --git a/tests/test_saltworker.py b/tests/test_saltworker.py index 04536b2ca..8bd08aadb 100644 --- a/tests/test_saltworker.py +++ b/tests/test_saltworker.py @@ -356,6 +356,93 @@ def test_linux_salt_content_none( assert mock_copytree.call_count > 1 +@pytest.mark.skipif(sys.version_info < (3, 4), + reason="Not supported in this Python version.") +@patch("codecs.open", autospec=True) +@patch("os.walk", autospec=True) +@patch("yaml.safe_dump", autospec=True) +@patch("yaml.safe_load", autospec=True) +@patch("watchmaker.utils.copytree", autospec=True) +@patch("glob.glob", autospec=True) +@patch("watchmaker.utils.copy_subdirectories", autospec=True) +def test_linux_salt_content_path_none( + mock_copysubdirs, mock_glob, mock_copytree, mock_yload, + mock_ydump, mock_os, mock_codec): + """Test that Pythonic None can be used without error rather than 'None'.""" + # setup ======================== + system_params = {} + salt_config = {} + system_params["prepdir"] = "4504257a-76d0-49bd-9d04-53c1459b7156" + system_params["logdir"] = "045143d6-0e87-497f-a11a-5eebb1ec7edf" + system_params["workingdir"] = "83f16e7b-c2cf-482b-93c8-32a558f6ded6" + + salt_config["salt_content"] = "33691f8e-e245-4be2-827b-2fa727600fb4.zip" + salt_config["salt_content_path"] = None + + # execution ==================== + saltworker_lx = SaltLinux(system_params, **salt_config) + saltworker_lx.working_dir = system_params["workingdir"] + + saltworker_lx.retrieve_file = MagicMock(return_value=None) + saltworker_lx.extract_contents = MagicMock(return_value=None) + + saltworker_lx._build_salt_formula("e8d7398e-49fa-4eb9-8f8b-22c9d3fdb7f7") + + # assertions =================== + assert saltworker_lx.retrieve_file.call_count == 1 + assert saltworker_lx.extract_contents.call_count == 1 + assert mock_copysubdirs.call_count == 1 + assert mock_codec.call_count == 1 + assert mock_os.call_count == 1 + assert mock_ydump.call_count == 1 + assert mock_yload.call_count == 1 + assert mock_copytree.call_count > 1 + assert mock_glob.call_count == 0 + + +@pytest.mark.skipif(sys.version_info < (3, 4), + reason="Not supported in this Python version.") +@patch("codecs.open", autospec=True) +@patch("os.walk", autospec=True) +@patch("yaml.safe_dump", autospec=True) +@patch("yaml.safe_load", autospec=True) +@patch("watchmaker.utils.copytree", autospec=True) +@patch("glob.glob", autospec=True) +def test_linux_salt_content_path( + mock_glob, mock_copytree, mock_yload, + mock_ydump, mock_os, mock_codec): + """Ensure that files from salt_content_path are retrieved correctly.""" + # setup ======================== + system_params = {} + salt_config = {} + system_params["prepdir"] = "96003f32-5808-4ef8-a573-763b7f47ba9d" + system_params["logdir"] = "0585f9d7-ed0e-4a1b-ac0d-2b10a245a0eb" + system_params["workingdir"] = "35f411db-355b-4953-ae31-d6f592753e58" + + salt_config["salt_content"] = "d002be6e-645d-4f58-97c9-8335df0ff5e4.zip" + salt_config["salt_content_path"] = "05628e08-f1be-474d-8c12-5bb6517fc5f9" + + # execution ==================== + saltworker_lx = SaltLinux(system_params, **salt_config) + + saltworker_lx.retrieve_file = MagicMock(return_value=None) + saltworker_lx.extract_contents = MagicMock(return_value=None) + saltworker_lx.working_dir = system_params["workingdir"] + mock_glob.return_value = ['05628e08-f1be-474d-8c12-5bb6517fc5f9/87a2324d'] + + saltworker_lx._build_salt_formula("8822e968-deea-410f-9b6e-d25a36c512d1") + + # assertions =================== + assert saltworker_lx.retrieve_file.call_count == 1 + assert saltworker_lx.extract_contents.call_count == 1 + assert mock_codec.call_count == 1 + assert mock_os.call_count == 3 + assert mock_ydump.call_count == 1 + assert mock_yload.call_count == 1 + assert mock_copytree.call_count > 1 + assert mock_glob.call_count == 1 + + def test_linux_ou_path_none(): """Test that Pythonic None can be used without error rather than 'None'.""" # setup ======================== diff --git a/tests/test_utils.py b/tests/test_utils.py index 332e676e7..1db68436f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -51,3 +51,21 @@ def test_clean_none(): assert not watchmaker.utils.clean_none('none') assert not watchmaker.utils.clean_none(None) assert watchmaker.utils.clean_none('not none') == 'not none' + + +@patch('os.path.exists', autospec=True) +@patch('watchmaker.utils.copytree', autospec=True) +@patch("os.walk", autospec=True) +def test_copy_subdirectories(mock_os, mock_copy, mock_exists): + """Test that copy_subdirectories executes expected calls.""" + random_src = '580a9176-20f6-4f64-b77a-75dbea14d74f' + random_dst = '6538965c-5131-414a-897f-b01f7dfb6c2b' + mock_exists.return_value = False + mock_os.return_value = [ + ('580a9176-20f6-4f64-b77a-75dbea14d74f', ('87a2a74d',)), + ('580a9176-20f6-4f64-b77a-75dbea14d74f/87a2a74d', (), + ('6274fd83', '1923c65a')), + ].__iter__() + + watchmaker.utils.copy_subdirectories(random_src, random_dst, None) + assert mock_copy.call_count == 1