Releases watchmaker version 0.19.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.18.2
+current_version = 0.19.0
 commit = False
 tag = False
 tag_name = {new_version}

CHANGELOG.md

@@ -1,5 +1,17 @@
 ## Changelog
 
+### 0.19.0
+
+**Commit Delta**: [Change from 0.18.2 release](https://github.com/plus3it/watchmaker/compare/0.18.2...0.19.0)
+
+**Released**: 2020.05.01
+
+**Summary**:
+
+*   Updates Watchmaker file permissions and makes them more restrictive
+*   Adds new SaltWorker optional argument `--salt-content-path` that allows specifying glob pattern for
+    salt files located within salt archive file
+
 ### 0.18.2
 
 **Commit Delta**: [Change from 0.18.1 release](https://github.com/plus3it/watchmaker/compare/0.18.1...0.18.2)

docs/configuration.md

@@ -134,6 +134,11 @@ Parameters supported by the Salt Worker:
 -   `salt_content` (_string_): URL to the Salt content file that contains
     further configuration specific to the salt install.
 
+-   `salt_content_path` (_string_): The path within the Salt content file
+    specified using `salt_content` where salt files are located.
+    Can be used to provide the path within the archive file where
+    the Salt configuration files are located.
+
 -   `install_method` (_string_): (Linux-only) The method used to install Salt.
     Currently supports: `yum`, `git`
 

requirements/basics.txt

@@ -2,7 +2,7 @@ setuptools==36.8.0;python_version<="2.6"
 setuptools==43.0.0;python_version<="3.4" and python_version>="2.7"
 setuptools==46.1.3;python_version>"3.4"
 virtualenv==15.2.0;python_version<="2.6"
-virtualenv==20.0.15;python_version>="2.7"
+virtualenv==20.0.18;python_version>="2.7"
 wheel==0.29.0;python_version<="2.6"
 wheel==0.33.6;python_version<="3.4" and python_version>="2.7"
 wheel==0.34.2;python_version>"3.4"

requirements/build.txt

@@ -1 +1 @@
-gravitybee==0.1.36
+gravitybee==0.1.40

requirements/check.txt

@@ -5,12 +5,12 @@ flake8==3.7.9
 flake8-bugbear==20.1.4
 flake8-builtins==1.5.2
 flake8-docstrings==1.5.0
-flake8-isort==2.9.1
+flake8-isort==3.0.0
 flake8-future-import==0.4.6
 flake8-print==3.1.4
 isort==4.3.21
 m2r==0.2.1
 pep8-naming==0.10.0
 pydocstyle==5.0.2
 pylint==2.4.4
-readme-renderer==25.0
+readme-renderer==26.0

requirements/docs.txt

@@ -2,5 +2,5 @@
 
 recommonmark==0.6.0
 setuptools==46.1.3
-sphinx==2.4.4
+sphinx==3.0.3
 sphinx-rtd-theme==0.4.3

requirements/pip.txt

@@ -1,3 +1,3 @@
 pip==9.0.3;python_version<="2.6"
 pip==19.1;python_version<="3.4" and python_version>="2.7"
-pip==20.0.2;python_version>"3.4"
+pip==20.1;python_version>"3.4"

requirements/test.txt

@@ -9,5 +9,6 @@ pytest-catchlog==1.2.2;python_version<"2.7"
 pytest-cov==2.5.1;python_version<"2.7"  # pyup: ==2.5.1
 pytest-cov==2.8.1;python_version>="2.7"
 pytest-mock==1.6.3;python_version<"2.7"  # pyup: ==1.6.3
-pytest-mock==2.0.0;python_version>="2.7"
+pytest-mock==2.0.0;python_version<"3.5" and python_version>="2.7"  # pyup: ==2.0.0
+pytest-mock==3.1.0;python_version>="3.5"
 wheel==0.29.0;python_version<="2.6"  # pyup: ==0.29.0

setup.cfg

@@ -3,7 +3,7 @@
 [metadata]
 name = watchmaker
 description = Applied Configuration Management
-version = 0.18.2
+version = 0.19.0
 long_description = file: README.md, CHANGELOG.md
 long_description_content_type = text/markdown
 author = Plus3IT Maintainers of Watchmaker
@@ -41,12 +41,13 @@ install_requires =
     defusedxml;platform_system=="Windows"
     futures;python_version<"3"
     six
-    pypiwin32;platform_system=="Windows"
+    pywin32;platform_system=="Windows"
     PyYAML;python_version>="3.5"
     PyYAML<=5.2;python_version<"3.5"
     PyYAML<4;python_version<"2.7"
     wheel<=0.29.0;python_version<"2.7"
     compatibleversion>=0.1.2
+    oschmod>=0.1.3
 packages = find:
 include_package_data = True
 

src/watchmaker/__init__.py

@@ -10,6 +10,7 @@
 import platform
 import subprocess
 
+import oschmod
 import pkg_resources
 import setuptools
 import yaml
@@ -407,6 +408,7 @@ def _set_system_params(self):
             self.system_drive = '/'
             self.workers_manager = LinuxWorkersManager
             self.system_params = self._get_linux_system_params()
+            os.umask(0o077)
         elif 'windows' in self.system:
             self.system_drive = os.environ['SYSTEMDRIVE']
             self.workers_manager = WindowsWorkersManager
@@ -431,6 +433,7 @@ def install(self):
         # Create watchmaker directories
         try:
             os.makedirs(self.system_params['workingdir'])
+            oschmod.set_mode(self.system_params['prepdir'], 0o700)
         except OSError:
             if not os.path.exists(self.system_params['workingdir']):
                 msg = (

src/watchmaker/logger/__init__.py

@@ -14,6 +14,8 @@
 import subprocess
 import xml.etree.ElementTree
 
+import oschmod
+
 IS_WINDOWS = platform.system() == 'Windows'
 MESSAGE_TYPES = ('Information', 'Warning', 'Error')
 
@@ -88,6 +90,7 @@ def make_log_dir(log_dir):
     """
     if not os.path.exists(log_dir):
         os.makedirs(log_dir)
+        oschmod.set_mode(log_dir, 0o700)
 
 
 def log_system_details(log):
@@ -141,6 +144,7 @@ def prepare_logging(log_dir, log_level):
         make_log_dir(log_dir)
         log_filename = os.sep.join((log_dir, 'watchmaker.log'))
         hdlr = logging.FileHandler(log_filename)
+        oschmod.set_mode(log_filename, 0o600)
         hdlr.setLevel(level)
         hdlr.setFormatter(logging.Formatter(logformat))
         logging.getLogger().addHandler(hdlr)

src/watchmaker/utils/__init__.py

@@ -119,3 +119,20 @@ def clean_none(value):
         return None
 
     return value
+
+
+def copy_subdirectories(src_dir, dest_dir, log=None):
+    """Copy subdirectories within given src dir into dest dir."""
+    for subdir in next(os.walk(src_dir))[1]:
+        if (
+            not subdir.startswith('.') and
+            not os.path.exists(os.sep.join((dest_dir, subdir)))
+        ):
+            copytree(
+                os.sep.join((src_dir, subdir)),
+                os.sep.join((dest_dir, subdir))
+            )
+            if log:
+                log.info('Copied from %s to %s',
+                         os.sep.join((src_dir, subdir)),
+                         os.sep.join((dest_dir, subdir)))

src/watchmaker/workers/salt.py

@@ -5,6 +5,7 @@
 
 import ast
 import codecs
+import glob
 import json
 import os
 import shutil
@@ -40,6 +41,16 @@ class SaltBase(WorkerBase, PlatformManagerBase):
             - *Linux*: ``/srv/watchmaker/salt``
             - *Windows*: ``C:\Watchmaker\Salt\srv``
 
+        salt_content_path: (:obj:`str`)
+            Used in conjunction with the "salt_content" arg.
+            Glob pattern for the location of salt content files inside the
+            provided salt_content archive. To be used when salt content files
+            are located within a sub-path of the archive, rather than at its
+            top-level. Multiple paths matching the given pattern will result
+            in error.
+            E.g. ``salt_content_path='*/'``
+            (*Default*: ``''``)
+
         salt_states: (:obj:`str`)
             Comma-separated string of salt states to execute. Accepts two
             special keywords (case-insensitive).
@@ -97,6 +108,7 @@ def __init__(self, *args, **kwargs):
         self.valid_envs = kwargs.pop('valid_environments', []) or []
         self.salt_debug_log = kwargs.pop('salt_debug_log', None) or ''
         self.salt_content = kwargs.pop('salt_content', None) or ''
+        self.salt_content_path = kwargs.pop('salt_content_path', None) or ''
         self.ou_path = kwargs.pop('ou_path', None) or ''
         self.admin_groups = kwargs.pop('admin_groups', None) or ''
         self.admin_users = kwargs.pop('admin_users', None) or ''
@@ -109,6 +121,8 @@ def __init__(self, *args, **kwargs):
             self.salt_debug_log, self.log)
         self.salt_content = watchmaker.utils.config_none_deprecate(
             self.salt_content, self.log)
+        self.salt_content_path = watchmaker.utils.config_none_deprecate(
+            self.salt_content_path, self.log)
         self.ou_path = watchmaker.utils.config_none_deprecate(
             self.ou_path, self.log)
         self.admin_groups = watchmaker.utils.config_none_deprecate(
@@ -262,27 +276,51 @@ def _build_salt_formula(self, extract_dir):
                 salt_content_filename
             ))
             self.retrieve_file(self.salt_content, salt_content_file)
-            self.extract_contents(
-                filepath=salt_content_file,
-                to_directory=extract_dir
-            )
+            if not self.salt_content_path:
+                self.extract_contents(
+                    filepath=salt_content_file,
+                    to_directory=extract_dir
+                )
+            else:
+                self.log.debug(
+                    'Using salt content path: %s',
+                    self.salt_content_path
+                )
+                temp_extract_dir = os.sep.join((
+                    self.working_dir, 'salt-archive'))
+                self.extract_contents(
+                    filepath=salt_content_file,
+                    to_directory=temp_extract_dir
+                )
+                salt_content_src = os.sep.join((
+                    temp_extract_dir, self.salt_content_path))
+                salt_content_glob = glob.glob(salt_content_src)
+                self.log.debug('salt_content_glob: %s', salt_content_glob)
+                if len(salt_content_glob) > 1:
+                    msg = 'Found multiple paths matching' \
+                          ' \'{0}\' in {1}'.format(
+                              self.salt_content_path,
+                              self.salt_content)
+                    self.log.critical(msg)
+                    raise WatchmakerException(msg)
+                try:
+                    salt_files_dir = salt_content_glob[0]
+                except IndexError:
+                    msg = 'Salt content glob path \'{0}\' not' \
+                          ' found in {1}'.format(
+                              self.salt_content_path,
+                              self.salt_content)
+                    self.log.critical(msg)
+                    raise WatchmakerException(msg)
+
+                watchmaker.utils.copy_subdirectories(
+                    salt_files_dir, extract_dir, self.log)
 
         bundled_content = os.sep.join(
             (static.__path__[0], 'salt', 'content')
         )
-        for subdir in next(os.walk(bundled_content))[1]:
-            if (
-                not subdir.startswith('.') and
-                not os.path.exists(os.sep.join((extract_dir, subdir)))
-            ):
-                watchmaker.utils.copytree(
-                    os.sep.join((bundled_content, subdir)),
-                    os.sep.join((extract_dir, subdir))
-                )
-                self.log.info(
-                    'Using bundled content from %s',
-                    os.sep.join((bundled_content, subdir))
-                )
+        watchmaker.utils.copy_subdirectories(
+            bundled_content, extract_dir, self.log)
 
         with codecs.open(
             os.path.join(self.salt_conf_path, 'minion'),

tests/test_logger.py

@@ -12,6 +12,7 @@
 import sys
 import xml.etree.ElementTree
 
+import oschmod
 import pytest
 
 from watchmaker import logger
@@ -65,21 +66,24 @@ def test_logger_handler():
     """
     Tests prepare_logging() use case.
 
-    Tests that prepare_logging() will set logger level appropriately
-    and attach a FileHandler if a directory is passed.
+    Tests that prepare_logging() will set logger level appropriately, attach a
+    FileHandler if a directory is passed, and set log file to the correct mode.
     """
-    logger.prepare_logging('./logfiles', 'debug')
+    logger.prepare_logging('logfiles', 'debug')
     this_logger = logging.getLogger()
 
     if logger.HAS_PYWIN32:
         log_hdlr = this_logger.handlers.pop()
-        assert type(log_hdlr) == logging.handlers.NTEventLogHandler
+        assert isinstance(log_hdlr, logging.handlers.NTEventLogHandler)
         assert log_hdlr.level == logging.DEBUG
 
     log_hdlr = this_logger.handlers.pop()
-    assert type(log_hdlr) == logging.FileHandler
+    assert isinstance(log_hdlr, logging.FileHandler)
     assert log_hdlr.level == logging.DEBUG
 
+    assert oschmod.get_mode(
+        os.path.join('logfiles', 'watchmaker.log')) == 0o600
+
 
 def test_log_if_no_log_directory_given(caplog):
     """