Merge pull request #890 from eemperor/0.16.0

Releases watchmaker version 0.16.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.15.2
+current_version = 0.16.0
 commit = False
 tag = False
 tag_name = {new_version}

.gitlab-ci.yml

@@ -1,8 +1,9 @@
 image: ${IMAGE}
 
 before_script:
-  - sudo yum -y install python34 python34-pip python34-libs python34-setuptools
-  - sudo python3 -m pip install --upgrade pip
+  - sudo yum -y install python36 python36-libs python36-setuptools
+  - sudo python3.6 -m ensurepip --upgrade
+  - test -f /usr/bin/pip3 || sudo ln -sf /usr/local/bin/pip3 /usr/bin/pip3
   - sudo pip3 install --upgrade -r requirements/docs.txt
   - sudo pip3 install -e .
 

.gitmodules

@@ -43,3 +43,6 @@
 [submodule "src/watchmaker/static/salt/formulas/amazon-inspector-formula"]
 	path = src/watchmaker/static/salt/formulas/amazon-inspector-formula
 	url = https://github.com/plus3it/amazon-inspector-formula.git
+[submodule "src/watchmaker/static/salt/content"]
+	path = src/watchmaker/static/salt/content
+	url = https://github.com/plus3it/watchmaker-salt-content.git

.travis.yml

@@ -54,6 +54,9 @@ notifications:
     on_failure: always
 jobs:
   include:
+    - stage: test
+      python: 3.7
+      dist: xenial
     - stage: test
       python: 3.6
       env:

CHANGELOG.md

@@ -1,5 +1,19 @@
 ## Changelog
 
+### 0.16.0
+
+**Commit Delta**: [Change from 0.15.2 release](https://github.com/plus3it/watchmaker/compare/0.15.2...0.16.0)
+
+**Released**: 2019.05.10
+
+**Summary**:
+
+*   Adds salt content locally as a submodule to better support Watchmaker standalone packages
+*   dotnet4-formula
+    -   Updates formula to support the use of Python3 versions of Salt
+*   join-domain-formula
+    -   Adds additional enhancements and logic to better handle the domin-join process in Linux
+
 ### 0.15.2
 
 **Commit Delta**: [Change from 0.15.1 release](https://github.com/plus3it/watchmaker/compare/0.15.1...0.15.2)

appveyor.yml

@@ -14,6 +14,41 @@ init:
 install:
   - git submodule update --init --recursive
 test_script:
+  # ----- python 3.7, x64
+  # python envs
+  - set PYTHON_HOME=C:\Python37-x64
+  - set PYTHON_VERSION=3.7
+  - set PYTHON_ARCH=64
+
+  # install
+  - '%PYTHON_HOME%\Scripts\pip install --upgrade tox setuptools virtualenv wheel'
+
+    # versions
+  - '%PYTHON_HOME%\Scripts\virtualenv --version'
+  - '%PYTHON_HOME%\Scripts\tox --version'
+  - '%PYTHON_HOME%\Scripts\pip --version'
+
+  # test
+  - set TOXENV=3.7-codecov
+  - '%WITH_ENV% %PYTHON_HOME%\Scripts\tox -v'
+
+  # ----- python 3.7, x32
+  # python envs
+  - set PYTHON_HOME=C:\Python37
+  - set PYTHON_ARCH=32
+
+  # install
+  - '%PYTHON_HOME%\Scripts\pip install --upgrade tox setuptools virtualenv wheel'
+
+    # versions
+  - '%PYTHON_HOME%\Scripts\virtualenv --version'
+  - '%PYTHON_HOME%\Scripts\tox --version'
+  - '%PYTHON_HOME%\Scripts\pip --version'
+
+  # test
+  - set TOXENV=3.7-codecov
+  - '%WITH_ENV% %PYTHON_HOME%\Scripts\tox -v'
+
   # ----- python 3.6, x64
   # python envs
   - set PYTHON_HOME=C:\Python36-x64

requirements/build.txt

@@ -1 +1 @@
-gravitybee==0.1.26
+gravitybee==0.1.27

requirements/check.txt

@@ -1,15 +1,15 @@
-check-manifest==0.37
+check-manifest==0.38
 flake8==3.7.7
 flake8-bugbear==19.3.0
 flake8-builtins==1.4.1
 flake8-docstrings==1.3.0
 flake8-isort==2.7.0
 flake8-future-import==0.4.5
 flake8-print==3.1.0
-isort==4.3.17
+isort==4.3.18
 m2r==0.2.1
 pep8-naming==0.8.2
 pydocstyle==3.0.0
-pygments==2.3.1
+pygments==2.4.0
 pylint==2.3.1
 readme-renderer==24.0

requirements/deploy.txt

@@ -1 +1 @@
-satsuki==0.1.13
+satsuki==0.1.15

requirements/docs.txt

@@ -1,4 +1,4 @@
 m2r==0.2.1
-setuptools==41.0.0
+setuptools==41.0.1
 sphinx==2.0.1
 sphinx-rtd-theme==0.4.3

requirements/pip.txt

@@ -1 +1 @@
-pip==19.0.3
+pip==19.1.1

requirements/test.txt

@@ -1,10 +1,11 @@
-mock==2.0.0
+mock==2.0.0;python_version=="2.6"  # pyup: ==2.0.0
+mock==3.0.5;python_version>="2.7"
 pytest==3.2.5;python_version=="2.6" or python_version=="3.3"  # pyup: ==3.2.5
-pytest==4.4.0;python_version=="2.7" or python_version>="3.4"
+pytest==4.4.2;python_version=="2.7" or python_version>="3.4"
 pytest-travis-fold==1.3.0
 pytest-catchlog==1.2.2;python_version=="2.6"
 pytest-cov==2.5.1;python_version=="2.6"  # pyup: ==2.5.1
-pytest-cov==2.6.1;python_version>="2.7"
+pytest-cov==2.7.1;python_version>="2.7"
 pytest-mock==1.6.3;python_version=="2.6"  # pyup: ==1.6.3
-pytest-mock==1.10.3;python_version>="2.7"
+pytest-mock==1.10.4;python_version>="2.7"
 wheel==0.29.0;python_version<="2.6"  # pyup: ==0.29.0

setup.cfg

@@ -3,7 +3,7 @@
 [metadata]
 name = watchmaker
 description = Applied Configuration Management
-version = 0.15.2
+version = 0.16.0
 author = Plus3IT Maintainers of Watchmaker
 author_email = [email protected]
 url = https://github.com/plus3it/watchmaker
@@ -31,15 +31,16 @@ classifiers =
 
 [options]
 install_requires =
-    backoff
+    backoff;python_version>="2.7"
+    backoff<1.7;python_version<"2.7"
     click;python_version>="2.7"
     click<7;python_version<"2.7"
     defusedxml;platform_system=="Windows"
     futures;python_version<"3"
     six
     pypiwin32;platform_system=="Windows"
     PyYAML;python_version>="2.7"
-    PyYAML<4;python_version<"2.7"    
+    PyYAML<4;python_version<"2.7"
     wheel<=0.29.0;python_version<"2.7"
 packages = find:
 include_package_data = True

src/watchmaker/static/config.yaml

@@ -5,7 +5,7 @@ all:
       computer_name: None
       environment: None
       ou_path: None
-      salt_content: https://s3.amazonaws.com/watchmaker/salt-content.zip
+      salt_content: None
       salt_states: Highstate
       user_formulas:
         # To add extra formulas, specify them as a map of

src/watchmaker/utils/__init__.py

@@ -4,6 +4,7 @@
                         unicode_literals, with_statement)
 
 import os
+import shutil
 import ssl
 
 import backoff
@@ -58,3 +59,26 @@ def urlopen_retry(uri):
         pass
 
     return urllib.request.urlopen(uri, **kwargs)
+
+
+def copytree(src, dst, force=False, **kwargs):
+    r"""
+    Copy OS directory trees from source to destination.
+
+    Args:
+        src: (:obj:`str`)
+            Source directory tree to be copied.
+            (*Default*: None)
+
+        dst: (:obj:`str`)
+            Destination where directory tree is to be copied.
+            (*Default*: None)
+
+        force: (:obj:`bool`)
+            Whether to delete destination prior to copy.
+            (*Default*: ``False``)
+    """
+    if force and os.path.exists(dst):
+        shutil.rmtree(dst)
+
+    shutil.copytree(src, dst, **kwargs)

src/watchmaker/workers/salt.py

@@ -169,15 +169,14 @@ def _prepare_for_install(self):
         ]
 
         for salt_dir in [
-            self.salt_base_env,
             self.salt_formula_root,
             self.salt_conf_path
         ]:
             try:
                 os.makedirs(salt_dir)
             except OSError:
                 if not os.path.isdir(salt_dir):
-                    msg = ('Unable create directory - {0}'.format(salt_dir))
+                    msg = ('Unable to create directory - {0}'.format(salt_dir))
                     self.log.error(msg)
                     raise SystemError(msg)
 
@@ -194,11 +193,11 @@ def _get_formulas_conf(self):
         formulas_path = os.sep.join((static.__path__[0], 'salt', 'formulas'))
         for formula in os.listdir(formulas_path):
             formula_path = os.path.join(self.salt_formula_root, '', formula)
-            if os.path.exists(formula_path):
-                shutil.rmtree(formula_path)
-            shutil.copytree(
+            watchmaker.utils.copytree(
                 os.sep.join((formulas_path, formula)),
-                formula_path)
+                formula_path,
+                force=True
+            )
 
         # Obtain & extract any Salt formulas specified in user_formulas.
         for formula_name, formula_url in self.user_formulas.items():
@@ -241,7 +240,7 @@ def _get_formulas_conf(self):
         ]
 
     def _build_salt_formula(self, extract_dir):
-        if self.salt_content:
+        if self.salt_content and self.salt_content != 'None':
             salt_content_filename = watchmaker.utils.basename_from_uri(
                 self.salt_content
             )
@@ -255,6 +254,23 @@ def _build_salt_formula(self, extract_dir):
                 to_directory=extract_dir
             )
 
+        bundled_content = os.sep.join(
+            (static.__path__[0], 'salt', 'content')
+        )
+        for subdir in next(os.walk(bundled_content))[1]:
+            if (
+                not subdir.startswith('.') and
+                not os.path.exists(os.sep.join((extract_dir, subdir)))
+            ):
+                watchmaker.utils.copytree(
+                    os.sep.join((bundled_content, subdir)),
+                    os.sep.join((extract_dir, subdir))
+                )
+                self.log.info(
+                    'Using bundled content from %s',
+                    os.sep.join((bundled_content, subdir))
+                )
+
         with codecs.open(
             os.path.join(self.salt_conf_path, 'minion'),
             'r+',