Merge branch 'master' into add-value-without-leak-in-history

.circleci/config.yml

@@ -1,5 +1,5 @@
 # Setup in CircleCI account the following ENV variables:
-# PACKAGECLOUD_ORGANIZATION (default: stackstorm) 
+# PACKAGECLOUD_ORGANIZATION (default: stackstorm)
 # PACKAGECLOUD_TOKEN
 version: 2
 jobs:

.git-blame-ignore-rev

@@ -0,0 +1,5 @@
+# Code was auto formatted using black
+8496bb2407b969f0937431992172b98b545f6756
+
+# Files were auto formatted to remove trailing whitespace
+969793f1fdbdd2c228e59ab112189166530d2680

.github/workflows/ci.yaml

@@ -113,7 +113,7 @@ jobs:
       - uses: actions/cache@v2
         with:
           path: |
-            .cache/pip
+            ~/.cache/pip
             virtualenv
           key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
           restore-keys: |

.pre-commit-config.yaml

@@ -0,0 +1,42 @@
+# pre-commit hook which runs all the various lint checks + black auto formatting on the added
+# files.
+# This hook relies on development virtual environment being present in virtualenv/.
+default_language_version:
+  python: python3.6
+
+exclude: '(build|dist)'
+repos:
+  - repo: local
+    hooks:
+    - id: black
+      name: black
+      entry: ./virtualenv/bin/python -m black --config pyproject.toml
+      language: script
+      types: [file, python]
+  - repo: local
+    hooks:
+    - id: flake8
+      name: flake8
+      entry: ./virtualenv/bin/python -m flake8 --config lint-configs/python/.flake8
+      language: script
+      types: [file, python]
+  - repo: local
+    hooks:
+    - id: pylint
+      name: pylint
+      entry: ./virtualenv/bin/python -m pylint -E --rcfile=./lint-configs/python/.pylintrc
+      language: script
+      types: [file, python]
+  - repo: local
+    hooks:
+    - id: bandit
+      name: bandit
+      entry: ./virtualenv/bin/python -m bandit -lll -x build,dist
+      language: script
+      types: [file, python]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.5.0
+    hooks:
+    - id: trailing-whitespace
+      exclude: (^conf/|^st2common/st2common/openapi.yaml|^st2client/tests/fixtures|^st2tests/st2tests/fixtures)
+    - id: check-yaml

CHANGELOG.rst

@@ -4,6 +4,17 @@ Changelog
 in development
 --------------
 
+Changed
+~~~~~~~
+
+* All the code has been refactored using black and black style is automatically enforced and
+  required for all the new code. (#5156)
+
+  Contributed by @Kami.
+
+3.4.0 - March 02, 2021
+----------------------
+
 Added
 ~~~~~
 
@@ -18,6 +29,10 @@ Added
 * Added st2-auth-ldap pip requirements for LDAP auth integartion. (new feature) #5082
   Contributed by @hnanchahal
 
+* Added --register-recreate-virtualenvs flag to st2ctl reload to recreate virtualenvs from scratch.
+  (part of upgrade instructions) [#5167]
+  Contributed by @winem and @blag
+
 Changed
 ~~~~~~~
 
@@ -27,7 +42,7 @@ Changed
 * Improve the st2-self-check script to echo to stderr and exit if it isn't run with a
   ST2_AUTH_TOKEN or ST2_API_KEY environment variable. (improvement) #5068
 
-* Added timeout parameter for packs.install action to help with long running installs that exceed the 
+* Added timeout parameter for packs.install action to help with long running installs that exceed the
   default timeout of 600 sec which is defined by the python_script action runner (improvement) #5084
 
   Contributed by @hnanchahal

Makefile

@@ -326,6 +326,73 @@ schemasgen: requirements .schemasgen
 	. $(VIRTUALENV_DIR)/bin/activate; pylint -j $(PYLINT_CONCURRENCY) -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models tools/*.py || exit 1;
 	. $(VIRTUALENV_DIR)/bin/activate; pylint -j $(PYLINT_CONCURRENCY) -E --rcfile=./lint-configs/python/.pylintrc pylint_plugins/*.py || exit 1;
 
+# Black task which checks if the code comforts to black code style
+.PHONY: black-check
+black: requirements .black-check
+
+.PHONY: .black-check
+.black-check:
+	@echo
+	@echo "================== black-check ===================="
+	@echo
+	# st2 components
+	@for component in $(COMPONENTS); do\
+		echo "==========================================================="; \
+		echo "Running black on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate ; black --check --config pyproject.toml $$component/ || exit 1; \
+	done
+	# runner modules and packages
+	@for component in $(COMPONENTS_RUNNERS); do\
+		echo "==========================================================="; \
+		echo "Running black on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate ; black --check --config pyproject.toml $$component/ || exit 1; \
+	done
+	. $(VIRTUALENV_DIR)/bin/activate; black --check --config pyproject.toml contrib/ || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --check --config pyproject.toml scripts/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --check --config pyproject.toml tools/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --check --config pyproject.toml pylint_plugins/*.py || exit 1;
+
+# Black task which reformats the code using black
+.PHONY: black-format
+black: requirements .black-format
+
+.PHONY: .black-format
+.black-format:
+	@echo
+	@echo "================== black ===================="
+	@echo
+	# st2 components
+	@for component in $(COMPONENTS); do\
+		echo "==========================================================="; \
+		echo "Running black on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml $$component/ || exit 1; \
+	done
+	# runner modules and packages
+	@for component in $(COMPONENTS_RUNNERS); do\
+		echo "==========================================================="; \
+		echo "Running black on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml  $$component/ || exit 1; \
+	done
+	. $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml contrib/ || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml scripts/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml tools/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml pylint_plugins/*.py || exit 1;
+
+.PHONY: pre-commit-checks
+black: requirements .pre-commit-checks
+
+# Ensure all files contain no trailing whitespace + that all YAML files are valid.
+.PHONY: .pre-commit-checks
+.pre-commit-checks:
+	@echo
+	@echo "================== pre-commit-checks ===================="
+	@echo
+	. $(VIRTUALENV_DIR)/bin/activate; pre-commit run trailing-whitespace --all --show-diff-on-failure
+	. $(VIRTUALENV_DIR)/bin/activate; pre-commit run check-yaml --all --show-diff-on-failure
 .PHONY: lint-api-spec
 lint-api-spec: requirements .lint-api-spec
 
@@ -418,7 +485,7 @@ bandit: requirements .bandit
 lint: requirements .lint
 
 .PHONY: .lint
-.lint: .generate-api-spec .flake8 .pylint .st2client-dependencies-check .st2common-circular-dependencies-check .rst-check .st2client-install-check
+.lint: .generate-api-spec .black-check .pre-commit-checks .flake8 .pylint .st2client-dependencies-check .st2common-circular-dependencies-check .rst-check .st2client-install-check
 
 .PHONY: clean
 clean: .cleanpycs
@@ -979,7 +1046,7 @@ debs:
 ci: ci-checks ci-unit ci-integration ci-packs-tests
 
 .PHONY: ci-checks
-ci-checks: .generated-files-check .pylint .flake8 check-requirements check-sdist-requirements .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check check-python-packages
+ci-checks: .generated-files-check .black-check .pre-commit-checks .pylint .flake8 check-requirements check-sdist-requirements .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check check-python-packages
 
 .PHONY: .rst-check
 .rst-check:

OWNERS.md

@@ -33,6 +33,8 @@ Being part of Technical Steering Committee (TSC) [@StackStorm/maintainers](https
   - Ansible, Core, deb/rpm packages, CI/CD, Deployments, StackStorm Exchange, Documentation.
 * JP Bourget ([@punkrokk](https://github.com/punkrokk)) <<[email protected]>>
   - Systems, deb/rpm, Deployments, Community, StackStorm Exchange, SecOps, CircleCI.
+* Marcel Weinberg ([@winem](https://github.com/winem)), _CoreMedia_ <<[email protected]>>
+  - Systems, Core, CI/CD, Docker, Community.
 * Mick McGrath ([@mickmcgrath13](https://github.com/mickmcgrath13)) <<[email protected]>>
   - Systems, ST2 Exchange. [Case Study](https://stackstorm.com/case-study-bitovi/).
 * Nick Maludy ([@nmaludy](https://github.com/nmaludy)) <<[email protected]>>
@@ -48,6 +50,7 @@ They're not part of the TSC voting process, but appreciated for their contributi
 * Carlos ([@nzlosh](https://github.com/nzlosh)) - Chatops, Errbot, Community, Discussions, StackStorm Exchange.
 * Harsh Nanchahal ([@hnanchahal](https://github.com/hnanchahal)), _Starbucks_ - Core, Docker, Kubernetes.
 * Hiroyasu Ohyama ([@userlocalhost](https://github.com/userlocalhost)) - Orquesta, Workflows, st2 Japan Community. [Case Study](https://stackstorm.com/case-study-dmm/).
+* Jacob Floyd ([@cognifloyd](https://github.com/cognifloyd/)), _Copart_ - StackStorm Exchange, ChatOps, Core, Discussions.
 * Jon Middleton ([@jjm](https://github.com/jjm)) - StackStorm Exchange, Core, Discussions.
 * Marcel Weinberg ([@winem](https://github.com/winem)) - Community, Docker, Core.
 * Sheshagiri Rao Mallipedhi ([@sheshagiri](https://github.com/sheshagiri)) - Docker, Core, StackStorm Exchange.
@@ -74,7 +77,7 @@ Thank you, Friends!
 * Johan Dahlberg ([@johandahlberg](https://github.com/johandahlberg)) - Using st2 for Bioinformatics/Science project, providing feedback & contributions in Ansible, Community, Workflows. [Case Study](https://stackstorm.com/case-study-scilifelab/).
 * Johan Hermansson ([@johanherman](https://github.com/johanherman)) - Using st2 for Bioinformatics/Science project, feedback & contributions in Ansible, Community, Workflows. [Case Study](https://stackstorm.com/case-study-scilifelab/).
 * Lakshmi Kannan ([@lakshmi-kannan](https://github.com/lakshmi-kannan)) - early Stormer. Initial Core platform architecture, scalability, reliability, Team Leadership during the project hard times.
-* Lindsay Hill ([@LindsayHill](https://github.com/LindsayHill)) - ex StackStorm product manager that made a significant impact building an ecosystem we see today. 
+* Lindsay Hill ([@LindsayHill](https://github.com/LindsayHill)) - ex StackStorm product manager that made a significant impact building an ecosystem we see today.
 * Manas Kelshikar ([@manasdk](https://github.com/manasdk)) - ex Stormer. Developed (well) early core platform features.
 * Vineesh Jain ([@VineeshJain](https://github.com/VineeshJain)) - ex Stormer. Community, Tests, Core, QA.
 * Warren Van Winckel ([@warrenvw](https://github.com/warrenvw)) - ex Stormer. Docker, Kubernetes, Vagrant, Infrastructure.

README.md

@@ -4,12 +4,12 @@
 
 [![Build Status](https://github.com/StackStorm/st2/workflows/ci-checks/badge.svg?branch=master)](https://github.com/StackStorm/st2/actions?query=branch%3Amaster)
 [![Travis Integration Tests Status](https://travis-ci.org/StackStorm/st2.svg?branch=master)](https://travis-ci.org/StackStorm/st2)
-[![Packages Build Status](https://circleci.com/gh/StackStorm/st2/tree/master.svg?style=shield)](https://circleci.com/gh/StackStorm/st2) 
-[![Codecov](https://codecov.io/github/StackStorm/st2/badge.svg?branch=master&service=github)](https://codecov.io/github/StackStorm/st2?branch=master) 
-[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1833/badge)](https://bestpractices.coreinfrastructure.org/projects/1833) 
-![Python 3.6](https://img.shields.io/badge/python-3.6-blue) 
-[![Apache Licensed](https://img.shields.io/github/license/StackStorm/st2)](LICENSE) 
-[![Join our community Slack](https://img.shields.io/badge/slack-stackstorm-success.svg?logo=slack)](https://stackstorm.com/community-signup) 
+[![Packages Build Status](https://circleci.com/gh/StackStorm/st2/tree/master.svg?style=shield)](https://circleci.com/gh/StackStorm/st2)
+[![Codecov](https://codecov.io/github/StackStorm/st2/badge.svg?branch=master&service=github)](https://codecov.io/github/StackStorm/st2?branch=master)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1833/badge)](https://bestpractices.coreinfrastructure.org/projects/1833)
+![Python 3.6](https://img.shields.io/badge/python-3.6-blue)
+[![Apache Licensed](https://img.shields.io/github/license/StackStorm/st2)](LICENSE)
+[![Join our community Slack](https://img.shields.io/badge/slack-stackstorm-success.svg?logo=slack)](https://stackstorm.com/community-signup)
 [![Forum](https://img.shields.io/discourse/https/forum.stackstorm.com/posts.svg)](https://forum.stackstorm.com/)
 
 ---

conf/st2.conf.sample

@@ -2,7 +2,7 @@
 # Note: This file is automatically generated using tools/config_gen.py - DO NOT UPDATE MANUALLY
 
 [action_sensor]
-# List of execution statuses for which a trigger will be emitted. 
+# List of execution statuses for which a trigger will be emitted.
 emit_when = succeeded,failed,timeout,canceled,abandoned # comma separated list allowed here.
 # Whether to enable or disable the ability to post a trigger on action.
 enable = True
@@ -170,7 +170,7 @@ trigger_instances_ttl = None
 # Allow encryption of values in key value stored qualified as "secret".
 enable_encryption = True
 # Location of the symmetric encryption key for encrypting values in kvstore. This key should be in JSON and should've been generated using st2-generate-symmetric-crypto-key tool.
-encryption_key_path = 
+encryption_key_path =
 
 [log]
 # Exclusion list of loggers to omit.

contrib/chatops/actions/format_execution_result.py

@@ -23,51 +23,50 @@
 
 class FormatResultAction(Action):
     def __init__(self, config=None, action_service=None):
-        super(FormatResultAction, self).__init__(config=config, action_service=action_service)
-        api_url = os.environ.get('ST2_ACTION_API_URL', None)
-        token = os.environ.get('ST2_ACTION_AUTH_TOKEN', None)
+        super(FormatResultAction, self).__init__(
+            config=config, action_service=action_service
+        )
+        api_url = os.environ.get("ST2_ACTION_API_URL", None)
+        token = os.environ.get("ST2_ACTION_AUTH_TOKEN", None)
         self.client = Client(api_url=api_url, token=token)
         self.jinja = jinja_utils.get_jinja_environment(allow_undefined=True)
-        self.jinja.tests['in'] = lambda item, list: item in list
+        self.jinja.tests["in"] = lambda item, list: item in list
 
         path = os.path.dirname(os.path.realpath(__file__))
-        with open(os.path.join(path, 'templates/default.j2'), 'r') as f:
+        with open(os.path.join(path, "templates/default.j2"), "r") as f:
             self.default_template = f.read()
 
     def run(self, execution_id):
         execution = self._get_execution(execution_id)
-        context = {
-            'six': six,
-            'execution': execution
-        }
+        context = {"six": six, "execution": execution}
         template = self.default_template
         result = {"enabled": True}
 
-        alias_id = execution['context'].get('action_alias_ref', {}).get('id', None)
+        alias_id = execution["context"].get("action_alias_ref", {}).get("id", None)
         if alias_id:
-            alias = self.client.managers['ActionAlias'].get_by_id(alias_id)
+            alias = self.client.managers["ActionAlias"].get_by_id(alias_id)
 
-            context.update({
-                'alias': alias
-            })
+            context.update({"alias": alias})
 
-            result_params = getattr(alias, 'result', None)
+            result_params = getattr(alias, "result", None)
             if result_params:
-                if not result_params.get('enabled', True):
+                if not result_params.get("enabled", True):
                     result["enabled"] = False
                 else:
-                    if 'format' in alias.result:
-                        template = alias.result['format']
-                    if 'extra' in alias.result:
-                        result['extra'] = jinja_utils.render_values(alias.result['extra'], context)
+                    if "format" in alias.result:
+                        template = alias.result["format"]
+                    if "extra" in alias.result:
+                        result["extra"] = jinja_utils.render_values(
+                            alias.result["extra"], context
+                        )
 
-        result['message'] = self.jinja.from_string(template).render(context)
+        result["message"] = self.jinja.from_string(template).render(context)
 
         return result
 
     def _get_execution(self, execution_id):
         if not execution_id:
-            raise ValueError('Invalid execution_id provided.')
+            raise ValueError("Invalid execution_id provided.")
         execution = self.client.liveactions.get_by_id(id=execution_id)
         if not execution:
             return None

contrib/chatops/actions/match.py

@@ -23,23 +23,16 @@
 class MatchAction(Action):
     def __init__(self, config=None):
         super(MatchAction, self).__init__(config=config)
-        api_url = os.environ.get('ST2_ACTION_API_URL', None)
-        token = os.environ.get('ST2_ACTION_AUTH_TOKEN', None)
+        api_url = os.environ.get("ST2_ACTION_API_URL", None)
+        token = os.environ.get("ST2_ACTION_AUTH_TOKEN", None)
         self.client = Client(api_url=api_url, token=token)
 
     def run(self, text):
         alias_match = ActionAliasMatch()
         alias_match.command = text
-        matches = self.client.managers['ActionAlias'].match(alias_match)
-        return {
-            'alias': _format_match(matches[0]),
-            'representation': matches[1]
-        }
+        matches = self.client.managers["ActionAlias"].match(alias_match)
+        return {"alias": _format_match(matches[0]), "representation": matches[1]}
 
 
 def _format_match(match):
-    return {
-        'name': match.name,
-        'pack': match.pack,
-        'action_ref': match.action_ref
-    }
+    return {"name": match.name, "pack": match.pack, "action_ref": match.action_ref}