Merge branch 'master' into pants-py-constraints

BUILD

@@ -1,5 +1,37 @@
 python_requirements(
-    name="root",
+    name="reqs",
+    source="requirements-pants.txt",
+    # module_mapping can be removed once pants is released with
+    # https://github.com/pantsbuild/pants/pull/17390
+    module_mapping={
+        "python-editor": ["editor"],
+        "python-json-logger": ["pythonjsonlogger"],
+        "python-statsd": ["statsd"],
+        "sseclient-py": ["sseclient"],
+        "oslo.config": ["oslo_config"],
+        "RandomWords": ["random_words"],
+    },
+    overrides={
+        # flex and stevedore uses pkg_resources w/o declaring the dep
+        ("flex", "stevedore"): {
+            "dependencies": [
+                "//:reqs#setuptools",
+            ]
+        },
+        # do not use the prance[flex] extra as that pulls in an old version of flex
+        "prance": {
+            "dependencies": [
+                "//:reqs#flex",
+            ]
+        },
+        # tooz needs one or more backends (tooz is used by the st2 coordination backend)
+        "tooz": {
+            "dependencies": [
+                "//:reqs#redis",
+                "//:reqs#zake",
+            ]
+        },
+    },
 )
 
 python_test_utils(

CHANGELOG.rst

@@ -10,7 +10,7 @@ Added
 * Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
-  #5795
+  #5778 #5789 #5817 #5795
   Contributed by @cognifloyd
 
 

contrib/core/BUILD

@@ -3,4 +3,7 @@ python_sources()
 python_requirements(
     name="reqs",
     source="requirements-tests.txt",
+    # module_mapping can be removed once pants is released with
+    # https://github.com/pantsbuild/pants/pull/17390
+    module_mapping={"mail-parser": ["mailparser"]},
 )

contrib/runners/action_chain_runner/action_chain_runner/__init__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.8dev"
+__version__ = "3.9dev"

contrib/runners/announcement_runner/announcement_runner/__init__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.8dev"
+__version__ = "3.9dev"

contrib/runners/http_runner/http_runner/__init__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.8dev"
+__version__ = "3.9dev"

contrib/runners/inquirer_runner/inquirer_runner/__init__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.8dev"
+__version__ = "3.9dev"

contrib/runners/local_runner/local_runner/__init__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.8dev"
+__version__ = "3.9dev"

contrib/runners/noop_runner/noop_runner/__init__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.8dev"
+__version__ = "3.9dev"

contrib/runners/orquesta_runner/orquesta_runner/__init__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.8dev"
+__version__ = "3.9dev"

contrib/runners/python_runner/python_runner/__init__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.8dev"
+__version__ = "3.9dev"

contrib/runners/remote_runner/remote_runner/__init__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.8dev"
+__version__ = "3.9dev"

contrib/runners/winrm_runner/BUILD

@@ -0,0 +1,7 @@
+python_requirement(
+    name="winrm",
+    requirements=["pywinrm"],
+    # modules can be removed once pants is released with
+    # https://github.com/pantsbuild/pants/pull/17390
+    modules=["winrm"],
+)

contrib/runners/winrm_runner/winrm_runner/__init__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.8dev"
+__version__ = "3.9dev"

lint-configs/regex-lint.yaml

@@ -0,0 +1,101 @@
+# Note that for values that are regexes, how YAML interprets backslashes and other
+# special characters matters. For example, an unquoted string is interpreted as a raw
+# string with no escape characters (so it's particularly useful for expressing regexes).
+# Adding quotes around these may change their meaning, so don't do so without thought.
+
+required_matches:
+  # If we decide to enable this, remove the st2flake8
+  #python_source:
+  #  - python_header
+  #build_files:
+  #  - python_header
+
+  # TODO: In the future pants should get `visibility` and possibly other
+  #       features to restrict imports for dependees or dependencies.
+  #       - https://github.com/pantsbuild/pants/issues/13393
+  #       - https://github.com/pantsbuild/pants/pull/15803
+  #       - https://github.com/pantsbuild/pants/pull/15836
+  #       When that happens, we can add that target metadata,
+  #       and remove these regex based dependency checks.
+
+  # st2client-dependencies-check
+  st2client:
+    - must_not_import_st2common
+
+  # st2common-circular-dependencies-check
+  st2common:
+    - must_not_import_st2reactor
+    - must_not_import_st2api
+    - must_not_import_st2auth
+    #- must_not_import_st2actions
+    #- must_not_import_st2stream
+  st2common_except_services_inquiry:
+    # The makefile excluded: runnersregistrar.py, compat.py, inquiry.py
+    # runnersregistrar does not have an st2actions ref since 2016.
+    # compat.py st2actions function was added and removed in 2017.
+    # services/inquiry.py still imports st2actions.
+    - must_not_import_st2actions
+  st2common_except_router:
+    # The makefile excluded router.py from st2stream check.
+    # In router.py, "st2stream" is a string, not an import.
+    - must_not_import_st2stream
+
+path_patterns:
+  #- name: python_source
+  #  pattern: (?<!__init__)\.py$
+  #- name: build_files
+  #  pattern: /BUILD$
+
+  - name: st2client
+    pattern: st2client/st2client/.*\.py$
+  - name: st2common
+    pattern: st2common/st2common/.*\.py$
+
+  - name: st2common_except_services_inquiry
+    pattern: st2common/st2common/(?!services/inquiry\.py).*\.py$
+
+  - name: st2common_except_router
+    pattern: st2common/st2common/(?!router\.py).*\.py$
+
+content_patterns:
+  #- name: python_header
+  #  pattern: |+
+  #    ^(?:#\!\/usr\/bin\/env python3
+  #    )?# Copyright 20\d\d The StackStorm Authors.
+  #    (?:# Copyright 20\d\d .*
+  #    )*#
+  #    # Licensed under the Apache License, Version 2.0 (the "License");
+  #    # you may not use this file except in compliance with the License.
+  #    # You may obtain a copy of the License at
+  #    #
+  #    #     http://www.apache.org/licenses/LICENSE-2.0
+  #    #
+  #    # Unless required by applicable law or agreed to in writing, software
+  #    # distributed under the License is distributed on an "AS IS" BASIS,
+  #    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  #    # See the License for the specific language governing permissions and
+  #    # limitations under the License.
+
+  - name: must_not_import_st2common
+    pattern: st2common
+    inverted: true
+
+  - name: must_not_import_st2reactor
+    pattern: st2reactor
+    inverted: true
+
+  - name: must_not_import_st2actions
+    pattern: st2actions
+    inverted: true
+
+  - name: must_not_import_st2api
+    pattern: st2api
+    inverted: true
+
+  - name: must_not_import_st2auth
+    pattern: st2auth
+    inverted: true
+
+  - name: must_not_import_st2stream
+    pattern: st2stream
+    inverted: true

lockfiles/st2-constraints.txt

@@ -0,0 +1,113 @@
+# Add/remove version constraints for transitive dependencies in this file
+# (transitive dependencies are dependencies of our direct dependencies).
+#
+# Direct dependencies should be recorded in `requirements-pants.txt`, not here.
+
+# please document each version constraint as follows:
+#
+# REQUIRED BY: <package>, <package>, ...
+# REASON: <why do we need to constrain this transitive dep?>
+# NOTE: <status of this constraint / when can we remove it?>
+# DROPS RESOLVED VERSION: <which version pip resolved without this constraint>
+#<package><version constraint>
+
+# ############################################ #
+# pinned transitive deps from requirements.txt #
+# ############################################ #
+
+# REQUIRED BY: jinja2
+# REASON: Fix MarkupSafe to < 2.1.0 as 2.1.0 removes soft_unicode >=0.23 was from jinja2
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: unknown
+MarkupSafe<2.1.0,>=0.23
+
+# REQUIRED BY: kombu
+# REASON: unknown -- this looks like a lockfile-style pin
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 5.1.1
+amqp==5.0.6
+
+# REQUIRED BY: cryptography, paramiko, passlib
+# REASON: unknown -- this looks like a lockfile-style pin
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 4.0.1
+bcrypt==3.2.0
+
+# REQUIRED BY: bcrypt, cryptography, pynacl, zstandard
+# REASON: unknown
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 1.15.1
+cffi<1.15.0
+
+# REQUIRED BY: orquesta, prance, requests
+# REASON: requests 2.23 requires chardet < 3.1.0
+# NOTE: orquesta already constrains this, so this is just documentation.
+# DROPS RESOLVED VERSION: 3.0.4
+#chardet<3.1.0
+
+# REQUIRED BY: jsonpath-rw, networkx
+# REASON:
+#     networkx requires decorator>=4.3,<5 which should resolve to version 4.4.2
+#     but the wheel on pypi does not say it supports python3.8, so pip gets
+#     confused. For now, pin decorator to work around pip's confusion.
+# NOTE: Since pants/pex use a newer version of pip, this is not an issue.
+# DROPS RESOLVED VERSION: 4.4.2
+#decorator==4.4.2
+
+# REQUIRED BY: eventlet, pymongo
+# REASON: 2.0 version breaks pymongo work with hosts
+# NOTE: try to remove this later
+# DROPS RESOLVED VERSION: 1.16
+dnspython>=1.16.0,<2.0.0
+
+# REQUIRED BY: eventlet
+# REASON: unknown -- this looks like a lockfile-style pin
+# NOTE: We are having a hard time upgrading eventlet, so this pin is commented
+#       out to see if that will help. If any tests fail, uncomment this.
+# DROPS RESOLVED VERSION: 1.1.3.post0
+#greenlet==1.0.0
+
+# REQUIRED BY: argcomplete, click, debtcollector, kombu, pluggy, prettytable,
+#             pytest, virtualenv
+# REASON: unknown
+# NOTE: This pinned version (3.10.1) actually conflicts with other requirements.
+#       So, it is commented out. If there are issues with newer versions,
+#       update this with a range of valid versions.
+# DROPS RESOLVED VERSION: 4.8.3
+#importlib-metadata==3.10.1
+
+# REQUIRED BY: tooz
+# REASON: unknown
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 4.13
+oslo.utils<5.0,>=4.0.0
+
+# REQUIRED BY: tooz
+# REASON: unknown
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 8.1
+tenacity>=3.2.1,<7.0.0
+
+# REQUIRED BY: st2-auth-backend-flat-file
+# REASON: unknown -- this looks like a lockfile-style pin
+# NOTE: st2-auth-backend-flat-file has a version range >=1.7.1,<1.8.0
+#       If we need to narrow that range, we should do so in:
+#       https://github.com/StackStorm/st2-auth-backend-flat-file/blob/master/requirements.txt
+# DROPS RESOLVED VERSION: 1.7.4
+#passlib==1.7.4
+
+# pyOpenSSL required by: pymongo[ocsp], redis[ocsp], urllib3[secure]
+# but we don't use any of those, so skip copying from fixed-requirements.txt
+
+# REQUIRED BY: httplib2, oslo.utils, packaging
+# REASON: unknown -- It looks like <3 was only needed for python2 compatibility.
+# NOTE: this is still here, commented, until we can validate that all test are
+#       passing without it.
+# DROPS RESOLVED VERSION: 3.0.7
+#pyparsing<3
+
+# REQUIRED BY: async-timeout, gitpython, importlib-metadata, redis
+# REASON: importlib-metadata requires typing-extensions but v4.2.0 requires py3.7+
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 4.1.1
+typing-extensions<4.2

pants.toml

@@ -35,9 +35,9 @@ pants_ignore.add = [
   "st2common/tests/fixtures/requirements-used-for-tests.txt",
   "/fixed-requirements.txt",
   "/test-requirements.txt",
-  # keep requirements.txt for now. We might ignore it if we need an alternate interrim
-  # file that is decoupled from our legacy requirements files generation.
-  # "/requirements.txt",
+  # ignore requirements.txt for now, preferring interim files that are decoupled from
+  # legacy requirements files generation: requirements-pants.txt & lockfiles/st2-constraints.txt
+  "/requirements.txt",
 ]
 
 [source]
@@ -91,6 +91,16 @@ interpreter_constraints = [
   "CPython>=3.6,<3.9",
 ]
 
+[python-infer]
+# https://www.pantsbuild.org/docs/reference-python-infer#unowned_dependency_behavior
+# The default changed from "ignore" to "warning" in pants 2.14.
+# Many of the new warnings however have been adressed via explicit deps,
+# so the warnings are not helpful. In pants 2.16, a "visibility" feature might help
+# us to disambiguate deps between files without those explicit BUILD dependencies,
+# and without adding "# pants: no-infer-dep" comments all over the codebase.
+# Revisit this in pants 2.16 to see if it is feasible to use the default "warning".
+unowned_dependency_behavior = "ignore"
+
 [bandit]
 lockfile = "lockfiles/bandit.lock"
 version = "bandit==1.7.0"
@@ -122,3 +132,6 @@ extra_requirements = [
   "st2flake8==0.1.0",  # TODO: remove in favor of regex-lint
 ]
 config = "lint-configs/python/.flake8"
+
+[regex-lint]
+config = "@lint-configs/regex-lint.yaml"