From f3d3e3ad1b5daa5bb0f03eaf9393299f7474e989 Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Tue, 13 Dec 2022 10:19:05 +0000 Subject: [PATCH 01/12] WIP: Fixing integration tests on windows --- qa/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qa/base.py b/qa/base.py index ce66fbf2..680f5618 100644 --- a/qa/base.py +++ b/qa/base.py @@ -14,7 +14,7 @@ from qa.shell import git, gitlint, RunningCommand -from qa.utils import DEFAULT_ENCODING +from qa.utils import DEFAULT_ENCODING, PLATFORM_IS_WINDOWS class BaseTestCase(TestCase): @@ -40,7 +40,8 @@ def tearDown(self): for tmpfile in self.tmpfiles: os.remove(tmpfile) for repo in self.tmp_git_repos: - shutil.rmtree(repo) + # On windows we need to ignore errors because git might still be holding on to some files + shutil.rmtree(repo, ignore_errors=PLATFORM_IS_WINDOWS) def assertEqualStdout(self, output, expected): # pylint: disable=invalid-name self.assertIsInstance(output, RunningCommand) From 9c0f623bad9ca87bf53f35bf16999ddd902cada4 Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Thu, 15 Dec 2022 10:14:33 +0000 Subject: [PATCH 02/12] WIndows: use ISO-8859-1 encoding --- qa/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/utils.py b/qa/utils.py index 89292cd7..5331d1b6 100644 --- a/qa/utils.py +++ b/qa/utils.py @@ -43,6 +43,7 @@ def getpreferredencoding(): # (on Linux/MacOS the `getpreferredencoding()` call will take care of this). # We fallback to UTF-8 if PLATFORM_IS_WINDOWS: + return "ISO-8859-1" default_encoding = "UTF-8" for env_var in ["LC_ALL", "LC_CTYPE", "LANG"]: encoding = os.environ.get(env_var, False) From b861fe0e4a5c61f75ed97aeeb754aee1af5d11ca Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Thu, 15 Dec 2022 10:28:25 +0000 Subject: [PATCH 03/12] Tweaks --- pyproject.toml | 5 +++-- qa/base.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 87e75a21..7db9cca1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,9 +108,10 @@ Run a set of integration tests against any gitlint binary (not just the one from """ detached = true dependencies = [ - "sh==1.14.3", "pytest==7.2.0", "arrow==1.2.3", + "sh==1.14.3; sys_platform != \"win32\"", + "pdbr==0.7.5; sys_platform != \"win32\"", ] [tool.hatch.envs.qa.scripts] @@ -118,7 +119,7 @@ dependencies = [ # This is why by default we don't install the local dev version of gitlint in the qa environment # To run integration tests against the dev version of gitlint, use install-local first install-local="pip install ./gitlint-core[trusted-deps]" -integration-tests = "pytest qa {args}" +integration-tests = "pytest -rw -s qa {args}" i = "integration-tests" diff --git a/qa/base.py b/qa/base.py index 680f5618..8e81a772 100644 --- a/qa/base.py +++ b/qa/base.py @@ -45,7 +45,7 @@ def tearDown(self): def assertEqualStdout(self, output, expected): # pylint: disable=invalid-name self.assertIsInstance(output, RunningCommand) - output = output.stdout.decode(DEFAULT_ENCODING) + output = output.stdout # .decode(DEFAULT_ENCODING) output = output.replace("\r", "") self.assertMultiLineEqual(output, expected) From b9e92a60671f6da41b4e25407f2e6f540f212bc8 Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Thu, 15 Dec 2022 10:54:39 +0000 Subject: [PATCH 04/12] Expected files are always UTF-8 --- pyproject.toml | 3 +++ qa/base.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7db9cca1..3f39e435 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,9 @@ dependencies = [ "pdbr==0.7.5; sys_platform != \"win32\"", ] +[tool.hatch.envs.qa.env-vars] +GITLINT_QA_USE_SH_LIB = "0" + [tool.hatch.envs.qa.scripts] # The integration tests can be ran against any gitlint binary, e.g. one installed from pypi (for post-release testing) # This is why by default we don't install the local dev version of gitlint in the qa environment diff --git a/qa/base.py b/qa/base.py index 8e81a772..5d50bcdb 100644 --- a/qa/base.py +++ b/qa/base.py @@ -182,7 +182,7 @@ def get_expected(filename="", variable_dict=None): specified by variable_dict.""" expected_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "expected") expected_path = os.path.join(expected_dir, filename) - with open(expected_path, encoding=DEFAULT_ENCODING) as file: + with open(expected_path, encoding="UTF-8") as file: expected = file.read() if variable_dict: From 41af46727d42fc0ce61a975a37544c31acc09ad2 Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Thu, 15 Dec 2022 10:56:18 +0000 Subject: [PATCH 05/12] fixup! Expected files are always UTF-8 --- qa/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/base.py b/qa/base.py index 5d50bcdb..c6306495 100644 --- a/qa/base.py +++ b/qa/base.py @@ -182,6 +182,7 @@ def get_expected(filename="", variable_dict=None): specified by variable_dict.""" expected_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "expected") expected_path = os.path.join(expected_dir, filename) + # Expected files are UTF-8 encoded (not dependent on the system's default encoding) with open(expected_path, encoding="UTF-8") as file: expected = file.read() From ad5d13d6f6897c8b5c847223f8d48ff4fabc0d4b Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Fri, 16 Dec 2022 11:59:57 +0000 Subject: [PATCH 06/12] Attempt to remove sh for many tests --- .github/workflows/checks.yml | 15 +++++++-- pyproject.toml | 5 +-- qa/base.py | 2 +- qa/shell.py | 62 ++++++++++++++++++++++++++++++------ qa/test_commits.py | 2 +- qa/test_config.py | 3 +- qa/test_gitlint.py | 2 +- qa/test_stdin.py | 6 ++-- qa/utils.py | 31 +++++++++++------- 9 files changed, 93 insertions(+), 35 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 4a2114a4..7a6a4f46 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -46,7 +46,13 @@ jobs: - name: Integration tests (GITLINT_USE_SH_LIB=1) run: | - hatch run qa:integration-tests + hatch run qa:integration-tests --ignore qa/test_gitlint.py qa + env: + GITLINT_USE_SH_LIB: 1 + + - name: Integration tests (GITLINT_QA_USE_SH_LIB=0) + run: | + hatch run qa:integration-tests --ignore qa/test_hooks.py --ignore qa/test_gitlint.py qa env: GITLINT_USE_SH_LIB: 1 @@ -125,11 +131,16 @@ jobs: - name: Code linting (pylint) run: hatch run test:lint + - name: Integration tests + run: | + hatch run qa:install-local + hatch run qa:integration-tests --ignore qa/test_hooks.py --ignore qa/test_gitlint.py qa + - name: Build test (gitlint) run: | hatch build hatch clean - + - name: Build test (gitlint-core) run: | hatch build diff --git a/pyproject.toml b/pyproject.toml index 3f39e435..52b70886 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,15 +114,12 @@ dependencies = [ "pdbr==0.7.5; sys_platform != \"win32\"", ] -[tool.hatch.envs.qa.env-vars] -GITLINT_QA_USE_SH_LIB = "0" - [tool.hatch.envs.qa.scripts] # The integration tests can be ran against any gitlint binary, e.g. one installed from pypi (for post-release testing) # This is why by default we don't install the local dev version of gitlint in the qa environment # To run integration tests against the dev version of gitlint, use install-local first install-local="pip install ./gitlint-core[trusted-deps]" -integration-tests = "pytest -rw -s qa {args}" +integration-tests = "pytest -rw -s {args:qa}" i = "integration-tests" diff --git a/qa/base.py b/qa/base.py index c6306495..5312c6fc 100644 --- a/qa/base.py +++ b/qa/base.py @@ -45,7 +45,7 @@ def tearDown(self): def assertEqualStdout(self, output, expected): # pylint: disable=invalid-name self.assertIsInstance(output, RunningCommand) - output = output.stdout # .decode(DEFAULT_ENCODING) + output = output.stdout.decode(DEFAULT_ENCODING) output = output.replace("\r", "") self.assertMultiLineEqual(output, expected) diff --git a/qa/shell.py b/qa/shell.py index 44716c0a..536042ce 100644 --- a/qa/shell.py +++ b/qa/shell.py @@ -29,13 +29,46 @@ def __init__(self, full_cmd, stdout, stderr="", exitcode=0): self.full_cmd = full_cmd # TODO(jorisroovers): The 'sh' library by default will merge stdout and stderr. We mimic this behavior # for now until we fully remove the 'sh' library. - self.stdout = stdout + stderr.decode(DEFAULT_ENCODING) - self.stderr = stderr + self._stdout = stdout + stderr # .decode(DEFAULT_ENCODING) + self._stderr = stderr self.exit_code = exitcode def __str__(self): + return self.stdout.decode(DEFAULT_ENCODING) + + def __unicode__(self): return self.stdout + @property + def stdout(self): + return self._stdout + + @property + def stderr(self): + return self._stderr + + def __getattr__(self, p): # pylint: disable=invalid-name + # https://github.com/amoffat/sh/blob/e0ed8e244e9d973ef4e0749b2b3c2695e7b5255b/sh.py#L952= + _unicode_methods = set(dir(str())) + + # # let these three attributes pass through to the OProc object + # if p in self._OProc_attr_whitelist: + # if self.process: + # return getattr(self.process, p) + # else: + # raise AttributeError + + # see if strings have what we're looking for. we're looking at the + # method names explicitly because we don't want to evaluate self unless + # we absolutely have to, the reason being, in python2, hasattr swallows + # exceptions, and if we try to run hasattr on a command that failed and + # is being run with _iter=True, the command will be evaluated, throw an + # exception, but hasattr will discard it + if p in _unicode_methods: + return getattr(str(self), p) + + raise AttributeError + class ErrorReturnCode(ShResult, Exception): """ShResult subclass for unexpected results (acts as an exception).""" @@ -56,26 +89,37 @@ def run_command(command, *args, **kwargs): # If we reach this point and the result has an exit_code that is larger than 0, this means that we didn't # get an exception (which is the default sh behavior for non-zero exit codes) and so the user is expecting # a non-zero exit code -> just return the entire result - if hasattr(result, "exit_code") and result.exit_code > 0: - return result - return str(result) + # if hasattr(result, "exit_code") and result.exit_code > 0: + # return result + return result def _exec(*args, **kwargs): - pipe = subprocess.PIPE - popen_kwargs = {"stdout": pipe, "stderr": pipe, "shell": kwargs.get("_tty_out", False)} + popen_kwargs = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "shell": kwargs.get("_tty_out", False)} if "_cwd" in kwargs: popen_kwargs["cwd"] = kwargs["_cwd"] if "_env" in kwargs: popen_kwargs["env"] = kwargs["_env"] + stdin_input = None + if len(args) > 1 and isinstance(args[1], ShResult): + stdin_input = args[1].stdout + # pop args[1] from the array and use it as stdin + args = list(args) + args.pop(1) + popen_kwargs["stdin"] = subprocess.PIPE + try: with subprocess.Popen(args, **popen_kwargs) as p: - result = p.communicate() + if stdin_input: + result = p.communicate(stdin_input) + else: + result = p.communicate() + except FileNotFoundError as exc: raise CommandNotFound from exc exit_code = p.returncode - stdout = result[0].decode(DEFAULT_ENCODING) + stdout = result[0] # .decode(DEFAULT_ENCODING) stderr = result[1] # 'sh' does not decode the stderr bytes to unicode full_cmd = "" if args is None else " ".join(args) diff --git a/qa/test_commits.py b/qa/test_commits.py index d40c211d..1449e933 100644 --- a/qa/test_commits.py +++ b/qa/test_commits.py @@ -129,7 +129,7 @@ def test_lint_single_commit(self): self.assertEqual(output.exit_code, 254) def test_lint_staged_stdin(self): - """Tests linting a staged commit. Gitint should lint the passed commit message andfetch additional meta-data + """Tests linting a staged commit. Gitint should lint the passed commit message and fetch additional meta-data from the underlying repository. The easiest way to test this is by inspecting `--debug` output. This is the equivalent of doing: echo "WIP: Pïpe test." | gitlint --staged --debug diff --git a/qa/test_config.py b/qa/test_config.py index 1225f6ab..819c0ed2 100644 --- a/qa/test_config.py +++ b/qa/test_config.py @@ -4,7 +4,6 @@ from qa.shell import gitlint from qa.base import BaseTestCase -from qa.utils import DEFAULT_ENCODING class ConfigTests(BaseTestCase): @@ -128,7 +127,7 @@ def test_config_from_env(self): # Extract date from actual output to insert it into the expected output # We have to do this since there's no way for us to deterministically know that date otherwise p = re.compile("Date: (.*)\n", re.UNICODE | re.MULTILINE) - result = p.search(output.stdout.decode(DEFAULT_ENCODING)) + result = p.search(str(output)) date = result.group(1).strip() expected_kwargs.update({"date": date}) diff --git a/qa/test_gitlint.py b/qa/test_gitlint.py index 6c451964..3f1e6e6f 100644 --- a/qa/test_gitlint.py +++ b/qa/test_gitlint.py @@ -252,7 +252,7 @@ def test_commit_binary_file(self): binary_filename = self.create_simple_commit("Sïmple commit", file_contents=bytes([0x48, 0x00, 0x49, 0x00])) output = gitlint( "--debug", - _ok_code=1, + _ok_code=[1], _cwd=self.tmp_git_repo, ) diff --git a/qa/test_stdin.py b/qa/test_stdin.py index 8ed4cb1b..f35741c4 100644 --- a/qa/test_stdin.py +++ b/qa/test_stdin.py @@ -2,7 +2,7 @@ import subprocess from qa.shell import echo, gitlint from qa.base import BaseTestCase -from qa.utils import DEFAULT_ENCODING +from qa.utils import FILE_ENCODING, DEFAULT_ENCODING class StdInTests(BaseTestCase): @@ -33,7 +33,7 @@ def test_stdin_pipe_empty(self): # http://amoffat.github.io/sh/sections/special_arguments.html?highlight=_tty_in#err-to-out output = gitlint(echo("-n", ""), _cwd=self.tmp_git_repo, _tty_in=False, _err_to_out=True, _ok_code=[3]) - self.assertEqual(output, self.get_expected("test_stdin/test_stdin_pipe_empty_1")) + self.assertEqualStdout(output, self.get_expected("test_stdin/test_stdin_pipe_empty_1")) def test_stdin_file(self): """Test the scenario where STDIN is a regular file (stat.S_ISREG = True) @@ -42,7 +42,7 @@ def test_stdin_file(self): """ tmp_commit_msg_file = self.create_tmpfile("WIP: STDIN ïs a file test.") - with open(tmp_commit_msg_file, encoding=DEFAULT_ENCODING) as file_handle: + with open(tmp_commit_msg_file, encoding=FILE_ENCODING) as file_handle: # We need to use subprocess.Popen() here instead of sh because when passing a file_handle to sh, it will # deal with reading the file itself instead of passing it on to gitlint as a STDIN. Since we're trying to # test for the condition where stat.S_ISREG == True that won't work for us here. diff --git a/qa/utils.py b/qa/utils.py index 5331d1b6..f23056fb 100644 --- a/qa/utils.py +++ b/qa/utils.py @@ -44,20 +44,27 @@ def getpreferredencoding(): # We fallback to UTF-8 if PLATFORM_IS_WINDOWS: return "ISO-8859-1" - default_encoding = "UTF-8" - for env_var in ["LC_ALL", "LC_CTYPE", "LANG"]: - encoding = os.environ.get(env_var, False) - if encoding: - # Support dotted (C.UTF-8) and non-dotted (C or UTF-8) charsets: - # If encoding contains a dot: split and use second part, otherwise use everything - dot_index = encoding.find(".") - if dot_index != -1: - default_encoding = encoding[dot_index + 1 :] - else: - default_encoding = encoding - break + # default_encoding = "UTF-8" + # for env_var in ["LC_ALL", "LC_CTYPE", "LANG"]: + # encoding = os.environ.get(env_var, False) + # if encoding: + # # Support dotted (C.UTF-8) and non-dotted (C or UTF-8) charsets: + # # If encoding contains a dot: split and use second part, otherwise use everything + # dot_index = encoding.find(".") + # if dot_index != -1: + # default_encoding = encoding[dot_index + 1 :] + # else: + # default_encoding = encoding + # break return default_encoding DEFAULT_ENCODING = getpreferredencoding() + + +######################################################################################################################## +# FILE_ENCODING + +# Encoding for reading/writing files within the tests, this is always UTF-8 +FILE_ENCODING = "UTF-8" From b04885c16ac2d4c1d11bcfbfcd413b85a7e5468e Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Sat, 17 Dec 2022 11:24:52 +0000 Subject: [PATCH 07/12] Fix for test_revert_commit --- .github/workflows/checks.yml | 8 ++++---- qa/shell.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7a6a4f46..409026f6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -46,15 +46,15 @@ jobs: - name: Integration tests (GITLINT_USE_SH_LIB=1) run: | - hatch run qa:integration-tests --ignore qa/test_gitlint.py qa + hatch run qa:integration-tests env: GITLINT_USE_SH_LIB: 1 - name: Integration tests (GITLINT_QA_USE_SH_LIB=0) run: | - hatch run qa:integration-tests --ignore qa/test_hooks.py --ignore qa/test_gitlint.py qa + hatch run qa:integration-tests --ignore qa/test_hooks.py qa env: - GITLINT_USE_SH_LIB: 1 + GITLINT_QA_USE_SH_LIB: 0 - name: Build test (gitlint) run: | @@ -134,7 +134,7 @@ jobs: - name: Integration tests run: | hatch run qa:install-local - hatch run qa:integration-tests --ignore qa/test_hooks.py --ignore qa/test_gitlint.py qa + hatch run qa:integration-tests --ignore qa/test_gitlint.py qa - name: Build test (gitlint) run: | diff --git a/qa/shell.py b/qa/shell.py index 536042ce..12528a8d 100644 --- a/qa/shell.py +++ b/qa/shell.py @@ -94,11 +94,14 @@ def run_command(command, *args, **kwargs): return result def _exec(*args, **kwargs): - popen_kwargs = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "shell": kwargs.get("_tty_out", False)} - if "_cwd" in kwargs: - popen_kwargs["cwd"] = kwargs["_cwd"] - if "_env" in kwargs: - popen_kwargs["env"] = kwargs["_env"] + popen_kwargs = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "stdin": subprocess.PIPE, + "shell": kwargs.get("_tty_out", False), + "cwd": kwargs.get("_cwd", None), + "env": kwargs.get("_env", None), + } stdin_input = None if len(args) > 1 and isinstance(args[1], ShResult): From 835689cfd215d63ff7a38fd30cfb549e5fdd5be6 Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Sat, 17 Dec 2022 13:42:25 +0000 Subject: [PATCH 08/12] Windows integration test fixes More to come --- pyproject.toml | 2 +- qa/base.py | 8 ++++---- qa/expected/test_commits/test_lint_staged_msg_filename_1 | 2 +- qa/expected/test_config/test_config_from_env_1 | 2 +- qa/expected/test_config/test_config_from_env_2 | 2 +- qa/expected/test_config/test_config_from_file_debug_1 | 2 +- qa/expected/test_gitlint/test_commit_binary_file_1 | 2 +- qa/test_config.py | 4 ++-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 52b70886..4d4dfffb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,7 +118,7 @@ dependencies = [ # The integration tests can be ran against any gitlint binary, e.g. one installed from pypi (for post-release testing) # This is why by default we don't install the local dev version of gitlint in the qa environment # To run integration tests against the dev version of gitlint, use install-local first -install-local="pip install ./gitlint-core[trusted-deps]" +install-local="pip install -e ./gitlint-core[trusted-deps]" integration-tests = "pytest -rw -s {args:qa}" i = "integration-tests" diff --git a/qa/base.py b/qa/base.py index 5312c6fc..33869865 100644 --- a/qa/base.py +++ b/qa/base.py @@ -14,7 +14,7 @@ from qa.shell import git, gitlint, RunningCommand -from qa.utils import DEFAULT_ENCODING, PLATFORM_IS_WINDOWS +from qa.utils import DEFAULT_ENCODING, FILE_ENCODING, PLATFORM_IS_WINDOWS class BaseTestCase(TestCase): @@ -85,13 +85,13 @@ def create_file(parent_dir, content=None): if isinstance(content, bytes): open_kwargs = {"mode": "wb"} else: - open_kwargs = {"mode": "w", "encoding": DEFAULT_ENCODING} + open_kwargs = {"mode": "w", "encoding": FILE_ENCODING} with open(full_path, **open_kwargs) as f: # pylint: disable=unspecified-encoding f.write(content) else: # pylint: disable=consider-using-with - open(full_path, "a", encoding=DEFAULT_ENCODING).close() + open(full_path, "a", encoding=FILE_ENCODING).close() return test_filename @@ -151,7 +151,7 @@ def create_tmpfile(self, content): if isinstance(content, bytes): open_kwargs = {"mode": "wb"} else: - open_kwargs = {"mode": "w", "encoding": DEFAULT_ENCODING} + open_kwargs = {"mode": "w", "encoding": FILE_ENCODING} with open(tmpfile, **open_kwargs) as f: # pylint: disable=unspecified-encoding f.write(content) diff --git a/qa/expected/test_commits/test_lint_staged_msg_filename_1 b/qa/expected/test_commits/test_lint_staged_msg_filename_1 index f2ab49e2..992a115b 100644 --- a/qa/expected/test_commits/test_lint_staged_msg_filename_1 +++ b/qa/expected/test_commits/test_lint_staged_msg_filename_1 @@ -5,7 +5,7 @@ DEBUG: gitlint.git ('--version',) DEBUG: gitlint.cli Git version: {git_version} DEBUG: gitlint.cli Gitlint version: {gitlint_version} DEBUG: gitlint.cli GITLINT_USE_SH_LIB: {GITLINT_USE_SH_LIB} -DEBUG: gitlint.cli DEFAULT_ENCODING: {DEFAULT_ENCODING} +DEBUG: gitlint.cli DEFAULT_ENCODING: UTF-8 DEBUG: gitlint.cli Configuration config-path: None [GENERAL] diff --git a/qa/expected/test_config/test_config_from_env_1 b/qa/expected/test_config/test_config_from_env_1 index 38fba21e..f3be5de7 100644 --- a/qa/expected/test_config/test_config_from_env_1 +++ b/qa/expected/test_config/test_config_from_env_1 @@ -5,7 +5,7 @@ DEBUG: gitlint.git ('--version',) DEBUG: gitlint.cli Git version: {git_version} DEBUG: gitlint.cli Gitlint version: {gitlint_version} DEBUG: gitlint.cli GITLINT_USE_SH_LIB: {GITLINT_USE_SH_LIB} -DEBUG: gitlint.cli DEFAULT_ENCODING: {DEFAULT_ENCODING} +DEBUG: gitlint.cli DEFAULT_ENCODING: UTF-8 DEBUG: gitlint.cli Configuration config-path: None [GENERAL] diff --git a/qa/expected/test_config/test_config_from_env_2 b/qa/expected/test_config/test_config_from_env_2 index 50d1e3f9..461d2773 100644 --- a/qa/expected/test_config/test_config_from_env_2 +++ b/qa/expected/test_config/test_config_from_env_2 @@ -5,7 +5,7 @@ DEBUG: gitlint.git ('--version',) DEBUG: gitlint.cli Git version: {git_version} DEBUG: gitlint.cli Gitlint version: {gitlint_version} DEBUG: gitlint.cli GITLINT_USE_SH_LIB: {GITLINT_USE_SH_LIB} -DEBUG: gitlint.cli DEFAULT_ENCODING: {DEFAULT_ENCODING} +DEBUG: gitlint.cli DEFAULT_ENCODING: UTF-8 DEBUG: gitlint.cli Configuration config-path: None [GENERAL] diff --git a/qa/expected/test_config/test_config_from_file_debug_1 b/qa/expected/test_config/test_config_from_file_debug_1 index 39bdf52a..ac8711f6 100644 --- a/qa/expected/test_config/test_config_from_file_debug_1 +++ b/qa/expected/test_config/test_config_from_file_debug_1 @@ -5,7 +5,7 @@ DEBUG: gitlint.git ('--version',) DEBUG: gitlint.cli Git version: {git_version} DEBUG: gitlint.cli Gitlint version: {gitlint_version} DEBUG: gitlint.cli GITLINT_USE_SH_LIB: {GITLINT_USE_SH_LIB} -DEBUG: gitlint.cli DEFAULT_ENCODING: {DEFAULT_ENCODING} +DEBUG: gitlint.cli DEFAULT_ENCODING: UTF-8 DEBUG: gitlint.cli Configuration config-path: {config_path} [GENERAL] diff --git a/qa/expected/test_gitlint/test_commit_binary_file_1 b/qa/expected/test_gitlint/test_commit_binary_file_1 index 6bc119b8..6ed11eb6 100644 --- a/qa/expected/test_gitlint/test_commit_binary_file_1 +++ b/qa/expected/test_gitlint/test_commit_binary_file_1 @@ -5,7 +5,7 @@ DEBUG: gitlint.git ('--version',) DEBUG: gitlint.cli Git version: {git_version} DEBUG: gitlint.cli Gitlint version: {gitlint_version} DEBUG: gitlint.cli GITLINT_USE_SH_LIB: {GITLINT_USE_SH_LIB} -DEBUG: gitlint.cli DEFAULT_ENCODING: {DEFAULT_ENCODING} +DEBUG: gitlint.cli DEFAULT_ENCODING: UTF-8 DEBUG: gitlint.cli Configuration config-path: None [GENERAL] diff --git a/qa/test_config.py b/qa/test_config.py index 819c0ed2..62cd2cf6 100644 --- a/qa/test_config.py +++ b/qa/test_config.py @@ -1,5 +1,5 @@ # pylint: disable=too-many-function-args,unexpected-keyword-arg - +import os import re from qa.shell import gitlint @@ -68,7 +68,7 @@ def test_config_from_file_debug(self): "This line of the body is here because we need it" ) filename = self.create_simple_commit(commit_msg, git_repo=target_repo) - config_path = self.get_sample_path("config/gitlintconfig") + config_path = self.get_sample_path(os.path.join("config", "gitlintconfig")) output = gitlint("--config", config_path, "--debug", _cwd=target_repo, _tty_in=True, _ok_code=[5]) expected_kwargs = self.get_debug_vars_last_commit(git_repo=target_repo) From 9888f86045ed758ae2f6951f2ddc4920107272e3 Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Sat, 17 Dec 2022 14:54:52 +0000 Subject: [PATCH 09/12] Ignore failing tests on windows --- .github/workflows/checks.yml | 2 +- qa/shell.py | 24 ++---------------------- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 409026f6..a04ee49a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -134,7 +134,7 @@ jobs: - name: Integration tests run: | hatch run qa:install-local - hatch run qa:integration-tests --ignore qa/test_gitlint.py qa + hatch run qa:integration-tests "not (HookTests or test_lint_staged_stdin or test_stdin_file or test_stdin_pipe_empty)" qa - name: Build test (gitlint) run: | diff --git a/qa/shell.py b/qa/shell.py index 12528a8d..508562d4 100644 --- a/qa/shell.py +++ b/qa/shell.py @@ -29,7 +29,7 @@ def __init__(self, full_cmd, stdout, stderr="", exitcode=0): self.full_cmd = full_cmd # TODO(jorisroovers): The 'sh' library by default will merge stdout and stderr. We mimic this behavior # for now until we fully remove the 'sh' library. - self._stdout = stdout + stderr # .decode(DEFAULT_ENCODING) + self._stdout = stdout + stderr self._stderr = stderr self.exit_code = exitcode @@ -50,20 +50,6 @@ def stderr(self): def __getattr__(self, p): # pylint: disable=invalid-name # https://github.com/amoffat/sh/blob/e0ed8e244e9d973ef4e0749b2b3c2695e7b5255b/sh.py#L952= _unicode_methods = set(dir(str())) - - # # let these three attributes pass through to the OProc object - # if p in self._OProc_attr_whitelist: - # if self.process: - # return getattr(self.process, p) - # else: - # raise AttributeError - - # see if strings have what we're looking for. we're looking at the - # method names explicitly because we don't want to evaluate self unless - # we absolutely have to, the reason being, in python2, hasattr swallows - # exceptions, and if we try to run hasattr on a command that failed and - # is being run with _iter=True, the command will be evaluated, throw an - # exception, but hasattr will discard it if p in _unicode_methods: return getattr(str(self), p) @@ -85,13 +71,7 @@ def gitlint(*command_parts, **kwargs): def run_command(command, *args, **kwargs): args = [command] + list(args) - result = _exec(*args, **kwargs) - # If we reach this point and the result has an exit_code that is larger than 0, this means that we didn't - # get an exception (which is the default sh behavior for non-zero exit codes) and so the user is expecting - # a non-zero exit code -> just return the entire result - # if hasattr(result, "exit_code") and result.exit_code > 0: - # return result - return result + return _exec(*args, **kwargs) def _exec(*args, **kwargs): popen_kwargs = { From 57116a43e82897854a37cb6fef9906071cc91689 Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Sat, 17 Dec 2022 15:02:18 +0000 Subject: [PATCH 10/12] fixup! Ignore failing tests on windows --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a04ee49a..86b5d940 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -134,7 +134,7 @@ jobs: - name: Integration tests run: | hatch run qa:install-local - hatch run qa:integration-tests "not (HookTests or test_lint_staged_stdin or test_stdin_file or test_stdin_pipe_empty)" qa + hatch run qa:integration-tests -k "not (HookTests or test_lint_staged_stdin or test_stdin_file or test_stdin_pipe_empty)" qa - name: Build test (gitlint) run: | From 8526efc22998cccaa819613e77e3eb3836a9a67f Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Mon, 19 Dec 2022 08:50:16 +0000 Subject: [PATCH 11/12] Code Cleanup, fallback to ISO-8859-1 encoding on windows --- qa/base.py | 2 +- qa/shell.py | 2 +- qa/utils.py | 27 +++++++++++++-------------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/qa/base.py b/qa/base.py index 33869865..dc403bc5 100644 --- a/qa/base.py +++ b/qa/base.py @@ -183,7 +183,7 @@ def get_expected(filename="", variable_dict=None): expected_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "expected") expected_path = os.path.join(expected_dir, filename) # Expected files are UTF-8 encoded (not dependent on the system's default encoding) - with open(expected_path, encoding="UTF-8") as file: + with open(expected_path, encoding=FILE_ENCODING) as file: expected = file.read() if variable_dict: diff --git a/qa/shell.py b/qa/shell.py index 508562d4..7cddf3bc 100644 --- a/qa/shell.py +++ b/qa/shell.py @@ -102,7 +102,7 @@ def _exec(*args, **kwargs): raise CommandNotFound from exc exit_code = p.returncode - stdout = result[0] # .decode(DEFAULT_ENCODING) + stdout = result[0] stderr = result[1] # 'sh' does not decode the stderr bytes to unicode full_cmd = "" if args is None else " ".join(args) diff --git a/qa/utils.py b/qa/utils.py index f23056fb..86eebabe 100644 --- a/qa/utils.py +++ b/qa/utils.py @@ -41,21 +41,20 @@ def getpreferredencoding(): # On Windows, we mimic git/linux by trying to read the LC_ALL, LC_CTYPE, LANG env vars manually # (on Linux/MacOS the `getpreferredencoding()` call will take care of this). - # We fallback to UTF-8 + # We fallback to ISO-8859-1 if PLATFORM_IS_WINDOWS: - return "ISO-8859-1" - # default_encoding = "UTF-8" - # for env_var in ["LC_ALL", "LC_CTYPE", "LANG"]: - # encoding = os.environ.get(env_var, False) - # if encoding: - # # Support dotted (C.UTF-8) and non-dotted (C or UTF-8) charsets: - # # If encoding contains a dot: split and use second part, otherwise use everything - # dot_index = encoding.find(".") - # if dot_index != -1: - # default_encoding = encoding[dot_index + 1 :] - # else: - # default_encoding = encoding - # break + default_encoding = "ISO-8859-1" + for env_var in ["LC_ALL", "LC_CTYPE", "LANG"]: + encoding = os.environ.get(env_var, False) + if encoding: + # Support dotted (C.UTF-8) and non-dotted (C or UTF-8) charsets: + # If encoding contains a dot: split and use second part, otherwise use everything + dot_index = encoding.find(".") + if dot_index != -1: + default_encoding = encoding[dot_index + 1 :] + else: + default_encoding = encoding + break return default_encoding From 011713983260dbdb9969b7d43a2046deb5cca95e Mon Sep 17 00:00:00 2001 From: Joris Roovers Date: Mon, 19 Dec 2022 09:07:42 +0000 Subject: [PATCH 12/12] Don't try to outsmart local.getprefferedencoding() --- qa/utils.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/qa/utils.py b/qa/utils.py index 86eebabe..9b3927dd 100644 --- a/qa/utils.py +++ b/qa/utils.py @@ -35,28 +35,8 @@ def use_sh_library(): def getpreferredencoding(): - """Modified version of local.getpreferredencoding() that takes into account LC_ALL, LC_CTYPE, LANG env vars - on windows and falls back to UTF-8.""" - default_encoding = locale.getpreferredencoding() or "UTF-8" - - # On Windows, we mimic git/linux by trying to read the LC_ALL, LC_CTYPE, LANG env vars manually - # (on Linux/MacOS the `getpreferredencoding()` call will take care of this). - # We fallback to ISO-8859-1 - if PLATFORM_IS_WINDOWS: - default_encoding = "ISO-8859-1" - for env_var in ["LC_ALL", "LC_CTYPE", "LANG"]: - encoding = os.environ.get(env_var, False) - if encoding: - # Support dotted (C.UTF-8) and non-dotted (C or UTF-8) charsets: - # If encoding contains a dot: split and use second part, otherwise use everything - dot_index = encoding.find(".") - if dot_index != -1: - default_encoding = encoding[dot_index + 1 :] - else: - default_encoding = encoding - break - - return default_encoding + """Use local.getpreferredencoding() or fallback to UTF-8.""" + return locale.getpreferredencoding() or "UTF-8" DEFAULT_ENCODING = getpreferredencoding()