From 5d925da6c5bce53850adf1ac2f287e3c03366fa1 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 27 Jun 2021 18:00:53 -0400 Subject: [PATCH 001/137] add integration requirements.txt --- tests/integration/requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/integration/requirements.txt diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt new file mode 100644 index 000000000..45f89aef7 --- /dev/null +++ b/tests/integration/requirements.txt @@ -0,0 +1,6 @@ +hvac >= 0.10.6, != 0.10.12, != 0.10.13, < 1.0.0 ; python_version == '2.7' # bugs in 0.10.12 and 0.10.13 prevent it from working in Python 2 +hvac >= 0.10.6 ; python_version >= '3.5' + +# these should be satisfied naturally by the requests versions required by hvac anyway +urllib3 >= 1.15 ; python_version >= '3.6' # we need raise_on_status for retry support to raise the correct exceptions https://github.com/urllib3/urllib3/blob/main/CHANGES.rst#115-2016-04-06 +urllib3 >= 1.15, <2.0.0 ; python_version < '3.6' # https://urllib3.readthedocs.io/en/latest/v2-roadmap.html#optimized-for-python-3-6 From e0e23ba949427660cde42c24c68b8f1a7d8a0ec8 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 27 Jun 2021 18:06:43 -0400 Subject: [PATCH 002/137] move some unneeded stuff temporarily --- tests/integration/targets/setup_old_stuff/aliases | 1 + .../files}/playbooks/download_vault.yml | 0 .../files}/playbooks/install_dependencies.yml | 0 .../files}/playbooks/test_lookup_hashi_vault.yml | 0 .../{lookup_hashi_vault => setup_old_stuff/files}/runme.sh | 0 .../{ => setup_old_stuff/files}/setup_pkg_mgr/tasks/main.yml | 0 .../{ => setup_old_stuff/files}/setup_remote_constraints/aliases | 0 .../files}/setup_remote_constraints/meta/main.yml | 0 .../files}/setup_remote_constraints/tasks/main.yml | 0 .../files}/setup_remote_tmp_dir/handlers/main.yml | 0 .../files}/setup_remote_tmp_dir/tasks/default-cleanup.yml | 0 .../files}/setup_remote_tmp_dir/tasks/default.yml | 0 .../files}/setup_remote_tmp_dir/tasks/main.yml | 0 13 files changed, 1 insertion(+) create mode 100644 tests/integration/targets/setup_old_stuff/aliases rename tests/integration/targets/{lookup_hashi_vault => setup_old_stuff/files}/playbooks/download_vault.yml (100%) rename tests/integration/targets/{lookup_hashi_vault => setup_old_stuff/files}/playbooks/install_dependencies.yml (100%) rename tests/integration/targets/{lookup_hashi_vault => setup_old_stuff/files}/playbooks/test_lookup_hashi_vault.yml (100%) rename tests/integration/targets/{lookup_hashi_vault => setup_old_stuff/files}/runme.sh (100%) rename tests/integration/targets/{ => setup_old_stuff/files}/setup_pkg_mgr/tasks/main.yml (100%) rename tests/integration/targets/{ => setup_old_stuff/files}/setup_remote_constraints/aliases (100%) rename tests/integration/targets/{ => setup_old_stuff/files}/setup_remote_constraints/meta/main.yml (100%) rename tests/integration/targets/{ => setup_old_stuff/files}/setup_remote_constraints/tasks/main.yml (100%) rename tests/integration/targets/{ => setup_old_stuff/files}/setup_remote_tmp_dir/handlers/main.yml (100%) rename tests/integration/targets/{ => setup_old_stuff/files}/setup_remote_tmp_dir/tasks/default-cleanup.yml (100%) rename tests/integration/targets/{ => setup_old_stuff/files}/setup_remote_tmp_dir/tasks/default.yml (100%) rename tests/integration/targets/{ => setup_old_stuff/files}/setup_remote_tmp_dir/tasks/main.yml (100%) diff --git a/tests/integration/targets/setup_old_stuff/aliases b/tests/integration/targets/setup_old_stuff/aliases new file mode 100644 index 000000000..136c05e0d --- /dev/null +++ b/tests/integration/targets/setup_old_stuff/aliases @@ -0,0 +1 @@ +hidden diff --git a/tests/integration/targets/lookup_hashi_vault/playbooks/download_vault.yml b/tests/integration/targets/setup_old_stuff/files/playbooks/download_vault.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/playbooks/download_vault.yml rename to tests/integration/targets/setup_old_stuff/files/playbooks/download_vault.yml diff --git a/tests/integration/targets/lookup_hashi_vault/playbooks/install_dependencies.yml b/tests/integration/targets/setup_old_stuff/files/playbooks/install_dependencies.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/playbooks/install_dependencies.yml rename to tests/integration/targets/setup_old_stuff/files/playbooks/install_dependencies.yml diff --git a/tests/integration/targets/lookup_hashi_vault/playbooks/test_lookup_hashi_vault.yml b/tests/integration/targets/setup_old_stuff/files/playbooks/test_lookup_hashi_vault.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/playbooks/test_lookup_hashi_vault.yml rename to tests/integration/targets/setup_old_stuff/files/playbooks/test_lookup_hashi_vault.yml diff --git a/tests/integration/targets/lookup_hashi_vault/runme.sh b/tests/integration/targets/setup_old_stuff/files/runme.sh similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/runme.sh rename to tests/integration/targets/setup_old_stuff/files/runme.sh diff --git a/tests/integration/targets/setup_pkg_mgr/tasks/main.yml b/tests/integration/targets/setup_old_stuff/files/setup_pkg_mgr/tasks/main.yml similarity index 100% rename from tests/integration/targets/setup_pkg_mgr/tasks/main.yml rename to tests/integration/targets/setup_old_stuff/files/setup_pkg_mgr/tasks/main.yml diff --git a/tests/integration/targets/setup_remote_constraints/aliases b/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/aliases similarity index 100% rename from tests/integration/targets/setup_remote_constraints/aliases rename to tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/aliases diff --git a/tests/integration/targets/setup_remote_constraints/meta/main.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/meta/main.yml similarity index 100% rename from tests/integration/targets/setup_remote_constraints/meta/main.yml rename to tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/meta/main.yml diff --git a/tests/integration/targets/setup_remote_constraints/tasks/main.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/tasks/main.yml similarity index 100% rename from tests/integration/targets/setup_remote_constraints/tasks/main.yml rename to tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/tasks/main.yml diff --git a/tests/integration/targets/setup_remote_tmp_dir/handlers/main.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/handlers/main.yml similarity index 100% rename from tests/integration/targets/setup_remote_tmp_dir/handlers/main.yml rename to tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/handlers/main.yml diff --git a/tests/integration/targets/setup_remote_tmp_dir/tasks/default-cleanup.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default-cleanup.yml similarity index 100% rename from tests/integration/targets/setup_remote_tmp_dir/tasks/default-cleanup.yml rename to tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default-cleanup.yml diff --git a/tests/integration/targets/setup_remote_tmp_dir/tasks/default.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default.yml similarity index 100% rename from tests/integration/targets/setup_remote_tmp_dir/tasks/default.yml rename to tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default.yml diff --git a/tests/integration/targets/setup_remote_tmp_dir/tasks/main.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/main.yml similarity index 100% rename from tests/integration/targets/setup_remote_tmp_dir/tasks/main.yml rename to tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/main.yml From 366953e309401aa78d3d50b5e03f723885cd581d Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 27 Jun 2021 18:07:44 -0400 Subject: [PATCH 003/137] Move existing tests into correct level --- .../lookup_hashi_vault/{lookup_hashi_vault => }/defaults/main.yml | 0 .../{lookup_hashi_vault => }/files/bin/.gitignore | 0 .../{lookup_hashi_vault => }/files/jwt_private.pem | 0 .../{lookup_hashi_vault => }/files/jwt_public.pem | 0 .../lookup_hashi_vault/{lookup_hashi_vault => }/files/token.jwt | 0 .../{lookup_hashi_vault => }/files/token_invalid.jwt | 0 .../lookup_hashi_vault/{lookup_hashi_vault => }/handlers/main.yml | 0 .../{lookup_hashi_vault => }/library/vault_ci_enable_auth.py | 0 .../{lookup_hashi_vault => }/library/vault_ci_enable_engine.py | 0 .../{lookup_hashi_vault => }/library/vault_ci_kv_put.py | 0 .../{lookup_hashi_vault => }/library/vault_ci_policy_put.py | 0 .../{lookup_hashi_vault => }/library/vault_ci_read.py | 0 .../{lookup_hashi_vault => }/library/vault_ci_token_create.py | 0 .../{lookup_hashi_vault => }/library/vault_ci_write.py | 0 .../tasks/approle_secret_id_less_setup.yml | 0 .../tasks/approle_secret_id_less_test.yml | 0 .../{lookup_hashi_vault => }/tasks/approle_setup.yml | 0 .../{lookup_hashi_vault => }/tasks/approle_test.yml | 0 .../{lookup_hashi_vault => }/tasks/jwt_setup.yml | 0 .../{lookup_hashi_vault => }/tasks/jwt_test.yml | 0 .../lookup_hashi_vault/{lookup_hashi_vault => }/tasks/main.yml | 0 .../{lookup_hashi_vault => }/tasks/none_test.yml | 0 .../lookup_hashi_vault/{lookup_hashi_vault => }/tasks/tests.yml | 0 .../{lookup_hashi_vault => }/tasks/tinyproxy_server.yml | 0 .../{lookup_hashi_vault => }/tasks/token_setup.yml | 0 .../{lookup_hashi_vault => }/tasks/token_test.yml | 0 .../{lookup_hashi_vault => }/tasks/vault_download.yml | 0 .../{lookup_hashi_vault => }/tasks/vault_server.yml | 0 .../{lookup_hashi_vault => }/tasks/vault_server_auth_setup.yml | 0 .../{lookup_hashi_vault => }/tasks/vault_server_configure.yml | 0 .../{lookup_hashi_vault => }/templates/vault_config.hcl.j2 | 0 .../lookup_hashi_vault/{lookup_hashi_vault => }/vars/.gitignore | 0 32 files changed, 0 insertions(+), 0 deletions(-) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/defaults/main.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/files/bin/.gitignore (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/files/jwt_private.pem (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/files/jwt_public.pem (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/files/token.jwt (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/files/token_invalid.jwt (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/handlers/main.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/library/vault_ci_enable_auth.py (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/library/vault_ci_enable_engine.py (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/library/vault_ci_kv_put.py (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/library/vault_ci_policy_put.py (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/library/vault_ci_read.py (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/library/vault_ci_token_create.py (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/library/vault_ci_write.py (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/approle_secret_id_less_setup.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/approle_secret_id_less_test.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/approle_setup.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/approle_test.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/jwt_setup.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/jwt_test.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/main.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/none_test.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/tests.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/tinyproxy_server.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/token_setup.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/token_test.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/vault_download.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/vault_server.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/vault_server_auth_setup.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/tasks/vault_server_configure.yml (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/templates/vault_config.hcl.j2 (100%) rename tests/integration/targets/lookup_hashi_vault/{lookup_hashi_vault => }/vars/.gitignore (100%) diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml rename to tests/integration/targets/lookup_hashi_vault/defaults/main.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/bin/.gitignore b/tests/integration/targets/lookup_hashi_vault/files/bin/.gitignore similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/bin/.gitignore rename to tests/integration/targets/lookup_hashi_vault/files/bin/.gitignore diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/jwt_private.pem b/tests/integration/targets/lookup_hashi_vault/files/jwt_private.pem similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/jwt_private.pem rename to tests/integration/targets/lookup_hashi_vault/files/jwt_private.pem diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/jwt_public.pem b/tests/integration/targets/lookup_hashi_vault/files/jwt_public.pem similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/jwt_public.pem rename to tests/integration/targets/lookup_hashi_vault/files/jwt_public.pem diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/token.jwt b/tests/integration/targets/lookup_hashi_vault/files/token.jwt similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/token.jwt rename to tests/integration/targets/lookup_hashi_vault/files/token.jwt diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/token_invalid.jwt b/tests/integration/targets/lookup_hashi_vault/files/token_invalid.jwt similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/token_invalid.jwt rename to tests/integration/targets/lookup_hashi_vault/files/token_invalid.jwt diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/handlers/main.yml b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/handlers/main.yml rename to tests/integration/targets/lookup_hashi_vault/handlers/main.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_enable_auth.py b/tests/integration/targets/lookup_hashi_vault/library/vault_ci_enable_auth.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_enable_auth.py rename to tests/integration/targets/lookup_hashi_vault/library/vault_ci_enable_auth.py diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_enable_engine.py b/tests/integration/targets/lookup_hashi_vault/library/vault_ci_enable_engine.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_enable_engine.py rename to tests/integration/targets/lookup_hashi_vault/library/vault_ci_enable_engine.py diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_kv_put.py b/tests/integration/targets/lookup_hashi_vault/library/vault_ci_kv_put.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_kv_put.py rename to tests/integration/targets/lookup_hashi_vault/library/vault_ci_kv_put.py diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_policy_put.py b/tests/integration/targets/lookup_hashi_vault/library/vault_ci_policy_put.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_policy_put.py rename to tests/integration/targets/lookup_hashi_vault/library/vault_ci_policy_put.py diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_read.py b/tests/integration/targets/lookup_hashi_vault/library/vault_ci_read.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_read.py rename to tests/integration/targets/lookup_hashi_vault/library/vault_ci_read.py diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_token_create.py b/tests/integration/targets/lookup_hashi_vault/library/vault_ci_token_create.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_token_create.py rename to tests/integration/targets/lookup_hashi_vault/library/vault_ci_token_create.py diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_write.py b/tests/integration/targets/lookup_hashi_vault/library/vault_ci_write.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/library/vault_ci_write.py rename to tests/integration/targets/lookup_hashi_vault/library/vault_ci_write.py diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_secret_id_less_setup.yml b/tests/integration/targets/lookup_hashi_vault/tasks/approle_secret_id_less_setup.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_secret_id_less_setup.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/approle_secret_id_less_setup.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_secret_id_less_test.yml b/tests/integration/targets/lookup_hashi_vault/tasks/approle_secret_id_less_test.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_secret_id_less_test.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/approle_secret_id_less_test.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_setup.yml b/tests/integration/targets/lookup_hashi_vault/tasks/approle_setup.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_setup.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/approle_setup.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_test.yml b/tests/integration/targets/lookup_hashi_vault/tasks/approle_test.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_test.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/approle_test.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/jwt_setup.yml b/tests/integration/targets/lookup_hashi_vault/tasks/jwt_setup.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/jwt_setup.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/jwt_setup.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/jwt_test.yml b/tests/integration/targets/lookup_hashi_vault/tasks/jwt_test.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/jwt_test.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/jwt_test.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/main.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/none_test.yml b/tests/integration/targets/lookup_hashi_vault/tasks/none_test.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/none_test.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/none_test.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/tests.yml b/tests/integration/targets/lookup_hashi_vault/tasks/tests.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/tests.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/tests.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/tinyproxy_server.yml b/tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/tinyproxy_server.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_setup.yml b/tests/integration/targets/lookup_hashi_vault/tasks/token_setup.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_setup.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/token_setup.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml b/tests/integration/targets/lookup_hashi_vault/tasks/token_test.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/token_test.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/vault_download.yml b/tests/integration/targets/lookup_hashi_vault/tasks/vault_download.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/vault_download.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/vault_download.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/vault_server.yml b/tests/integration/targets/lookup_hashi_vault/tasks/vault_server.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/vault_server.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/vault_server.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/vault_server_auth_setup.yml b/tests/integration/targets/lookup_hashi_vault/tasks/vault_server_auth_setup.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/vault_server_auth_setup.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/vault_server_auth_setup.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/vault_server_configure.yml b/tests/integration/targets/lookup_hashi_vault/tasks/vault_server_configure.yml similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/vault_server_configure.yml rename to tests/integration/targets/lookup_hashi_vault/tasks/vault_server_configure.yml diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/templates/vault_config.hcl.j2 b/tests/integration/targets/lookup_hashi_vault/templates/vault_config.hcl.j2 similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/templates/vault_config.hcl.j2 rename to tests/integration/targets/lookup_hashi_vault/templates/vault_config.hcl.j2 diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/vars/.gitignore b/tests/integration/targets/lookup_hashi_vault/vars/.gitignore similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/vars/.gitignore rename to tests/integration/targets/lookup_hashi_vault/vars/.gitignore From 13d0bbde32d379f9ce6c2dd2e8ff39819104b0c6 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 28 Jun 2021 21:05:49 -0400 Subject: [PATCH 004/137] add setup target for downloading vault --- .../tasks/vault_download.yml | 36 ------------- .../setup_vault_server_download/aliases | 1 + .../defaults/main.yml | 16 ++++++ .../files/.gitignore | 1 + .../tasks/main.yml | 52 +++++++++++++++++++ 5 files changed, 70 insertions(+), 36 deletions(-) delete mode 100644 tests/integration/targets/lookup_hashi_vault/tasks/vault_download.yml create mode 100644 tests/integration/targets/setup_vault_server_download/aliases create mode 100644 tests/integration/targets/setup_vault_server_download/defaults/main.yml create mode 100644 tests/integration/targets/setup_vault_server_download/files/.gitignore create mode 100644 tests/integration/targets/setup_vault_server_download/tasks/main.yml diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/vault_download.yml b/tests/integration/targets/lookup_hashi_vault/tasks/vault_download.yml deleted file mode 100644 index 092098909..000000000 --- a/tests/integration/targets/lookup_hashi_vault/tasks/vault_download.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -# NOTE: 'package' does not work properly with Ubuntu/Debian (like the 'default' docker image), -# if you're running in a version of Python other than the "system" Python, due to system libraries -# needed for the python 'apt' package. See https://stackoverflow.com/q/13708180/3905079 -# So for those OSes, we'll set the Python interpreter to the symlink in /usr/bin which should -# always be the correct one that corresponds to the system libraries. -# -# All this just for unzip, which is only needed to unzip the vault binary to set up for testing. -# TODO: revisit how we set up vault in the first place or how we host the binary (.gz?) -- name: 'Install unzip' - vars: - # by assuming python3 here we're probably condeming this to not work on older Ubuntu/Debian (from like 2014?) - # but the alternative is probably reimplementing parts of interpreter_discovery.py - ansible_python_interpreter: "{{ - '/usr/bin/python3' if ansible_distribution in ['Ubuntu', 'Debian'] else ansible_python.executable - }}" - package: - name: unzip - when: ansible_distribution != "MacOSX" # unzip already installed (#TODO: get MacOSX tests working again) - -- name: "Create bin directory" - file: - path: '{{ vault_bin }}' - state: directory - -- name: 'Download vault binary' - get_url: - url: '{{ vault_uri }}' - dest: '{{ vault_zip }}' - -- name: 'Extract vault binary' - unarchive: - src: '{{ vault_zip }}' - dest: '{{ vault_bin }}' - remote_src: yes - creates: '{{ vault_cmd }}' diff --git a/tests/integration/targets/setup_vault_server_download/aliases b/tests/integration/targets/setup_vault_server_download/aliases new file mode 100644 index 000000000..136c05e0d --- /dev/null +++ b/tests/integration/targets/setup_vault_server_download/aliases @@ -0,0 +1 @@ +hidden diff --git a/tests/integration/targets/setup_vault_server_download/defaults/main.yml b/tests/integration/targets/setup_vault_server_download/defaults/main.yml new file mode 100644 index 000000000..dad4c5c31 --- /dev/null +++ b/tests/integration/targets/setup_vault_server_download/defaults/main.yml @@ -0,0 +1,16 @@ +--- +vault_ansible_arch_table: + 'x86_64': 'amd64' # Linux + 'amd64': 'amd64' # FreeBSD + 'i386': '386' + +vault_arch: "{{ vault_ansible_arch_table[ansible_architecture] }}" + +vault_version: '1.7.3' +vault_bin: '{{ role_path }}/files/bin/{{ vault_slug }}' +vault_slug: 'vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}' +vault_zip: '{{ vault_bin }}/{{ vault_slug }}.zip' +vault_uri: 'https://releases.hashicorp.com/vault/{{ vault_version }}/{{ vault_slug }}.zip' +vault_cmd: '{{ vault_bin }}/vault' + +vault_server_download_force: False diff --git a/tests/integration/targets/setup_vault_server_download/files/.gitignore b/tests/integration/targets/setup_vault_server_download/files/.gitignore new file mode 100644 index 000000000..e660fd93d --- /dev/null +++ b/tests/integration/targets/setup_vault_server_download/files/.gitignore @@ -0,0 +1 @@ +bin/ diff --git a/tests/integration/targets/setup_vault_server_download/tasks/main.yml b/tests/integration/targets/setup_vault_server_download/tasks/main.yml new file mode 100644 index 000000000..4d7f2d78a --- /dev/null +++ b/tests/integration/targets/setup_vault_server_download/tasks/main.yml @@ -0,0 +1,52 @@ +--- +- name: "Check if vault binary exists" + stat: + path: '{{ vault_cmd }}' + follow: yes + get_attributes: no + get_checksum: no + get_mime: no + register: bin_status + +- name: "Download vault if not local" + when: not bin_status.stat.exists or vault_server_download_force | bool + block: + # NOTE: 'package' does not work properly with Ubuntu/Debian (like the 'default' docker image), + # if you're running in a version of Python other than the "system" Python, due to system libraries + # needed for the python 'apt' package. See https://stackoverflow.com/q/13708180/3905079 + # So for those OSes, we'll set the Python interpreter to the symlink in /usr/bin which should + # always be the correct one that corresponds to the system libraries. + # + # All this just for unzip, which is only needed to unzip the vault binary to set up for testing. + # TODO: revisit how we set up vault in the first place or how we host the binary (.gz?) + - name: 'Install unzip' + vars: + # by assuming python3 here we're probably condeming this to not work on older Ubuntu/Debian (from like 2014?) + # but the alternative is probably reimplementing parts of interpreter_discovery.py + ansible_python_interpreter: "{{ + '/usr/bin/python3' if ansible_distribution in ['Ubuntu', 'Debian'] else ansible_python.executable + }}" + package: + name: unzip + # when: ansible_distribution != "MacOSX" # unzip already installed (#TODO: get MacOSX tests working again) + + - name: "Create bin directory" + file: + path: '{{ vault_bin }}' + state: directory + + - name: 'Download vault binary' + get_url: + url: '{{ vault_uri }}' + dest: '{{ vault_zip }}' + + - name: 'Extract vault binary' + unarchive: + src: '{{ vault_zip }}' + dest: '{{ vault_bin }}' + remote_src: yes + creates: '{{ vault_cmd }}' + +- name: 'Export Vault command' + set_fact: + vault_cmd: '{{ vault_cmd }}' From 3e23b1f41eb12df6f475e48f80fea939863d08dd Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 28 Jun 2021 21:08:18 -0400 Subject: [PATCH 005/137] add setup targets for cert mgmt --- .../lookup_hashi_vault/defaults/main.yml | 4 +-- .../targets/setup_cert_content/aliases | 1 + .../setup_cert_content/defaults/main.yml | 4 +++ .../setup_cert_content/files/.gitignore | 2 ++ .../targets/setup_cert_content/tasks/main.yml | 12 +++++++++ .../targets/setup_vault_server_cert/aliases | 1 + .../setup_vault_server_cert/tasks/main.yml | 27 +++++++++++++++++++ 7 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 tests/integration/targets/setup_cert_content/aliases create mode 100644 tests/integration/targets/setup_cert_content/defaults/main.yml create mode 100644 tests/integration/targets/setup_cert_content/files/.gitignore create mode 100644 tests/integration/targets/setup_cert_content/tasks/main.yml create mode 100644 tests/integration/targets/setup_vault_server_cert/aliases create mode 100644 tests/integration/targets/setup_vault_server_cert/tasks/main.yml diff --git a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml index 54a034fe6..908d27f03 100644 --- a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml @@ -106,8 +106,8 @@ vault_test_server_configure: True # when False the tests requiring a valid SSL connection to Vault will be skipped vault_run_https_tests: True -vault_cert_file: '{{ local_temp_dir }}/cert.pem' -vault_key_file: '{{ local_temp_dir }}/privatekey.pem' +# vault_cert_file: '{{ local_temp_dir }}/cert.pem' +# vault_key_file: '{{ local_temp_dir }}/privatekey.pem' vault_proxy_server: 'http://127.0.0.1:8001' vault_proxy_external: False diff --git a/tests/integration/targets/setup_cert_content/aliases b/tests/integration/targets/setup_cert_content/aliases new file mode 100644 index 000000000..136c05e0d --- /dev/null +++ b/tests/integration/targets/setup_cert_content/aliases @@ -0,0 +1 @@ +hidden diff --git a/tests/integration/targets/setup_cert_content/defaults/main.yml b/tests/integration/targets/setup_cert_content/defaults/main.yml new file mode 100644 index 000000000..29f4aff6e --- /dev/null +++ b/tests/integration/targets/setup_cert_content/defaults/main.yml @@ -0,0 +1,4 @@ +--- +cert_output_dir: '{{ role_path }}' +cert_file_name: cert.pem +cert_location: '{{ cert_output_dir }}/{{ cert_file_name }}' diff --git a/tests/integration/targets/setup_cert_content/files/.gitignore b/tests/integration/targets/setup_cert_content/files/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/tests/integration/targets/setup_cert_content/files/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/integration/targets/setup_cert_content/tasks/main.yml b/tests/integration/targets/setup_cert_content/tasks/main.yml new file mode 100644 index 000000000..79c847b74 --- /dev/null +++ b/tests/integration/targets/setup_cert_content/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- name: "Cert Content Block" + when: vault_cert_content is defined + block: + - name: "Write Certificate" + copy: + dest: '{{ cert_location }}' + content: '{{ vault_cert_content }}' + + - name: "Register the Cert Location" + set_fact: + vault_cert_file: '{{ cert_location }}' diff --git a/tests/integration/targets/setup_vault_server_cert/aliases b/tests/integration/targets/setup_vault_server_cert/aliases new file mode 100644 index 000000000..136c05e0d --- /dev/null +++ b/tests/integration/targets/setup_vault_server_cert/aliases @@ -0,0 +1 @@ +hidden diff --git a/tests/integration/targets/setup_vault_server_cert/tasks/main.yml b/tests/integration/targets/setup_vault_server_cert/tasks/main.yml new file mode 100644 index 000000000..1aa16ab5c --- /dev/null +++ b/tests/integration/targets/setup_vault_server_cert/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: "Cert generation tasks" + vars: + vault_csr_file: '{{ vault_key_file | dirname }}/csr.csr' + block: + - name: Generate privatekey + community.crypto.openssl_privatekey: + mode: 'o=r' + path: '{{ vault_key_file }}' + + - name: Generate CSR + community.crypto.openssl_csr: + mode: 'o=r' + path: '{{ vault_csr_file }}' + privatekey_path: '{{ vault_key_file }}' + subject: + commonName: '{{ vault_hostname }}' + + - name: Generate selfsigned certificate + community.crypto.x509_certificate: + mode: 'o=r' + path: '{{ vault_cert_file }}' + csr_path: '{{ vault_csr_file }}' + privatekey_path: '{{ vault_key_file }}' + provider: selfsigned + selfsigned_digest: sha256 + register: selfsigned_certificate From 5ca5c783e03c64d7a95f05e264dff33259c8726e Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 28 Jun 2021 21:10:04 -0400 Subject: [PATCH 006/137] move vault server setup to setup target --- .../targets/lookup_hashi_vault/aliases | 1 + .../lookup_hashi_vault/defaults/main.yml | 38 +++++------ .../lookup_hashi_vault/handlers/main.yml | 8 +-- .../targets/lookup_hashi_vault/meta/main.yml | 3 + .../targets/lookup_hashi_vault/tasks/main.yml | 7 ++- .../lookup_hashi_vault/tasks/tests.yml | 6 +- .../lookup_hashi_vault/tasks/vault_server.yml | 53 ---------------- .../targets/setup_vault_server/aliases | 3 + .../setup_vault_server/defaults/main.yml | 34 ++++++++++ .../setup_vault_server/handlers/main.yml | 9 +++ .../targets/setup_vault_server/tasks/main.yml | 9 +++ .../setup_vault_server/tasks/vault_server.yml | 63 +++++++++++++++++++ .../templates/vault_config.hcl.j2 | 10 +++ 13 files changed, 163 insertions(+), 81 deletions(-) create mode 100644 tests/integration/targets/lookup_hashi_vault/meta/main.yml delete mode 100644 tests/integration/targets/lookup_hashi_vault/tasks/vault_server.yml create mode 100644 tests/integration/targets/setup_vault_server/aliases create mode 100644 tests/integration/targets/setup_vault_server/defaults/main.yml create mode 100644 tests/integration/targets/setup_vault_server/handlers/main.yml create mode 100644 tests/integration/targets/setup_vault_server/tasks/main.yml create mode 100644 tests/integration/targets/setup_vault_server/tasks/vault_server.yml create mode 100644 tests/integration/targets/setup_vault_server/templates/vault_config.hcl.j2 diff --git a/tests/integration/targets/lookup_hashi_vault/aliases b/tests/integration/targets/lookup_hashi_vault/aliases index c8dd03553..953081c26 100644 --- a/tests/integration/targets/lookup_hashi_vault/aliases +++ b/tests/integration/targets/lookup_hashi_vault/aliases @@ -4,3 +4,4 @@ needs/file/tests/utils/constraints.txt skip/aix skip/python2.6 # lookups are controller only, and we no longer support Python 2.6 on the controller skip/macos # FIXME seems to be always unstable +needs/target/setup_vault_server diff --git a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml index 908d27f03..75ef0b214 100644 --- a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml @@ -76,34 +76,34 @@ auth_methods: -## vars for vault server +# ## vars for vault server -vault_ansible_arch_table: - 'x86_64': 'amd64' # Linux - 'amd64': 'amd64' # FreeBSD - 'i386': '386' +# vault_ansible_arch_table: +# 'x86_64': 'amd64' # Linux +# 'amd64': 'amd64' # FreeBSD +# 'i386': '386' -vault_arch: "{{ vault_ansible_arch_table[ansible_architecture] }}" +# vault_arch: "{{ vault_ansible_arch_table[ansible_architecture] }}" -vault_version: '1.7.3' -vault_bin: '{{ role_path }}/files/bin/{{ vault_slug }}' -vault_slug: 'vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}' -vault_zip: '{{ vault_bin }}/{{ vault_slug }}.zip' -vault_uri: 'https://releases.hashicorp.com/vault/{{ vault_version }}/{{ vault_slug }}.zip' -vault_cmd: '{{ vault_bin }}/vault' +# vault_version: '1.7.3' +# vault_bin: '{{ role_path }}/files/bin/{{ vault_slug }}' +# vault_slug: 'vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}' +# vault_zip: '{{ vault_bin }}/{{ vault_slug }}.zip' +# vault_uri: 'https://releases.hashicorp.com/vault/{{ vault_version }}/{{ vault_slug }}.zip' +# vault_cmd: '{{ vault_bin }}/vault' vault_dev_root_token_id: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' -vault_test_server_https: 'https://localhost:8201' -vault_test_server_http: 'http://localhost:8200' +# vault_test_server_https: 'https://localhost:8201' +# vault_test_server_http: 'http://localhost:8200' -# this means "don't download and start a Vault server", instead -# just use the addresses above to connect to one that's already running -vault_test_server_external: False +# # this means "don't download and start a Vault server", instead +# # just use the addresses above to connect to one that's already running +# vault_test_server_external: False -# WIP +# # WIP vault_test_server_configure: True -# when False the tests requiring a valid SSL connection to Vault will be skipped +# # when False the tests requiring a valid SSL connection to Vault will be skipped vault_run_https_tests: True # vault_cert_file: '{{ local_temp_dir }}/cert.pem' diff --git a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml index 19e556d3f..8f5038dc8 100644 --- a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml @@ -2,10 +2,10 @@ # notify 'test_managed_vault_cleanup' for tasks related to the # vault server that is started by these tests # (those tasks should skip if the vault server is external to the test run) -- name: 'Kill vault process' - shell: "kill $(cat {{ local_temp_dir }}/vault.pid)" - ignore_errors: true - listen: test_managed_vault_cleanup +# - name: 'Kill vault process' +# shell: "kill $(cat {{ local_temp_dir }}/vault.pid)" +# ignore_errors: true +# listen: test_managed_vault_cleanup - name: Stop tinyproxy shell: diff --git a/tests/integration/targets/lookup_hashi_vault/meta/main.yml b/tests/integration/targets/lookup_hashi_vault/meta/main.yml new file mode 100644 index 000000000..5ce43771d --- /dev/null +++ b/tests/integration/targets/lookup_hashi_vault/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - setup_cert_content diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml index fa1bade00..fadf115b4 100644 --- a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml @@ -13,8 +13,11 @@ - set_fact: local_temp_dir: '{{ tempfile_result.path }}' -- include_tasks: vault_server.yml - when: not vault_test_server_external | bool +# - include_tasks: vault_server.yml +# when: not vault_test_server_external | bool +- include_role: + name: setup_vault_server + when: vault_server_legacy | default(True) | bool - import_tasks: vault_server_configure.yml when: vault_test_server_configure | bool diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/tests.yml b/tests/integration/targets/lookup_hashi_vault/tasks/tests.yml index b0ddd068f..aa0a3fc35 100644 --- a/tests/integration/targets/lookup_hashi_vault/tasks/tests.yml +++ b/tests/integration/targets/lookup_hashi_vault/tasks/tests.yml @@ -19,7 +19,7 @@ - name: 'test {{ auth_type }} auth without SSL (lookup parameters, with string proxy)' include_tasks: '{{ auth_type }}_test.yml' vars: - conn_params: 'proxies={{ vault_proxy_server }} ' + conn_params: 'url={{ vault_proxy_alt_vault_http | default(vault_test_server_http) }} proxies={{ vault_proxy_server }} ' - name: 'test {{ auth_type }} auth without SSL (ansible variable)' include_tasks: '{{ auth_type }}_test.yml' @@ -43,7 +43,7 @@ - name: 'test {{ auth_type }} auth with certs (validation enabled, lookup parameters, with string proxy)' include_tasks: '{{ auth_type }}_test.yml' vars: - conn_params: 'ca_cert={{ vault_cert_file }} validate_certs=True proxies={{ vault_proxy_server }} ' + conn_params: 'url={{ vault_proxy_alt_vault_https | default(vault_test_server_https) }} ca_cert={{ vault_cert_file }} validate_certs=True proxies={{ vault_proxy_server }} ' - name: Set proxies variable set_fact: @@ -54,7 +54,7 @@ - name: 'test {{ auth_type }} auth with certs (validation enabled, lookup parameters, with dict proxy via ansible vars)' include_tasks: '{{ auth_type }}_test.yml' vars: - conn_params: 'url={{ vault_test_server_https }} ca_cert={{ vault_cert_file }} validate_certs=True ' + conn_params: 'url={{ vault_proxy_alt_vault_https | default(vault_test_server_https) }} ca_cert={{ vault_cert_file }} validate_certs=True ' - name: Reset proxies variable set_fact: diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/vault_server.yml b/tests/integration/targets/lookup_hashi_vault/tasks/vault_server.yml deleted file mode 100644 index 7f01f8abe..000000000 --- a/tests/integration/targets/lookup_hashi_vault/tasks/vault_server.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -- name: Install Hashi Vault on controlled node and test - block: - - - when: vault_run_https_tests | bool - vars: - vault_csr_file: '{{ vault_key_file | dirname }}/csr.csr' - block: - - name: Generate privatekey - community.crypto.openssl_privatekey: - path: '{{ vault_key_file }}' - - - name: Generate CSR - community.crypto.openssl_csr: - path: '{{ vault_csr_file }}' - privatekey_path: '{{ vault_key_file }}' - subject: - commonName: localhost - - - name: Generate selfsigned certificate - community.crypto.openssl_certificate: - path: '{{ vault_cert_file }}' - csr_path: '{{ vault_csr_file }}' - privatekey_path: '{{ vault_key_file }}' - provider: selfsigned - selfsigned_digest: sha256 - register: selfsigned_certificate - - - name: "Check if vault binary exists" - stat: - path: '{{ vault_cmd }}' - follow: yes - get_attributes: no - get_checksum: no - get_mime: no - register: bin_status - - - name: "Download vault if not local" - when: not bin_status.stat.exists - import_tasks: vault_download.yml - - - environment: - # used by vault command - VAULT_DEV_ROOT_TOKEN_ID: '{{ vault_dev_root_token_id }}' - block: - - name: 'Create configuration file' - template: - src: vault_config.hcl.j2 - dest: '{{ local_temp_dir }}/vault_config.hcl' - - - name: 'Start vault server (dev mode enabled)' - shell: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl /dev/null 2>&1 &' - notify: test_managed_vault_cleanup diff --git a/tests/integration/targets/setup_vault_server/aliases b/tests/integration/targets/setup_vault_server/aliases new file mode 100644 index 000000000..42eba2ced --- /dev/null +++ b/tests/integration/targets/setup_vault_server/aliases @@ -0,0 +1,3 @@ +hidden +needs/target/setup_vault_server_cert +needs/target/setup_vault_server_download diff --git a/tests/integration/targets/setup_vault_server/defaults/main.yml b/tests/integration/targets/setup_vault_server/defaults/main.yml new file mode 100644 index 000000000..97811be26 --- /dev/null +++ b/tests/integration/targets/setup_vault_server/defaults/main.yml @@ -0,0 +1,34 @@ +--- +## vars for vault server + +vault_ansible_arch_table: + 'x86_64': 'amd64' # Linux + 'amd64': 'amd64' # FreeBSD + 'i386': '386' + +vault_arch: "{{ vault_ansible_arch_table[ansible_architecture] }}" + +vault_version: '1.7.3' +vault_bin: '{{ role_path }}/files/bin/{{ vault_slug }}' +vault_slug: 'vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}' +vault_zip: '{{ vault_bin }}/{{ vault_slug }}.zip' +vault_uri: 'https://releases.hashicorp.com/vault/{{ vault_version }}/{{ vault_slug }}.zip' +vault_cmd: '{{ vault_bin }}/vault' + +vault_dev_root_token_id: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' +vault_test_server_https: 'https://localhost:8201' +vault_test_server_http: 'http://localhost:8200' + +# this means "don't download and start a Vault server", instead +# just use the addresses above to connect to one that's already running +vault_test_server_external: False + +# WIP +vault_test_server_configure: True + +# when False the tests requiring a valid SSL connection to Vault will be skipped +vault_run_https_tests: True + +local_temp_dir: /tmp +vault_cert_file: '{{ local_temp_dir }}/cert.pem' +vault_key_file: '{{ local_temp_dir }}/privatekey.pem' diff --git a/tests/integration/targets/setup_vault_server/handlers/main.yml b/tests/integration/targets/setup_vault_server/handlers/main.yml new file mode 100644 index 000000000..3b77e6710 --- /dev/null +++ b/tests/integration/targets/setup_vault_server/handlers/main.yml @@ -0,0 +1,9 @@ +--- +# notify 'test_managed_vault_cleanup' for tasks related to the +# vault server that is started by these tests +# (those tasks should skip if the vault server is external to the test run) +- name: 'Kill vault process' + shell: "kill $(cat {{ local_temp_dir }}/vault.pid)" + ignore_errors: true + listen: test_managed_vault_cleanup + when: vault_server_legacy | default(True) | bool diff --git a/tests/integration/targets/setup_vault_server/tasks/main.yml b/tests/integration/targets/setup_vault_server/tasks/main.yml new file mode 100644 index 000000000..22b11ad81 --- /dev/null +++ b/tests/integration/targets/setup_vault_server/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- import_tasks: vault_server.yml + # when: not vault_test_server_external | bool + +- set_fact: + vault_test_server_http: "{{ vault_test_server_http }}" + vault_test_server_https: "{{ vault_test_server_https }}" + vault_cert_file: "{{ vault_cert_file }}" + vault_key_file: "{{ vault_key_file }}" diff --git a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml new file mode 100644 index 000000000..a333494db --- /dev/null +++ b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml @@ -0,0 +1,63 @@ +--- +- name: Install Hashi Vault on controlled node and test + block: + + - when: vault_run_https_tests | bool + vars: + vault_hostname: localhost + include_role: + name: setup_vault_server_cert + + # vars: + # vault_csr_file: '{{ vault_key_file | dirname }}/csr.csr' + # block: + # - name: Generate privatekey + # community.crypto.openssl_privatekey: + # path: '{{ vault_key_file }}' + + # - name: Generate CSR + # community.crypto.openssl_csr: + # path: '{{ vault_csr_file }}' + # privatekey_path: '{{ vault_key_file }}' + # subject: + # commonName: localhost + + # - name: Generate selfsigned certificate + # community.crypto.openssl_certificate: + # path: '{{ vault_cert_file }}' + # csr_path: '{{ vault_csr_file }}' + # privatekey_path: '{{ vault_key_file }}' + # provider: selfsigned + # selfsigned_digest: sha256 + # register: selfsigned_certificate + + # - name: "Check if vault binary exists" + # stat: + # path: '{{ vault_cmd }}' + # follow: yes + # get_attributes: no + # get_checksum: no + # get_mime: no + # register: bin_status + + # - name: "Download vault if not local" + # when: not bin_status.stat.exists + # import_tasks: vault_download.yml + - import_role: + name: setup_vault_server_download + + - environment: + # used by vault command + VAULT_DEV_ROOT_TOKEN_ID: '{{ vault_dev_root_token_id }}' + block: + - name: 'Create configuration file' + template: + src: vault_config.hcl.j2 + dest: '{{ local_temp_dir }}/vault_config.hcl' + + - debug: + msg: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl /dev/null 2>&1 &' + + - name: 'Start vault server (dev mode enabled)' + shell: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl /dev/null 2>&1 &' + notify: test_managed_vault_cleanup diff --git a/tests/integration/targets/setup_vault_server/templates/vault_config.hcl.j2 b/tests/integration/targets/setup_vault_server/templates/vault_config.hcl.j2 new file mode 100644 index 000000000..26bf47f15 --- /dev/null +++ b/tests/integration/targets/setup_vault_server/templates/vault_config.hcl.j2 @@ -0,0 +1,10 @@ +# {{ ansible_managed }} +pid_file = "{{ local_temp_dir }}/vault.pid" +{% if vault_run_https_tests | bool %} +listener "tcp" { + tls_key_file = "{{ vault_key_file }}" + tls_cert_file = "{{ vault_cert_file }}" + tls_disable = false + address = "{{ vault_test_server_https | regex_replace('^https://([^:]+):(\\d+).*?$', '\\1:\\2') }}" +} +{% endif %} From 5539715368a8b34c1d0cdd2db432379253efe863 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 28 Jun 2021 21:10:39 -0400 Subject: [PATCH 007/137] add a localenv setup target docker --- .../targets/setup_localenv_docker/aliases | 1 + .../setup_localenv_docker/defaults/main.yml | 23 +++++++++ .../files/.output/.gitignore | 2 + .../setup_localenv_docker/tasks/docker.yml | 50 +++++++++++++++++++ .../templates/docker-compose.yml.j2 | 21 ++++++++ .../templates/integration_config.yml.j2 | 5 ++ .../templates/vault_config.hcl.j2 | 14 ++++++ .../setup_localenv_docker/vars/.gitignore | 3 ++ .../setup_localenv_docker/vars/main.yml | 12 +++++ 9 files changed, 131 insertions(+) create mode 100644 tests/integration/targets/setup_localenv_docker/aliases create mode 100644 tests/integration/targets/setup_localenv_docker/defaults/main.yml create mode 100644 tests/integration/targets/setup_localenv_docker/files/.output/.gitignore create mode 100644 tests/integration/targets/setup_localenv_docker/tasks/docker.yml create mode 100644 tests/integration/targets/setup_localenv_docker/templates/docker-compose.yml.j2 create mode 100644 tests/integration/targets/setup_localenv_docker/templates/integration_config.yml.j2 create mode 100644 tests/integration/targets/setup_localenv_docker/templates/vault_config.hcl.j2 create mode 100644 tests/integration/targets/setup_localenv_docker/vars/.gitignore create mode 100644 tests/integration/targets/setup_localenv_docker/vars/main.yml diff --git a/tests/integration/targets/setup_localenv_docker/aliases b/tests/integration/targets/setup_localenv_docker/aliases new file mode 100644 index 000000000..136c05e0d --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/aliases @@ -0,0 +1 @@ +hidden diff --git a/tests/integration/targets/setup_localenv_docker/defaults/main.yml b/tests/integration/targets/setup_localenv_docker/defaults/main.yml new file mode 100644 index 000000000..4356372cf --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/defaults/main.yml @@ -0,0 +1,23 @@ +--- +vault_version: '1.7.3' + +docker_compose: clean +# up +# down + +docker_compose_project_name: hashi_vault + +vault_port_http: 8200 +vault_port_https: 8300 +vault_container_name: vault +vault_dev_root_token_id: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a +proxy_port: 8888 +proxy_container_name: tinyproxy + +output_dir: '{{ role_path }}/files/.output' + +docker_compose_output: '{{ output_dir }}/{{ docker_compose_project_name }}' +vault_config_output: '{{ output_dir }}/vault_config' + +vault_cert_file: '{{ vault_config_output }}/cert.pem' +vault_key_file: '{{ vault_config_output }}/key.pem' diff --git a/tests/integration/targets/setup_localenv_docker/files/.output/.gitignore b/tests/integration/targets/setup_localenv_docker/files/.output/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/files/.output/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/integration/targets/setup_localenv_docker/tasks/docker.yml b/tests/integration/targets/setup_localenv_docker/tasks/docker.yml new file mode 100644 index 000000000..6d3f62ead --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/tasks/docker.yml @@ -0,0 +1,50 @@ +--- +- name: "Ensure output dirs exist" + file: + state: directory + path: '{{ item }}' + loop: + - '{{ docker_compose_output }}' + - '{{ vault_config_output }}' + +- name: "Create the docker-compose definition" + template: + src: docker-compose.yml.j2 + dest: '{{ docker_compose_output }}/docker-compose.yml' + +- name: "Persist vars (role_path is relative)" + set_fact: + vault_cert_file: '{{ vault_cert_file }}' + vault_key_file: '{{ vault_key_file }}' + +- name: "Generate certs" + import_role: + name: setup_vault_server_cert + vars: + vault_hostname: '{{ vault_container_name }}' + +- name: "Template vault config" + template: + src: vault_config.hcl.j2 + dest: '{{ vault_config_output }}/vault_config.hcl' + +- name: "Shut down" + when: docker_compose in ['clean', 'down'] + community.docker.docker_compose: + project_name: hashi_vault + state: absent + # definition: '{{ docker_compose_definition }}' + project_src: '{{ docker_compose_output }}' + +- name: "Bring up" + when: docker_compose in ['clean', 'up'] + community.docker.docker_compose: + project_name: hashi_vault + # definition: '{{ docker_compose_definition }}' + project_src: '{{ docker_compose_output }}' + +- name: "Template integration_config" + template: + src: integration_config.yml.j2 + dest: '{{ output_dir }}/integration_config.yml' + force: yes diff --git a/tests/integration/targets/setup_localenv_docker/templates/docker-compose.yml.j2 b/tests/integration/targets/setup_localenv_docker/templates/docker-compose.yml.j2 new file mode 100644 index 000000000..ae9ea8c65 --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/templates/docker-compose.yml.j2 @@ -0,0 +1,21 @@ +# {{ ansible_managed }} +--- +version: '3' +services: + vault: + image: 'vault:{{ vault_version }}' + container_name: '{{ vault_container_name }}' + ports: + - '{{ vault_port_http }}:{{ vault_port_http }}' + - '{{ vault_port_https }}:{{ vault_port_https }}' + volumes: + - '{{ vault_config_output }}:/vault/config' + environment: + VAULT_DEV_ROOT_TOKEN_ID: '{{ vault_dev_root_token_id }}' + SKIP_CHOWN: 1 + tinyproxy: + image: 'monokal/tinyproxy' + container_name: '{{ proxy_container_name }}' + ports: + - '{{ proxy_port }}:{{ proxy_port }}' + command: ANY diff --git a/tests/integration/targets/setup_localenv_docker/templates/integration_config.yml.j2 b/tests/integration/targets/setup_localenv_docker/templates/integration_config.yml.j2 new file mode 100644 index 000000000..e57e3fc03 --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/templates/integration_config.yml.j2 @@ -0,0 +1,5 @@ +# {{ ansible_managed }} +--- +# can be uncommented once vault configuration has been run once (WIP/broken) +# vault_server_configure: False +{{ integration_config | to_nice_yaml }} diff --git a/tests/integration/targets/setup_localenv_docker/templates/vault_config.hcl.j2 b/tests/integration/targets/setup_localenv_docker/templates/vault_config.hcl.j2 new file mode 100644 index 000000000..ddc42c4b5 --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/templates/vault_config.hcl.j2 @@ -0,0 +1,14 @@ +# {{ ansible_managed }} +# pid_file = "{{ local_temp_dir | default('/tmp') }}/vault.pid" +# % if vault_run_https_tests | bool % +listener "tcp" { + tls_key_file = "/vault/config/{{ vault_key_file | basename }}" + tls_cert_file = "/vault/config/{{ vault_cert_file | basename }}" + tls_disable = false + # address = " vault_test_server_https | regex_replace('^https://([^:]+):(\\d+).*?$', '\\1:\\2') " + address = "{{ + integration_config.vault_test_server_https + | regex_replace('^https://([^:]+):(\\d+).*?$', '\\1:\\2') + }}" +} +# % endif % diff --git a/tests/integration/targets/setup_localenv_docker/vars/.gitignore b/tests/integration/targets/setup_localenv_docker/vars/.gitignore new file mode 100644 index 000000000..88d5df19c --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/vars/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!main.yml diff --git a/tests/integration/targets/setup_localenv_docker/vars/main.yml b/tests/integration/targets/setup_localenv_docker/vars/main.yml new file mode 100644 index 000000000..e5a5bb333 --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/vars/main.yml @@ -0,0 +1,12 @@ +--- +integration_config: + vault_version: '{{ vault_version }}' + vault_test_server_http: 'http://{{ vault_container_name }}:{{ vault_port_http }}' + vault_test_server_https: 'https://{{ vault_container_name }}:{{ vault_port_https}}' + vault_dev_root_token_id: '{{ vault_dev_root_token_id }}' + vault_proxy_server: 'http://{{ proxy_container_name }}:{{ proxy_port }}' + vault_test_server_external: True + vault_proxy_external: True + vault_cert_content: "{{ lookup('file', vault_cert_file) }}" + # vault_cert_file: '{{ vault_cert_file }}' + # vault_key_file: '{{ vault_key_file }}' From daeb80fb2fe451364ec817abe940693756d4759a Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 30 Jun 2021 15:17:51 -0400 Subject: [PATCH 008/137] gha tidbit --- .github/workflows/ansible-test.yml | 108 +++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 676f3f9f7..665bdab73 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -240,3 +240,111 @@ jobs: - uses: codecov/codecov-action@v1 with: fail_ci_if_error: false + + classic_integration_ubuntu: + runs-on: ubuntu-latest + name: Local - I (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) + strategy: + fail-fast: false + matrix: + ansible: + - stable-2.11 + python: + - 3.8 + # vault: + # - 1.7.3 + # exclude: + # - ansible: stable-2.9 + # python: 3.9 + # include: + # - ansible: devel + # vault: 1.7.3 + # python: '3.10' + + steps: + - name: Initialize env vars + run: | + COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} + LOOKUP_HASHI_VAULT_PATH=${COLLECTION_PATH}/tests/integration/targets/lookup_hashi_vault + LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin + LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars + + echo "COLLECTION_PATH=${COLLECTION_PATH}" >> ${GITHUB_ENV} + echo "LOOKUP_HASHI_VAULT_PATH=${LOOKUP_HASHI_VAULT_PATH}" >> ${GITHUB_ENV} + echo "LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_BIN}" >> ${GITHUB_ENV} + echo "LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_VARS}" >> ${GITHUB_ENV} + + - name: Check out code + uses: actions/checkout@v2 + with: + path: ${{ env.COLLECTION_PATH }} + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + # - name: Cache for vault binaries + # id: cache-vault + # uses: actions/cache@v2 + # with: + # path: ${{ env.LOOKUP_HASHI_VAULT_BIN }} + # key: ${{ runner.os }}-vault${{ matrix.vault }} # future: include version/arch when configurable + + # removing .gitignore lets the files in those dirs be sent to the container via ansible-test + # the files/bin dir will contain the vault binary downloaded a few steps later (or retrieved from cache) + # the vars/ dir will be used to write a file overriding role defaults (for Vault version) + # - name: Prepare for Vault version and caching + # run: | + # rm -f "${LOOKUP_HASHI_VAULT_BIN}/.gitignore" + # rm -f "${LOOKUP_HASHI_VAULT_VARS}/.gitignore" + # echo '{ "vault_version": "${{ matrix.vault }}" }' > "${LOOKUP_HASHI_VAULT_VARS}/main.json" + + - name: Install ansible-base (${{ matrix.ansible }}) + run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check + + #TODO: remove dependency on community.crypto + - name: Find the latest version of community.crypto + run: | + # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 + + cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) + echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV + + - name: Cache for community.crypto + id: cache-cc + uses: actions/cache@v2 + with: + path: ansible_collections/community/crypto + key: collection-community.crypto-${{ env.CC_LATEST }} + + - name: Install collection dependencies + if: steps.cache-cc.outputs.cache-hit != 'true' + uses: actions/checkout@v2 + with: + repository: ansible-collections/community.crypto + ref: refs/tags/${{ env.CC_LATEST }} + path: ansible_collections/community/crypto + + # this will populate files/bin with the selected vault version binary + # - name: Pre-download Vault + # if: steps.cache-vault.outputs.cache-hit != 'true' + # env: + # ANSIBLE_ROLES_PATH: ../ + # working-directory: ${{ env.LOOKUP_HASHI_VAULT_PATH }}/playbooks + # run: ansible-playbook "download_vault.yml" -v + + # Run the integration tests + - name: Run integration test + run: ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --docker #--coverage + working-directory: ${{ env.COLLECTION_PATH }} + + # ansible-test support producing code coverage date + # - name: Generate coverage report + # run: ansible-test coverage xml -v --requirements --group-by command --group-by version + # working-directory: ${{ env.COLLECTION_PATH }} + + # See the reports at https://codecov.io/gh/ansible-collections/community.hashi_vault + # - uses: codecov/codecov-action@v1 + # with: + # fail_ci_if_error: false From 90599bf90f4e5818511a90e9fe59ee77932a7eb4 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 30 Jun 2021 17:28:06 -0400 Subject: [PATCH 009/137] try out macos? --- .github/workflows/ansible-test.yml | 107 +++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 665bdab73..28b23b21e 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -284,21 +284,94 @@ jobs: with: python-version: ${{ matrix.python }} - # - name: Cache for vault binaries - # id: cache-vault - # uses: actions/cache@v2 + - name: Install ansible-base (${{ matrix.ansible }}) + run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check + + #TODO: remove dependency on community.crypto + - name: Find the latest version of community.crypto + run: | + # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 + + cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) + echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV + + - name: Cache for community.crypto + id: cache-cc + uses: actions/cache@v2 + with: + path: ansible_collections/community/crypto + key: collection-community.crypto-${{ env.CC_LATEST }} + + - name: Install collection dependencies + if: steps.cache-cc.outputs.cache-hit != 'true' + uses: actions/checkout@v2 + with: + repository: ansible-collections/community.crypto + ref: refs/tags/${{ env.CC_LATEST }} + path: ansible_collections/community/crypto + + # Run the integration tests + - name: Run integration test + run: ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --docker #--coverage + working-directory: ${{ env.COLLECTION_PATH }} + + # ansible-test support producing code coverage date + # - name: Generate coverage report + # run: ansible-test coverage xml -v --requirements --group-by command --group-by version + # working-directory: ${{ env.COLLECTION_PATH }} + + # See the reports at https://codecov.io/gh/ansible-collections/community.hashi_vault + # - uses: codecov/codecov-action@v1 # with: - # path: ${{ env.LOOKUP_HASHI_VAULT_BIN }} - # key: ${{ runner.os }}-vault${{ matrix.vault }} # future: include version/arch when configurable + # fail_ci_if_error: false - # removing .gitignore lets the files in those dirs be sent to the container via ansible-test - # the files/bin dir will contain the vault binary downloaded a few steps later (or retrieved from cache) - # the vars/ dir will be used to write a file overriding role defaults (for Vault version) - # - name: Prepare for Vault version and caching - # run: | - # rm -f "${LOOKUP_HASHI_VAULT_BIN}/.gitignore" - # rm -f "${LOOKUP_HASHI_VAULT_VARS}/.gitignore" - # echo '{ "vault_version": "${{ matrix.vault }}" }' > "${LOOKUP_HASHI_VAULT_VARS}/main.json" + classic_integration_macos: + runs-on: macos-latest + name: Local - I (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) + strategy: + fail-fast: false + matrix: + ansible: + - stable-2.11 + python: + - 3.8 + # vault: + # - 1.7.3 + # exclude: + # - ansible: stable-2.9 + # python: 3.9 + # include: + # - ansible: devel + # vault: 1.7.3 + # python: '3.10' + + steps: + - name: Initialize env vars + run: | + COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} + LOOKUP_HASHI_VAULT_PATH=${COLLECTION_PATH}/tests/integration/targets/lookup_hashi_vault + LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin + LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars + + echo "COLLECTION_PATH=${COLLECTION_PATH}" >> ${GITHUB_ENV} + echo "LOOKUP_HASHI_VAULT_PATH=${LOOKUP_HASHI_VAULT_PATH}" >> ${GITHUB_ENV} + echo "LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_BIN}" >> ${GITHUB_ENV} + echo "LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_VARS}" >> ${GITHUB_ENV} + + - name: Check out code + uses: actions/checkout@v2 + with: + path: ${{ env.COLLECTION_PATH }} + + - name: Install Docker + run: | + brew install docker-machine docker + sudo docker –version + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check @@ -326,14 +399,6 @@ jobs: ref: refs/tags/${{ env.CC_LATEST }} path: ansible_collections/community/crypto - # this will populate files/bin with the selected vault version binary - # - name: Pre-download Vault - # if: steps.cache-vault.outputs.cache-hit != 'true' - # env: - # ANSIBLE_ROLES_PATH: ../ - # working-directory: ${{ env.LOOKUP_HASHI_VAULT_PATH }}/playbooks - # run: ansible-playbook "download_vault.yml" -v - # Run the integration tests - name: Run integration test run: ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --docker #--coverage From 1ae0984da25abcd60c42e7fd045b866a57eb3616 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 30 Jun 2021 18:23:33 -0400 Subject: [PATCH 010/137] matrix runner? --- .github/workflows/ansible-test.yml | 96 +++--------------------------- 1 file changed, 8 insertions(+), 88 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 28b23b21e..1ed183f86 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -241,8 +241,8 @@ jobs: with: fail_ci_if_error: false - classic_integration_ubuntu: - runs-on: ubuntu-latest + classic_integration: + runs-on: ${{ matrix.runner }} name: Local - I (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) strategy: fail-fast: false @@ -251,6 +251,9 @@ jobs: - stable-2.11 python: - 3.8 + runner: + - ubuntu-latest + - macos-latest # vault: # - 1.7.3 # exclude: @@ -284,94 +287,11 @@ jobs: with: python-version: ${{ matrix.python }} - - name: Install ansible-base (${{ matrix.ansible }}) - run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check - - #TODO: remove dependency on community.crypto - - name: Find the latest version of community.crypto - run: | - # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 - - cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) - echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV - - - name: Cache for community.crypto - id: cache-cc - uses: actions/cache@v2 - with: - path: ansible_collections/community/crypto - key: collection-community.crypto-${{ env.CC_LATEST }} - - - name: Install collection dependencies - if: steps.cache-cc.outputs.cache-hit != 'true' - uses: actions/checkout@v2 - with: - repository: ansible-collections/community.crypto - ref: refs/tags/${{ env.CC_LATEST }} - path: ansible_collections/community/crypto - - # Run the integration tests - - name: Run integration test - run: ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --docker #--coverage - working-directory: ${{ env.COLLECTION_PATH }} - - # ansible-test support producing code coverage date - # - name: Generate coverage report - # run: ansible-test coverage xml -v --requirements --group-by command --group-by version - # working-directory: ${{ env.COLLECTION_PATH }} - - # See the reports at https://codecov.io/gh/ansible-collections/community.hashi_vault - # - uses: codecov/codecov-action@v1 - # with: - # fail_ci_if_error: false - - classic_integration_macos: - runs-on: macos-latest - name: Local - I (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) - strategy: - fail-fast: false - matrix: - ansible: - - stable-2.11 - python: - - 3.8 - # vault: - # - 1.7.3 - # exclude: - # - ansible: stable-2.9 - # python: 3.9 - # include: - # - ansible: devel - # vault: 1.7.3 - # python: '3.10' - - steps: - - name: Initialize env vars - run: | - COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} - LOOKUP_HASHI_VAULT_PATH=${COLLECTION_PATH}/tests/integration/targets/lookup_hashi_vault - LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin - LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars - - echo "COLLECTION_PATH=${COLLECTION_PATH}" >> ${GITHUB_ENV} - echo "LOOKUP_HASHI_VAULT_PATH=${LOOKUP_HASHI_VAULT_PATH}" >> ${GITHUB_ENV} - echo "LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_BIN}" >> ${GITHUB_ENV} - echo "LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_VARS}" >> ${GITHUB_ENV} - - - name: Check out code - uses: actions/checkout@v2 - with: - path: ${{ env.COLLECTION_PATH }} - - - name: Install Docker + - name: Install Docker on MacOS + if: ${{ matrix.runner }} == 'macos-latest' run: | brew install docker-machine docker - sudo docker –version - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} + sudo docker -–version - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check From 62e55bdfd9e03f9bcfd15686a8710e408de95e73 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 30 Jun 2021 18:24:48 -0400 Subject: [PATCH 011/137] fix hyphen/dash --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 1ed183f86..fd6dfee7d 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -291,7 +291,7 @@ jobs: if: ${{ matrix.runner }} == 'macos-latest' run: | brew install docker-machine docker - sudo docker -–version + sudo docker --version - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check From 719e90ff6bc2a33ba9ab1a04f1011ea51b060c13 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 30 Jun 2021 18:42:14 -0400 Subject: [PATCH 012/137] woop --- .github/workflows/ansible-test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index fd6dfee7d..2924460cd 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -292,6 +292,9 @@ jobs: run: | brew install docker-machine docker sudo docker --version + docker-machine create --driver virtualbox default + docker-machine env default + docker-machine env default >> $GITHUB_ENV - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check @@ -321,7 +324,9 @@ jobs: # Run the integration tests - name: Run integration test - run: ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --docker #--coverage + run: | + docker ps || true + ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --docker #--coverage working-directory: ${{ env.COLLECTION_PATH }} # ansible-test support producing code coverage date From 2e7f1ab295e3a3c09d8e5c76830e1d639251778d Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 30 Jun 2021 18:46:40 -0400 Subject: [PATCH 013/137] boot2docker yay --- .github/workflows/ansible-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 2924460cd..f6cc0b888 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -290,6 +290,8 @@ jobs: - name: Install Docker on MacOS if: ${{ matrix.runner }} == 'macos-latest' run: | + mkdir -p ~/.docker/machine/cache + curl -Lo ~/.docker/machine/cache/boot2docker.iso https://github.com/boot2docker/boot2docker/releases/download/v19.03.12/boot2docker.iso brew install docker-machine docker sudo docker --version docker-machine create --driver virtualbox default From 346eed029627b2326570979c0804aee55e5011e2 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 30 Jun 2021 18:56:00 -0400 Subject: [PATCH 014/137] fix conditional --- .github/workflows/ansible-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index f6cc0b888..8d70f109b 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -243,7 +243,7 @@ jobs: classic_integration: runs-on: ${{ matrix.runner }} - name: Local - I (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) + name: Local I - ${{ matrix.runner }} (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) strategy: fail-fast: false matrix: @@ -288,7 +288,7 @@ jobs: python-version: ${{ matrix.python }} - name: Install Docker on MacOS - if: ${{ matrix.runner }} == 'macos-latest' + if: ${{ matrix.runner == 'macos-latest' }} run: | mkdir -p ~/.docker/machine/cache curl -Lo ~/.docker/machine/cache/boot2docker.iso https://github.com/boot2docker/boot2docker/releases/download/v19.03.12/boot2docker.iso From dd6aca775c009d97dd9f4d0f498c9a8b7a58f2b6 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 30 Jun 2021 19:19:43 -0400 Subject: [PATCH 015/137] right sed fred --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 8d70f109b..5dbdf7f59 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -296,7 +296,7 @@ jobs: sudo docker --version docker-machine create --driver virtualbox default docker-machine env default - docker-machine env default >> $GITHUB_ENV + docker-machine env default | sed 's/^export //;/^#/d;s/^#.*//' >> $GITHUB_ENV - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check From 96da4fb046c80b61ba9f1d90c903a1f68085fc15 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 30 Jun 2021 19:30:16 -0400 Subject: [PATCH 016/137] don't tr -d on me --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 5dbdf7f59..f966de55e 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -296,7 +296,7 @@ jobs: sudo docker --version docker-machine create --driver virtualbox default docker-machine env default - docker-machine env default | sed 's/^export //;/^#/d;s/^#.*//' >> $GITHUB_ENV + docker-machine env default | sed 's/^export //;/^#/d;s/^#.*//' | tr -d '"' >> $GITHUB_ENV - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check From 9742c1d165d8ab1cba7033a5b4e86b056afdaca7 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 15:18:38 -0400 Subject: [PATCH 017/137] housekeeping --- .../targets/setup_localenv_docker/tasks/{docker.yml => main.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/integration/targets/setup_localenv_docker/tasks/{docker.yml => main.yml} (100%) diff --git a/tests/integration/targets/setup_localenv_docker/tasks/docker.yml b/tests/integration/targets/setup_localenv_docker/tasks/main.yml similarity index 100% rename from tests/integration/targets/setup_localenv_docker/tasks/docker.yml rename to tests/integration/targets/setup_localenv_docker/tasks/main.yml From 733e038f8b4c40b6ef0ea8c21683d35b12153d4d Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 15:19:01 -0400 Subject: [PATCH 018/137] add localenv setup for github actions --- .../targets/setup_localenv_gha/aliases | 1 + .../files/.output/integration_config.yml | 37 ++++++++++++++ .../files/.output/vault_config/cert.pem | 29 +++++++++++ .../files/.output/vault_config/key.pem | 51 +++++++++++++++++++ .../.output/vault_config/vault_config.hcl | 6 +++ 5 files changed, 124 insertions(+) create mode 100644 tests/integration/targets/setup_localenv_gha/aliases create mode 100644 tests/integration/targets/setup_localenv_gha/files/.output/integration_config.yml create mode 100644 tests/integration/targets/setup_localenv_gha/files/.output/vault_config/cert.pem create mode 100644 tests/integration/targets/setup_localenv_gha/files/.output/vault_config/key.pem create mode 100644 tests/integration/targets/setup_localenv_gha/files/.output/vault_config/vault_config.hcl diff --git a/tests/integration/targets/setup_localenv_gha/aliases b/tests/integration/targets/setup_localenv_gha/aliases new file mode 100644 index 000000000..136c05e0d --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/aliases @@ -0,0 +1 @@ +hidden diff --git a/tests/integration/targets/setup_localenv_gha/files/.output/integration_config.yml b/tests/integration/targets/setup_localenv_gha/files/.output/integration_config.yml new file mode 100644 index 000000000..d874e7290 --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/files/.output/integration_config.yml @@ -0,0 +1,37 @@ +--- +vault_dev_root_token_id: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a +vault_proxy_external: true +vault_proxy_server: http://tinyproxy:8888 +vault_test_server_external: true +vault_test_server_http: http://vault:8200 +vault_test_server_https: https://vault:8300 +vault_cert_content: | + -----BEGIN CERTIFICATE----- + MIIE4TCCAsmgAwIBAgIUIL77ChgfiJcegZViZh8TV9KviB8wDQYJKoZIhvcNAQEL + BQAwEDEOMAwGA1UEAwwFdmF1bHQwHhcNMjEwNzAxMTgyNjIzWhcNMzEwNjI5MTgy + NjIzWjAQMQ4wDAYDVQQDDAV2YXVsdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC + AgoCggIBAK1mI97qE0Pijd0vYrepaoZen/7yuayVslJ1uKzNQWCwHCpaAG9QuGof + ym2N4UnCnBAmpcOH8N+xVzFR5oQVJwYLodfPeI8VDcHvN6Cj011Lb40o2fAbQz+v + 2Xz4MOpEzRbCrSlMZa7M10+iiZJpU379Yqn6JDiCkwrq4dIBVUnp8Wg5ykV+qdUz + ypB6jJF67WRisXqYp7hACL67FRlj/r8+76FSUg/oAo7g0rkrahWb+SvNMPWS1hS7 + Tk8Tjf6qb2ZO2Iwx0eEY2GNoziUzQu1xTkZQhbQt3vt8ZiWvddCRxCz5W1cfKWO/ + 0XQlMlqlW2RmfxEouMUqhQlH1NSYCKas12RFxxSaiPI2Idq2Kve2xDLwQuIOoCLD + joqSQmyF88f9Jxb0l2sXXsro/YwmYJ+qwz81QW4BO5LzX7BHT7EuiV97m8kLuBzE + 0pjat0XVJ9fzmfGYX89uiwnO/fb7jZubjQLus0cmZXHMB6wK0fo8bTXgi9TLXwbi + wymDG+A+jlSShf7aE3vZtr6fRcacjwh6Y6DFbfxdVV9Vxzv1aHaMUYwtIu+d+uVC + cTQbwou6B7hS0BUdXQKtM1mjPDAwcdgz//TLzk26tIsx69AXtOREKb9W1ffIbfGh + B8nrdmI6+80tlc8KL6s+/cMLEMya3K2GZPloAw+CO00ihO7SGC6VAgMBAAGjMzAx + MBAGA1UdEQQJMAeCBXZhdWx0MB0GA1UdDgQWBBRbYPX83fHK2QUTsWExMQESmS6B + yTANBgkqhkiG9w0BAQsFAAOCAgEAc9slz9up7xd3bsr+q/kCoDt+w6rm/dc6ONSJ + PITZAbuWtRBtCJStQuie5ZICnh1X0IajhczIFVcD9CjxOIxfxA7S49gL9vDHVpiJ + K4nW0KR3Zviq2XwtHYAs99CZH63EUTVqz0nEuMu10H/0PCFPtTHcXFpgovCLRAGH + HqnM7LVeM7a0g85Zt+HXuPJ2MThlEyIBy64MBPIczSiGDVx0cQwe1LJREkQJgB4F + +3iAOPIsHAWkApFfx2cyq+L4sEd0EdxUFk1mw4sni/VFzK8wcd3L7gEgseKSk2Kz + Z3JZiAXessjoa26JL0/KBSN6LTB3/pdn/dG7lz7DUr17PymbggRLVjdbSsbAFD9S + BTGgt3kFjrXIeNAyqGodK673R5jACXjz0vuEePJh4Vk/ffl953bH8Xhs/BZyNXBC + meOu/sU93MGPi0vqE+Jdjplvj5smLzOW9Y6HAAitDqHTQ0sNdZP2DdT6FkSLSzMx + ErPFBePgxhQFXvQt8h11Gadox5vsm1Ca2nLGClKWRt458goFEWgmmDA+mOD2/sJ0 + eYdGfBgN/ZnTzD2y2z18Sd9H2Zb4HZbfpPvvFwR+5oYMqE72Rz3oj8APt2f/Eq5B + WYNpi1fRCGPXhM2wNr2DKPKcoqbRcIVTxD/E0MATmxQRGtPaO/JbFcY5v37qcLWf + Jb3iD/U= + -----END CERTIFICATE----- diff --git a/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/cert.pem b/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/cert.pem new file mode 100644 index 000000000..42c72e891 --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/cert.pem @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE4TCCAsmgAwIBAgIUIL77ChgfiJcegZViZh8TV9KviB8wDQYJKoZIhvcNAQEL +BQAwEDEOMAwGA1UEAwwFdmF1bHQwHhcNMjEwNzAxMTgyNjIzWhcNMzEwNjI5MTgy +NjIzWjAQMQ4wDAYDVQQDDAV2YXVsdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC +AgoCggIBAK1mI97qE0Pijd0vYrepaoZen/7yuayVslJ1uKzNQWCwHCpaAG9QuGof +ym2N4UnCnBAmpcOH8N+xVzFR5oQVJwYLodfPeI8VDcHvN6Cj011Lb40o2fAbQz+v +2Xz4MOpEzRbCrSlMZa7M10+iiZJpU379Yqn6JDiCkwrq4dIBVUnp8Wg5ykV+qdUz +ypB6jJF67WRisXqYp7hACL67FRlj/r8+76FSUg/oAo7g0rkrahWb+SvNMPWS1hS7 +Tk8Tjf6qb2ZO2Iwx0eEY2GNoziUzQu1xTkZQhbQt3vt8ZiWvddCRxCz5W1cfKWO/ +0XQlMlqlW2RmfxEouMUqhQlH1NSYCKas12RFxxSaiPI2Idq2Kve2xDLwQuIOoCLD +joqSQmyF88f9Jxb0l2sXXsro/YwmYJ+qwz81QW4BO5LzX7BHT7EuiV97m8kLuBzE +0pjat0XVJ9fzmfGYX89uiwnO/fb7jZubjQLus0cmZXHMB6wK0fo8bTXgi9TLXwbi +wymDG+A+jlSShf7aE3vZtr6fRcacjwh6Y6DFbfxdVV9Vxzv1aHaMUYwtIu+d+uVC +cTQbwou6B7hS0BUdXQKtM1mjPDAwcdgz//TLzk26tIsx69AXtOREKb9W1ffIbfGh +B8nrdmI6+80tlc8KL6s+/cMLEMya3K2GZPloAw+CO00ihO7SGC6VAgMBAAGjMzAx +MBAGA1UdEQQJMAeCBXZhdWx0MB0GA1UdDgQWBBRbYPX83fHK2QUTsWExMQESmS6B +yTANBgkqhkiG9w0BAQsFAAOCAgEAc9slz9up7xd3bsr+q/kCoDt+w6rm/dc6ONSJ +PITZAbuWtRBtCJStQuie5ZICnh1X0IajhczIFVcD9CjxOIxfxA7S49gL9vDHVpiJ +K4nW0KR3Zviq2XwtHYAs99CZH63EUTVqz0nEuMu10H/0PCFPtTHcXFpgovCLRAGH +HqnM7LVeM7a0g85Zt+HXuPJ2MThlEyIBy64MBPIczSiGDVx0cQwe1LJREkQJgB4F ++3iAOPIsHAWkApFfx2cyq+L4sEd0EdxUFk1mw4sni/VFzK8wcd3L7gEgseKSk2Kz +Z3JZiAXessjoa26JL0/KBSN6LTB3/pdn/dG7lz7DUr17PymbggRLVjdbSsbAFD9S +BTGgt3kFjrXIeNAyqGodK673R5jACXjz0vuEePJh4Vk/ffl953bH8Xhs/BZyNXBC +meOu/sU93MGPi0vqE+Jdjplvj5smLzOW9Y6HAAitDqHTQ0sNdZP2DdT6FkSLSzMx +ErPFBePgxhQFXvQt8h11Gadox5vsm1Ca2nLGClKWRt458goFEWgmmDA+mOD2/sJ0 +eYdGfBgN/ZnTzD2y2z18Sd9H2Zb4HZbfpPvvFwR+5oYMqE72Rz3oj8APt2f/Eq5B +WYNpi1fRCGPXhM2wNr2DKPKcoqbRcIVTxD/E0MATmxQRGtPaO/JbFcY5v37qcLWf +Jb3iD/U= +-----END CERTIFICATE----- diff --git a/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/key.pem b/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/key.pem new file mode 100644 index 000000000..cdfc143fa --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJJwIBAAKCAgEArWYj3uoTQ+KN3S9it6lqhl6f/vK5rJWyUnW4rM1BYLAcKloA +b1C4ah/KbY3hScKcECalw4fw37FXMVHmhBUnBguh1894jxUNwe83oKPTXUtvjSjZ +8BtDP6/ZfPgw6kTNFsKtKUxlrszXT6KJkmlTfv1iqfokOIKTCurh0gFVSenxaDnK +RX6p1TPKkHqMkXrtZGKxepinuEAIvrsVGWP+vz7voVJSD+gCjuDSuStqFZv5K80w +9ZLWFLtOTxON/qpvZk7YjDHR4RjYY2jOJTNC7XFORlCFtC3e+3xmJa910JHELPlb +Vx8pY7/RdCUyWqVbZGZ/ESi4xSqFCUfU1JgIpqzXZEXHFJqI8jYh2rYq97bEMvBC +4g6gIsOOipJCbIXzx/0nFvSXaxdeyuj9jCZgn6rDPzVBbgE7kvNfsEdPsS6JX3ub +yQu4HMTSmNq3RdUn1/OZ8Zhfz26LCc799vuNm5uNAu6zRyZlccwHrArR+jxtNeCL +1MtfBuLDKYMb4D6OVJKF/toTe9m2vp9FxpyPCHpjoMVt/F1VX1XHO/VodoxRjC0i +75365UJxNBvCi7oHuFLQFR1dAq0zWaM8MDBx2DP/9MvOTbq0izHr0Be05EQpv1bV +98ht8aEHyet2Yjr7zS2Vzwovqz79wwsQzJrcrYZk+WgDD4I7TSKE7tIYLpUCAwEA +AQKCAgBRFMLPOJs7khOOCttZUEH8hQDBoVRLLFCPGHb7bpsUkCULdLxhUNh631Sz +t7VR3UtGjhvS+50ZrH4+FLL6rj1qKURLertABLNDLQ3Q8uoh5OVLl3+ZM9ZVUHAd +bJzK3tMbwnpgJlYhz60aksFLki614dwh6VLIjd4eK8jefzsXbeoxN1yq9FiL1kTx +HCzg1h9tYmzlC77ZfC/ap/ZswgJcARziTXEL+QxBLUW7yl43rpBr9+3d1wR9+zvZ +5CXwoFeo+lNt8tnWLNz6VwaWBItWJjZY0MxEfXKYHApTsuEfXxnhgdRBP9QFzasu +aTpLgqjioL7oEiecIo7E0S7nHhRkvmhl/yZlI8PcneK7kRbobJJ7mNr65PA2yQIj +a4hMNr9bvYMZqkhqkXe5vFxmUBhc7MfoOFKP1wHHrEc1Gv4hNl/RXO8G72pN/otf +OdRCVluveEdUfjfjo10ptN/hqmnTs5ryz2QfISetpnDbebgPB9pgBYOEb27pdxEf +mLPH+dLVDAEFQRnLqAHIB7P1NtfsCgSmhbrIHvDO6qm39NOtmeJBvHzm/aC0JEfI +CeDMA80zB5nDsHKztL65VTDQ5okTtfMkAuzXhDEGprEnk2hi4LYWurvEgdE01rL0 +Yk/FCH4Ae+YSd/yqBjTaDJ2kkwGeyR562i7AOf4KoECSdwdqnQKCAQEA35dXubjy +BXfbNewCiK4I0wkkD88QFiSAZBQi3mK+1ZI6eB58Rvrq/dE226fw2fnzuV6mYQoD +/mok3hpHM7MTtCg9vMvMq4sW47cpjYsMxHLIYvpQxfJ74Ja/vlpxAa9EJDhClKQu +1vmkNqpxdmqOLM7EQgjJ7q5WVm528wPBMcoO/MaVNGLj0Mw63NIiX7i0QldVnc1r +oMRN6HVpXrPqBNUjUtOxXAFdfVc/ecm8PbS0UCGQxS0rL/m0/nFuxDwPQ6sdgCsr +FgMSvRS/aWuDvHfwicCLSOSAwtLD7gyk9/li3R62sP9ZmH54eqMaOOYbYZIVrRDZ +fIdfi67dwz1hSwKCAQEAxohY3tqmDUspah/UHofn2gqIuK8RIkRzrHvDXu919fzn +LEbN+aeWR6nFPPU3N3YCSS8GRxC6BK0AiCTtjoSpzc7PY1X7EZ2LVNYvLTLwG2Q5 +JEkk2z7Sh3Ckb3C4RXlYGO/VueS8uduJKllC8gDILaWt4CSDFt76ftwW1Ykd0z6Y +ttpcEkeOXwfvq0MKTh60584A7+WeWN2SnQ5Nd6yZ2UdMvHs9AA7Z2pI0cPqKZTMJ +/C0F/hjVlK3IzKNXhySITCPfLCc8zG+NdcTcMq9oORg4iSWi6vfdBCKAoxl/tZSd +NMBSIXF2Et8F2dGOQXctoktot1DJ3D/ZDaQp8FKjnwKCAQA/ptUJgTYdCmb0bdC2 +2lyWv3ZtrbPVqeSHGRhoGNJc+Hj+sycMCeiWA1ZLp/6v/zE8J07UN0a1yb/fHroA +ZM2KLx6MPbRxbXJBt2Xz7o7e6hJT4xDVmVdcs224ogSjxyJt8To7GJZhCsaN7W3J +mUIKTX6fkeLyuNzJVD82pW39X5luqpXVSQWz7kAflw7EE0/1xukPuGgYZ2oKcEpj +9HH9mDOrI6rNF3Jm+UyPvjeBlYzzdurAx0ARM1QsT1yKLtLln7QRRbjJVmyiOWLY +fwwMvlRS0uaUBMEHNXsjrKPWGD/l1RWLQ9ZczDw+JWqUoiVPXIghMbfIdL1lhY2K +RFILAoIBAFcM8erFdInfqwkda6T3cnHorIpOOPUNjqrSmZfG57I0cii34xjUotBJ +YHKaEtv+ooH0XIxMiUQDl3gauQ8EnG/hfo3P5YaTzcQlJgpri1x3VZbTe5Wmtiks ++uziP+o2iqpkfxJDeX0FBd/Lw87ZCOz6+IKPf+tHWg7F8j0vIiS+Dbgfhr99ILAK +isxNWPdn/2qkqUSHKidNADxdSRpwVAUxfjQm1VhxKpTsBpKDSKuW9YndNAN2YDAR +Azq3ZGmdWyYHlJOHoOEDCMPazbJrtwdR13hLSGQ+ympFtqHNTHIR99xpd5myZlNf +9vDLMdUCrL+80O2QPvKUDFgu4zS9B4ECggEAEx1/ZRmiVuWNYtQcAvK4Ub95gitY +1EG6AsdNVrv5AwMK7lab+WWhIxASHhge9yPGuQ0DAEVGkGkSsEOjHlnydT6L9lfE +s3vx1oT6xdDQsgRpNSocbrE8RP4p+Uvce7PfDofnCzjWnfv7YsRtASPxjjZe70uP +4RweQaXq34dPoJ0ZeVI46FfIfNz8AicbRUyS7oow9cWkyhq4GD8Ch8a7DruP7KDX +2UCF9iwv6wwoqvxIeL3fHkdpsZ26+bie65RPsdxPY6F83vJa9AXr//grAAd44EKz +x/nAN0TijTDbbqBbYipUOMghC8sJlmPQNpypqNklXb0TywNvgcuxw6IzAg== +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/vault_config.hcl b/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/vault_config.hcl new file mode 100644 index 000000000..e39bb1470 --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/vault_config.hcl @@ -0,0 +1,6 @@ +listener "tcp" { + tls_key_file = "/vault/config/key.pem" + tls_cert_file = "/vault/config/cert.pem" + tls_disable = false + address = "vault:8300" +} From d7a610c19eda217ecc48f17618d7d290f61497b7 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 15:19:13 -0400 Subject: [PATCH 019/137] try out service containers --- .github/workflows/ansible-test.yml | 100 ++++++++++++++++------------- 1 file changed, 57 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index f966de55e..afe91b737 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -152,16 +152,28 @@ jobs: - ansible: devel vault: 1.7.3 python: '3.10' + services: + vault: + image: vault:${{ matrix.vault }} + volumes: + - ansible_collections/${{ env.NAMESPACE }}/${{ env.COLLECTION_NAME }}/tests/integration/targets/setup_localenv_gha/files/.output/vault_config:/vault/config + env: + VAULT_DEV_ROOT_TOKEN_ID: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' + SKIP_CHOWN: 1 + tinyproxy: + image: monokal/tinyproxy steps: - name: Initialize env vars run: | COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} - LOOKUP_HASHI_VAULT_PATH=${COLLECTION_PATH}/tests/integration/targets/lookup_hashi_vault + COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets + LOOKUP_HASHI_VAULT_PATH=${COLLECTION_INTEGRATION_TARGETS}/lookup_hashi_vault LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars echo "COLLECTION_PATH=${COLLECTION_PATH}" >> ${GITHUB_ENV} + echo "COLLECTION_INTEGRATION_TARGETS=${COLLECTION_INTEGRATION_TARGETS}" >> ${GITHUB_ENV} echo "LOOKUP_HASHI_VAULT_PATH=${LOOKUP_HASHI_VAULT_PATH}" >> ${GITHUB_ENV} echo "LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_BIN}" >> ${GITHUB_ENV} echo "LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_VARS}" >> ${GITHUB_ENV} @@ -176,59 +188,61 @@ jobs: with: python-version: 3.8 - - name: Cache for vault binaries - id: cache-vault - uses: actions/cache@v2 - with: - path: ${{ env.LOOKUP_HASHI_VAULT_BIN }} - key: ${{ runner.os }}-vault${{ matrix.vault }} # future: include version/arch when configurable - - # removing .gitignore lets the files in those dirs be sent to the container via ansible-test - # the files/bin dir will contain the vault binary downloaded a few steps later (or retrieved from cache) - # the vars/ dir will be used to write a file overriding role defaults (for Vault version) - - name: Prepare for Vault version and caching - run: | - rm -f "${LOOKUP_HASHI_VAULT_BIN}/.gitignore" - rm -f "${LOOKUP_HASHI_VAULT_VARS}/.gitignore" - echo '{ "vault_version": "${{ matrix.vault }}" }' > "${LOOKUP_HASHI_VAULT_VARS}/main.json" + # - name: Cache for vault binaries + # id: cache-vault + # uses: actions/cache@v2 + # with: + # path: ${{ env.LOOKUP_HASHI_VAULT_BIN }} + # key: ${{ runner.os }}-vault${{ matrix.vault }} # future: include version/arch when configurable + + # # removing .gitignore lets the files in those dirs be sent to the container via ansible-test + # # the files/bin dir will contain the vault binary downloaded a few steps later (or retrieved from cache) + # # the vars/ dir will be used to write a file overriding role defaults (for Vault version) + # - name: Prepare for Vault version and caching + # run: | + # rm -f "${LOOKUP_HASHI_VAULT_BIN}/.gitignore" + # rm -f "${LOOKUP_HASHI_VAULT_VARS}/.gitignore" + # echo '{ "vault_version": "${{ matrix.vault }}" }' > "${LOOKUP_HASHI_VAULT_VARS}/main.json" - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check - #TODO: remove dependency on community.crypto - - name: Find the latest version of community.crypto - run: | - # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 + # #TODO: remove dependency on community.crypto + # - name: Find the latest version of community.crypto + # run: | + # # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 - cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) - echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV + # cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) + # echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV - - name: Cache for community.crypto - id: cache-cc - uses: actions/cache@v2 - with: - path: ansible_collections/community/crypto - key: collection-community.crypto-${{ env.CC_LATEST }} + # - name: Cache for community.crypto + # id: cache-cc + # uses: actions/cache@v2 + # with: + # path: ansible_collections/community/crypto + # key: collection-community.crypto-${{ env.CC_LATEST }} - - name: Install collection dependencies - if: steps.cache-cc.outputs.cache-hit != 'true' - uses: actions/checkout@v2 - with: - repository: ansible-collections/community.crypto - ref: refs/tags/${{ env.CC_LATEST }} - path: ansible_collections/community/crypto + # - name: Install collection dependencies + # if: steps.cache-cc.outputs.cache-hit != 'true' + # uses: actions/checkout@v2 + # with: + # repository: ansible-collections/community.crypto + # ref: refs/tags/${{ env.CC_LATEST }} + # path: ansible_collections/community/crypto # this will populate files/bin with the selected vault version binary - - name: Pre-download Vault - if: steps.cache-vault.outputs.cache-hit != 'true' - env: - ANSIBLE_ROLES_PATH: ../ - working-directory: ${{ env.LOOKUP_HASHI_VAULT_PATH }}/playbooks - run: ansible-playbook "download_vault.yml" -v + # - name: Pre-download Vault + # if: steps.cache-vault.outputs.cache-hit != 'true' + # env: + # ANSIBLE_ROLES_PATH: ../ + # working-directory: ${{ env.LOOKUP_HASHI_VAULT_PATH }}/playbooks + # run: ansible-playbook "download_vault.yml" -v # Run the integration tests - name: Run integration test - run: ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --docker --coverage + run: | + cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_gha/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_TARGETS}" + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network ${{ job.services.vault.network }} working-directory: ${{ env.COLLECTION_PATH }} # ansible-test support producing code coverage date @@ -325,7 +339,7 @@ jobs: path: ansible_collections/community/crypto # Run the integration tests - - name: Run integration test + - name: Run integration test in docker run: | docker ps || true ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --docker #--coverage From 8820612e16cd2a117e4e100bf7295089de56b1c5 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 15:23:40 -0400 Subject: [PATCH 020/137] fix dumb workflow syntax --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index afe91b737..aba33dc15 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -156,7 +156,7 @@ jobs: vault: image: vault:${{ matrix.vault }} volumes: - - ansible_collections/${{ env.NAMESPACE }}/${{ env.COLLECTION_NAME }}/tests/integration/targets/setup_localenv_gha/files/.output/vault_config:/vault/config + - ansible_collections/community/hashi_vault/tests/integration/targets/setup_localenv_gha/files/.output/vault_config:/vault/config env: VAULT_DEV_ROOT_TOKEN_ID: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' SKIP_CHOWN: 1 From 05950131e2b8a816ef85f8a8a87e84e727de55b7 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 15:26:30 -0400 Subject: [PATCH 021/137] abs path --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index aba33dc15..34bc67fc2 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -156,7 +156,7 @@ jobs: vault: image: vault:${{ matrix.vault }} volumes: - - ansible_collections/community/hashi_vault/tests/integration/targets/setup_localenv_gha/files/.output/vault_config:/vault/config + - ${{ github.workspace }}/ansible_collections/community/hashi_vault/tests/integration/targets/setup_localenv_gha/files/.output/vault_config:/vault/config env: VAULT_DEV_ROOT_TOKEN_ID: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' SKIP_CHOWN: 1 From 415733814566bae0f05bccb935d4b1ec83792caa Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 15:46:33 -0400 Subject: [PATCH 022/137] ugh wth --- .github/workflows/ansible-test.yml | 84 +++++++++++++++++++----------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 34bc67fc2..6cb53c0b8 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -152,16 +152,16 @@ jobs: - ansible: devel vault: 1.7.3 python: '3.10' - services: - vault: - image: vault:${{ matrix.vault }} - volumes: - - ${{ github.workspace }}/ansible_collections/community/hashi_vault/tests/integration/targets/setup_localenv_gha/files/.output/vault_config:/vault/config - env: - VAULT_DEV_ROOT_TOKEN_ID: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' - SKIP_CHOWN: 1 - tinyproxy: - image: monokal/tinyproxy + # services: + # vault: + # image: vault:${{ matrix.vault }} + # volumes: + # - ${{ github.workspace }}/ansible_collections/community/hashi_vault/tests/integration/targets/setup_localenv_gha/files/.output/vault_config:/vault/config + # env: + # VAULT_DEV_ROOT_TOKEN_ID: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' + # SKIP_CHOWN: 1 + # tinyproxy: + # image: monokal/tinyproxy steps: - name: Initialize env vars @@ -207,28 +207,46 @@ jobs: - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check - # #TODO: remove dependency on community.crypto - # - name: Find the latest version of community.crypto - # run: | - # # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 + #TODO: remove dependency on community.crypto + - name: Find the latest version of community.crypto & community.docker + run: | + # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 - # cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) - # echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV + cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) + echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV - # - name: Cache for community.crypto - # id: cache-cc - # uses: actions/cache@v2 - # with: - # path: ansible_collections/community/crypto - # key: collection-community.crypto-${{ env.CC_LATEST }} + cdlu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.docker/releases/latest) + echo "CD_LATEST=$(basename ${cdlu})" >> $GITHUB_ENV - # - name: Install collection dependencies - # if: steps.cache-cc.outputs.cache-hit != 'true' - # uses: actions/checkout@v2 - # with: - # repository: ansible-collections/community.crypto - # ref: refs/tags/${{ env.CC_LATEST }} - # path: ansible_collections/community/crypto + - name: Cache for community.crypto + id: cache-cc + uses: actions/cache@v2 + with: + path: ansible_collections/community/crypto + key: collection-community.crypto-${{ env.CC_LATEST }} + + - name: Cache for community.docker + id: cache-cd + uses: actions/cache@v2 + with: + path: ansible_collections/community/docker + key: collection-community.docker-${{ env.CD_LATEST }} + + - name: Install community.crypto + if: steps.cache-cc.outputs.cache-hit != 'true' + uses: actions/checkout@v2 + with: + repository: ansible-collections/community.crypto + ref: refs/tags/${{ env.CC_LATEST }} + path: ansible_collections/community/crypto + + - name: Install community.docker + if: steps.cache-cd.outputs.cache-hit != 'true' + uses: actions/checkout@v2 + with: + repository: ansible-collections/community.docker + ref: refs/tags/${{ env.CD_LATEST }} + path: ansible_collections/community/docker # this will populate files/bin with the selected vault version binary # - name: Pre-download Vault @@ -239,9 +257,15 @@ jobs: # run: ansible-playbook "download_vault.yml" -v # Run the integration tests + - name: Prepare docker dependencies + env: + ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} + run: | + ansible localhost -m include_role -a "name=setup_localenv_docker" + - name: Run integration test run: | - cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_gha/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_TARGETS}" + cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_TARGETS}" ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network ${{ job.services.vault.network }} working-directory: ${{ env.COLLECTION_PATH }} From 7eaa1f05bfbfd628c035f30926da761e249ff26f Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 15:49:57 -0400 Subject: [PATCH 023/137] cols paff --- .github/workflows/ansible-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 6cb53c0b8..3436a7e67 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -260,6 +260,7 @@ jobs: - name: Prepare docker dependencies env: ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} + ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} run: | ansible localhost -m include_role -a "name=setup_localenv_docker" From 4b3615534aca3a621879a5062acc539789910911 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 15:53:12 -0400 Subject: [PATCH 024/137] pip pip, hooray? --- .github/workflows/ansible-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 3436a7e67..ddf21ac40 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -262,6 +262,7 @@ jobs: ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} run: | + pip install 'docker>5' ansible localhost -m include_role -a "name=setup_localenv_docker" - name: Run integration test From 2a0f19abf1cb96024e079802f0ff2101455f5918 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 15:54:52 -0400 Subject: [PATCH 025/137] err errday --- .github/workflows/ansible-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index ddf21ac40..1aa8f67b7 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -262,8 +262,8 @@ jobs: ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} run: | - pip install 'docker>5' - ansible localhost -m include_role -a "name=setup_localenv_docker" + pip install 'docker>=5' + ansible localhost -m include_role -a "name=setup_localenv_docker" -e vault_version=${{ matrix.vault }} - name: Run integration test run: | From 965e7b47640ed38931de268aa631bc28745a2d2b Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 15:58:31 -0400 Subject: [PATCH 026/137] need six too? --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 1aa8f67b7..ef95e1dc0 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -262,7 +262,7 @@ jobs: ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} run: | - pip install 'docker>=5' + pip install six 'docker>=5' ansible localhost -m include_role -a "name=setup_localenv_docker" -e vault_version=${{ matrix.vault }} - name: Run integration test From 5a161aad582bee0c4e4a5c1f87f56cdf74d4d440 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 15:59:57 -0400 Subject: [PATCH 027/137] and docker-compose --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index ef95e1dc0..a72236cc2 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -262,7 +262,7 @@ jobs: ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} run: | - pip install six 'docker>=5' + pip install docker-compose six 'docker>=5' ansible localhost -m include_role -a "name=setup_localenv_docker" -e vault_version=${{ matrix.vault }} - name: Run integration test From 75b5cc13368bcf4f50fbb0ae87d893eb2f081c98 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 16:04:51 -0400 Subject: [PATCH 028/137] more path fun! --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index a72236cc2..4f7b5f070 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -267,7 +267,7 @@ jobs: - name: Run integration test run: | - cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_TARGETS}" + cp "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_TARGETS}" ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network ${{ job.services.vault.network }} working-directory: ${{ env.COLLECTION_PATH }} From 0e399cc16589ed1eb26df3216cf5f6b7667e1e39 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 16:07:28 -0400 Subject: [PATCH 029/137] paths, as far as the eye can see! --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 4f7b5f070..fc2efd70e 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -267,7 +267,7 @@ jobs: - name: Run integration test run: | - cp "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_TARGETS}" + cp "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}" ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network ${{ job.services.vault.network }} working-directory: ${{ env.COLLECTION_PATH }} From 20b7e2be9eb53f8f28e3717566df5f91ff36b1d6 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 16:10:09 -0400 Subject: [PATCH 030/137] netwerkz --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index fc2efd70e..3f3ab30b2 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -268,7 +268,7 @@ jobs: - name: Run integration test run: | cp "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}" - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network ${{ job.services.vault.network }} + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network hashi_vault_default working-directory: ${{ env.COLLECTION_PATH }} # ansible-test support producing code coverage date From 247f5b2258773ac837e47cb7d83c92b26fc0fcf2 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 1 Jul 2021 16:13:43 -0400 Subject: [PATCH 031/137] poopiters legacy --- tests/integration/targets/setup_localenv_docker/vars/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/targets/setup_localenv_docker/vars/main.yml b/tests/integration/targets/setup_localenv_docker/vars/main.yml index e5a5bb333..faa52fd6e 100644 --- a/tests/integration/targets/setup_localenv_docker/vars/main.yml +++ b/tests/integration/targets/setup_localenv_docker/vars/main.yml @@ -8,5 +8,6 @@ integration_config: vault_test_server_external: True vault_proxy_external: True vault_cert_content: "{{ lookup('file', vault_cert_file) }}" + vault_server_legacy: False # vault_cert_file: '{{ vault_cert_file }}' # vault_key_file: '{{ vault_key_file }}' From 466e8727b6fde0bf94a79eb560b0b564f424c02e Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 15:04:29 -0400 Subject: [PATCH 032/137] whats goin on --- .github/workflows/ansible-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 3f3ab30b2..38441fa54 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -268,6 +268,9 @@ jobs: - name: Run integration test run: | cp "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}" + ls -alh "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}" + cat "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/integration_config.yml" + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network hashi_vault_default working-directory: ${{ env.COLLECTION_PATH }} From e76c1fbbf3b6325e486c76e52f372599c2dc8a15 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 17:30:37 -0400 Subject: [PATCH 033/137] fixee --- .github/workflows/ansible-test.yml | 4 +-- .../integration/integration_config.yml.sample | 13 ++++++++++ tests/integration/scripts/hv-test.sh | 1 + .../lookup_hashi_vault/defaults/main.yml | 2 ++ .../targets/lookup_hashi_vault/tasks/main.yml | 2 +- .../files/requirements/constraints.txt | 2 ++ .../files/requirements/requirements.txt | 3 +++ .../setup_localenv_docker/vars/main.yml | 2 +- .../targets/setup_vault_server/aliases | 1 + .../setup_vault_server/defaults/main.yml | 26 +++++++++---------- .../setup_vault_server/handlers/main.yml | 2 +- .../setup_vault_server_download/aliases | 1 + 12 files changed, 41 insertions(+), 18 deletions(-) create mode 100644 tests/integration/integration_config.yml.sample create mode 100644 tests/integration/scripts/hv-test.sh create mode 100644 tests/integration/targets/setup_localenv_docker/files/requirements/constraints.txt create mode 100644 tests/integration/targets/setup_localenv_docker/files/requirements/requirements.txt diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 38441fa54..67892f147 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -262,12 +262,12 @@ jobs: ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} run: | - pip install docker-compose six 'docker>=5' + pip install -r "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/requirements/requirements.txt" -c "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/requirements/constraints.txt" ansible localhost -m include_role -a "name=setup_localenv_docker" -e vault_version=${{ matrix.vault }} - name: Run integration test run: | - cp "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}" + cp "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/.." ls -alh "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}" cat "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/integration_config.yml" diff --git a/tests/integration/integration_config.yml.sample b/tests/integration/integration_config.yml.sample new file mode 100644 index 000000000..1e30235d4 --- /dev/null +++ b/tests/integration/integration_config.yml.sample @@ -0,0 +1,13 @@ +# +--- +vault_integration_legacy: true +vault_version: 1.7.2 + +vault_dev_root_token_id: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a + +vault_proxy_server: 'http://127.0.0.1:8001' +vault_proxy_external: false + +vault_test_server_external: false +vault_test_server_http: http://localhost:8200 +vault_test_server_https: https://localhost:8300 diff --git a/tests/integration/scripts/hv-test.sh b/tests/integration/scripts/hv-test.sh new file mode 100644 index 000000000..f1f641af1 --- /dev/null +++ b/tests/integration/scripts/hv-test.sh @@ -0,0 +1 @@ +#!/usr/bin/env bash diff --git a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml index 75ef0b214..f5acdb09a 100644 --- a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml @@ -115,3 +115,5 @@ vault_proxy_external: False # if any connections are taking longer than this to complete there's probably something really wrong # with the integration tests, so it'd be better to fail faster than the 30s default ansible_hashi_vault_timeout: 5 + +vault_integration_legacy: False diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml index fadf115b4..3cab37978 100644 --- a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml @@ -17,7 +17,7 @@ # when: not vault_test_server_external | bool - include_role: name: setup_vault_server - when: vault_server_legacy | default(True) | bool + when: vault_integration_legacy | bool - import_tasks: vault_server_configure.yml when: vault_test_server_configure | bool diff --git a/tests/integration/targets/setup_localenv_docker/files/requirements/constraints.txt b/tests/integration/targets/setup_localenv_docker/files/requirements/constraints.txt new file mode 100644 index 000000000..8706ac186 --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/files/requirements/constraints.txt @@ -0,0 +1,2 @@ +docker >= 5.0.0 ; python_version >= 3.6 +docker < 5.0.0 ; python_version == 2.7 diff --git a/tests/integration/targets/setup_localenv_docker/files/requirements/requirements.txt b/tests/integration/targets/setup_localenv_docker/files/requirements/requirements.txt new file mode 100644 index 000000000..de536a9e5 --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/files/requirements/requirements.txt @@ -0,0 +1,3 @@ +docker +docker-compose +six # https://github.com/ansible-collections/community.docker/issues/171 diff --git a/tests/integration/targets/setup_localenv_docker/vars/main.yml b/tests/integration/targets/setup_localenv_docker/vars/main.yml index faa52fd6e..985f0899d 100644 --- a/tests/integration/targets/setup_localenv_docker/vars/main.yml +++ b/tests/integration/targets/setup_localenv_docker/vars/main.yml @@ -8,6 +8,6 @@ integration_config: vault_test_server_external: True vault_proxy_external: True vault_cert_content: "{{ lookup('file', vault_cert_file) }}" - vault_server_legacy: False + # vault_server_legacy: False # vault_cert_file: '{{ vault_cert_file }}' # vault_key_file: '{{ vault_key_file }}' diff --git a/tests/integration/targets/setup_vault_server/aliases b/tests/integration/targets/setup_vault_server/aliases index 42eba2ced..a93ae517d 100644 --- a/tests/integration/targets/setup_vault_server/aliases +++ b/tests/integration/targets/setup_vault_server/aliases @@ -1,3 +1,4 @@ hidden needs/target/setup_vault_server_cert needs/target/setup_vault_server_download +destructive diff --git a/tests/integration/targets/setup_vault_server/defaults/main.yml b/tests/integration/targets/setup_vault_server/defaults/main.yml index 97811be26..97a1b9a32 100644 --- a/tests/integration/targets/setup_vault_server/defaults/main.yml +++ b/tests/integration/targets/setup_vault_server/defaults/main.yml @@ -1,19 +1,19 @@ --- ## vars for vault server -vault_ansible_arch_table: - 'x86_64': 'amd64' # Linux - 'amd64': 'amd64' # FreeBSD - 'i386': '386' - -vault_arch: "{{ vault_ansible_arch_table[ansible_architecture] }}" - -vault_version: '1.7.3' -vault_bin: '{{ role_path }}/files/bin/{{ vault_slug }}' -vault_slug: 'vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}' -vault_zip: '{{ vault_bin }}/{{ vault_slug }}.zip' -vault_uri: 'https://releases.hashicorp.com/vault/{{ vault_version }}/{{ vault_slug }}.zip' -vault_cmd: '{{ vault_bin }}/vault' +# vault_ansible_arch_table: +# 'x86_64': 'amd64' # Linux +# 'amd64': 'amd64' # FreeBSD +# 'i386': '386' + +# vault_arch: "{{ vault_ansible_arch_table[ansible_architecture] }}" + +# vault_version: '1.7.3' +# vault_bin: '{{ role_path }}/files/bin/{{ vault_slug }}' +# vault_slug: 'vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}' +# vault_zip: '{{ vault_bin }}/{{ vault_slug }}.zip' +# vault_uri: 'https://releases.hashicorp.com/vault/{{ vault_version }}/{{ vault_slug }}.zip' +# vault_cmd: '{{ vault_bin }}/vault' vault_dev_root_token_id: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' vault_test_server_https: 'https://localhost:8201' diff --git a/tests/integration/targets/setup_vault_server/handlers/main.yml b/tests/integration/targets/setup_vault_server/handlers/main.yml index 3b77e6710..3be1c63cb 100644 --- a/tests/integration/targets/setup_vault_server/handlers/main.yml +++ b/tests/integration/targets/setup_vault_server/handlers/main.yml @@ -6,4 +6,4 @@ shell: "kill $(cat {{ local_temp_dir }}/vault.pid)" ignore_errors: true listen: test_managed_vault_cleanup - when: vault_server_legacy | default(True) | bool + when: vault_integration_legacy | bool diff --git a/tests/integration/targets/setup_vault_server_download/aliases b/tests/integration/targets/setup_vault_server_download/aliases index 136c05e0d..b305f06e7 100644 --- a/tests/integration/targets/setup_vault_server_download/aliases +++ b/tests/integration/targets/setup_vault_server_download/aliases @@ -1 +1,2 @@ hidden +destructive # installls unzip From 091090d9368a98d9f143ab2b7a310d2c94580440 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 17:31:53 -0400 Subject: [PATCH 034/137] paths.. of course --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 67892f147..fdcb50cf7 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -262,7 +262,7 @@ jobs: ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} run: | - pip install -r "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/requirements/requirements.txt" -c "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/requirements/constraints.txt" + pip install -r "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/requirements.txt" -c "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/constraints.txt" ansible localhost -m include_role -a "name=setup_localenv_docker" -e vault_version=${{ matrix.vault }} - name: Run integration test From 9a17a2d4b5e81b2c7f5429b60abb61fbae246705 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 17:34:10 -0400 Subject: [PATCH 035/137] verz --- .../setup_localenv_docker/files/requirements/constraints.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/targets/setup_localenv_docker/files/requirements/constraints.txt b/tests/integration/targets/setup_localenv_docker/files/requirements/constraints.txt index 8706ac186..16f5a6645 100644 --- a/tests/integration/targets/setup_localenv_docker/files/requirements/constraints.txt +++ b/tests/integration/targets/setup_localenv_docker/files/requirements/constraints.txt @@ -1,2 +1,2 @@ -docker >= 5.0.0 ; python_version >= 3.6 -docker < 5.0.0 ; python_version == 2.7 +docker >= 5.0.0 ; python_version >= '3.6' +docker < 5.0.0 ; python_version == '2.7' From 5d9c33c70d057e078b332dfba1bb5e8209dacb1c Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 17:36:13 -0400 Subject: [PATCH 036/137] pathHHHSSSS --- .github/workflows/ansible-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index fdcb50cf7..14c0c2259 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -268,8 +268,8 @@ jobs: - name: Run integration test run: | cp "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/.." - ls -alh "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}" - cat "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/integration_config.yml" + # ls -alh "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}" + cat "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/../integration_config.yml" ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network hashi_vault_default working-directory: ${{ env.COLLECTION_PATH }} From ce2c54df3634f7efac196ab130087b2c8a62bd3c Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 17:55:38 -0400 Subject: [PATCH 037/137] the locals --- .github/workflows/ansible-test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 14c0c2259..6fe62941f 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -368,12 +368,17 @@ jobs: path: ansible_collections/community/crypto # Run the integration tests - - name: Run integration test in docker + - name: Run integration tests in docker run: | docker ps || true + cp "tests/integration/integration_config.yml.sample" "tests/integration/integration_config.yml" ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --docker #--coverage working-directory: ${{ env.COLLECTION_PATH }} + - name: Run the integration tests in a venv + run: | + ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --venv --requirements #--coverage + # ansible-test support producing code coverage date # - name: Generate coverage report # run: ansible-test coverage xml -v --requirements --group-by command --group-by version From 384107e20fbfcfc641b37a8d8bd3f4df686fbf9e Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 17:59:47 -0400 Subject: [PATCH 038/137] work it, directory --- .github/workflows/ansible-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 6fe62941f..ffb9c1c65 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -378,6 +378,7 @@ jobs: - name: Run the integration tests in a venv run: | ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --venv --requirements #--coverage + working-directory: ${{ env.COLLECTION_PATH }} # ansible-test support producing code coverage date # - name: Generate coverage report From 825f9a655d5a9076cd9b47f8d0ec7fca90522954 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 18:12:26 -0400 Subject: [PATCH 039/137] destroy! whats the diff --- .github/workflows/ansible-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index ffb9c1c65..6923536c2 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -372,12 +372,12 @@ jobs: run: | docker ps || true cp "tests/integration/integration_config.yml.sample" "tests/integration/integration_config.yml" - ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --docker #--coverage + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker #--coverage working-directory: ${{ env.COLLECTION_PATH }} - name: Run the integration tests in a venv run: | - ansible-test integration -v --color --retry-on-error --continue-on-error --diff --python ${{ matrix.python }} --venv --requirements #--coverage + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive #--coverage working-directory: ${{ env.COLLECTION_PATH }} # ansible-test support producing code coverage date From b872a56040197c57e07367c9ff3ed28461592084 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 19:32:42 -0400 Subject: [PATCH 040/137] add easyenv action --- .github/actions/easyenv/action.yml | 13 +++++++++++++ .github/actions/easyenv/easyenv.sh | 6 ++++++ .github/workflows/ansible-test.yml | 27 +++++++++++++++++++-------- 3 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 .github/actions/easyenv/action.yml create mode 100755 .github/actions/easyenv/easyenv.sh diff --git a/.github/actions/easyenv/action.yml b/.github/actions/easyenv/action.yml new file mode 100644 index 000000000..f23566482 --- /dev/null +++ b/.github/actions/easyenv/action.yml @@ -0,0 +1,13 @@ +--- +name: Easy Env Vars +description: Set many persistent env vars, which can reference previous vars +inputs: + env: + description: Multi-line string ENV_NAME=VALUE + required: true +runs: + using: composite + steps: + - shell: bash + run: |- + '${{ inputs.env }}' | easyenv.sh diff --git a/.github/actions/easyenv/easyenv.sh b/.github/actions/easyenv/easyenv.sh new file mode 100755 index 000000000..b988b387b --- /dev/null +++ b/.github/actions/easyenv/easyenv.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +while IFS= read -r line ; do + eval "$line" + __var=${line%%=*} + echo "${__var}=${!__var}" >> ${GITHUB_ENV} +done diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 6923536c2..09923cc10 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -167,22 +167,33 @@ jobs: - name: Initialize env vars run: | COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} - COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets - LOOKUP_HASHI_VAULT_PATH=${COLLECTION_INTEGRATION_TARGETS}/lookup_hashi_vault - LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin - LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars + # COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets + # LOOKUP_HASHI_VAULT_PATH=${COLLECTION_INTEGRATION_TARGETS}/lookup_hashi_vault + # LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin + # LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars echo "COLLECTION_PATH=${COLLECTION_PATH}" >> ${GITHUB_ENV} - echo "COLLECTION_INTEGRATION_TARGETS=${COLLECTION_INTEGRATION_TARGETS}" >> ${GITHUB_ENV} - echo "LOOKUP_HASHI_VAULT_PATH=${LOOKUP_HASHI_VAULT_PATH}" >> ${GITHUB_ENV} - echo "LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_BIN}" >> ${GITHUB_ENV} - echo "LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_VARS}" >> ${GITHUB_ENV} + # echo "COLLECTION_INTEGRATION_TARGETS=${COLLECTION_INTEGRATION_TARGETS}" >> ${GITHUB_ENV} + # echo "LOOKUP_HASHI_VAULT_PATH=${LOOKUP_HASHI_VAULT_PATH}" >> ${GITHUB_ENV} + # echo "LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_BIN}" >> ${GITHUB_ENV} + # echo "LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_VARS}" >> ${GITHUB_ENV} - name: Check out code uses: actions/checkout@v2 with: path: ${{ env.COLLECTION_PATH }} + - name: Initialize other env vars + uses: ${{ env.COLLECTION_PATH }}/.github/actions/easyenv + with: + env: | + COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets + LOOKUP_HASHI_VAULT_PATH=${COLLECTION_INTEGRATION_TARGETS}/lookup_hashi_vault + LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin + LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars + + - run: env sort + - name: Set up Python uses: actions/setup-python@v2 with: From 07263e5b859e750645bfd63f745382b6bef3435f Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 19:38:49 -0400 Subject: [PATCH 041/137] hmm --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 09923cc10..0814c1bd9 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -184,7 +184,7 @@ jobs: path: ${{ env.COLLECTION_PATH }} - name: Initialize other env vars - uses: ${{ env.COLLECTION_PATH }}/.github/actions/easyenv + uses: briantist/hashi_vault/.github/actions/easyenv@tests/breakup with: env: | COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets From e752322a09a3fcae5e1f5ca8e20b7802ca0c0e05 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 19:39:54 -0400 Subject: [PATCH 042/137] duh --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 0814c1bd9..da7bcf7b8 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -184,7 +184,7 @@ jobs: path: ${{ env.COLLECTION_PATH }} - name: Initialize other env vars - uses: briantist/hashi_vault/.github/actions/easyenv@tests/breakup + uses: briantist/community.hashi_vault/.github/actions/easyenv@tests/breakup with: env: | COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets From d1efacd825d6b563cc9f9bf695d57939796913c1 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 19:42:09 -0400 Subject: [PATCH 043/137] action path --- .github/actions/easyenv/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/easyenv/action.yml b/.github/actions/easyenv/action.yml index f23566482..32e3d9c98 100644 --- a/.github/actions/easyenv/action.yml +++ b/.github/actions/easyenv/action.yml @@ -10,4 +10,4 @@ runs: steps: - shell: bash run: |- - '${{ inputs.env }}' | easyenv.sh + '${{ inputs.env }}' | ${{ github.action_path }}/easyenv.sh From 0329b519dc66745ee50abd0ec698844649eb199b Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 20:06:51 -0400 Subject: [PATCH 044/137] ok --- .github/actions/easyenv/easyenv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/easyenv/easyenv.sh b/.github/actions/easyenv/easyenv.sh index b988b387b..d62722624 100755 --- a/.github/actions/easyenv/easyenv.sh +++ b/.github/actions/easyenv/easyenv.sh @@ -2,5 +2,5 @@ while IFS= read -r line ; do eval "$line" __var=${line%%=*} - echo "${__var}=${!__var}" >> ${GITHUB_ENV} + echo "${__var}=${!__var}" # >> ${GITHUB_ENV} done From cf4d948f21bcf63903a031402261cc8c2cf94a03 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 20:08:44 -0400 Subject: [PATCH 045/137] ooof --- .github/actions/easyenv/action.yml | 2 +- .github/actions/easyenv/easyenv.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/easyenv/action.yml b/.github/actions/easyenv/action.yml index 32e3d9c98..2d2750579 100644 --- a/.github/actions/easyenv/action.yml +++ b/.github/actions/easyenv/action.yml @@ -10,4 +10,4 @@ runs: steps: - shell: bash run: |- - '${{ inputs.env }}' | ${{ github.action_path }}/easyenv.sh + ${{ github.action_path }}/easyenv.sh <<<'${{ inputs.env }}' diff --git a/.github/actions/easyenv/easyenv.sh b/.github/actions/easyenv/easyenv.sh index d62722624..dfd1a7176 100755 --- a/.github/actions/easyenv/easyenv.sh +++ b/.github/actions/easyenv/easyenv.sh @@ -2,5 +2,5 @@ while IFS= read -r line ; do eval "$line" __var=${line%%=*} - echo "${__var}=${!__var}" # >> ${GITHUB_ENV} + echo "${__var}=${!__var}" >> ${GITHUB_ENV} done From 16aad32369a1c291886b5cda16226f747d44e465 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 20:09:57 -0400 Subject: [PATCH 046/137] nope --- .github/actions/easyenv/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/easyenv/action.yml b/.github/actions/easyenv/action.yml index 2d2750579..0003ca529 100644 --- a/.github/actions/easyenv/action.yml +++ b/.github/actions/easyenv/action.yml @@ -9,5 +9,4 @@ runs: using: composite steps: - shell: bash - run: |- - ${{ github.action_path }}/easyenv.sh <<<'${{ inputs.env }}' + run: echo '${{ inputs.env }}' | ${{ github.action_path }}/easyenv.sh From cf53c495fb02c331301196fee018609efbda2fd4 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 20:17:16 -0400 Subject: [PATCH 047/137] uneasy env --- .github/actions/easyenv/easyenv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/easyenv/easyenv.sh b/.github/actions/easyenv/easyenv.sh index dfd1a7176..05e1e92fb 100755 --- a/.github/actions/easyenv/easyenv.sh +++ b/.github/actions/easyenv/easyenv.sh @@ -2,5 +2,5 @@ while IFS= read -r line ; do eval "$line" __var=${line%%=*} - echo "${__var}=${!__var}" >> ${GITHUB_ENV} + echo "${__var}=${!__var}" >> "${GITHUB_ENV}" done From 6658fc5f2a1a4d19ec6d52a35d384556b5ec8ffa Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 20:18:42 -0400 Subject: [PATCH 048/137] trying --- .github/actions/easyenv/easyenv.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/easyenv/easyenv.sh b/.github/actions/easyenv/easyenv.sh index 05e1e92fb..e050983ba 100755 --- a/.github/actions/easyenv/easyenv.sh +++ b/.github/actions/easyenv/easyenv.sh @@ -2,5 +2,8 @@ while IFS= read -r line ; do eval "$line" __var=${line%%=*} + echo "$line" + echo "${__var}" + echo "${!__var}" echo "${__var}=${!__var}" >> "${GITHUB_ENV}" done From 539eca41d99e6853c54c5a338129734a41e21918 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 20:43:00 -0400 Subject: [PATCH 049/137] and now --- .github/actions/easyenv/action.yml | 2 +- .github/actions/easyenv/easyenv.sh | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/actions/easyenv/action.yml b/.github/actions/easyenv/action.yml index 0003ca529..a02b50ead 100644 --- a/.github/actions/easyenv/action.yml +++ b/.github/actions/easyenv/action.yml @@ -9,4 +9,4 @@ runs: using: composite steps: - shell: bash - run: echo '${{ inputs.env }}' | ${{ github.action_path }}/easyenv.sh + run: ${{ github.action_path }}/easyenv.sh <<<'${{ inputs.env }}' diff --git a/.github/actions/easyenv/easyenv.sh b/.github/actions/easyenv/easyenv.sh index e050983ba..a8c6d08d9 100755 --- a/.github/actions/easyenv/easyenv.sh +++ b/.github/actions/easyenv/easyenv.sh @@ -1,9 +1,8 @@ #!/usr/bin/env bash while IFS= read -r line ; do - eval "$line" - __var=${line%%=*} - echo "$line" - echo "${__var}" - echo "${!__var}" - echo "${__var}=${!__var}" >> "${GITHUB_ENV}" + if [[ "$line" =~ [^[:space:]] ]] ; then + eval "$line" + __var=${line%%=*} + echo "${__var}=${!__var}" >> "${GITHUB_ENV}" + fi done From 2100101c1944cae44417e4e75652a2376a053fe0 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 22:29:00 -0400 Subject: [PATCH 050/137] -ezenvA B C --- .github/actions/easyenv/action.yml | 12 ------------ .github/actions/easyenv/easyenv.sh | 8 -------- .github/workflows/ansible-test.yml | 2 +- 3 files changed, 1 insertion(+), 21 deletions(-) delete mode 100644 .github/actions/easyenv/action.yml delete mode 100755 .github/actions/easyenv/easyenv.sh diff --git a/.github/actions/easyenv/action.yml b/.github/actions/easyenv/action.yml deleted file mode 100644 index a02b50ead..000000000 --- a/.github/actions/easyenv/action.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -name: Easy Env Vars -description: Set many persistent env vars, which can reference previous vars -inputs: - env: - description: Multi-line string ENV_NAME=VALUE - required: true -runs: - using: composite - steps: - - shell: bash - run: ${{ github.action_path }}/easyenv.sh <<<'${{ inputs.env }}' diff --git a/.github/actions/easyenv/easyenv.sh b/.github/actions/easyenv/easyenv.sh deleted file mode 100755 index a8c6d08d9..000000000 --- a/.github/actions/easyenv/easyenv.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -while IFS= read -r line ; do - if [[ "$line" =~ [^[:space:]] ]] ; then - eval "$line" - __var=${line%%=*} - echo "${__var}=${!__var}" >> "${GITHUB_ENV}" - fi -done diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index da7bcf7b8..83214751f 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -184,7 +184,7 @@ jobs: path: ${{ env.COLLECTION_PATH }} - name: Initialize other env vars - uses: briantist/community.hashi_vault/.github/actions/easyenv@tests/breakup + uses: briantist/ezenv@main with: env: | COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets From c6536de38ea479414271dd977fac88fdf41ead1e Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 22:32:56 -0400 Subject: [PATCH 051/137] use v1 --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 83214751f..a820a0083 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -184,7 +184,7 @@ jobs: path: ${{ env.COLLECTION_PATH }} - name: Initialize other env vars - uses: briantist/ezenv@main + uses: briantist/ezenv@v1 with: env: | COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets From b2aaa1b0163e80d9db96dbd25afa1d2ce194706e Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 22:54:37 -0400 Subject: [PATCH 052/137] =?UTF-8?q?=F0=9F=98=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ansible-test.yml | 63 +++++++++++++++--------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index a820a0083..c7dd2767f 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -143,7 +143,7 @@ jobs: - 3.8 - 3.9 vault: - - 1.7.3 + # - 1.7.3 - 1.6.5 exclude: - ansible: stable-2.9 @@ -165,34 +165,36 @@ jobs: steps: - name: Initialize env vars - run: | - COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} - # COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets - # LOOKUP_HASHI_VAULT_PATH=${COLLECTION_INTEGRATION_TARGETS}/lookup_hashi_vault - # LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin - # LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars - - echo "COLLECTION_PATH=${COLLECTION_PATH}" >> ${GITHUB_ENV} - # echo "COLLECTION_INTEGRATION_TARGETS=${COLLECTION_INTEGRATION_TARGETS}" >> ${GITHUB_ENV} - # echo "LOOKUP_HASHI_VAULT_PATH=${LOOKUP_HASHI_VAULT_PATH}" >> ${GITHUB_ENV} - # echo "LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_BIN}" >> ${GITHUB_ENV} - # echo "LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_VARS}" >> ${GITHUB_ENV} - - - name: Check out code - uses: actions/checkout@v2 - with: - path: ${{ env.COLLECTION_PATH }} - - - name: Initialize other env vars uses: briantist/ezenv@v1 with: env: | - COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets + COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} + COLLECTION_INTEGRATION_PATH=${COLLECTION_PATH}/tests/integration + COLLECTION_INTEGRATION_TARGETS=${COLLECTION_INTEGRATION_PATH}/targets LOOKUP_HASHI_VAULT_PATH=${COLLECTION_INTEGRATION_TARGETS}/lookup_hashi_vault LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars - - run: env sort + # run: | + # COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} + # # COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets + # # LOOKUP_HASHI_VAULT_PATH=${COLLECTION_INTEGRATION_TARGETS}/lookup_hashi_vault + # # LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin + # # LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars + + # echo "COLLECTION_PATH=${COLLECTION_PATH}" >> ${GITHUB_ENV} + # # echo "COLLECTION_INTEGRATION_TARGETS=${COLLECTION_INTEGRATION_TARGETS}" >> ${GITHUB_ENV} + # # echo "LOOKUP_HASHI_VAULT_PATH=${LOOKUP_HASHI_VAULT_PATH}" >> ${GITHUB_ENV} + # # echo "LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_BIN}" >> ${GITHUB_ENV} + # # echo "LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_VARS}" >> ${GITHUB_ENV} + + - name: Check out code + uses: actions/checkout@v2 + # with: + # path: ${{ env.COLLECTION_PATH }} + + - name: Linkit + run: ln -s "${COLLECTION_PATH}" $(pwd) - name: Set up Python uses: actions/setup-python@v2 @@ -320,16 +322,13 @@ jobs: steps: - name: Initialize env vars - run: | - COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} - LOOKUP_HASHI_VAULT_PATH=${COLLECTION_PATH}/tests/integration/targets/lookup_hashi_vault - LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin - LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars - - echo "COLLECTION_PATH=${COLLECTION_PATH}" >> ${GITHUB_ENV} - echo "LOOKUP_HASHI_VAULT_PATH=${LOOKUP_HASHI_VAULT_PATH}" >> ${GITHUB_ENV} - echo "LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_BIN}" >> ${GITHUB_ENV} - echo "LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_VARS}" >> ${GITHUB_ENV} + uses: briantist/ezenv@v1 + with: + env: | + COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} + LOOKUP_HASHI_VAULT_PATH=${COLLECTION_PATH}/tests/integration/targets/lookup_hashi_vault + LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin + LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars - name: Check out code uses: actions/checkout@v2 From 1ce06d47bda7b0869806df10e1e43cf826a98727 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sat, 3 Jul 2021 23:11:25 -0400 Subject: [PATCH 053/137] squeakle --- .github/workflows/ansible-test.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index c7dd2767f..be0741ff3 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -190,11 +190,11 @@ jobs: - name: Check out code uses: actions/checkout@v2 - # with: - # path: ${{ env.COLLECTION_PATH }} + with: + path: ${{ env.COLLECTION_PATH }} - name: Linkit - run: ln -s "${COLLECTION_PATH}" $(pwd) + run: ln -s "${COLLECTION_PATH}/.github" .github - name: Set up Python uses: actions/setup-python@v2 @@ -275,15 +275,12 @@ jobs: ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} run: | - pip install -r "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/requirements.txt" -c "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/constraints.txt" + pip install -r "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/requirements.txt" -c "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/constraints.txt" ansible localhost -m include_role -a "name=setup_localenv_docker" -e vault_version=${{ matrix.vault }} + cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_PATH}" - name: Run integration test run: | - cp "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/.." - # ls -alh "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}" - cat "${{ github.workspace }}/${COLLECTION_INTEGRATION_TARGETS}/../integration_config.yml" - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network hashi_vault_default working-directory: ${{ env.COLLECTION_PATH }} From f63a59172cc8fee7885d2fedfc73f6489eb41db9 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 5 Jul 2021 13:31:06 -0400 Subject: [PATCH 054/137] rework --- .../targets/setup_localenv_docker/aliases | 1 + .../setup_localenv_docker/defaults/main.yml | 3 ++ .../setup_localenv_docker/tasks/docker.yml | 13 +++++++ .../setup_localenv_docker/tasks/main.yml | 39 +++++++++++-------- .../targets/setup_localenv_gha/aliases | 1 + .../setup_localenv_gha/defaults/main.yml | 3 ++ .../targets/setup_localenv_gha/tasks/main.yml | 13 +++++++ .../setup_localenv_gha/templates/launch.sh.j2 | 2 + 8 files changed, 59 insertions(+), 16 deletions(-) create mode 100644 tests/integration/targets/setup_localenv_docker/tasks/docker.yml create mode 100644 tests/integration/targets/setup_localenv_gha/defaults/main.yml create mode 100644 tests/integration/targets/setup_localenv_gha/tasks/main.yml create mode 100644 tests/integration/targets/setup_localenv_gha/templates/launch.sh.j2 diff --git a/tests/integration/targets/setup_localenv_docker/aliases b/tests/integration/targets/setup_localenv_docker/aliases index 136c05e0d..fbc7f5062 100644 --- a/tests/integration/targets/setup_localenv_docker/aliases +++ b/tests/integration/targets/setup_localenv_docker/aliases @@ -1 +1,2 @@ hidden +needs/target/setup_vault_server_cert diff --git a/tests/integration/targets/setup_localenv_docker/defaults/main.yml b/tests/integration/targets/setup_localenv_docker/defaults/main.yml index 4356372cf..f02deec5f 100644 --- a/tests/integration/targets/setup_localenv_docker/defaults/main.yml +++ b/tests/integration/targets/setup_localenv_docker/defaults/main.yml @@ -4,6 +4,7 @@ vault_version: '1.7.3' docker_compose: clean # up # down +# none docker_compose_project_name: hashi_vault @@ -17,6 +18,8 @@ proxy_container_name: tinyproxy output_dir: '{{ role_path }}/files/.output' docker_compose_output: '{{ output_dir }}/{{ docker_compose_project_name }}' +docker_compose_file: '{{ docker_compose_output }}/docker-compose.yml' + vault_config_output: '{{ output_dir }}/vault_config' vault_cert_file: '{{ vault_config_output }}/cert.pem' diff --git a/tests/integration/targets/setup_localenv_docker/tasks/docker.yml b/tests/integration/targets/setup_localenv_docker/tasks/docker.yml new file mode 100644 index 000000000..dc7e0ea63 --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/tasks/docker.yml @@ -0,0 +1,13 @@ +--- +- name: "Shut down" + when: docker_compose in ['clean', 'down'] + community.docker.docker_compose: + project_name: '{{ docker_compose_project_name }}' + state: absent + project_src: '{{ docker_compose_output }}' + +- name: "Bring up" + when: docker_compose in ['clean', 'up'] + community.docker.docker_compose: + project_name: '{{ docker_compose_project_name }}' + project_src: '{{ docker_compose_output }}' diff --git a/tests/integration/targets/setup_localenv_docker/tasks/main.yml b/tests/integration/targets/setup_localenv_docker/tasks/main.yml index 6d3f62ead..d97ae221e 100644 --- a/tests/integration/targets/setup_localenv_docker/tasks/main.yml +++ b/tests/integration/targets/setup_localenv_docker/tasks/main.yml @@ -10,15 +10,34 @@ - name: "Create the docker-compose definition" template: src: docker-compose.yml.j2 - dest: '{{ docker_compose_output }}/docker-compose.yml' + dest: '{{ docker_compose_file }}' - name: "Persist vars (role_path is relative)" set_fact: vault_cert_file: '{{ vault_cert_file }}' vault_key_file: '{{ vault_key_file }}' +- name: "Check if cert already exists" + stat: + path: '{{ vault_cert_file }}' + follow: yes + get_attributes: no + get_checksum: no + get_mime: no + register: cert_status + +- name: "Check if key already exists" + stat: + path: '{{ vault_key_file }}' + follow: yes + get_attributes: no + get_checksum: no + get_mime: no + register: key_status + - name: "Generate certs" - import_role: + when: not (key_status.stat.exists and cert_status.stat.exists) + include_role: name: setup_vault_server_cert vars: vault_hostname: '{{ vault_container_name }}' @@ -28,20 +47,8 @@ src: vault_config.hcl.j2 dest: '{{ vault_config_output }}/vault_config.hcl' -- name: "Shut down" - when: docker_compose in ['clean', 'down'] - community.docker.docker_compose: - project_name: hashi_vault - state: absent - # definition: '{{ docker_compose_definition }}' - project_src: '{{ docker_compose_output }}' - -- name: "Bring up" - when: docker_compose in ['clean', 'up'] - community.docker.docker_compose: - project_name: hashi_vault - # definition: '{{ docker_compose_definition }}' - project_src: '{{ docker_compose_output }}' +- include_tasks: docker.yml + when: docker_compose != 'none' - name: "Template integration_config" template: diff --git a/tests/integration/targets/setup_localenv_gha/aliases b/tests/integration/targets/setup_localenv_gha/aliases index 136c05e0d..80fa188f1 100644 --- a/tests/integration/targets/setup_localenv_gha/aliases +++ b/tests/integration/targets/setup_localenv_gha/aliases @@ -1 +1,2 @@ hidden +needs/target/setup_localenv_docker diff --git a/tests/integration/targets/setup_localenv_gha/defaults/main.yml b/tests/integration/targets/setup_localenv_gha/defaults/main.yml new file mode 100644 index 000000000..615ddd6ee --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/defaults/main.yml @@ -0,0 +1,3 @@ +--- +output_dir: '{{ role_path }}/files/.output' +docker_compose_project_name: hashi_vault diff --git a/tests/integration/targets/setup_localenv_gha/tasks/main.yml b/tests/integration/targets/setup_localenv_gha/tasks/main.yml new file mode 100644 index 000000000..b44a308dc --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: "Force output path to be in this role (which has existing files)" + set_fact: + output_dir: '{{ output_dir }}' + +- import_role: + name: setup_localenv_docker + +- name: "Template the launch script" + template: + src: launch.sh.j2 + dest: '{{ output_dir }}/launch.sh' + mode: '+x' diff --git a/tests/integration/targets/setup_localenv_gha/templates/launch.sh.j2 b/tests/integration/targets/setup_localenv_gha/templates/launch.sh.j2 new file mode 100644 index 000000000..6b89f2f26 --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/templates/launch.sh.j2 @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +docker-compose -f "{{ docker_compose_file }}" up -d From 748dd4d16fc3a78b6bbcdb90de1e0d6dd432b587 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 5 Jul 2021 13:33:33 -0400 Subject: [PATCH 055/137] docker none --- tests/integration/targets/setup_localenv_gha/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/targets/setup_localenv_gha/defaults/main.yml b/tests/integration/targets/setup_localenv_gha/defaults/main.yml index 615ddd6ee..5f04a155c 100644 --- a/tests/integration/targets/setup_localenv_gha/defaults/main.yml +++ b/tests/integration/targets/setup_localenv_gha/defaults/main.yml @@ -1,3 +1,4 @@ --- output_dir: '{{ role_path }}/files/.output' docker_compose_project_name: hashi_vault +docker_compose: none From 18bb3e0745e888b966af3097f358e6269258b786 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 5 Jul 2021 13:36:23 -0400 Subject: [PATCH 056/137] use gha --- .github/workflows/ansible-test.yml | 77 ++++++++++++++++-------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index be0741ff3..66944d94c 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -220,46 +220,46 @@ jobs: - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check - #TODO: remove dependency on community.crypto - - name: Find the latest version of community.crypto & community.docker - run: | - # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 + # #TODO: remove dependency on community.crypto + # - name: Find the latest version of community.crypto & community.docker + # run: | + # # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 - cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) - echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV + # cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) + # echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV - cdlu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.docker/releases/latest) - echo "CD_LATEST=$(basename ${cdlu})" >> $GITHUB_ENV + # cdlu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.docker/releases/latest) + # echo "CD_LATEST=$(basename ${cdlu})" >> $GITHUB_ENV - - name: Cache for community.crypto - id: cache-cc - uses: actions/cache@v2 - with: - path: ansible_collections/community/crypto - key: collection-community.crypto-${{ env.CC_LATEST }} + # - name: Cache for community.crypto + # id: cache-cc + # uses: actions/cache@v2 + # with: + # path: ansible_collections/community/crypto + # key: collection-community.crypto-${{ env.CC_LATEST }} - - name: Cache for community.docker - id: cache-cd - uses: actions/cache@v2 - with: - path: ansible_collections/community/docker - key: collection-community.docker-${{ env.CD_LATEST }} + # - name: Cache for community.docker + # id: cache-cd + # uses: actions/cache@v2 + # with: + # path: ansible_collections/community/docker + # key: collection-community.docker-${{ env.CD_LATEST }} - - name: Install community.crypto - if: steps.cache-cc.outputs.cache-hit != 'true' - uses: actions/checkout@v2 - with: - repository: ansible-collections/community.crypto - ref: refs/tags/${{ env.CC_LATEST }} - path: ansible_collections/community/crypto + # - name: Install community.crypto + # if: steps.cache-cc.outputs.cache-hit != 'true' + # uses: actions/checkout@v2 + # with: + # repository: ansible-collections/community.crypto + # ref: refs/tags/${{ env.CC_LATEST }} + # path: ansible_collections/community/crypto - - name: Install community.docker - if: steps.cache-cd.outputs.cache-hit != 'true' - uses: actions/checkout@v2 - with: - repository: ansible-collections/community.docker - ref: refs/tags/${{ env.CD_LATEST }} - path: ansible_collections/community/docker + # - name: Install community.docker + # if: steps.cache-cd.outputs.cache-hit != 'true' + # uses: actions/checkout@v2 + # with: + # repository: ansible-collections/community.docker + # ref: refs/tags/${{ env.CD_LATEST }} + # path: ansible_collections/community/docker # this will populate files/bin with the selected vault version binary # - name: Pre-download Vault @@ -275,9 +275,12 @@ jobs: ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} run: | - pip install -r "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/requirements.txt" -c "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/constraints.txt" - ansible localhost -m include_role -a "name=setup_localenv_docker" -e vault_version=${{ matrix.vault }} - cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_PATH}" + # pip install -r "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/requirements.txt" -c "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/constraints.txt" + # ansible localhost -m include_role -a "name=setup_localenv_docker" -e vault_version=${{ matrix.vault }} + ansible localhost -m include_role -a "name=setup_localenv_gha" -e vault_version=${{ matrix.vault }} + # cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_PATH}" + cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_gha/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_PATH}" + "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_gha/files/.output/launch.sh" - name: Run integration test run: | From e3144e988c6af900630df48d2120c5dc464c7c15 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 5 Jul 2021 13:40:07 -0400 Subject: [PATCH 057/137] try to get local working i guess --- .../targets/setup_vault_server_download/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/targets/setup_vault_server_download/tasks/main.yml b/tests/integration/targets/setup_vault_server_download/tasks/main.yml index 4d7f2d78a..3e1949b97 100644 --- a/tests/integration/targets/setup_vault_server_download/tasks/main.yml +++ b/tests/integration/targets/setup_vault_server_download/tasks/main.yml @@ -20,6 +20,7 @@ # All this just for unzip, which is only needed to unzip the vault binary to set up for testing. # TODO: revisit how we set up vault in the first place or how we host the binary (.gz?) - name: 'Install unzip' + become: yes vars: # by assuming python3 here we're probably condeming this to not work on older Ubuntu/Debian (from like 2014?) # but the alternative is probably reimplementing parts of interpreter_discovery.py @@ -28,7 +29,7 @@ }}" package: name: unzip - # when: ansible_distribution != "MacOSX" # unzip already installed (#TODO: get MacOSX tests working again) + when: ansible_distribution != "MacOSX" # unzip already installed (#TODO: get MacOSX tests working again) - name: "Create bin directory" file: From 3ef29f3cf200b46d84da053617327b5ec9b56924 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 5 Jul 2021 13:55:22 -0400 Subject: [PATCH 058/137] oop --- .../tasks/tinyproxy_server.yml | 1 + .../setup_localenv_gha/files/.gitignore | 2 + .../files/.output/integration_config.yml | 37 ------------------- .../.output/vault_config/vault_config.hcl | 6 --- .../targets/setup_localenv_gha/tasks/main.yml | 4 +- 5 files changed, 6 insertions(+), 44 deletions(-) create mode 100644 tests/integration/targets/setup_localenv_gha/files/.gitignore delete mode 100644 tests/integration/targets/setup_localenv_gha/files/.output/integration_config.yml delete mode 100644 tests/integration/targets/setup_localenv_gha/files/.output/vault_config/vault_config.hcl diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml b/tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml index 7b7091f18..5b6a0fcbc 100644 --- a/tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml +++ b/tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml @@ -1,5 +1,6 @@ --- - name: Install tinyproxy + become: yes vars: # check 'Install unzip' task to know why we set ansible_python_interpreter ansible_python_interpreter: "{{ diff --git a/tests/integration/targets/setup_localenv_gha/files/.gitignore b/tests/integration/targets/setup_localenv_gha/files/.gitignore new file mode 100644 index 000000000..bcb1622af --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/files/.gitignore @@ -0,0 +1,2 @@ +.output/ +!.output/vault_config/*.pem diff --git a/tests/integration/targets/setup_localenv_gha/files/.output/integration_config.yml b/tests/integration/targets/setup_localenv_gha/files/.output/integration_config.yml deleted file mode 100644 index d874e7290..000000000 --- a/tests/integration/targets/setup_localenv_gha/files/.output/integration_config.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -vault_dev_root_token_id: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a -vault_proxy_external: true -vault_proxy_server: http://tinyproxy:8888 -vault_test_server_external: true -vault_test_server_http: http://vault:8200 -vault_test_server_https: https://vault:8300 -vault_cert_content: | - -----BEGIN CERTIFICATE----- - MIIE4TCCAsmgAwIBAgIUIL77ChgfiJcegZViZh8TV9KviB8wDQYJKoZIhvcNAQEL - BQAwEDEOMAwGA1UEAwwFdmF1bHQwHhcNMjEwNzAxMTgyNjIzWhcNMzEwNjI5MTgy - NjIzWjAQMQ4wDAYDVQQDDAV2YXVsdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC - AgoCggIBAK1mI97qE0Pijd0vYrepaoZen/7yuayVslJ1uKzNQWCwHCpaAG9QuGof - ym2N4UnCnBAmpcOH8N+xVzFR5oQVJwYLodfPeI8VDcHvN6Cj011Lb40o2fAbQz+v - 2Xz4MOpEzRbCrSlMZa7M10+iiZJpU379Yqn6JDiCkwrq4dIBVUnp8Wg5ykV+qdUz - ypB6jJF67WRisXqYp7hACL67FRlj/r8+76FSUg/oAo7g0rkrahWb+SvNMPWS1hS7 - Tk8Tjf6qb2ZO2Iwx0eEY2GNoziUzQu1xTkZQhbQt3vt8ZiWvddCRxCz5W1cfKWO/ - 0XQlMlqlW2RmfxEouMUqhQlH1NSYCKas12RFxxSaiPI2Idq2Kve2xDLwQuIOoCLD - joqSQmyF88f9Jxb0l2sXXsro/YwmYJ+qwz81QW4BO5LzX7BHT7EuiV97m8kLuBzE - 0pjat0XVJ9fzmfGYX89uiwnO/fb7jZubjQLus0cmZXHMB6wK0fo8bTXgi9TLXwbi - wymDG+A+jlSShf7aE3vZtr6fRcacjwh6Y6DFbfxdVV9Vxzv1aHaMUYwtIu+d+uVC - cTQbwou6B7hS0BUdXQKtM1mjPDAwcdgz//TLzk26tIsx69AXtOREKb9W1ffIbfGh - B8nrdmI6+80tlc8KL6s+/cMLEMya3K2GZPloAw+CO00ihO7SGC6VAgMBAAGjMzAx - MBAGA1UdEQQJMAeCBXZhdWx0MB0GA1UdDgQWBBRbYPX83fHK2QUTsWExMQESmS6B - yTANBgkqhkiG9w0BAQsFAAOCAgEAc9slz9up7xd3bsr+q/kCoDt+w6rm/dc6ONSJ - PITZAbuWtRBtCJStQuie5ZICnh1X0IajhczIFVcD9CjxOIxfxA7S49gL9vDHVpiJ - K4nW0KR3Zviq2XwtHYAs99CZH63EUTVqz0nEuMu10H/0PCFPtTHcXFpgovCLRAGH - HqnM7LVeM7a0g85Zt+HXuPJ2MThlEyIBy64MBPIczSiGDVx0cQwe1LJREkQJgB4F - +3iAOPIsHAWkApFfx2cyq+L4sEd0EdxUFk1mw4sni/VFzK8wcd3L7gEgseKSk2Kz - Z3JZiAXessjoa26JL0/KBSN6LTB3/pdn/dG7lz7DUr17PymbggRLVjdbSsbAFD9S - BTGgt3kFjrXIeNAyqGodK673R5jACXjz0vuEePJh4Vk/ffl953bH8Xhs/BZyNXBC - meOu/sU93MGPi0vqE+Jdjplvj5smLzOW9Y6HAAitDqHTQ0sNdZP2DdT6FkSLSzMx - ErPFBePgxhQFXvQt8h11Gadox5vsm1Ca2nLGClKWRt458goFEWgmmDA+mOD2/sJ0 - eYdGfBgN/ZnTzD2y2z18Sd9H2Zb4HZbfpPvvFwR+5oYMqE72Rz3oj8APt2f/Eq5B - WYNpi1fRCGPXhM2wNr2DKPKcoqbRcIVTxD/E0MATmxQRGtPaO/JbFcY5v37qcLWf - Jb3iD/U= - -----END CERTIFICATE----- diff --git a/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/vault_config.hcl b/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/vault_config.hcl deleted file mode 100644 index e39bb1470..000000000 --- a/tests/integration/targets/setup_localenv_gha/files/.output/vault_config/vault_config.hcl +++ /dev/null @@ -1,6 +0,0 @@ -listener "tcp" { - tls_key_file = "/vault/config/key.pem" - tls_cert_file = "/vault/config/cert.pem" - tls_disable = false - address = "vault:8300" -} diff --git a/tests/integration/targets/setup_localenv_gha/tasks/main.yml b/tests/integration/targets/setup_localenv_gha/tasks/main.yml index b44a308dc..f241aafaa 100644 --- a/tests/integration/targets/setup_localenv_gha/tasks/main.yml +++ b/tests/integration/targets/setup_localenv_gha/tasks/main.yml @@ -1,7 +1,9 @@ --- -- name: "Force output path to be in this role (which has existing files)" +- name: "Persist defaults" set_fact: output_dir: '{{ output_dir }}' + docker_compose: '{{ docker_compose }}' + docker_compose_project_name: '{{ docker_compose_project_name }}' - import_role: name: setup_localenv_docker From 9896b49acf678cb26e51ecd4e0fc8c352a3d6b12 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 5 Jul 2021 14:17:32 -0400 Subject: [PATCH 059/137] fixmemememe --- .../targets/lookup_hashi_vault/handlers/main.yml | 2 ++ .../setup_vault_server/tasks/vault_server.yml | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml index 8f5038dc8..f2a8d4951 100644 --- a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml @@ -15,6 +15,7 @@ removes: "{{ local_temp_dir }}/tinyproxy.pid" - name: Uninstall tinyproxy + become: yes vars: # check 'Install unzip' task to know why we set ansible_python_interpreter ansible_python_interpreter: "{{ @@ -23,6 +24,7 @@ package: name: tinyproxy state: absent + ignore_errors: yes # notify 'cleanup' for any handlers that should always run at the end of tests - name: 'Delete temp dir' diff --git a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml index a333494db..42033323d 100644 --- a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml +++ b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml @@ -61,3 +61,18 @@ - name: 'Start vault server (dev mode enabled)' shell: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl /dev/null 2>&1 &' notify: test_managed_vault_cleanup + + - name: 'Ensure it succeeded' + block: + - name: 'Check Vault status' + shell: '{{ vault_cmd }} status' + + rescue: + - name: 'dump the config' + debug: + msg: "{{ lookup('file', local_temp_dir ~ '/vault_config.hcl') }}" + + # vault wasn't up, let's run the launch command with output, which we expect to fail + - name: 'Re-launch Vault' + shell: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl' + notify: test_managed_vault_cleanup From b1da5f788d9fe4cf5fcd07a873fe9405ea4ee7dd Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 5 Jul 2021 14:23:01 -0400 Subject: [PATCH 060/137] env --- .../targets/setup_vault_server/tasks/vault_server.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml index 42033323d..253059b05 100644 --- a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml +++ b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml @@ -65,6 +65,8 @@ - name: 'Ensure it succeeded' block: - name: 'Check Vault status' + environment: + VAULT_ADDR: '{{ vault_test_server_http }}' shell: '{{ vault_cmd }} status' rescue: From 8f0a725dd5e670cbda62f6319f04adb13813b403 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 5 Jul 2021 14:38:56 -0400 Subject: [PATCH 061/137] macoses --- tests/integration/targets/lookup_hashi_vault/handlers/main.yml | 2 +- .../targets/lookup_hashi_vault/tasks/tinyproxy_server.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml index f2a8d4951..c9da1d10d 100644 --- a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml @@ -15,7 +15,7 @@ removes: "{{ local_temp_dir }}/tinyproxy.pid" - name: Uninstall tinyproxy - become: yes + become: '{{ ansible_distribution != "MacOSX" }}' vars: # check 'Install unzip' task to know why we set ansible_python_interpreter ansible_python_interpreter: "{{ diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml b/tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml index 5b6a0fcbc..ca53f0121 100644 --- a/tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml +++ b/tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml @@ -1,6 +1,6 @@ --- - name: Install tinyproxy - become: yes + become: '{{ ansible_distribution != "MacOSX" }}' vars: # check 'Install unzip' task to know why we set ansible_python_interpreter ansible_python_interpreter: "{{ From 8cca27123e25bd475ebd9ed3659f5570d85bd4d6 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 5 Jul 2021 14:39:53 -0400 Subject: [PATCH 062/137] retries --- .../targets/setup_vault_server/tasks/vault_server.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml index 253059b05..6f679cfcc 100644 --- a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml +++ b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml @@ -68,6 +68,8 @@ environment: VAULT_ADDR: '{{ vault_test_server_http }}' shell: '{{ vault_cmd }} status' + retries: 10 + delay: 1 rescue: - name: 'dump the config' From 1c44a33ac5354f2fc90987da0d0e5c8873b2dc2a Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 5 Jul 2021 14:58:25 -0400 Subject: [PATCH 063/137] cg macos --- .github/workflows/ansible-test.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 66944d94c..8a20ccc03 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -362,6 +362,9 @@ jobs: cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV + cglu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.general/releases/latest) + echo "CG_LATEST=$(basename ${cglu})" >> $GITHUB_ENV + - name: Cache for community.crypto id: cache-cc uses: actions/cache@v2 @@ -377,6 +380,14 @@ jobs: ref: refs/tags/${{ env.CC_LATEST }} path: ansible_collections/community/crypto + - name: Install collection dependencies + # if: steps.cache-cc.outputs.cache-hit != 'true' + uses: actions/checkout@v2 + with: + repository: ansible-collections/community.general + ref: refs/tags/${{ env.CG_LATEST }} + path: ansible_collections/community/general + # Run the integration tests - name: Run integration tests in docker run: | From 83bd614741c8aaaafb6862ac372332f1f441d7a7 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 6 Jul 2021 15:11:47 -0400 Subject: [PATCH 064/137] fixyfixy --- .github/workflows/ansible-test.yml | 37 +++++++++++++++++++ tests/integration/scripts/hv-test.sh | 1 - .../targets/lookup_hashi_vault/aliases | 1 + .../lookup_hashi_vault/handlers/main.yml | 7 ++++ .../targets/lookup_hashi_vault/tasks/main.yml | 2 - .../templates/vault_config.hcl.j2 | 10 ----- .../setup_vault_server/defaults/main.yml | 10 +++-- .../setup_vault_server/handlers/main.yml | 2 +- .../setup_vault_server/tasks/vault_server.yml | 11 +++--- .../templates/vault_config.hcl.j2 | 2 +- 10 files changed, 60 insertions(+), 23 deletions(-) delete mode 100644 tests/integration/scripts/hv-test.sh delete mode 100644 tests/integration/targets/lookup_hashi_vault/templates/vault_config.hcl.j2 diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 8a20ccc03..c4127e17b 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -351,6 +351,43 @@ jobs: docker-machine env default docker-machine env default | sed 's/^export //;/^#/d;s/^#.*//' | tr -d '"' >> $GITHUB_ENV + - name: Fix forks/maxfiles on MacOS + if: ${{ matrix.runner == 'macos-latest' }} + env: + PLIST: /Library/LaunchDaemons/limit.maxfiles.plist + MAXFILES: 524588 + run: | + ulimit -S -n $MAXFILES + # sudo launchctl limit maxfiles $MAXFILES $MAXFILES + # cat >${PLIST} < + # + # + # + # Label + # limit.maxfiles + # ProgramArguments + # + # launchctl + # limit + # maxfiles + # 524288 + # 524288 + # + # RunAtLoad + # + # ServiceIPC + # + # + # + # EOF + + # sudo chmod 644 ${PLIST} + + + + - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check diff --git a/tests/integration/scripts/hv-test.sh b/tests/integration/scripts/hv-test.sh deleted file mode 100644 index f1f641af1..000000000 --- a/tests/integration/scripts/hv-test.sh +++ /dev/null @@ -1 +0,0 @@ -#!/usr/bin/env bash diff --git a/tests/integration/targets/lookup_hashi_vault/aliases b/tests/integration/targets/lookup_hashi_vault/aliases index 953081c26..4bccb5550 100644 --- a/tests/integration/targets/lookup_hashi_vault/aliases +++ b/tests/integration/targets/lookup_hashi_vault/aliases @@ -5,3 +5,4 @@ skip/aix skip/python2.6 # lookups are controller only, and we no longer support Python 2.6 on the controller skip/macos # FIXME seems to be always unstable needs/target/setup_vault_server +needs/target/setup_localenv_gha diff --git a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml index c9da1d10d..0602fc008 100644 --- a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml @@ -6,6 +6,13 @@ # shell: "kill $(cat {{ local_temp_dir }}/vault.pid)" # ignore_errors: true # listen: test_managed_vault_cleanup +- shell: ls -alh /tmp + listen: test_managed_vault_cleanup + +- shell: "cat {{ vault_pid_file }}" + listen: test_managed_vault_cleanup + ignore_errors: yes + - name: Stop tinyproxy shell: diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml index 3cab37978..588385bd3 100644 --- a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml @@ -13,8 +13,6 @@ - set_fact: local_temp_dir: '{{ tempfile_result.path }}' -# - include_tasks: vault_server.yml -# when: not vault_test_server_external | bool - include_role: name: setup_vault_server when: vault_integration_legacy | bool diff --git a/tests/integration/targets/lookup_hashi_vault/templates/vault_config.hcl.j2 b/tests/integration/targets/lookup_hashi_vault/templates/vault_config.hcl.j2 deleted file mode 100644 index 26bf47f15..000000000 --- a/tests/integration/targets/lookup_hashi_vault/templates/vault_config.hcl.j2 +++ /dev/null @@ -1,10 +0,0 @@ -# {{ ansible_managed }} -pid_file = "{{ local_temp_dir }}/vault.pid" -{% if vault_run_https_tests | bool %} -listener "tcp" { - tls_key_file = "{{ vault_key_file }}" - tls_cert_file = "{{ vault_cert_file }}" - tls_disable = false - address = "{{ vault_test_server_https | regex_replace('^https://([^:]+):(\\d+).*?$', '\\1:\\2') }}" -} -{% endif %} diff --git a/tests/integration/targets/setup_vault_server/defaults/main.yml b/tests/integration/targets/setup_vault_server/defaults/main.yml index 97a1b9a32..d13e6e398 100644 --- a/tests/integration/targets/setup_vault_server/defaults/main.yml +++ b/tests/integration/targets/setup_vault_server/defaults/main.yml @@ -29,6 +29,10 @@ vault_test_server_configure: True # when False the tests requiring a valid SSL connection to Vault will be skipped vault_run_https_tests: True -local_temp_dir: /tmp -vault_cert_file: '{{ local_temp_dir }}/cert.pem' -vault_key_file: '{{ local_temp_dir }}/privatekey.pem' +vault_server_temp_dir: /tmp +vault_cert_file: '{{ vault_server_temp_dir }}/cert.pem' +vault_key_file: '{{ vault_server_temp_dir }}/privatekey.pem' +vault_config_file: '{{ vault_server_temp_dir }}/vault_config.hcl' +vault_pid_file: '{{ vault_server_temp_dir }}/vault.pid' + +vault_launch_cmd: 'nohup {{ vault_cmd }} server -dev -config {{ vault_config_file }}' diff --git a/tests/integration/targets/setup_vault_server/handlers/main.yml b/tests/integration/targets/setup_vault_server/handlers/main.yml index 3be1c63cb..2292ac4cb 100644 --- a/tests/integration/targets/setup_vault_server/handlers/main.yml +++ b/tests/integration/targets/setup_vault_server/handlers/main.yml @@ -3,7 +3,7 @@ # vault server that is started by these tests # (those tasks should skip if the vault server is external to the test run) - name: 'Kill vault process' - shell: "kill $(cat {{ local_temp_dir }}/vault.pid)" + shell: "kill $(cat {{ vault_pid_file }})" ignore_errors: true listen: test_managed_vault_cleanup when: vault_integration_legacy | bool diff --git a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml index 6f679cfcc..eba48a2e5 100644 --- a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml +++ b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml @@ -53,13 +53,13 @@ - name: 'Create configuration file' template: src: vault_config.hcl.j2 - dest: '{{ local_temp_dir }}/vault_config.hcl' + dest: '{{ vault_config_file }}' - debug: - msg: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl /dev/null 2>&1 &' + msg: '{{ vault_launch_cmd }} /dev/null 2>&1 &' - name: 'Start vault server (dev mode enabled)' - shell: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl /dev/null 2>&1 &' + shell: '{{ vault_launch_cmd }} /dev/null 2>&1 &' notify: test_managed_vault_cleanup - name: 'Ensure it succeeded' @@ -74,9 +74,10 @@ rescue: - name: 'dump the config' debug: - msg: "{{ lookup('file', local_temp_dir ~ '/vault_config.hcl') }}" + msg: "{{ lookup('file', vault_config_file) }}" # vault wasn't up, let's run the launch command with output, which we expect to fail - name: 'Re-launch Vault' - shell: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl' + shell: '{{ vault_launch_cmd }}' notify: test_managed_vault_cleanup + # timeout: 10 # when we drop 2.9 support we can enable this just in case diff --git a/tests/integration/targets/setup_vault_server/templates/vault_config.hcl.j2 b/tests/integration/targets/setup_vault_server/templates/vault_config.hcl.j2 index 26bf47f15..55f6ef49f 100644 --- a/tests/integration/targets/setup_vault_server/templates/vault_config.hcl.j2 +++ b/tests/integration/targets/setup_vault_server/templates/vault_config.hcl.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -pid_file = "{{ local_temp_dir }}/vault.pid" +pid_file = "{{ vault_pid_file }}" {% if vault_run_https_tests | bool %} listener "tcp" { tls_key_file = "{{ vault_key_file }}" From b44a9d34f466b321de6e9b5370565ac3242841ce Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 6 Jul 2021 15:17:40 -0400 Subject: [PATCH 065/137] do the thing --- .github/workflows/ansible-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index c4127e17b..b166d249b 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -357,7 +357,7 @@ jobs: PLIST: /Library/LaunchDaemons/limit.maxfiles.plist MAXFILES: 524588 run: | - ulimit -S -n $MAXFILES + ulimit -S -n $MAXFILES $MAXFILES # sudo launchctl limit maxfiles $MAXFILES $MAXFILES # cat >${PLIST} < @@ -386,8 +386,6 @@ jobs: # sudo chmod 644 ${PLIST} - - - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check From eef7bf3e26ba074930f8140e2e7a3ba294649209 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 6 Jul 2021 15:27:27 -0400 Subject: [PATCH 066/137] limits --- .github/workflows/ansible-test.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index b166d249b..e8f1ec288 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -340,25 +340,14 @@ jobs: with: python-version: ${{ matrix.python }} - - name: Install Docker on MacOS - if: ${{ matrix.runner == 'macos-latest' }} - run: | - mkdir -p ~/.docker/machine/cache - curl -Lo ~/.docker/machine/cache/boot2docker.iso https://github.com/boot2docker/boot2docker/releases/download/v19.03.12/boot2docker.iso - brew install docker-machine docker - sudo docker --version - docker-machine create --driver virtualbox default - docker-machine env default - docker-machine env default | sed 's/^export //;/^#/d;s/^#.*//' | tr -d '"' >> $GITHUB_ENV - - name: Fix forks/maxfiles on MacOS if: ${{ matrix.runner == 'macos-latest' }} env: PLIST: /Library/LaunchDaemons/limit.maxfiles.plist MAXFILES: 524588 run: | - ulimit -S -n $MAXFILES $MAXFILES - # sudo launchctl limit maxfiles $MAXFILES $MAXFILES + sudo launchctl limit maxfiles $MAXFILES $MAXFILES + ulimit -n $MAXFILES $MAXFILES # cat >${PLIST} < # > $GITHUB_ENV - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check From 6b28ac99dbade85b0e5ca3dcbb2a14e5e3d895d5 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 6 Jul 2021 15:29:30 -0400 Subject: [PATCH 067/137] softly --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index e8f1ec288..cbbebbb7c 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -347,7 +347,7 @@ jobs: MAXFILES: 524588 run: | sudo launchctl limit maxfiles $MAXFILES $MAXFILES - ulimit -n $MAXFILES $MAXFILES + ulimit -S -n $MAXFILES $MAXFILES # cat >${PLIST} < # Date: Tue, 6 Jul 2021 15:33:27 -0400 Subject: [PATCH 068/137] wot now --- .github/workflows/ansible-test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index cbbebbb7c..1344ccaac 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -346,8 +346,18 @@ jobs: PLIST: /Library/LaunchDaemons/limit.maxfiles.plist MAXFILES: 524588 run: | + ulimit -n + ulimit -Hn + ulimit -Sn + sudo launchctl limit maxfiles $MAXFILES $MAXFILES + + ulimit -n + ulimit -Hn + ulimit -Sn + ulimit -S -n $MAXFILES $MAXFILES + # cat >${PLIST} < # Date: Tue, 6 Jul 2021 15:38:13 -0400 Subject: [PATCH 069/137] two88tho --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 1344ccaac..4226422d4 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -344,7 +344,7 @@ jobs: if: ${{ matrix.runner == 'macos-latest' }} env: PLIST: /Library/LaunchDaemons/limit.maxfiles.plist - MAXFILES: 524588 + MAXFILES: 524288 run: | ulimit -n ulimit -Hn From e9d7ffdb3be15ce20bc54b0a7c899e839723bd1d Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 6 Jul 2021 15:51:07 -0400 Subject: [PATCH 070/137] what now --- .github/workflows/ansible-test.yml | 35 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 4226422d4..5afcfc434 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -384,17 +384,6 @@ jobs: # sudo chmod 644 ${PLIST} - - name: Install Docker on MacOS - if: ${{ matrix.runner == 'macos-latest' }} - run: | - mkdir -p ~/.docker/machine/cache - curl -Lo ~/.docker/machine/cache/boot2docker.iso https://github.com/boot2docker/boot2docker/releases/download/v19.03.12/boot2docker.iso - brew install docker-machine docker - sudo docker --version - docker-machine create --driver virtualbox default - docker-machine env default - docker-machine env default | sed 's/^export //;/^#/d;s/^#.*//' | tr -d '"' >> $GITHUB_ENV - - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check @@ -432,7 +421,24 @@ jobs: ref: refs/tags/${{ env.CG_LATEST }} path: ansible_collections/community/general - # Run the integration tests + - name: Run the integration tests in a venv + run: | + ulimit -S -n $(ulimit -Hn) + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive #--coverage + working-directory: ${{ env.COLLECTION_PATH }} + + - name: Install Docker on MacOS + if: ${{ matrix.runner == 'macos-latest' }} + run: | + mkdir -p ~/.docker/machine/cache + curl -Lo ~/.docker/machine/cache/boot2docker.iso https://github.com/boot2docker/boot2docker/releases/download/v19.03.12/boot2docker.iso + brew install docker-machine docker + sudo docker --version + docker-machine create --driver virtualbox default + docker-machine env default + docker-machine env default | sed 's/^export //;/^#/d;s/^#.*//' | tr -d '"' >> $GITHUB_ENV + + # Run the integration tests - name: Run integration tests in docker run: | docker ps || true @@ -440,11 +446,6 @@ jobs: ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker #--coverage working-directory: ${{ env.COLLECTION_PATH }} - - name: Run the integration tests in a venv - run: | - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive #--coverage - working-directory: ${{ env.COLLECTION_PATH }} - # ansible-test support producing code coverage date # - name: Generate coverage report # run: ansible-test coverage xml -v --requirements --group-by command --group-by version From d1324d61ef052c87c20864532865ab8ffc2a60aa Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 6 Jul 2021 15:55:14 -0400 Subject: [PATCH 071/137] oops --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 5afcfc434..00aa4fdfa 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -424,6 +424,7 @@ jobs: - name: Run the integration tests in a venv run: | ulimit -S -n $(ulimit -Hn) + cp "tests/integration/integration_config.yml.sample" "tests/integration/integration_config.yml" ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive #--coverage working-directory: ${{ env.COLLECTION_PATH }} @@ -442,7 +443,6 @@ jobs: - name: Run integration tests in docker run: | docker ps || true - cp "tests/integration/integration_config.yml.sample" "tests/integration/integration_config.yml" ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker #--coverage working-directory: ${{ env.COLLECTION_PATH }} From 220c433aa39927754a1c7f64910986b993bdf961 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 6 Jul 2021 16:39:58 -0400 Subject: [PATCH 072/137] fork safety --- .github/workflows/ansible-test.yml | 90 +++++++++++++++--------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 00aa4fdfa..dd65ea243 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -300,6 +300,8 @@ jobs: classic_integration: runs-on: ${{ matrix.runner }} name: Local I - ${{ matrix.runner }} (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) + env: + OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES' strategy: fail-fast: false matrix: @@ -340,49 +342,49 @@ jobs: with: python-version: ${{ matrix.python }} - - name: Fix forks/maxfiles on MacOS - if: ${{ matrix.runner == 'macos-latest' }} - env: - PLIST: /Library/LaunchDaemons/limit.maxfiles.plist - MAXFILES: 524288 - run: | - ulimit -n - ulimit -Hn - ulimit -Sn - - sudo launchctl limit maxfiles $MAXFILES $MAXFILES - - ulimit -n - ulimit -Hn - ulimit -Sn - - ulimit -S -n $MAXFILES $MAXFILES - - # cat >${PLIST} < - # - # - # - # Label - # limit.maxfiles - # ProgramArguments - # - # launchctl - # limit - # maxfiles - # 524288 - # 524288 - # - # RunAtLoad - # - # ServiceIPC - # - # - # - # EOF - - # sudo chmod 644 ${PLIST} + # - name: Fix forks/maxfiles on MacOS + # if: ${{ matrix.runner == 'macos-latest' }} + # env: + # PLIST: /Library/LaunchDaemons/limit.maxfiles.plist + # MAXFILES: 524288 + # run: | + # ulimit -n + # ulimit -Hn + # ulimit -Sn + + # sudo launchctl limit maxfiles $MAXFILES $MAXFILES + + # ulimit -n + # ulimit -Hn + # ulimit -Sn + + # ulimit -S -n $MAXFILES $MAXFILES + + # # cat >${PLIST} < + # # + # # + # # + # # Label + # # limit.maxfiles + # # ProgramArguments + # # + # # launchctl + # # limit + # # maxfiles + # # 524288 + # # 524288 + # # + # # RunAtLoad + # # + # # ServiceIPC + # # + # # + # # + # # EOF + + # # sudo chmod 644 ${PLIST} - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check @@ -423,7 +425,7 @@ jobs: - name: Run the integration tests in a venv run: | - ulimit -S -n $(ulimit -Hn) + # ulimit -S -n $(ulimit -Hn) cp "tests/integration/integration_config.yml.sample" "tests/integration/integration_config.yml" ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive #--coverage working-directory: ${{ env.COLLECTION_PATH }} From 0ae2cddfbef88965237ef4b458f9ff66c444bf58 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 6 Jul 2021 16:45:44 -0400 Subject: [PATCH 073/137] dress for succeeded --- .../targets/lookup_hashi_vault/handlers/main.yml | 7 ------- .../targets/setup_vault_server/tasks/vault_server.yml | 3 +++ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml index 0602fc008..c9da1d10d 100644 --- a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml @@ -6,13 +6,6 @@ # shell: "kill $(cat {{ local_temp_dir }}/vault.pid)" # ignore_errors: true # listen: test_managed_vault_cleanup -- shell: ls -alh /tmp - listen: test_managed_vault_cleanup - -- shell: "cat {{ vault_pid_file }}" - listen: test_managed_vault_cleanup - ignore_errors: yes - - name: Stop tinyproxy shell: diff --git a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml index eba48a2e5..dbca7cae6 100644 --- a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml +++ b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml @@ -68,8 +68,11 @@ environment: VAULT_ADDR: '{{ vault_test_server_http }}' shell: '{{ vault_cmd }} status' + register: vault_status retries: 10 delay: 1 + until: vault_status is succeeded + rescue: - name: 'dump the config' From a5bb68ba0367cba85360916958c628fce60ef410 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 7 Jul 2021 21:44:48 -0400 Subject: [PATCH 074/137] move tinyproxy to its own setup role --- tests/integration/!integration_config.yml | 68 +++++++++++++++++++ .../targets/lookup_hashi_vault/aliases | 10 +-- .../lookup_hashi_vault/defaults/main.yml | 6 +- .../lookup_hashi_vault/handlers/main.yml | 44 ++++++------ .../targets/lookup_hashi_vault/tasks/main.yml | 22 +++--- .../integration/targets/lookup_thing/aliases | 2 + .../targets/lookup_thing/meta/main.yml | 3 + .../targets/setup_tinyproxy_server/aliases | 2 + .../setup_tinyproxy_server/defaults/main.yml | 7 ++ .../setup_tinyproxy_server/handlers/main.yml | 20 ++++++ .../tasks/main.yml} | 12 ++-- 11 files changed, 149 insertions(+), 47 deletions(-) create mode 100644 tests/integration/!integration_config.yml create mode 100644 tests/integration/targets/lookup_thing/aliases create mode 100644 tests/integration/targets/lookup_thing/meta/main.yml create mode 100644 tests/integration/targets/setup_tinyproxy_server/aliases create mode 100644 tests/integration/targets/setup_tinyproxy_server/defaults/main.yml create mode 100644 tests/integration/targets/setup_tinyproxy_server/handlers/main.yml rename tests/integration/targets/{lookup_hashi_vault/tasks/tinyproxy_server.yml => setup_tinyproxy_server/tasks/main.yml} (65%) diff --git a/tests/integration/!integration_config.yml b/tests/integration/!integration_config.yml new file mode 100644 index 000000000..38c97ddfd --- /dev/null +++ b/tests/integration/!integration_config.yml @@ -0,0 +1,68 @@ +# Ansible managed +--- +# can be uncommented once vault configuration has been run once (WIP/broken) +# vault_server_configure: False +vault_cert_content: '-----BEGIN CERTIFICATE----- + + MIIE4TCCAsmgAwIBAgIUIL77ChgfiJcegZViZh8TV9KviB8wDQYJKoZIhvcNAQEL + + BQAwEDEOMAwGA1UEAwwFdmF1bHQwHhcNMjEwNzAxMTgyNjIzWhcNMzEwNjI5MTgy + + NjIzWjAQMQ4wDAYDVQQDDAV2YXVsdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC + + AgoCggIBAK1mI97qE0Pijd0vYrepaoZen/7yuayVslJ1uKzNQWCwHCpaAG9QuGof + + ym2N4UnCnBAmpcOH8N+xVzFR5oQVJwYLodfPeI8VDcHvN6Cj011Lb40o2fAbQz+v + + 2Xz4MOpEzRbCrSlMZa7M10+iiZJpU379Yqn6JDiCkwrq4dIBVUnp8Wg5ykV+qdUz + + ypB6jJF67WRisXqYp7hACL67FRlj/r8+76FSUg/oAo7g0rkrahWb+SvNMPWS1hS7 + + Tk8Tjf6qb2ZO2Iwx0eEY2GNoziUzQu1xTkZQhbQt3vt8ZiWvddCRxCz5W1cfKWO/ + + 0XQlMlqlW2RmfxEouMUqhQlH1NSYCKas12RFxxSaiPI2Idq2Kve2xDLwQuIOoCLD + + joqSQmyF88f9Jxb0l2sXXsro/YwmYJ+qwz81QW4BO5LzX7BHT7EuiV97m8kLuBzE + + 0pjat0XVJ9fzmfGYX89uiwnO/fb7jZubjQLus0cmZXHMB6wK0fo8bTXgi9TLXwbi + + wymDG+A+jlSShf7aE3vZtr6fRcacjwh6Y6DFbfxdVV9Vxzv1aHaMUYwtIu+d+uVC + + cTQbwou6B7hS0BUdXQKtM1mjPDAwcdgz//TLzk26tIsx69AXtOREKb9W1ffIbfGh + + B8nrdmI6+80tlc8KL6s+/cMLEMya3K2GZPloAw+CO00ihO7SGC6VAgMBAAGjMzAx + + MBAGA1UdEQQJMAeCBXZhdWx0MB0GA1UdDgQWBBRbYPX83fHK2QUTsWExMQESmS6B + + yTANBgkqhkiG9w0BAQsFAAOCAgEAc9slz9up7xd3bsr+q/kCoDt+w6rm/dc6ONSJ + + PITZAbuWtRBtCJStQuie5ZICnh1X0IajhczIFVcD9CjxOIxfxA7S49gL9vDHVpiJ + + K4nW0KR3Zviq2XwtHYAs99CZH63EUTVqz0nEuMu10H/0PCFPtTHcXFpgovCLRAGH + + HqnM7LVeM7a0g85Zt+HXuPJ2MThlEyIBy64MBPIczSiGDVx0cQwe1LJREkQJgB4F + + +3iAOPIsHAWkApFfx2cyq+L4sEd0EdxUFk1mw4sni/VFzK8wcd3L7gEgseKSk2Kz + + Z3JZiAXessjoa26JL0/KBSN6LTB3/pdn/dG7lz7DUr17PymbggRLVjdbSsbAFD9S + + BTGgt3kFjrXIeNAyqGodK673R5jACXjz0vuEePJh4Vk/ffl953bH8Xhs/BZyNXBC + + meOu/sU93MGPi0vqE+Jdjplvj5smLzOW9Y6HAAitDqHTQ0sNdZP2DdT6FkSLSzMx + + ErPFBePgxhQFXvQt8h11Gadox5vsm1Ca2nLGClKWRt458goFEWgmmDA+mOD2/sJ0 + + eYdGfBgN/ZnTzD2y2z18Sd9H2Zb4HZbfpPvvFwR+5oYMqE72Rz3oj8APt2f/Eq5B + + WYNpi1fRCGPXhM2wNr2DKPKcoqbRcIVTxD/E0MATmxQRGtPaO/JbFcY5v37qcLWf + + Jb3iD/U= + + -----END CERTIFICATE-----' +vault_dev_root_token_id: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a +vault_proxy_external: true +vault_proxy_server: http://tinyproxy:8888 +vault_test_server_external: true +vault_test_server_http: http://vault:8200 +vault_test_server_https: https://vault:8300 +vault_version: 1.7.3 diff --git a/tests/integration/targets/lookup_hashi_vault/aliases b/tests/integration/targets/lookup_hashi_vault/aliases index 4bccb5550..87649fca9 100644 --- a/tests/integration/targets/lookup_hashi_vault/aliases +++ b/tests/integration/targets/lookup_hashi_vault/aliases @@ -1,8 +1,4 @@ -shippable/posix/group2 -destructive -needs/file/tests/utils/constraints.txt -skip/aix -skip/python2.6 # lookups are controller only, and we no longer support Python 2.6 on the controller -skip/macos # FIXME seems to be always unstable +skip/python2.6 # Python 2.6 is not supported by the collection needs/target/setup_vault_server -needs/target/setup_localenv_gha +needs/target/setup_tinyproxy_server +# needs/target/setup_localenv_gha diff --git a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml index f5acdb09a..94025354e 100644 --- a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml @@ -92,7 +92,7 @@ auth_methods: # vault_uri: 'https://releases.hashicorp.com/vault/{{ vault_version }}/{{ vault_slug }}.zip' # vault_cmd: '{{ vault_bin }}/vault' -vault_dev_root_token_id: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' +# vault_dev_root_token_id: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' # vault_test_server_https: 'https://localhost:8201' # vault_test_server_http: 'http://localhost:8200' @@ -109,8 +109,8 @@ vault_run_https_tests: True # vault_cert_file: '{{ local_temp_dir }}/cert.pem' # vault_key_file: '{{ local_temp_dir }}/privatekey.pem' -vault_proxy_server: 'http://127.0.0.1:8001' -vault_proxy_external: False +# vault_proxy_server: 'http://127.0.0.1:8001' +# vault_proxy_external: False # if any connections are taking longer than this to complete there's probably something really wrong # with the integration tests, so it'd be better to fail faster than the 30s default diff --git a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml index c9da1d10d..e6be2a140 100644 --- a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml @@ -7,28 +7,28 @@ # ignore_errors: true # listen: test_managed_vault_cleanup -- name: Stop tinyproxy - shell: - cmd: | - kill $(cat "{{ local_temp_dir }}/tinyproxy.pid") - rm -f "{{ local_temp_dir }}/tinyproxy.pid" - removes: "{{ local_temp_dir }}/tinyproxy.pid" +# - name: Stop tinyproxy +# shell: +# cmd: | +# kill $(cat "{{ local_temp_dir }}/tinyproxy.pid") +# rm -f "{{ local_temp_dir }}/tinyproxy.pid" +# removes: "{{ local_temp_dir }}/tinyproxy.pid" -- name: Uninstall tinyproxy - become: '{{ ansible_distribution != "MacOSX" }}' - vars: - # check 'Install unzip' task to know why we set ansible_python_interpreter - ansible_python_interpreter: "{{ - '/usr/bin/python3' if ansible_distribution in ['Ubuntu', 'Debian'] else ansible_python.executable - }}" - package: - name: tinyproxy - state: absent - ignore_errors: yes +# - name: Uninstall tinyproxy +# become: '{{ ansible_distribution != "MacOSX" }}' +# vars: +# # check 'Install unzip' task to know why we set ansible_python_interpreter +# ansible_python_interpreter: "{{ +# '/usr/bin/python3' if ansible_distribution in ['Ubuntu', 'Debian'] else ansible_python.executable +# }}" +# package: +# name: tinyproxy +# state: absent +# ignore_errors: yes # notify 'cleanup' for any handlers that should always run at the end of tests -- name: 'Delete temp dir' - file: - path: '{{ local_temp_dir }}' - state: absent - listen: cleanup +# - name: 'Delete temp dir' +# file: +# path: '{{ local_temp_dir }}' +# state: absent +# listen: cleanup diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml index 588385bd3..b72226246 100644 --- a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml @@ -4,24 +4,28 @@ # and should not be used as examples of how to write Ansible roles # #################################################################### -- name: Create a local temporary directory - tempfile: - state: directory - register: tempfile_result - notify: cleanup +# - name: Create a local temporary directory +# tempfile: +# state: directory +# register: tempfile_result +# notify: cleanup -- set_fact: - local_temp_dir: '{{ tempfile_result.path }}' +# - set_fact: +# local_temp_dir: '{{ tempfile_result.path }}' - include_role: name: setup_vault_server when: vault_integration_legacy | bool +- include_role: + name: setup_tinyproxy_server + when: vault_integration_legacy | bool + - import_tasks: vault_server_configure.yml when: vault_test_server_configure | bool -- include_tasks: tinyproxy_server.yml - when: not vault_proxy_external | bool +# - include_tasks: tinyproxy_server.yml +# when: not vault_proxy_external | bool - import_tasks: tests.yml vars: diff --git a/tests/integration/targets/lookup_thing/aliases b/tests/integration/targets/lookup_thing/aliases new file mode 100644 index 000000000..4351e915b --- /dev/null +++ b/tests/integration/targets/lookup_thing/aliases @@ -0,0 +1,2 @@ +needs/target/lookup_hashi_vault +hidden diff --git a/tests/integration/targets/lookup_thing/meta/main.yml b/tests/integration/targets/lookup_thing/meta/main.yml new file mode 100644 index 000000000..9f29d69ab --- /dev/null +++ b/tests/integration/targets/lookup_thing/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - lookup_hashi_vault diff --git a/tests/integration/targets/setup_tinyproxy_server/aliases b/tests/integration/targets/setup_tinyproxy_server/aliases new file mode 100644 index 000000000..cdf2936ca --- /dev/null +++ b/tests/integration/targets/setup_tinyproxy_server/aliases @@ -0,0 +1,2 @@ +hidden +destructive diff --git a/tests/integration/targets/setup_tinyproxy_server/defaults/main.yml b/tests/integration/targets/setup_tinyproxy_server/defaults/main.yml new file mode 100644 index 000000000..86a28b508 --- /dev/null +++ b/tests/integration/targets/setup_tinyproxy_server/defaults/main.yml @@ -0,0 +1,7 @@ +--- +vault_proxy_server: 'http://127.0.0.1:8001' +# vault_proxy_external: False + +vault_proxy_tmp: /tmp +vault_proxy_pid: '{{ vault_proxy_tmp }}/tinyproxy.pid' +vault_proxy_conf: '{{ vault_proxy_tmp }}/tinyproxy.conf' diff --git a/tests/integration/targets/setup_tinyproxy_server/handlers/main.yml b/tests/integration/targets/setup_tinyproxy_server/handlers/main.yml new file mode 100644 index 000000000..c517c4b5e --- /dev/null +++ b/tests/integration/targets/setup_tinyproxy_server/handlers/main.yml @@ -0,0 +1,20 @@ +--- +- name: Stop tinyproxy + shell: + cmd: | + kill $(cat "{{ vault_proxy_pid }}") + rm -f "{{ vault_proxy_pid }}" + removes: "{{ vault_proxy_pid }}" + listen: proxy_cleanup + +- name: Uninstall tinyproxy + become: '{{ ansible_distribution != "MacOSX" }}' + vars: + ansible_python_interpreter: "{{ + '/usr/bin/python3' if ansible_distribution in ['Ubuntu', 'Debian'] else ansible_python.executable + }}" + package: + name: tinyproxy + state: absent + ignore_errors: yes + listen: proxy_cleanup diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml b/tests/integration/targets/setup_tinyproxy_server/tasks/main.yml similarity index 65% rename from tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml rename to tests/integration/targets/setup_tinyproxy_server/tasks/main.yml index ca53f0121..8b133a668 100644 --- a/tests/integration/targets/lookup_hashi_vault/tasks/tinyproxy_server.yml +++ b/tests/integration/targets/setup_tinyproxy_server/tasks/main.yml @@ -8,7 +8,7 @@ }}" package: name: tinyproxy - notify: Uninstall tinyproxy + notify: proxy_cleanup - name: Configure tinyproxy copy: @@ -16,11 +16,11 @@ Port 8001 MaxClients 100 StartServers 10 - PidFile "{{ local_temp_dir }}/tinyproxy.pid" - dest: "{{ local_temp_dir }}/tinyproxy.conf" + PidFile "{{ vault_proxy_pid }}" + dest: "{{ vault_proxy_conf }}" - name: Start tinyproxy shell: - cmd: tinyproxy -c "{{ local_temp_dir }}/tinyproxy.conf" - creates: "{{ local_temp_dir }}/tinyproxy.pid" - notify: Stop tinyproxy + cmd: tinyproxy -c "{{ vault_proxy_conf }}" + creates: "{{ vault_proxy_pid }}" + notify: proxy_cleanup From 730dc57d956fd771eb0b378d1a87d837a3fdfc29 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 7 Jul 2021 21:55:42 -0400 Subject: [PATCH 075/137] speed up unzip when it doesn\'t need to be installed --- .../tasks/main.yml | 62 +++++++++++-------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/tests/integration/targets/setup_vault_server_download/tasks/main.yml b/tests/integration/targets/setup_vault_server_download/tasks/main.yml index 3e1949b97..e041b0922 100644 --- a/tests/integration/targets/setup_vault_server_download/tasks/main.yml +++ b/tests/integration/targets/setup_vault_server_download/tasks/main.yml @@ -11,26 +11,6 @@ - name: "Download vault if not local" when: not bin_status.stat.exists or vault_server_download_force | bool block: - # NOTE: 'package' does not work properly with Ubuntu/Debian (like the 'default' docker image), - # if you're running in a version of Python other than the "system" Python, due to system libraries - # needed for the python 'apt' package. See https://stackoverflow.com/q/13708180/3905079 - # So for those OSes, we'll set the Python interpreter to the symlink in /usr/bin which should - # always be the correct one that corresponds to the system libraries. - # - # All this just for unzip, which is only needed to unzip the vault binary to set up for testing. - # TODO: revisit how we set up vault in the first place or how we host the binary (.gz?) - - name: 'Install unzip' - become: yes - vars: - # by assuming python3 here we're probably condeming this to not work on older Ubuntu/Debian (from like 2014?) - # but the alternative is probably reimplementing parts of interpreter_discovery.py - ansible_python_interpreter: "{{ - '/usr/bin/python3' if ansible_distribution in ['Ubuntu', 'Debian'] else ansible_python.executable - }}" - package: - name: unzip - when: ansible_distribution != "MacOSX" # unzip already installed (#TODO: get MacOSX tests working again) - - name: "Create bin directory" file: path: '{{ vault_bin }}' @@ -41,12 +21,42 @@ url: '{{ vault_uri }}' dest: '{{ vault_zip }}' - - name: 'Extract vault binary' - unarchive: - src: '{{ vault_zip }}' - dest: '{{ vault_bin }}' - remote_src: yes - creates: '{{ vault_cmd }}' + - block: + # because installing unzip is so slow, even when it already exists, we're going to ask for + # forgiveness rather than permission, and try to unzip first. If it fails, then we'll try to + # install it and extract again. + - name: 'Extract vault binary' + unarchive: + src: '{{ vault_zip }}' + dest: '{{ vault_bin }}' + remote_src: yes + creates: '{{ vault_cmd }}' + rescue: + # NOTE: 'package' does not work properly with Ubuntu/Debian (like the 'default' docker image), + # if you're running in a version of Python other than the "system" Python, due to system libraries + # needed for the python 'apt' package. See https://stackoverflow.com/q/13708180/3905079 + # So for those OSes, we'll set the Python interpreter to the symlink in /usr/bin which should + # always be the correct one that corresponds to the system libraries. + # + # All this just for unzip, which is only needed to unzip the vault binary to set up for testing. + - name: 'Install unzip' + become: yes + vars: + # by assuming python3 here we're probably condeming this to not work on older Ubuntu/Debian (from like 2014?) + # but the alternative is probably reimplementing parts of interpreter_discovery.py + ansible_python_interpreter: "{{ + '/usr/bin/python3' if ansible_distribution in ['Ubuntu', 'Debian'] else ansible_python.executable + }}" + package: + name: unzip + when: ansible_distribution != "MacOSX" # unzip already installed + + - name: 'Extract vault binary' + unarchive: + src: '{{ vault_zip }}' + dest: '{{ vault_bin }}' + remote_src: yes + creates: '{{ vault_cmd }}' - name: 'Export Vault command' set_fact: From 62a1321a0c8b1c0114bd215d8ab414e6074b8cdb Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 7 Jul 2021 23:41:01 -0400 Subject: [PATCH 076/137] more docker role cleanup --- .../targets/setup_localenv_docker/defaults/main.yml | 9 +++++---- .../files/playbooks/vault_docker.yml | 5 +++++ .../files/requirements/requirements.yml | 6 ++++++ tests/integration/targets/setup_localenv_docker/setup.sh | 8 ++++++++ .../setup_vault_server_download/defaults/main.yml | 1 - 5 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 tests/integration/targets/setup_localenv_docker/files/playbooks/vault_docker.yml create mode 100644 tests/integration/targets/setup_localenv_docker/files/requirements/requirements.yml create mode 100755 tests/integration/targets/setup_localenv_docker/setup.sh diff --git a/tests/integration/targets/setup_localenv_docker/defaults/main.yml b/tests/integration/targets/setup_localenv_docker/defaults/main.yml index f02deec5f..19b328dd0 100644 --- a/tests/integration/targets/setup_localenv_docker/defaults/main.yml +++ b/tests/integration/targets/setup_localenv_docker/defaults/main.yml @@ -1,17 +1,18 @@ --- vault_version: '1.7.3' +vault_dev_root_token_id: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a docker_compose: clean -# up -# down -# none +# clean - down, then up +# up - bring up the configuration +# down - destroy the configuration +# none - do not take any docker actions (templating of docker-compose.yml still happens) docker_compose_project_name: hashi_vault vault_port_http: 8200 vault_port_https: 8300 vault_container_name: vault -vault_dev_root_token_id: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a proxy_port: 8888 proxy_container_name: tinyproxy diff --git a/tests/integration/targets/setup_localenv_docker/files/playbooks/vault_docker.yml b/tests/integration/targets/setup_localenv_docker/files/playbooks/vault_docker.yml new file mode 100644 index 000000000..6f6ae5ab9 --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/files/playbooks/vault_docker.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + gather_facts: no + roles: + - setup_localenv_docker diff --git a/tests/integration/targets/setup_localenv_docker/files/requirements/requirements.yml b/tests/integration/targets/setup_localenv_docker/files/requirements/requirements.yml new file mode 100644 index 000000000..8114e6169 --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/files/requirements/requirements.yml @@ -0,0 +1,6 @@ +--- +collections: + # community.docker is not required if using docker_compose=none + - community.docker + # community.crypto is not required the certificate and key files specified already exist + - community.crypto diff --git a/tests/integration/targets/setup_localenv_docker/setup.sh b/tests/integration/targets/setup_localenv_docker/setup.sh new file mode 100755 index 000000000..d6600f35c --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/setup.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +pushd "${BASH_SOURCE%/*}/files/playbooks" + +ANSIBLE_ROLES_PATH="../../../" \ + ansible-playbook vault_docker.yml $@ + +popd diff --git a/tests/integration/targets/setup_vault_server_download/defaults/main.yml b/tests/integration/targets/setup_vault_server_download/defaults/main.yml index dad4c5c31..f387dd0cc 100644 --- a/tests/integration/targets/setup_vault_server_download/defaults/main.yml +++ b/tests/integration/targets/setup_vault_server_download/defaults/main.yml @@ -6,7 +6,6 @@ vault_ansible_arch_table: vault_arch: "{{ vault_ansible_arch_table[ansible_architecture] }}" -vault_version: '1.7.3' vault_bin: '{{ role_path }}/files/bin/{{ vault_slug }}' vault_slug: 'vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}' vault_zip: '{{ vault_bin }}/{{ vault_slug }}.zip' From 30d0be7639b5141d9350ee107c12be5280bb42dd Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 7 Jul 2021 23:49:14 -0400 Subject: [PATCH 077/137] more fixups, gha cleanup --- .../integration/targets/setup_localenv_docker/setup.sh | 6 +++--- .../targets/setup_localenv_gha/files/playbooks/gha.yml | 5 +++++ tests/integration/targets/setup_localenv_gha/setup.sh | 10 ++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 tests/integration/targets/setup_localenv_gha/files/playbooks/gha.yml create mode 100755 tests/integration/targets/setup_localenv_gha/setup.sh diff --git a/tests/integration/targets/setup_localenv_docker/setup.sh b/tests/integration/targets/setup_localenv_docker/setup.sh index d6600f35c..b2e8dc1e0 100755 --- a/tests/integration/targets/setup_localenv_docker/setup.sh +++ b/tests/integration/targets/setup_localenv_docker/setup.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -pushd "${BASH_SOURCE%/*}/files/playbooks" +pushd "${BASH_SOURCE%/*}/files" -ANSIBLE_ROLES_PATH="../../../" \ - ansible-playbook vault_docker.yml $@ +ANSIBLE_ROLES_PATH="../../" \ + ansible-playbook playbooks/vault_docker.yml "${@}" popd diff --git a/tests/integration/targets/setup_localenv_gha/files/playbooks/gha.yml b/tests/integration/targets/setup_localenv_gha/files/playbooks/gha.yml new file mode 100644 index 000000000..aea370d3f --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/files/playbooks/gha.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + gather_facts: no + roles: + - setup_localenv_gha diff --git a/tests/integration/targets/setup_localenv_gha/setup.sh b/tests/integration/targets/setup_localenv_gha/setup.sh new file mode 100755 index 000000000..04003609f --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/setup.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +pushd "${BASH_SOURCE%/*}/files" + +ANSIBLE_ROLES_PATH=../../ \ + ansible-playbook playbooks/gha.yml "${@}" + +.output/launch.sh + +popd From 3ce6825829b5c15750491cfa7ce224af934491f1 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 00:04:38 -0400 Subject: [PATCH 078/137] localenv workflows --- .github/workflows/ansible-test.yml | 43 ++++++++++++------- .../targets/setup_localenv_docker/setup.sh | 3 ++ .../targets/setup_localenv_gha/setup.sh | 4 ++ 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index dd65ea243..06d5a6837 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -270,17 +270,19 @@ jobs: # run: ansible-playbook "download_vault.yml" -v # Run the integration tests + # env: + # ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} + # ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} - name: Prepare docker dependencies - env: - ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} - ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} - run: | + run: setup.sh + working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_gha + # pip install -r "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/requirements.txt" -c "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/constraints.txt" # ansible localhost -m include_role -a "name=setup_localenv_docker" -e vault_version=${{ matrix.vault }} - ansible localhost -m include_role -a "name=setup_localenv_gha" -e vault_version=${{ matrix.vault }} + # ansible localhost -m include_role -a "name=setup_localenv_gha" -e vault_version=${{ matrix.vault }} # cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_PATH}" - cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_gha/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_PATH}" - "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_gha/files/.output/launch.sh" + # cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_gha/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_PATH}" + #"${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_gha/files/.output/launch.sh" - name: Run integration test run: | @@ -427,7 +429,7 @@ jobs: run: | # ulimit -S -n $(ulimit -Hn) cp "tests/integration/integration_config.yml.sample" "tests/integration/integration_config.yml" - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive #--coverage + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive --coverage working-directory: ${{ env.COLLECTION_PATH }} - name: Install Docker on MacOS @@ -445,15 +447,26 @@ jobs: - name: Run integration tests in docker run: | docker ps || true - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker #--coverage + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker --coverage + working-directory: ${{ env.COLLECTION_PATH }} + + - name: Prepare local docker setup + run: | + rm -f "${COLLECTION_INTEGRATION_TARGETS}/integration_config.yml" + setup.sh + working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_docker + + - name: Run integration test + run: | + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network hashi_vault_default working-directory: ${{ env.COLLECTION_PATH }} # ansible-test support producing code coverage date - # - name: Generate coverage report - # run: ansible-test coverage xml -v --requirements --group-by command --group-by version - # working-directory: ${{ env.COLLECTION_PATH }} + - name: Generate coverage report + run: ansible-test coverage xml -v --requirements --group-by command --group-by version + working-directory: ${{ env.COLLECTION_PATH }} # See the reports at https://codecov.io/gh/ansible-collections/community.hashi_vault - # - uses: codecov/codecov-action@v1 - # with: - # fail_ci_if_error: false + - uses: codecov/codecov-action@v1 + with: + fail_ci_if_error: false diff --git a/tests/integration/targets/setup_localenv_docker/setup.sh b/tests/integration/targets/setup_localenv_docker/setup.sh index b2e8dc1e0..6c130db28 100755 --- a/tests/integration/targets/setup_localenv_docker/setup.sh +++ b/tests/integration/targets/setup_localenv_docker/setup.sh @@ -5,4 +5,7 @@ pushd "${BASH_SOURCE%/*}/files" ANSIBLE_ROLES_PATH="../../" \ ansible-playbook playbooks/vault_docker.yml "${@}" +# copy generated integration_config.yml if it doesn't exist +cp -n .output/integration_config.yml ../../../ || true + popd diff --git a/tests/integration/targets/setup_localenv_gha/setup.sh b/tests/integration/targets/setup_localenv_gha/setup.sh index 04003609f..f1685ccc6 100755 --- a/tests/integration/targets/setup_localenv_gha/setup.sh +++ b/tests/integration/targets/setup_localenv_gha/setup.sh @@ -5,6 +5,10 @@ pushd "${BASH_SOURCE%/*}/files" ANSIBLE_ROLES_PATH=../../ \ ansible-playbook playbooks/gha.yml "${@}" +# launch containers .output/launch.sh +# copy generated integration_config.yml if it doesn't exist +cp -n .output/integration_config.yml ../../../ || true + popd From 2c9b995b5479b9a1e1ec63ce1f504a561e4d900c Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 00:34:24 -0400 Subject: [PATCH 079/137] welp --- .github/workflows/ansible-test.yml | 64 ++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 06d5a6837..405ac2375 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -29,14 +29,18 @@ jobs: sanity: name: Sanity (Ⓐ${{ matrix.ansible }}) + runs-on: ${{ matrix.runner }} strategy: matrix: + runner: + - ubuntu-latest + test_container: + - default ansible: - stable-2.9 - stable-2.10 - stable-2.11 - devel - runs-on: ubuntu-latest steps: # ansible-test requires the collection to be in a directory in the form @@ -62,16 +66,20 @@ jobs: # The docker container has all the pinned dependencies that are required # and all python versions ansible supports. - name: Run sanity tests - run: ansible-test sanity --docker -v --color + run: ansible-test sanity --docker ${{ matrix.test_container }} -v --color working-directory: ./ansible_collections/${{ env.NAMESPACE }}/${{ env.COLLECTION_NAME }} units: - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} name: Units (Ⓐ${{ matrix.ansible }}) strategy: # As soon as the first unit test fails, cancel the others to free up the CI queue fail-fast: true matrix: + runner: + - ubuntu-latest + test_container: + - default ansible: - stable-2.9 - stable-2.10 @@ -101,7 +109,7 @@ jobs: # Run the unit tests - name: Run unit test - run: ansible-test units -v --color --docker --coverage + run: ansible-test units -v --color --docker ${{ matrix.test_container }} --coverage working-directory: ./ansible_collections/${{ env.NAMESPACE }}/${{ env.COLLECTION_NAME }} # ansible-test support producing code coverage date @@ -125,11 +133,15 @@ jobs: # https://github.com/ansible-collections/community.zabbix/tree/master/.github/workflows integration: - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} name: I (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }} | Vault ${{ matrix.vault }}) strategy: fail-fast: false matrix: + runner: + - ubuntu-latest + test_container: + - default ansible: - stable-2.9 - stable-2.10 @@ -274,7 +286,9 @@ jobs: # ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} # ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} - name: Prepare docker dependencies - run: setup.sh + run: | + pwd + ./setup.sh working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_gha # pip install -r "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/requirements.txt" -c "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/constraints.txt" @@ -286,7 +300,7 @@ jobs: - name: Run integration test run: | - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network hashi_vault_default + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage --docker-network hashi_vault_default working-directory: ${{ env.COLLECTION_PATH }} # ansible-test support producing code coverage date @@ -299,10 +313,11 @@ jobs: with: fail_ci_if_error: false - classic_integration: + local_test_invocation: runs-on: ${{ matrix.runner }} name: Local I - ${{ matrix.runner }} (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) env: + # needed to prevent Ansible crashing on MacOS OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES' strategy: fail-fast: false @@ -314,6 +329,8 @@ jobs: runner: - ubuntu-latest - macos-latest + test_container: + - ubuntu1804 # vault: # - 1.7.3 # exclude: @@ -330,7 +347,9 @@ jobs: with: env: | COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} - LOOKUP_HASHI_VAULT_PATH=${COLLECTION_PATH}/tests/integration/targets/lookup_hashi_vault + COLLECTION_INTEGRATION_PATH=${COLLECTION_PATH}/tests/integration + COLLECTION_INTEGRATION_TARGETS=${COLLECTION_INTEGRATION_PATH}/targets + LOOKUP_HASHI_VAULT_PATH=${COLLECTION_INTEGRATION_TARGETS}/lookup_hashi_vault LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars @@ -428,8 +447,8 @@ jobs: - name: Run the integration tests in a venv run: | # ulimit -S -n $(ulimit -Hn) - cp "tests/integration/integration_config.yml.sample" "tests/integration/integration_config.yml" - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive --coverage + cp "${COLLECTION_INTEGRATION_PATH}/integration_config.yml.sample" "${COLLECTION_INTEGRATION_PATH}/integration_config.yml" + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --local --requirements --allow-destructive --coverage working-directory: ${{ env.COLLECTION_PATH }} - name: Install Docker on MacOS @@ -443,22 +462,23 @@ jobs: docker-machine env default docker-machine env default | sed 's/^export //;/^#/d;s/^#.*//' | tr -d '"' >> $GITHUB_ENV - # Run the integration tests - - name: Run integration tests in docker + - name: localenv_docker - setup run: | - docker ps || true - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker --coverage - working-directory: ${{ env.COLLECTION_PATH }} + pwd + rm -f "${COLLECTION_INTEGRATION_PATH}/integration_config.yml" + ./setup.sh + working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_docker - - name: Prepare local docker setup + - name: localenv_docker - Run integration test (in docker) run: | - rm -f "${COLLECTION_INTEGRATION_TARGETS}/integration_config.yml" - setup.sh - working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_docker + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage --docker-network hashi_vault_default + working-directory: ${{ env.COLLECTION_PATH }} - - name: Run integration test + # Run the integration tests + - name: Run legacy integration tests (in docker) run: | - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker default --coverage --docker-network hashi_vault_default + docker ps || true + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage working-directory: ${{ env.COLLECTION_PATH }} # ansible-test support producing code coverage date From 0b085467780343586aa19324717313db6355651d Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 00:43:30 -0400 Subject: [PATCH 080/137] ummmm --- .github/workflows/ansible-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 405ac2375..157581ca2 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -444,10 +444,13 @@ jobs: ref: refs/tags/${{ env.CG_LATEST }} path: ansible_collections/community/general + - name: Use Sample integration_config + working-directory: ${COLLECTION_INTEGRATION_PATH} + run: cp "integration_config.yml.sample" "integration_config.yml" + - name: Run the integration tests in a venv run: | # ulimit -S -n $(ulimit -Hn) - cp "${COLLECTION_INTEGRATION_PATH}/integration_config.yml.sample" "${COLLECTION_INTEGRATION_PATH}/integration_config.yml" ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --local --requirements --allow-destructive --coverage working-directory: ${{ env.COLLECTION_PATH }} From 03f912c87d3fbbf03fa47225167831c434d7f6e3 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 00:57:01 -0400 Subject: [PATCH 081/137] matrix include is tricky --- .github/workflows/ansible-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 157581ca2..1c41b12ab 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -164,6 +164,8 @@ jobs: - ansible: devel vault: 1.7.3 python: '3.10' + runner: ubuntu-latest + test_container: default # services: # vault: # image: vault:${{ matrix.vault }} @@ -446,7 +448,9 @@ jobs: - name: Use Sample integration_config working-directory: ${COLLECTION_INTEGRATION_PATH} - run: cp "integration_config.yml.sample" "integration_config.yml" + run: | + pwd + cp "integration_config.yml.sample" "integration_config.yml" - name: Run the integration tests in a venv run: | From 6348820b0936015bc2840fc367ae7a5541bfff98 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 01:03:57 -0400 Subject: [PATCH 082/137] wrong path --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 1c41b12ab..8813f784e 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -447,7 +447,7 @@ jobs: path: ansible_collections/community/general - name: Use Sample integration_config - working-directory: ${COLLECTION_INTEGRATION_PATH} + working-directory: ${{ env.COLLECTION_INTEGRATION_PATH }} run: | pwd cp "integration_config.yml.sample" "integration_config.yml" From 49708967eb294cee9facfd47bb26c3c1e6f426c9 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 01:14:21 -0400 Subject: [PATCH 083/137] covz --- .github/workflows/ansible-test.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 8813f784e..560194c6a 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -360,6 +360,12 @@ jobs: with: path: ${{ env.COLLECTION_PATH }} + - name: Install coverage in Ubuntu system python + if: startsWith( ${{ matrix.runner }}, 'ubuntu' ) + run: | + /usr/bin/python2 -m pip install coverage + /usr/bin/python3 -m pip install coverage + - name: Set up Python uses: actions/setup-python@v2 with: @@ -455,7 +461,7 @@ jobs: - name: Run the integration tests in a venv run: | # ulimit -S -n $(ulimit -Hn) - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --local --requirements --allow-destructive --coverage + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive --coverage working-directory: ${{ env.COLLECTION_PATH }} - name: Install Docker on MacOS From bf7a6b7e2ea6f8d62f70530fe297389fa04ed5d7 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 01:15:46 -0400 Subject: [PATCH 084/137] pip3dake --- .github/workflows/ansible-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 560194c6a..5bd5c7f83 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -362,9 +362,7 @@ jobs: - name: Install coverage in Ubuntu system python if: startsWith( ${{ matrix.runner }}, 'ubuntu' ) - run: | - /usr/bin/python2 -m pip install coverage - /usr/bin/python3 -m pip install coverage + run: /usr/bin/python3 -m pip install coverage - name: Set up Python uses: actions/setup-python@v2 From 69db78b10f075e9eccc469c26e431fcde472ce43 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 01:20:51 -0400 Subject: [PATCH 085/137] 1 more cov? --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 5bd5c7f83..71f90a915 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -362,7 +362,7 @@ jobs: - name: Install coverage in Ubuntu system python if: startsWith( ${{ matrix.runner }}, 'ubuntu' ) - run: /usr/bin/python3 -m pip install coverage + run: /usr/bin/python3 -m pip install 'coverage>=4,<=5' - name: Set up Python uses: actions/setup-python@v2 From a6754654b25f6b2effbbcc8a03963c256136251e Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 01:26:21 -0400 Subject: [PATCH 086/137] no cov for local/venv --- .github/workflows/ansible-test.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 71f90a915..2f714df95 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -360,10 +360,6 @@ jobs: with: path: ${{ env.COLLECTION_PATH }} - - name: Install coverage in Ubuntu system python - if: startsWith( ${{ matrix.runner }}, 'ubuntu' ) - run: /usr/bin/python3 -m pip install 'coverage>=4,<=5' - - name: Set up Python uses: actions/setup-python@v2 with: @@ -459,7 +455,7 @@ jobs: - name: Run the integration tests in a venv run: | # ulimit -S -n $(ulimit -Hn) - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive --coverage + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive #--coverage working-directory: ${{ env.COLLECTION_PATH }} - name: Install Docker on MacOS From 091c89804b2a4819fa955890f75737a21d5567ed Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 14:29:37 -0400 Subject: [PATCH 087/137] buttoning up --- .github/workflows/ansible-test.yml | 2 +- .../targets/setup_localenv_docker/setup.sh | 11 +++++++---- tests/integration/targets/setup_localenv_gha/setup.sh | 11 +++++++---- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 2f714df95..68a8d3106 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -332,7 +332,7 @@ jobs: - ubuntu-latest - macos-latest test_container: - - ubuntu1804 + - default # vault: # - 1.7.3 # exclude: diff --git a/tests/integration/targets/setup_localenv_docker/setup.sh b/tests/integration/targets/setup_localenv_docker/setup.sh index 6c130db28..c67cea08b 100755 --- a/tests/integration/targets/setup_localenv_docker/setup.sh +++ b/tests/integration/targets/setup_localenv_docker/setup.sh @@ -1,11 +1,14 @@ #!/usr/bin/env bash -pushd "${BASH_SOURCE%/*}/files" +set -eux -ANSIBLE_ROLES_PATH="../../" \ - ansible-playbook playbooks/vault_docker.yml "${@}" +pushd "${BASH_SOURCE%/*}" + +ANSIBLE_COLLECTIONS_PATHS="../../../../" \ +ANSIBLE_ROLES_PATH="../" \ + ansible-playbook files/playbooks/vault_docker.yml "${@}" # copy generated integration_config.yml if it doesn't exist -cp -n .output/integration_config.yml ../../../ || true +cp -n .output/integration_config.yml ../../ || true popd diff --git a/tests/integration/targets/setup_localenv_gha/setup.sh b/tests/integration/targets/setup_localenv_gha/setup.sh index f1685ccc6..d61902172 100755 --- a/tests/integration/targets/setup_localenv_gha/setup.sh +++ b/tests/integration/targets/setup_localenv_gha/setup.sh @@ -1,14 +1,17 @@ #!/usr/bin/env bash -pushd "${BASH_SOURCE%/*}/files" +set -eux -ANSIBLE_ROLES_PATH=../../ \ - ansible-playbook playbooks/gha.yml "${@}" +pushd "${BASH_SOURCE%/*}" + +ANSIBLE_COLLECTIONS_PATHS="../../../../" \ +ANSIBLE_ROLES_PATH="../" \ + ansible-playbook files/playbooks/gha.yml "${@}" # launch containers .output/launch.sh # copy generated integration_config.yml if it doesn't exist -cp -n .output/integration_config.yml ../../../ || true +cp -n .output/integration_config.yml ../../ || true popd From a1a8ede1cde076113e9b2c07e38fc2f777c94672 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 14:31:09 -0400 Subject: [PATCH 088/137] files!files!files! --- tests/integration/targets/setup_localenv_docker/setup.sh | 2 +- tests/integration/targets/setup_localenv_gha/setup.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/targets/setup_localenv_docker/setup.sh b/tests/integration/targets/setup_localenv_docker/setup.sh index c67cea08b..672cd05eb 100755 --- a/tests/integration/targets/setup_localenv_docker/setup.sh +++ b/tests/integration/targets/setup_localenv_docker/setup.sh @@ -9,6 +9,6 @@ ANSIBLE_ROLES_PATH="../" \ ansible-playbook files/playbooks/vault_docker.yml "${@}" # copy generated integration_config.yml if it doesn't exist -cp -n .output/integration_config.yml ../../ || true +cp -n files/.output/integration_config.yml ../../ || true popd diff --git a/tests/integration/targets/setup_localenv_gha/setup.sh b/tests/integration/targets/setup_localenv_gha/setup.sh index d61902172..54ec03091 100755 --- a/tests/integration/targets/setup_localenv_gha/setup.sh +++ b/tests/integration/targets/setup_localenv_gha/setup.sh @@ -9,9 +9,9 @@ ANSIBLE_ROLES_PATH="../" \ ansible-playbook files/playbooks/gha.yml "${@}" # launch containers -.output/launch.sh +files/.output/launch.sh # copy generated integration_config.yml if it doesn't exist -cp -n .output/integration_config.yml ../../ || true +cp -n files/.output/integration_config.yml ../../ || true popd From f1a38510b0757492ea07c06a34c558d7ebb811cd Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 14:39:37 -0400 Subject: [PATCH 089/137] paths are the death of me --- tests/integration/targets/setup_localenv_docker/setup.sh | 2 +- tests/integration/targets/setup_localenv_gha/setup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/targets/setup_localenv_docker/setup.sh b/tests/integration/targets/setup_localenv_docker/setup.sh index 672cd05eb..c49ba7f82 100755 --- a/tests/integration/targets/setup_localenv_docker/setup.sh +++ b/tests/integration/targets/setup_localenv_docker/setup.sh @@ -4,7 +4,7 @@ set -eux pushd "${BASH_SOURCE%/*}" -ANSIBLE_COLLECTIONS_PATHS="../../../../" \ +ANSIBLE_COLLECTIONS_PATHS="../../../../../../../" \ ANSIBLE_ROLES_PATH="../" \ ansible-playbook files/playbooks/vault_docker.yml "${@}" diff --git a/tests/integration/targets/setup_localenv_gha/setup.sh b/tests/integration/targets/setup_localenv_gha/setup.sh index 54ec03091..b5dfb0ac3 100755 --- a/tests/integration/targets/setup_localenv_gha/setup.sh +++ b/tests/integration/targets/setup_localenv_gha/setup.sh @@ -4,7 +4,7 @@ set -eux pushd "${BASH_SOURCE%/*}" -ANSIBLE_COLLECTIONS_PATHS="../../../../" \ +ANSIBLE_COLLECTIONS_PATHS="../../../../../../../" \ ANSIBLE_ROLES_PATH="../" \ ansible-playbook files/playbooks/gha.yml "${@}" From 4daccb954938068b81ef0ac807da439cf41f9364 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 14:47:11 -0400 Subject: [PATCH 090/137] community.docker req --- .github/workflows/ansible-test.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 68a8d3106..a7a76d17f 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -423,6 +423,9 @@ jobs: cglu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.general/releases/latest) echo "CG_LATEST=$(basename ${cglu})" >> $GITHUB_ENV + cdlu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.docker/releases/latest) + echo "CD_LATEST=$(basename ${cdlu})" >> $GITHUB_ENV + - name: Cache for community.crypto id: cache-cc uses: actions/cache@v2 @@ -446,6 +449,14 @@ jobs: ref: refs/tags/${{ env.CG_LATEST }} path: ansible_collections/community/general + - name: Install collection dependencies + # if: steps.cache-cc.outputs.cache-hit != 'true' + uses: actions/checkout@v2 + with: + repository: ansible-collections/community.docker + ref: refs/tags/${{ env.CD_LATEST }} + path: ansible_collections/community/docker + - name: Use Sample integration_config working-directory: ${{ env.COLLECTION_INTEGRATION_PATH }} run: | From 257b6ca044b2b2bee15815da64589ca0e27cc65f Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 14:56:49 -0400 Subject: [PATCH 091/137] requirements! --- .github/workflows/ansible-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index a7a76d17f..010513f9f 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -484,6 +484,7 @@ jobs: run: | pwd rm -f "${COLLECTION_INTEGRATION_PATH}/integration_config.yml" + pip install -r files/requirements/requirements.txt ./setup.sh working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_docker From 27c119612d677a3f3112490635e1a07d19740b81 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 15:15:23 -0400 Subject: [PATCH 092/137] more cleanup and fixes --- .github/workflows/ansible-test.yml | 31 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 010513f9f..051a1423c 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -457,16 +457,12 @@ jobs: ref: refs/tags/${{ env.CD_LATEST }} path: ansible_collections/community/docker - - name: Use Sample integration_config + - name: legacy integration - use sample integration_config working-directory: ${{ env.COLLECTION_INTEGRATION_PATH }} - run: | - pwd - cp "integration_config.yml.sample" "integration_config.yml" + run: cp "integration_config.yml.sample" "integration_config.yml" - - name: Run the integration tests in a venv - run: | - # ulimit -S -n $(ulimit -Hn) - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive #--coverage + - name: legacy integration - venv + run: ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive #--coverage working-directory: ${{ env.COLLECTION_PATH }} - name: Install Docker on MacOS @@ -480,11 +476,20 @@ jobs: docker-machine env default docker-machine env default | sed 's/^export //;/^#/d;s/^#.*//' | tr -d '"' >> $GITHUB_ENV + - name: legacy integration - docker + run: | + docker ps || true + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage + working-directory: ${{ env.COLLECTION_PATH }} + + - name: legacy integration - remove integration_config + working-directory: ${{ env.COLLECTION_INTEGRATION_PATH }} + run: rm "integration_config.yml" + - name: localenv_docker - setup run: | pwd - rm -f "${COLLECTION_INTEGRATION_PATH}/integration_config.yml" - pip install -r files/requirements/requirements.txt + pip install -r files/requirements/requirements.txt -c files/requirements/constraints.txt ./setup.sh working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_docker @@ -493,12 +498,6 @@ jobs: ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage --docker-network hashi_vault_default working-directory: ${{ env.COLLECTION_PATH }} - # Run the integration tests - - name: Run legacy integration tests (in docker) - run: | - docker ps || true - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage - working-directory: ${{ env.COLLECTION_PATH }} # ansible-test support producing code coverage date - name: Generate coverage report From 1e54a065ab3a691f73310c2fb57166b9e62ff505 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 15:55:32 -0400 Subject: [PATCH 093/137] macos has old ass bash --- tests/integration/targets/setup_localenv_docker/setup.sh | 2 +- tests/integration/targets/setup_localenv_gha/setup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/targets/setup_localenv_docker/setup.sh b/tests/integration/targets/setup_localenv_docker/setup.sh index c49ba7f82..29b17bab8 100755 --- a/tests/integration/targets/setup_localenv_docker/setup.sh +++ b/tests/integration/targets/setup_localenv_docker/setup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -eux +set -ex pushd "${BASH_SOURCE%/*}" diff --git a/tests/integration/targets/setup_localenv_gha/setup.sh b/tests/integration/targets/setup_localenv_gha/setup.sh index b5dfb0ac3..ec93d8d64 100755 --- a/tests/integration/targets/setup_localenv_gha/setup.sh +++ b/tests/integration/targets/setup_localenv_gha/setup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -eux +set -ex pushd "${BASH_SOURCE%/*}" From b12a5f9568440a499a952f4c311543fa4621b539 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 17:16:19 -0400 Subject: [PATCH 094/137] move collection installs into action --- .github/actions/collection-via-git/action.yml | 50 +++++++++++ .github/workflows/ansible-test.yml | 83 +++++++++++-------- 2 files changed, 99 insertions(+), 34 deletions(-) create mode 100644 .github/actions/collection-via-git/action.yml diff --git a/.github/actions/collection-via-git/action.yml b/.github/actions/collection-via-git/action.yml new file mode 100644 index 000000000..872d80307 --- /dev/null +++ b/.github/actions/collection-via-git/action.yml @@ -0,0 +1,50 @@ +--- +name: Ansible Collection via GitHub +description: Install Ansible collections direct from GitHub repositories without using ansible-galaxy. +branding: + icon: git-branch + color: yellow +inputs: + collection: + description: The name of the collection in namespace.collection_name form. + required: true + ref: + description: The git ref to install. Defaults to the latest release as listed in GitHub releases. Only supports branches and tags. + required: false + path: + description: The path to clone it to. Defaults to ansible_collections/namespace/collection_name. + required: false +runs: + using: composite + steps: + - shell: bash + run: | + COLLECTION="${{ inputs.collection }}" + P_PATH="${{ inputs.path }}" + P_REF="${{ inputs.ref }}" + + NS="${COLLECTION%.*}" + CN="${COLLECTION#*.}" + + # only collections in the ansible-collections organization are supported right now + $URLBASE="https://github.com/ansible-collections/${COLLECTION}" + $URLCLONE="${URLBASE}.git" + $URLLATEST="${URLBASE}/releases/latest" + + if [[ -n "${P_PATH}" ]] + then + OUTPATH="${P_PATH}" + else + OUTPATH="ansible_collections/${NS}/${CN}" + fi + + if [[ -n "${P_REF}" ]] + then + REF="${P_REF}" + else + # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 + latest=$(curl -fs -o/dev/null -w %{redirect_url} "${URLLATEST}") + REF=$(basename ${latest}) + fi + + git clone --depth=1 --branch "${REF}" "${URLCLONE}" "${OUTPATH}" diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 051a1423c..5c2f2bb56 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -360,6 +360,9 @@ jobs: with: path: ${{ env.COLLECTION_PATH }} + - name: Linkit + run: ln -s "${COLLECTION_PATH}/.github" .github + - name: Set up Python uses: actions/setup-python@v2 with: @@ -412,50 +415,62 @@ jobs: - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check - #TODO: remove dependency on community.crypto - - name: Find the latest version of community.crypto - run: | - # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 + # - name: Find the latest version of community.crypto + # run: | + # # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 - cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) - echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV + # cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) + # echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV - cglu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.general/releases/latest) - echo "CG_LATEST=$(basename ${cglu})" >> $GITHUB_ENV + # cglu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.general/releases/latest) + # echo "CG_LATEST=$(basename ${cglu})" >> $GITHUB_ENV - cdlu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.docker/releases/latest) - echo "CD_LATEST=$(basename ${cdlu})" >> $GITHUB_ENV + # cdlu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.docker/releases/latest) + # echo "CD_LATEST=$(basename ${cdlu})" >> $GITHUB_ENV - - name: Cache for community.crypto - id: cache-cc - uses: actions/cache@v2 - with: - path: ansible_collections/community/crypto - key: collection-community.crypto-${{ env.CC_LATEST }} + # - name: Cache for community.crypto + # id: cache-cc + # uses: actions/cache@v2 + # with: + # path: ansible_collections/community/crypto + # key: collection-community.crypto-${{ env.CC_LATEST }} - - name: Install collection dependencies - if: steps.cache-cc.outputs.cache-hit != 'true' - uses: actions/checkout@v2 + # - name: Install collection dependencies + # if: steps.cache-cc.outputs.cache-hit != 'true' + # uses: actions/checkout@v2 + # with: + # repository: ansible-collections/community.crypto + # ref: refs/tags/${{ env.CC_LATEST }} + # path: ansible_collections/community/crypto + + # - name: Install collection dependencies + # # if: steps.cache-cc.outputs.cache-hit != 'true' + # uses: actions/checkout@v2 + # with: + # repository: ansible-collections/community.general + # ref: refs/tags/${{ env.CG_LATEST }} + # path: ansible_collections/community/general + + # - name: Install collection dependencies + # # if: steps.cache-cc.outputs.cache-hit != 'true' + # uses: actions/checkout@v2 + # with: + # repository: ansible-collections/community.docker + # ref: refs/tags/${{ env.CD_LATEST }} + # path: ansible_collections/community/docker + + - uses: .github/actions/collection-via-git with: - repository: ansible-collections/community.crypto - ref: refs/tags/${{ env.CC_LATEST }} - path: ansible_collections/community/crypto + collection: community.crypto - - name: Install collection dependencies - # if: steps.cache-cc.outputs.cache-hit != 'true' - uses: actions/checkout@v2 + - uses: .github/actions/collection-via-git with: - repository: ansible-collections/community.general - ref: refs/tags/${{ env.CG_LATEST }} - path: ansible_collections/community/general + collection: community.docker - - name: Install collection dependencies - # if: steps.cache-cc.outputs.cache-hit != 'true' - uses: actions/checkout@v2 + - uses: .github/actions/collection-via-git + if: ${{ matrix.runner == 'macos-latest' }} # we need community.general.homebrew with: - repository: ansible-collections/community.docker - ref: refs/tags/${{ env.CD_LATEST }} - path: ansible_collections/community/docker + collection: community.general - name: legacy integration - use sample integration_config working-directory: ${{ env.COLLECTION_INTEGRATION_PATH }} From 4cecf7e7f59a444a1f016eccbd47a832b138754f Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 17:22:03 -0400 Subject: [PATCH 095/137] more fricken path bs --- .github/workflows/ansible-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 5c2f2bb56..147d1260d 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -459,15 +459,15 @@ jobs: # ref: refs/tags/${{ env.CD_LATEST }} # path: ansible_collections/community/docker - - uses: .github/actions/collection-via-git + - uses: ./.github/actions/collection-via-git with: collection: community.crypto - - uses: .github/actions/collection-via-git + - uses: ./.github/actions/collection-via-git with: collection: community.docker - - uses: .github/actions/collection-via-git + - uses: ./.github/actions/collection-via-git if: ${{ matrix.runner == 'macos-latest' }} # we need community.general.homebrew with: collection: community.general From 8068707edf9718844c48f17a7f9ae2642c84bf18 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 17:23:28 -0400 Subject: [PATCH 096/137] real dumb one, this one, meaning me --- .github/actions/collection-via-git/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/collection-via-git/action.yml b/.github/actions/collection-via-git/action.yml index 872d80307..43895af44 100644 --- a/.github/actions/collection-via-git/action.yml +++ b/.github/actions/collection-via-git/action.yml @@ -27,9 +27,9 @@ runs: CN="${COLLECTION#*.}" # only collections in the ansible-collections organization are supported right now - $URLBASE="https://github.com/ansible-collections/${COLLECTION}" - $URLCLONE="${URLBASE}.git" - $URLLATEST="${URLBASE}/releases/latest" + URLBASE="https://github.com/ansible-collections/${COLLECTION}" + URLCLONE="${URLBASE}.git" + URLLATEST="${URLBASE}/releases/latest" if [[ -n "${P_PATH}" ]] then From 8de52fb00303747f95d6e12e9e1115881d614966 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 17:45:32 -0400 Subject: [PATCH 097/137] Add action for macos docker setup --- .github/actions/macos-docker/action.yml | 18 ++++++++++++++++++ .github/workflows/ansible-test.yml | 23 +++++++++-------------- 2 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 .github/actions/macos-docker/action.yml diff --git a/.github/actions/macos-docker/action.yml b/.github/actions/macos-docker/action.yml new file mode 100644 index 000000000..aacd52387 --- /dev/null +++ b/.github/actions/macos-docker/action.yml @@ -0,0 +1,18 @@ +--- +name: Install Docker on MacOS GitHub Runner +description: Install and configure docker for a MacOS GitHub runner, and export the environment variables. +branding: + icon: command + color: white +runs: + using: composite + steps: + - shell: bash + run: | + mkdir -p ~/.docker/machine/cache + curl -Lo ~/.docker/machine/cache/boot2docker.iso https://github.com/boot2docker/boot2docker/releases/download/v19.03.12/boot2docker.iso + brew install docker-machine docker + docker --version + docker-machine create --driver virtualbox default + docker-machine env default + docker-machine env default | sed 's/^export //;/^#/d;s/^#.*//' | tr -d '"' >> $GITHUB_ENV diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 147d1260d..78403db39 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -207,7 +207,7 @@ jobs: with: path: ${{ env.COLLECTION_PATH }} - - name: Linkit + - name: Link to .github # easier access to local actions run: ln -s "${COLLECTION_PATH}/.github" .github - name: Set up Python @@ -360,7 +360,7 @@ jobs: with: path: ${{ env.COLLECTION_PATH }} - - name: Linkit + - name: Link to .github # easier access to local actions run: ln -s "${COLLECTION_PATH}/.github" .github - name: Set up Python @@ -459,15 +459,18 @@ jobs: # ref: refs/tags/${{ env.CD_LATEST }} # path: ansible_collections/community/docker - - uses: ./.github/actions/collection-via-git + - name: Install community.crypto + uses: ./.github/actions/collection-via-git with: collection: community.crypto - - uses: ./.github/actions/collection-via-git + - name: Install community.docker + uses: ./.github/actions/collection-via-git with: collection: community.docker - - uses: ./.github/actions/collection-via-git + - name: Install community.general + uses: ./.github/actions/collection-via-git if: ${{ matrix.runner == 'macos-latest' }} # we need community.general.homebrew with: collection: community.general @@ -482,18 +485,10 @@ jobs: - name: Install Docker on MacOS if: ${{ matrix.runner == 'macos-latest' }} - run: | - mkdir -p ~/.docker/machine/cache - curl -Lo ~/.docker/machine/cache/boot2docker.iso https://github.com/boot2docker/boot2docker/releases/download/v19.03.12/boot2docker.iso - brew install docker-machine docker - sudo docker --version - docker-machine create --driver virtualbox default - docker-machine env default - docker-machine env default | sed 's/^export //;/^#/d;s/^#.*//' | tr -d '"' >> $GITHUB_ENV + uses: ./.github/actions/macos-docker - name: legacy integration - docker run: | - docker ps || true ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage working-directory: ${{ env.COLLECTION_PATH }} From 7152ae0661b5f14a3bf53954abd8b053c8f4da79 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 18:12:36 -0400 Subject: [PATCH 098/137] reversing course on collections path... path... path... ptha.. phththththththt --- .github/workflows/ansible-test.yml | 1 + tests/integration/targets/setup_localenv_docker/setup.sh | 1 - tests/integration/targets/setup_localenv_gha/setup.sh | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 78403db39..6c7f912f8 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -19,6 +19,7 @@ env: NAMESPACE: community COLLECTION_NAME: hashi_vault ANSIBLE_FORCE_COLOR: true + ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} jobs: diff --git a/tests/integration/targets/setup_localenv_docker/setup.sh b/tests/integration/targets/setup_localenv_docker/setup.sh index 29b17bab8..f2fab1d8b 100755 --- a/tests/integration/targets/setup_localenv_docker/setup.sh +++ b/tests/integration/targets/setup_localenv_docker/setup.sh @@ -4,7 +4,6 @@ set -ex pushd "${BASH_SOURCE%/*}" -ANSIBLE_COLLECTIONS_PATHS="../../../../../../../" \ ANSIBLE_ROLES_PATH="../" \ ansible-playbook files/playbooks/vault_docker.yml "${@}" diff --git a/tests/integration/targets/setup_localenv_gha/setup.sh b/tests/integration/targets/setup_localenv_gha/setup.sh index ec93d8d64..5317aa407 100755 --- a/tests/integration/targets/setup_localenv_gha/setup.sh +++ b/tests/integration/targets/setup_localenv_gha/setup.sh @@ -4,7 +4,6 @@ set -ex pushd "${BASH_SOURCE%/*}" -ANSIBLE_COLLECTIONS_PATHS="../../../../../../../" \ ANSIBLE_ROLES_PATH="../" \ ansible-playbook files/playbooks/gha.yml "${@}" From c471edb17f10a7ed1190a955b6ad80fb2260d5ac Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 21:51:22 -0400 Subject: [PATCH 099/137] more tweaks --- .../setup_localenv_docker/defaults/main.yml | 5 +++++ .../setup_localenv_docker/tasks/main.yml | 6 ++++-- .../setup_localenv_docker/vars/.gitignore | 3 --- .../setup_localenv_docker/vars/local_client.yml | 17 +++++++++++++++++ .../targets/setup_localenv_docker/vars/main.yml | 9 +++------ .../setup_vault_server_cert/tasks/main.yml | 5 +++-- 6 files changed, 32 insertions(+), 13 deletions(-) delete mode 100644 tests/integration/targets/setup_localenv_docker/vars/.gitignore create mode 100644 tests/integration/targets/setup_localenv_docker/vars/local_client.yml diff --git a/tests/integration/targets/setup_localenv_docker/defaults/main.yml b/tests/integration/targets/setup_localenv_docker/defaults/main.yml index 19b328dd0..d597ecc18 100644 --- a/tests/integration/targets/setup_localenv_docker/defaults/main.yml +++ b/tests/integration/targets/setup_localenv_docker/defaults/main.yml @@ -13,8 +13,11 @@ docker_compose_project_name: hashi_vault vault_port_http: 8200 vault_port_https: 8300 vault_container_name: vault +vault_target_name: '{{ vault_container_name }}' + proxy_port: 8888 proxy_container_name: tinyproxy +proxy_target_name: '{{ proxy_container_name }}' output_dir: '{{ role_path }}/files/.output' @@ -25,3 +28,5 @@ vault_config_output: '{{ output_dir }}/vault_config' vault_cert_file: '{{ vault_config_output }}/cert.pem' vault_key_file: '{{ vault_config_output }}/key.pem' + +vault_crypto_force: False diff --git a/tests/integration/targets/setup_localenv_docker/tasks/main.yml b/tests/integration/targets/setup_localenv_docker/tasks/main.yml index d97ae221e..3480adfef 100644 --- a/tests/integration/targets/setup_localenv_docker/tasks/main.yml +++ b/tests/integration/targets/setup_localenv_docker/tasks/main.yml @@ -36,11 +36,13 @@ register: key_status - name: "Generate certs" - when: not (key_status.stat.exists and cert_status.stat.exists) + when: >- + vault_crypto_force | bool + or not (key_status.stat.exists and cert_status.stat.exists) include_role: name: setup_vault_server_cert vars: - vault_hostname: '{{ vault_container_name }}' + vault_dns_names: '{{ [vault_target_name, vault_container_name] | unique }}' - name: "Template vault config" template: diff --git a/tests/integration/targets/setup_localenv_docker/vars/.gitignore b/tests/integration/targets/setup_localenv_docker/vars/.gitignore deleted file mode 100644 index 88d5df19c..000000000 --- a/tests/integration/targets/setup_localenv_docker/vars/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!.gitignore -!main.yml diff --git a/tests/integration/targets/setup_localenv_docker/vars/local_client.yml b/tests/integration/targets/setup_localenv_docker/vars/local_client.yml new file mode 100644 index 000000000..0cf3a9ddf --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/vars/local_client.yml @@ -0,0 +1,17 @@ +# WIP: experimental/broken +# a configuration for running the tests locally (outside the container network) +--- +vault_target_name: localhost +proxy_target_name: localhost + +integration_config: + vault_version: '{{ vault_version }}' + vault_test_server_http: 'http://{{ vault_target_name }}:{{ vault_port_http }}' + vault_test_server_https: 'https://{{ vault_target_name }}:{{ vault_port_https}}' + vault_dev_root_token_id: '{{ vault_dev_root_token_id }}' + vault_proxy_server: 'http://{{ proxy_target_name }}:{{ proxy_port }}' + vault_test_server_external: True + vault_proxy_external: True + vault_cert_content: "{{ lookup('file', vault_cert_file) }}" + vault_proxy_alt_vault_http: 'http://{{ vault_container_name }}:{{ vault_port_http }}' + vault_proxy_alt_vault_https: 'https://{{ vault_container_name }}:{{ vault_port_https }}' diff --git a/tests/integration/targets/setup_localenv_docker/vars/main.yml b/tests/integration/targets/setup_localenv_docker/vars/main.yml index 985f0899d..0d8ef3a48 100644 --- a/tests/integration/targets/setup_localenv_docker/vars/main.yml +++ b/tests/integration/targets/setup_localenv_docker/vars/main.yml @@ -1,13 +1,10 @@ --- integration_config: vault_version: '{{ vault_version }}' - vault_test_server_http: 'http://{{ vault_container_name }}:{{ vault_port_http }}' - vault_test_server_https: 'https://{{ vault_container_name }}:{{ vault_port_https}}' + vault_test_server_http: 'http://{{ vault_target_name }}:{{ vault_port_http }}' + vault_test_server_https: 'https://{{ vault_target_name }}:{{ vault_port_https}}' vault_dev_root_token_id: '{{ vault_dev_root_token_id }}' - vault_proxy_server: 'http://{{ proxy_container_name }}:{{ proxy_port }}' + vault_proxy_server: 'http://{{ proxy_target_name }}:{{ proxy_port }}' vault_test_server_external: True vault_proxy_external: True vault_cert_content: "{{ lookup('file', vault_cert_file) }}" - # vault_server_legacy: False - # vault_cert_file: '{{ vault_cert_file }}' - # vault_key_file: '{{ vault_key_file }}' diff --git a/tests/integration/targets/setup_vault_server_cert/tasks/main.yml b/tests/integration/targets/setup_vault_server_cert/tasks/main.yml index 1aa16ab5c..6c7a56ac4 100644 --- a/tests/integration/targets/setup_vault_server_cert/tasks/main.yml +++ b/tests/integration/targets/setup_vault_server_cert/tasks/main.yml @@ -9,12 +9,13 @@ path: '{{ vault_key_file }}' - name: Generate CSR + vars: + vault_dns_names: '{{ [vault_hostname] + (vault_alternate_hostnames | default([]) }}' community.crypto.openssl_csr: mode: 'o=r' path: '{{ vault_csr_file }}' privatekey_path: '{{ vault_key_file }}' - subject: - commonName: '{{ vault_hostname }}' + subject_alt_name: "{{ vault_dns_names | map('regex_replace', '^', 'DNS:') | list }}" - name: Generate selfsigned certificate community.crypto.x509_certificate: From 283dd66fb4edf96442f6f42a393e3d1a65b2afcf Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 21:54:14 -0400 Subject: [PATCH 100/137] errant ) --- .../integration/targets/setup_vault_server_cert/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/setup_vault_server_cert/tasks/main.yml b/tests/integration/targets/setup_vault_server_cert/tasks/main.yml index 6c7a56ac4..a2997bed5 100644 --- a/tests/integration/targets/setup_vault_server_cert/tasks/main.yml +++ b/tests/integration/targets/setup_vault_server_cert/tasks/main.yml @@ -10,7 +10,7 @@ - name: Generate CSR vars: - vault_dns_names: '{{ [vault_hostname] + (vault_alternate_hostnames | default([]) }}' + vault_dns_names: '{{ [vault_hostname] + (vault_alternate_hostnames | default([])) }}' community.crypto.openssl_csr: mode: 'o=r' path: '{{ vault_csr_file }}' From 67d41478437e2b960398163ed54d9f5c01a58666 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 22:36:12 -0400 Subject: [PATCH 101/137] integration_config cleanup --- tests/integration/!integration_config.yml | 68 ------------------- .../integration/integration_config.yml.sample | 6 +- 2 files changed, 4 insertions(+), 70 deletions(-) delete mode 100644 tests/integration/!integration_config.yml diff --git a/tests/integration/!integration_config.yml b/tests/integration/!integration_config.yml deleted file mode 100644 index 38c97ddfd..000000000 --- a/tests/integration/!integration_config.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Ansible managed ---- -# can be uncommented once vault configuration has been run once (WIP/broken) -# vault_server_configure: False -vault_cert_content: '-----BEGIN CERTIFICATE----- - - MIIE4TCCAsmgAwIBAgIUIL77ChgfiJcegZViZh8TV9KviB8wDQYJKoZIhvcNAQEL - - BQAwEDEOMAwGA1UEAwwFdmF1bHQwHhcNMjEwNzAxMTgyNjIzWhcNMzEwNjI5MTgy - - NjIzWjAQMQ4wDAYDVQQDDAV2YXVsdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC - - AgoCggIBAK1mI97qE0Pijd0vYrepaoZen/7yuayVslJ1uKzNQWCwHCpaAG9QuGof - - ym2N4UnCnBAmpcOH8N+xVzFR5oQVJwYLodfPeI8VDcHvN6Cj011Lb40o2fAbQz+v - - 2Xz4MOpEzRbCrSlMZa7M10+iiZJpU379Yqn6JDiCkwrq4dIBVUnp8Wg5ykV+qdUz - - ypB6jJF67WRisXqYp7hACL67FRlj/r8+76FSUg/oAo7g0rkrahWb+SvNMPWS1hS7 - - Tk8Tjf6qb2ZO2Iwx0eEY2GNoziUzQu1xTkZQhbQt3vt8ZiWvddCRxCz5W1cfKWO/ - - 0XQlMlqlW2RmfxEouMUqhQlH1NSYCKas12RFxxSaiPI2Idq2Kve2xDLwQuIOoCLD - - joqSQmyF88f9Jxb0l2sXXsro/YwmYJ+qwz81QW4BO5LzX7BHT7EuiV97m8kLuBzE - - 0pjat0XVJ9fzmfGYX89uiwnO/fb7jZubjQLus0cmZXHMB6wK0fo8bTXgi9TLXwbi - - wymDG+A+jlSShf7aE3vZtr6fRcacjwh6Y6DFbfxdVV9Vxzv1aHaMUYwtIu+d+uVC - - cTQbwou6B7hS0BUdXQKtM1mjPDAwcdgz//TLzk26tIsx69AXtOREKb9W1ffIbfGh - - B8nrdmI6+80tlc8KL6s+/cMLEMya3K2GZPloAw+CO00ihO7SGC6VAgMBAAGjMzAx - - MBAGA1UdEQQJMAeCBXZhdWx0MB0GA1UdDgQWBBRbYPX83fHK2QUTsWExMQESmS6B - - yTANBgkqhkiG9w0BAQsFAAOCAgEAc9slz9up7xd3bsr+q/kCoDt+w6rm/dc6ONSJ - - PITZAbuWtRBtCJStQuie5ZICnh1X0IajhczIFVcD9CjxOIxfxA7S49gL9vDHVpiJ - - K4nW0KR3Zviq2XwtHYAs99CZH63EUTVqz0nEuMu10H/0PCFPtTHcXFpgovCLRAGH - - HqnM7LVeM7a0g85Zt+HXuPJ2MThlEyIBy64MBPIczSiGDVx0cQwe1LJREkQJgB4F - - +3iAOPIsHAWkApFfx2cyq+L4sEd0EdxUFk1mw4sni/VFzK8wcd3L7gEgseKSk2Kz - - Z3JZiAXessjoa26JL0/KBSN6LTB3/pdn/dG7lz7DUr17PymbggRLVjdbSsbAFD9S - - BTGgt3kFjrXIeNAyqGodK673R5jACXjz0vuEePJh4Vk/ffl953bH8Xhs/BZyNXBC - - meOu/sU93MGPi0vqE+Jdjplvj5smLzOW9Y6HAAitDqHTQ0sNdZP2DdT6FkSLSzMx - - ErPFBePgxhQFXvQt8h11Gadox5vsm1Ca2nLGClKWRt458goFEWgmmDA+mOD2/sJ0 - - eYdGfBgN/ZnTzD2y2z18Sd9H2Zb4HZbfpPvvFwR+5oYMqE72Rz3oj8APt2f/Eq5B - - WYNpi1fRCGPXhM2wNr2DKPKcoqbRcIVTxD/E0MATmxQRGtPaO/JbFcY5v37qcLWf - - Jb3iD/U= - - -----END CERTIFICATE-----' -vault_dev_root_token_id: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a -vault_proxy_external: true -vault_proxy_server: http://tinyproxy:8888 -vault_test_server_external: true -vault_test_server_http: http://vault:8200 -vault_test_server_https: https://vault:8300 -vault_version: 1.7.3 diff --git a/tests/integration/integration_config.yml.sample b/tests/integration/integration_config.yml.sample index 1e30235d4..4179a04a3 100644 --- a/tests/integration/integration_config.yml.sample +++ b/tests/integration/integration_config.yml.sample @@ -1,7 +1,9 @@ -# +# copy this file to integration_config.yml to use the legacy style all-in-one setup +# for best results, consider a different localenv setup. +# See the additional documentation at https://docs.ansible.com/ansible/devel/collections/community/hashi_vault/ --- vault_integration_legacy: true -vault_version: 1.7.2 +vault_version: 1.7.3 vault_dev_root_token_id: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a From 2c4768e20ea56367fc818ef187b49d8bf7c0b817 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 22:37:41 -0400 Subject: [PATCH 102/137] add READMEs for setup roles --- tests/integration/targets/setup_cert_content/README.md | 2 ++ tests/integration/targets/setup_localenv_docker/README.md | 7 +++++++ tests/integration/targets/setup_localenv_gha/README.md | 4 ++++ tests/integration/targets/setup_tinyproxy_server/README.md | 5 +++++ tests/integration/targets/setup_vault_server/README.md | 6 ++++++ .../integration/targets/setup_vault_server_cert/README.md | 5 +++++ .../targets/setup_vault_server_download/README.md | 5 +++++ 7 files changed, 34 insertions(+) create mode 100644 tests/integration/targets/setup_cert_content/README.md create mode 100644 tests/integration/targets/setup_localenv_docker/README.md create mode 100644 tests/integration/targets/setup_localenv_gha/README.md create mode 100644 tests/integration/targets/setup_tinyproxy_server/README.md create mode 100644 tests/integration/targets/setup_vault_server/README.md create mode 100644 tests/integration/targets/setup_vault_server_cert/README.md create mode 100644 tests/integration/targets/setup_vault_server_download/README.md diff --git a/tests/integration/targets/setup_cert_content/README.md b/tests/integration/targets/setup_cert_content/README.md new file mode 100644 index 000000000..48c2b10ba --- /dev/null +++ b/tests/integration/targets/setup_cert_content/README.md @@ -0,0 +1,2 @@ +# `setup_cert_content` +Tiny role used for writing out the certificate when it was supplied as a string in vars (usually from `integration_config.yml`). Any target testing TLS connectivity that needs to verify the cert will need this. diff --git a/tests/integration/targets/setup_localenv_docker/README.md b/tests/integration/targets/setup_localenv_docker/README.md new file mode 100644 index 000000000..934df7acc --- /dev/null +++ b/tests/integration/targets/setup_localenv_docker/README.md @@ -0,0 +1,7 @@ +# `setup_localenv_docker` +Uses `docker-compose` to set up required external dependencies for integration tests. + +See the guides in the [devel documentation for the latest information](https://docs.ansible.com/ansible/devel/collections/community/hashi_vault/). + +## Notes +* For requirements, see the files in `files/requirements/`. diff --git a/tests/integration/targets/setup_localenv_gha/README.md b/tests/integration/targets/setup_localenv_gha/README.md new file mode 100644 index 000000000..1f9641e3b --- /dev/null +++ b/tests/integration/targets/setup_localenv_gha/README.md @@ -0,0 +1,4 @@ +# `setup_localenv_gha` +A special case "localenv" role that isn't for end-user use, but rather specifically for use in the collection's GitHub Actions CI. + +It uses `setup_localenv_docker` but with pre-generated PKI and directly uses the `docker-compose` CLI in CI, in order to avoid additional dependencies and cut down execution time. diff --git a/tests/integration/targets/setup_tinyproxy_server/README.md b/tests/integration/targets/setup_tinyproxy_server/README.md new file mode 100644 index 000000000..dc1fb2a58 --- /dev/null +++ b/tests/integration/targets/setup_tinyproxy_server/README.md @@ -0,0 +1,5 @@ +# `setup_tinyproxy_server` +Responsible for installing and running a `tinyproxy` server. + +## Notes +* Installs `tinyproxy` via the `ansible.builtin.package` module, which will in turn require the correct packaging module and its required libraries to be available. On MacOS that means the [`community.general` collection](https://galaxy.ansible.com/community/general) is required for homebrew support. diff --git a/tests/integration/targets/setup_vault_server/README.md b/tests/integration/targets/setup_vault_server/README.md new file mode 100644 index 000000000..e3402be83 --- /dev/null +++ b/tests/integration/targets/setup_vault_server/README.md @@ -0,0 +1,6 @@ +# `setup_vault_server` +Responsible for ensuring a Vault server is running locally, with or without TLS. + +## Notes +* Uses `setup_vault_server_download`. +* Uses `setup_vault_server_cert`. diff --git a/tests/integration/targets/setup_vault_server_cert/README.md b/tests/integration/targets/setup_vault_server_cert/README.md new file mode 100644 index 000000000..1d12c29d1 --- /dev/null +++ b/tests/integration/targets/setup_vault_server_cert/README.md @@ -0,0 +1,5 @@ +# `setup_vault_server_cert` +Generates and key and self-signed certificate for the Vault server. + +## Notes +* Requires the [`community.crypto` collection](https://galaxy.ansible.com/community/crypto). diff --git a/tests/integration/targets/setup_vault_server_download/README.md b/tests/integration/targets/setup_vault_server_download/README.md new file mode 100644 index 000000000..4a73ad379 --- /dev/null +++ b/tests/integration/targets/setup_vault_server_download/README.md @@ -0,0 +1,5 @@ +# `setup_vault_server_download` +This role downloads a specified version of Vault and sets `vault_cmd` to the full path of the binary. + +## Notes +* Installs `unzip` via the `ansible.builtin.package` module, which will in turn require the correct packaging module and its required libraries to be available. MacOS is assumed to have `unzip` already. From 87b92cd2f2db3dd8643058444183c17e0ccb7cac Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 22:38:11 -0400 Subject: [PATCH 103/137] setup_vault_server cleanup --- .../setup_vault_server/handlers/main.yml | 3 -- .../targets/setup_vault_server/tasks/main.yml | 1 - .../setup_vault_server/tasks/vault_server.yml | 38 +------------------ 3 files changed, 1 insertion(+), 41 deletions(-) diff --git a/tests/integration/targets/setup_vault_server/handlers/main.yml b/tests/integration/targets/setup_vault_server/handlers/main.yml index 2292ac4cb..eaab6a803 100644 --- a/tests/integration/targets/setup_vault_server/handlers/main.yml +++ b/tests/integration/targets/setup_vault_server/handlers/main.yml @@ -1,7 +1,4 @@ --- -# notify 'test_managed_vault_cleanup' for tasks related to the -# vault server that is started by these tests -# (those tasks should skip if the vault server is external to the test run) - name: 'Kill vault process' shell: "kill $(cat {{ vault_pid_file }})" ignore_errors: true diff --git a/tests/integration/targets/setup_vault_server/tasks/main.yml b/tests/integration/targets/setup_vault_server/tasks/main.yml index 22b11ad81..832a3701a 100644 --- a/tests/integration/targets/setup_vault_server/tasks/main.yml +++ b/tests/integration/targets/setup_vault_server/tasks/main.yml @@ -1,6 +1,5 @@ --- - import_tasks: vault_server.yml - # when: not vault_test_server_external | bool - set_fact: vault_test_server_http: "{{ vault_test_server_http }}" diff --git a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml index dbca7cae6..7d6f2edce 100644 --- a/tests/integration/targets/setup_vault_server/tasks/vault_server.yml +++ b/tests/integration/targets/setup_vault_server/tasks/vault_server.yml @@ -1,5 +1,5 @@ --- -- name: Install Hashi Vault on controlled node and test +- name: Install HashiCorp Vault locally block: - when: vault_run_https_tests | bool @@ -7,42 +7,6 @@ vault_hostname: localhost include_role: name: setup_vault_server_cert - - # vars: - # vault_csr_file: '{{ vault_key_file | dirname }}/csr.csr' - # block: - # - name: Generate privatekey - # community.crypto.openssl_privatekey: - # path: '{{ vault_key_file }}' - - # - name: Generate CSR - # community.crypto.openssl_csr: - # path: '{{ vault_csr_file }}' - # privatekey_path: '{{ vault_key_file }}' - # subject: - # commonName: localhost - - # - name: Generate selfsigned certificate - # community.crypto.openssl_certificate: - # path: '{{ vault_cert_file }}' - # csr_path: '{{ vault_csr_file }}' - # privatekey_path: '{{ vault_key_file }}' - # provider: selfsigned - # selfsigned_digest: sha256 - # register: selfsigned_certificate - - # - name: "Check if vault binary exists" - # stat: - # path: '{{ vault_cmd }}' - # follow: yes - # get_attributes: no - # get_checksum: no - # get_mime: no - # register: bin_status - - # - name: "Download vault if not local" - # when: not bin_status.stat.exists - # import_tasks: vault_download.yml - import_role: name: setup_vault_server_download From d9cf90188d2c4cf76f7c39fb61c72f2bb50a3689 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 22:38:31 -0400 Subject: [PATCH 104/137] setup_tinyproxy_server cleanup --- .../targets/setup_tinyproxy_server/defaults/main.yml | 1 - tests/integration/targets/setup_tinyproxy_server/tasks/main.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/targets/setup_tinyproxy_server/defaults/main.yml b/tests/integration/targets/setup_tinyproxy_server/defaults/main.yml index 86a28b508..4f65590fc 100644 --- a/tests/integration/targets/setup_tinyproxy_server/defaults/main.yml +++ b/tests/integration/targets/setup_tinyproxy_server/defaults/main.yml @@ -1,6 +1,5 @@ --- vault_proxy_server: 'http://127.0.0.1:8001' -# vault_proxy_external: False vault_proxy_tmp: /tmp vault_proxy_pid: '{{ vault_proxy_tmp }}/tinyproxy.pid' diff --git a/tests/integration/targets/setup_tinyproxy_server/tasks/main.yml b/tests/integration/targets/setup_tinyproxy_server/tasks/main.yml index 8b133a668..c897ed2b1 100644 --- a/tests/integration/targets/setup_tinyproxy_server/tasks/main.yml +++ b/tests/integration/targets/setup_tinyproxy_server/tasks/main.yml @@ -2,7 +2,7 @@ - name: Install tinyproxy become: '{{ ansible_distribution != "MacOSX" }}' vars: - # check 'Install unzip' task to know why we set ansible_python_interpreter + # check 'Install unzip' task in setup_vault_server_download to know why we set ansible_python_interpreter ansible_python_interpreter: "{{ '/usr/bin/python3' if ansible_distribution in ['Ubuntu', 'Debian'] else ansible_python.executable }}" From 40d104d7fc58eb08082a99ff66f6f26ba4d5d5ff Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 8 Jul 2021 22:43:16 -0400 Subject: [PATCH 105/137] tidying up --- .../targets/lookup_hashi_vault/aliases | 1 - .../lookup_hashi_vault/defaults/main.yml | 36 ++----------------- .../lookup_hashi_vault/files/bin/.gitignore | 3 -- .../lookup_hashi_vault/handlers/main.yml | 34 ------------------ .../targets/lookup_hashi_vault/tasks/main.yml | 12 ------- .../lookup_hashi_vault/vars/.gitignore | 3 -- .../integration/targets/lookup_thing/aliases | 2 -- .../targets/lookup_thing/meta/main.yml | 3 -- .../targets/setup_old_stuff/aliases | 1 - .../files/playbooks/download_vault.yml | 9 ----- .../files/playbooks/install_dependencies.yml | 13 ------- .../playbooks/test_lookup_hashi_vault.yml | 10 ------ .../targets/setup_old_stuff/files/runme.sh | 26 -------------- .../files/setup_pkg_mgr/tasks/main.yml | 17 --------- .../files/setup_remote_constraints/aliases | 1 - .../setup_remote_constraints/meta/main.yml | 2 -- .../setup_remote_constraints/tasks/main.yml | 13 ------- .../setup_remote_tmp_dir/handlers/main.yml | 5 --- .../tasks/default-cleanup.yml | 5 --- .../setup_remote_tmp_dir/tasks/default.yml | 11 ------ .../files/setup_remote_tmp_dir/tasks/main.yml | 15 -------- 21 files changed, 2 insertions(+), 220 deletions(-) delete mode 100644 tests/integration/targets/lookup_hashi_vault/files/bin/.gitignore delete mode 100644 tests/integration/targets/lookup_hashi_vault/handlers/main.yml delete mode 100644 tests/integration/targets/lookup_hashi_vault/vars/.gitignore delete mode 100644 tests/integration/targets/lookup_thing/aliases delete mode 100644 tests/integration/targets/lookup_thing/meta/main.yml delete mode 100644 tests/integration/targets/setup_old_stuff/aliases delete mode 100644 tests/integration/targets/setup_old_stuff/files/playbooks/download_vault.yml delete mode 100644 tests/integration/targets/setup_old_stuff/files/playbooks/install_dependencies.yml delete mode 100644 tests/integration/targets/setup_old_stuff/files/playbooks/test_lookup_hashi_vault.yml delete mode 100755 tests/integration/targets/setup_old_stuff/files/runme.sh delete mode 100644 tests/integration/targets/setup_old_stuff/files/setup_pkg_mgr/tasks/main.yml delete mode 100644 tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/aliases delete mode 100644 tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/meta/main.yml delete mode 100644 tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/tasks/main.yml delete mode 100644 tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/handlers/main.yml delete mode 100644 tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default-cleanup.yml delete mode 100644 tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default.yml delete mode 100644 tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/main.yml diff --git a/tests/integration/targets/lookup_hashi_vault/aliases b/tests/integration/targets/lookup_hashi_vault/aliases index 87649fca9..4d013e3b0 100644 --- a/tests/integration/targets/lookup_hashi_vault/aliases +++ b/tests/integration/targets/lookup_hashi_vault/aliases @@ -1,4 +1,3 @@ skip/python2.6 # Python 2.6 is not supported by the collection needs/target/setup_vault_server needs/target/setup_tinyproxy_server -# needs/target/setup_localenv_gha diff --git a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml index 94025354e..dd1f85c73 100644 --- a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml @@ -74,44 +74,12 @@ auth_methods: - approle - approle-alt - - -# ## vars for vault server - -# vault_ansible_arch_table: -# 'x86_64': 'amd64' # Linux -# 'amd64': 'amd64' # FreeBSD -# 'i386': '386' - -# vault_arch: "{{ vault_ansible_arch_table[ansible_architecture] }}" - -# vault_version: '1.7.3' -# vault_bin: '{{ role_path }}/files/bin/{{ vault_slug }}' -# vault_slug: 'vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}' -# vault_zip: '{{ vault_bin }}/{{ vault_slug }}.zip' -# vault_uri: 'https://releases.hashicorp.com/vault/{{ vault_version }}/{{ vault_slug }}.zip' -# vault_cmd: '{{ vault_bin }}/vault' - -# vault_dev_root_token_id: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' -# vault_test_server_https: 'https://localhost:8201' -# vault_test_server_http: 'http://localhost:8200' - -# # this means "don't download and start a Vault server", instead -# # just use the addresses above to connect to one that's already running -# vault_test_server_external: False - -# # WIP +# WIP vault_test_server_configure: True -# # when False the tests requiring a valid SSL connection to Vault will be skipped +# when False the tests requiring a valid SSL connection to Vault will be skipped vault_run_https_tests: True -# vault_cert_file: '{{ local_temp_dir }}/cert.pem' -# vault_key_file: '{{ local_temp_dir }}/privatekey.pem' - -# vault_proxy_server: 'http://127.0.0.1:8001' -# vault_proxy_external: False - # if any connections are taking longer than this to complete there's probably something really wrong # with the integration tests, so it'd be better to fail faster than the 30s default ansible_hashi_vault_timeout: 5 diff --git a/tests/integration/targets/lookup_hashi_vault/files/bin/.gitignore b/tests/integration/targets/lookup_hashi_vault/files/bin/.gitignore deleted file mode 100644 index 7c9d611b5..000000000 --- a/tests/integration/targets/lookup_hashi_vault/files/bin/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!.gitignore -!README.md diff --git a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml b/tests/integration/targets/lookup_hashi_vault/handlers/main.yml deleted file mode 100644 index e6be2a140..000000000 --- a/tests/integration/targets/lookup_hashi_vault/handlers/main.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -# notify 'test_managed_vault_cleanup' for tasks related to the -# vault server that is started by these tests -# (those tasks should skip if the vault server is external to the test run) -# - name: 'Kill vault process' -# shell: "kill $(cat {{ local_temp_dir }}/vault.pid)" -# ignore_errors: true -# listen: test_managed_vault_cleanup - -# - name: Stop tinyproxy -# shell: -# cmd: | -# kill $(cat "{{ local_temp_dir }}/tinyproxy.pid") -# rm -f "{{ local_temp_dir }}/tinyproxy.pid" -# removes: "{{ local_temp_dir }}/tinyproxy.pid" - -# - name: Uninstall tinyproxy -# become: '{{ ansible_distribution != "MacOSX" }}' -# vars: -# # check 'Install unzip' task to know why we set ansible_python_interpreter -# ansible_python_interpreter: "{{ -# '/usr/bin/python3' if ansible_distribution in ['Ubuntu', 'Debian'] else ansible_python.executable -# }}" -# package: -# name: tinyproxy -# state: absent -# ignore_errors: yes - -# notify 'cleanup' for any handlers that should always run at the end of tests -# - name: 'Delete temp dir' -# file: -# path: '{{ local_temp_dir }}' -# state: absent -# listen: cleanup diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml index b72226246..839e0a4e5 100644 --- a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml @@ -4,15 +4,6 @@ # and should not be used as examples of how to write Ansible roles # #################################################################### -# - name: Create a local temporary directory -# tempfile: -# state: directory -# register: tempfile_result -# notify: cleanup - -# - set_fact: -# local_temp_dir: '{{ tempfile_result.path }}' - - include_role: name: setup_vault_server when: vault_integration_legacy | bool @@ -24,9 +15,6 @@ - import_tasks: vault_server_configure.yml when: vault_test_server_configure | bool -# - include_tasks: tinyproxy_server.yml -# when: not vault_proxy_external | bool - - import_tasks: tests.yml vars: auth_type: none diff --git a/tests/integration/targets/lookup_hashi_vault/vars/.gitignore b/tests/integration/targets/lookup_hashi_vault/vars/.gitignore deleted file mode 100644 index 7c9d611b5..000000000 --- a/tests/integration/targets/lookup_hashi_vault/vars/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!.gitignore -!README.md diff --git a/tests/integration/targets/lookup_thing/aliases b/tests/integration/targets/lookup_thing/aliases deleted file mode 100644 index 4351e915b..000000000 --- a/tests/integration/targets/lookup_thing/aliases +++ /dev/null @@ -1,2 +0,0 @@ -needs/target/lookup_hashi_vault -hidden diff --git a/tests/integration/targets/lookup_thing/meta/main.yml b/tests/integration/targets/lookup_thing/meta/main.yml deleted file mode 100644 index 9f29d69ab..000000000 --- a/tests/integration/targets/lookup_thing/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - lookup_hashi_vault diff --git a/tests/integration/targets/setup_old_stuff/aliases b/tests/integration/targets/setup_old_stuff/aliases deleted file mode 100644 index 136c05e0d..000000000 --- a/tests/integration/targets/setup_old_stuff/aliases +++ /dev/null @@ -1 +0,0 @@ -hidden diff --git a/tests/integration/targets/setup_old_stuff/files/playbooks/download_vault.yml b/tests/integration/targets/setup_old_stuff/files/playbooks/download_vault.yml deleted file mode 100644 index d5664c776..000000000 --- a/tests/integration/targets/setup_old_stuff/files/playbooks/download_vault.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- hosts: localhost - force_handlers: yes - gather_facts: yes - gather_subset: '!min,!all,distribution' - tasks: - - import_role: - name: lookup_hashi_vault - tasks_from: vault_download diff --git a/tests/integration/targets/setup_old_stuff/files/playbooks/install_dependencies.yml b/tests/integration/targets/setup_old_stuff/files/playbooks/install_dependencies.yml deleted file mode 100644 index bfc533a60..000000000 --- a/tests/integration/targets/setup_old_stuff/files/playbooks/install_dependencies.yml +++ /dev/null @@ -1,13 +0,0 @@ -- hosts: localhost - vars: - remote_constraints: '{{ playbook_dir }}/../../../../utils/constraints.txt' - tasks: - - name: Install cryptography - pip: - name: cryptography - extra_args: '-c {{ remote_constraints }}' - - - name: 'Install hvac Python package' - pip: - name: "{{ hvac_package|default('hvac') }}" - extra_args: '-c {{ remote_constraints }}' diff --git a/tests/integration/targets/setup_old_stuff/files/playbooks/test_lookup_hashi_vault.yml b/tests/integration/targets/setup_old_stuff/files/playbooks/test_lookup_hashi_vault.yml deleted file mode 100644 index 090c19df4..000000000 --- a/tests/integration/targets/setup_old_stuff/files/playbooks/test_lookup_hashi_vault.yml +++ /dev/null @@ -1,10 +0,0 @@ -- hosts: localhost - force_handlers: yes - tasks: - - name: register cryptography version - command: "{{ ansible_python.executable }} -c 'import cryptography; print(cryptography.__version__)'" - register: cryptography_version - - - name: Test lookup hashi_vault - import_role: - name: lookup_hashi_vault/lookup_hashi_vault diff --git a/tests/integration/targets/setup_old_stuff/files/runme.sh b/tests/integration/targets/setup_old_stuff/files/runme.sh deleted file mode 100755 index 4aec084e3..000000000 --- a/tests/integration/targets/setup_old_stuff/files/runme.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -export ANSIBLE_TEST_PREFER_VENV=1 # see https://github.com/ansible/ansible/pull/73000#issuecomment-757012395; can be removed once Ansible 2.9 and ansible-base 2.10 support has been dropped -source virtualenv.sh - -# First install pyOpenSSL, then test lookup in a second playbook in order to -# workaround this error which occurs on OS X 10.11 only: -# -# TASK [lookup_hashi_vault : test token auth with certs (validation enabled, lookup parameters)] *** -# included: lookup_hashi_vault/tasks/token_test.yml for testhost -# -# TASK [lookup_hashi_vault : Fetch secrets using "hashi_vault" lookup] *** -# From cffi callback : -# Traceback (most recent call last): -# File "/usr/local/lib/python2.7/site-packages/OpenSSL/SSL.py", line 309, in wrapper -# _lib.X509_up_ref(x509) -# AttributeError: 'module' object has no attribute 'X509_up_ref' -# fatal: [testhost]: FAILED! => { "msg": "An unhandled exception occurred while running the lookup plugin 'hashi_vault'. Error was a , original message: HTTPSConnectionPool(host='localhost', port=8201): Max retries exceeded with url: /v1/auth/token/lookup-self (Caused by SSLError(SSLError(\"bad handshake: Error([('SSL routines', 'ssl3_get_server_certificate', 'certificate verify failed')],)\",),))"} - -ANSIBLE_ROLES_PATH=../ \ - ansible-playbook playbooks/install_dependencies.yml -v "$@" - -ANSIBLE_ROLES_PATH=../ \ - ansible-playbook playbooks/test_lookup_hashi_vault.yml -v "$@" diff --git a/tests/integration/targets/setup_old_stuff/files/setup_pkg_mgr/tasks/main.yml b/tests/integration/targets/setup_old_stuff/files/setup_pkg_mgr/tasks/main.yml deleted file mode 100644 index 24d022286..000000000 --- a/tests/integration/targets/setup_old_stuff/files/setup_pkg_mgr/tasks/main.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -#################################################################### -# WARNING: These are designed specifically for Ansible tests # -# and should not be used as examples of how to write Ansible roles # -#################################################################### - -- set_fact: - pkg_mgr: community.general.pkgng - ansible_pkg_mgr: community.general.pkgng - cacheable: yes - when: ansible_os_family == "FreeBSD" - -- set_fact: - pkg_mgr: community.general.zypper - ansible_pkg_mgr: community.general.zypper - cacheable: yes - when: ansible_os_family == "Suse" diff --git a/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/aliases b/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/aliases deleted file mode 100644 index 1ad133ba0..000000000 --- a/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/aliases +++ /dev/null @@ -1 +0,0 @@ -needs/file/tests/utils/constraints.txt diff --git a/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/meta/main.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/meta/main.yml deleted file mode 100644 index 1810d4bec..000000000 --- a/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/meta/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -dependencies: - - setup_remote_tmp_dir diff --git a/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/tasks/main.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/tasks/main.yml deleted file mode 100644 index d4f8148c8..000000000 --- a/tests/integration/targets/setup_old_stuff/files/setup_remote_constraints/tasks/main.yml +++ /dev/null @@ -1,13 +0,0 @@ -#################################################################### -# WARNING: These are designed specifically for Ansible tests # -# and should not be used as examples of how to write Ansible roles # -#################################################################### - -- name: record constraints.txt path on remote host - set_fact: - remote_constraints: "{{ remote_tmp_dir }}/constraints.txt" - -- name: copy constraints.txt to remote host - copy: - src: "{{ role_path }}/../../../utils/constraints.txt" - dest: "{{ remote_constraints }}" diff --git a/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/handlers/main.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/handlers/main.yml deleted file mode 100644 index 229037c8b..000000000 --- a/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ -- name: delete temporary directory - include_tasks: default-cleanup.yml - -- name: delete temporary directory (windows) - include_tasks: windows-cleanup.yml diff --git a/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default-cleanup.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default-cleanup.yml deleted file mode 100644 index 39872d749..000000000 --- a/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default-cleanup.yml +++ /dev/null @@ -1,5 +0,0 @@ -- name: delete temporary directory - file: - path: "{{ remote_tmp_dir }}" - state: absent - no_log: yes diff --git a/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default.yml deleted file mode 100644 index 1e0f51b89..000000000 --- a/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/default.yml +++ /dev/null @@ -1,11 +0,0 @@ -- name: create temporary directory - tempfile: - state: directory - suffix: .test - register: remote_tmp_dir - notify: - - delete temporary directory - -- name: record temporary directory - set_fact: - remote_tmp_dir: "{{ remote_tmp_dir.path }}" diff --git a/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/main.yml b/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/main.yml deleted file mode 100644 index 93d786f0f..000000000 --- a/tests/integration/targets/setup_old_stuff/files/setup_remote_tmp_dir/tasks/main.yml +++ /dev/null @@ -1,15 +0,0 @@ -#################################################################### -# WARNING: These are designed specifically for Ansible tests # -# and should not be used as examples of how to write Ansible roles # -#################################################################### - -- name: make sure we have the ansible_os_family and ansible_distribution_version facts - setup: - gather_subset: distribution - when: ansible_facts == {} - -- include_tasks: "{{ lookup('first_found', files)}}" - vars: - files: - - "{{ ansible_os_family | lower }}.yml" - - "default.yml" From 6a5fec6d17cad5fc0da0e2d40a2c5fcfa0af7682 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 14:20:40 -0400 Subject: [PATCH 106/137] move test plugins into dedicated target --- tests/integration/targets/lookup_hashi_vault/aliases | 1 + tests/integration/targets/lookup_hashi_vault/meta/main.yml | 1 + tests/integration/targets/setup_vault_test_plugins/README.md | 2 ++ tests/integration/targets/setup_vault_test_plugins/aliases | 1 + .../library/vault_ci_enable_auth.py | 0 .../library/vault_ci_enable_engine.py | 0 .../library/vault_ci_kv_put.py | 0 .../library/vault_ci_policy_put.py | 0 .../library/vault_ci_read.py | 0 .../library/vault_ci_token_create.py | 0 .../library/vault_ci_write.py | 0 11 files changed, 5 insertions(+) create mode 100644 tests/integration/targets/setup_vault_test_plugins/README.md create mode 100644 tests/integration/targets/setup_vault_test_plugins/aliases rename tests/integration/targets/{lookup_hashi_vault => setup_vault_test_plugins}/library/vault_ci_enable_auth.py (100%) rename tests/integration/targets/{lookup_hashi_vault => setup_vault_test_plugins}/library/vault_ci_enable_engine.py (100%) rename tests/integration/targets/{lookup_hashi_vault => setup_vault_test_plugins}/library/vault_ci_kv_put.py (100%) rename tests/integration/targets/{lookup_hashi_vault => setup_vault_test_plugins}/library/vault_ci_policy_put.py (100%) rename tests/integration/targets/{lookup_hashi_vault => setup_vault_test_plugins}/library/vault_ci_read.py (100%) rename tests/integration/targets/{lookup_hashi_vault => setup_vault_test_plugins}/library/vault_ci_token_create.py (100%) rename tests/integration/targets/{lookup_hashi_vault => setup_vault_test_plugins}/library/vault_ci_write.py (100%) diff --git a/tests/integration/targets/lookup_hashi_vault/aliases b/tests/integration/targets/lookup_hashi_vault/aliases index 4d013e3b0..78cba09ad 100644 --- a/tests/integration/targets/lookup_hashi_vault/aliases +++ b/tests/integration/targets/lookup_hashi_vault/aliases @@ -1,3 +1,4 @@ skip/python2.6 # Python 2.6 is not supported by the collection +needs/target/setup_vault_test_plugins needs/target/setup_vault_server needs/target/setup_tinyproxy_server diff --git a/tests/integration/targets/lookup_hashi_vault/meta/main.yml b/tests/integration/targets/lookup_hashi_vault/meta/main.yml index 5ce43771d..2cc0486bd 100644 --- a/tests/integration/targets/lookup_hashi_vault/meta/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/meta/main.yml @@ -1,3 +1,4 @@ --- dependencies: + - setup_vault_test_plugins - setup_cert_content diff --git a/tests/integration/targets/setup_vault_test_plugins/README.md b/tests/integration/targets/setup_vault_test_plugins/README.md new file mode 100644 index 000000000..50fc9cbe2 --- /dev/null +++ b/tests/integration/targets/setup_vault_test_plugins/README.md @@ -0,0 +1,2 @@ +# `setup_vault_test_plugins` +Contains plugins/modules that are used only in testing. diff --git a/tests/integration/targets/setup_vault_test_plugins/aliases b/tests/integration/targets/setup_vault_test_plugins/aliases new file mode 100644 index 000000000..136c05e0d --- /dev/null +++ b/tests/integration/targets/setup_vault_test_plugins/aliases @@ -0,0 +1 @@ +hidden diff --git a/tests/integration/targets/lookup_hashi_vault/library/vault_ci_enable_auth.py b/tests/integration/targets/setup_vault_test_plugins/library/vault_ci_enable_auth.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/library/vault_ci_enable_auth.py rename to tests/integration/targets/setup_vault_test_plugins/library/vault_ci_enable_auth.py diff --git a/tests/integration/targets/lookup_hashi_vault/library/vault_ci_enable_engine.py b/tests/integration/targets/setup_vault_test_plugins/library/vault_ci_enable_engine.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/library/vault_ci_enable_engine.py rename to tests/integration/targets/setup_vault_test_plugins/library/vault_ci_enable_engine.py diff --git a/tests/integration/targets/lookup_hashi_vault/library/vault_ci_kv_put.py b/tests/integration/targets/setup_vault_test_plugins/library/vault_ci_kv_put.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/library/vault_ci_kv_put.py rename to tests/integration/targets/setup_vault_test_plugins/library/vault_ci_kv_put.py diff --git a/tests/integration/targets/lookup_hashi_vault/library/vault_ci_policy_put.py b/tests/integration/targets/setup_vault_test_plugins/library/vault_ci_policy_put.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/library/vault_ci_policy_put.py rename to tests/integration/targets/setup_vault_test_plugins/library/vault_ci_policy_put.py diff --git a/tests/integration/targets/lookup_hashi_vault/library/vault_ci_read.py b/tests/integration/targets/setup_vault_test_plugins/library/vault_ci_read.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/library/vault_ci_read.py rename to tests/integration/targets/setup_vault_test_plugins/library/vault_ci_read.py diff --git a/tests/integration/targets/lookup_hashi_vault/library/vault_ci_token_create.py b/tests/integration/targets/setup_vault_test_plugins/library/vault_ci_token_create.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/library/vault_ci_token_create.py rename to tests/integration/targets/setup_vault_test_plugins/library/vault_ci_token_create.py diff --git a/tests/integration/targets/lookup_hashi_vault/library/vault_ci_write.py b/tests/integration/targets/setup_vault_test_plugins/library/vault_ci_write.py similarity index 100% rename from tests/integration/targets/lookup_hashi_vault/library/vault_ci_write.py rename to tests/integration/targets/setup_vault_test_plugins/library/vault_ci_write.py From 1b33d1529fe64f9b65b6056bda42079c3c6c581c Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 15:18:53 -0400 Subject: [PATCH 107/137] Move legacy logic to dedicated setup target --- tests/integration/targets/lookup_hashi_vault/aliases | 3 +-- .../targets/lookup_hashi_vault/defaults/main.yml | 2 -- .../integration/targets/lookup_hashi_vault/meta/main.yml | 1 + .../integration/targets/lookup_hashi_vault/tasks/main.yml | 8 -------- tests/integration/targets/setup_vault_legacy/README.md | 5 +++++ tests/integration/targets/setup_vault_legacy/aliases | 4 ++++ .../targets/setup_vault_legacy/defaults/main.yml | 2 ++ .../integration/targets/setup_vault_legacy/tasks/main.yml | 8 ++++++++ 8 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 tests/integration/targets/setup_vault_legacy/README.md create mode 100644 tests/integration/targets/setup_vault_legacy/aliases create mode 100644 tests/integration/targets/setup_vault_legacy/defaults/main.yml create mode 100644 tests/integration/targets/setup_vault_legacy/tasks/main.yml diff --git a/tests/integration/targets/lookup_hashi_vault/aliases b/tests/integration/targets/lookup_hashi_vault/aliases index 78cba09ad..8ddf02953 100644 --- a/tests/integration/targets/lookup_hashi_vault/aliases +++ b/tests/integration/targets/lookup_hashi_vault/aliases @@ -1,4 +1,3 @@ skip/python2.6 # Python 2.6 is not supported by the collection +needs/target/setup_vault_legacy needs/target/setup_vault_test_plugins -needs/target/setup_vault_server -needs/target/setup_tinyproxy_server diff --git a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml index dd1f85c73..4d70cead2 100644 --- a/tests/integration/targets/lookup_hashi_vault/defaults/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/defaults/main.yml @@ -83,5 +83,3 @@ vault_run_https_tests: True # if any connections are taking longer than this to complete there's probably something really wrong # with the integration tests, so it'd be better to fail faster than the 30s default ansible_hashi_vault_timeout: 5 - -vault_integration_legacy: False diff --git a/tests/integration/targets/lookup_hashi_vault/meta/main.yml b/tests/integration/targets/lookup_hashi_vault/meta/main.yml index 2cc0486bd..eb4a205b6 100644 --- a/tests/integration/targets/lookup_hashi_vault/meta/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/meta/main.yml @@ -2,3 +2,4 @@ dependencies: - setup_vault_test_plugins - setup_cert_content + - setup_vault_legacy diff --git a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml index 839e0a4e5..0e972ebef 100644 --- a/tests/integration/targets/lookup_hashi_vault/tasks/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/tasks/main.yml @@ -4,14 +4,6 @@ # and should not be used as examples of how to write Ansible roles # #################################################################### -- include_role: - name: setup_vault_server - when: vault_integration_legacy | bool - -- include_role: - name: setup_tinyproxy_server - when: vault_integration_legacy | bool - - import_tasks: vault_server_configure.yml when: vault_test_server_configure | bool diff --git a/tests/integration/targets/setup_vault_legacy/README.md b/tests/integration/targets/setup_vault_legacy/README.md new file mode 100644 index 000000000..4b71a87ce --- /dev/null +++ b/tests/integration/targets/setup_vault_legacy/README.md @@ -0,0 +1,5 @@ +# `setup_vault_legacy` +Single role to use for a particular target to support legacy dependency setup. + +## Notes +* Uses the other setup roles for setting up Vault, proxy, etc. so will require any dependencies that those do. diff --git a/tests/integration/targets/setup_vault_legacy/aliases b/tests/integration/targets/setup_vault_legacy/aliases new file mode 100644 index 000000000..fef2b0646 --- /dev/null +++ b/tests/integration/targets/setup_vault_legacy/aliases @@ -0,0 +1,4 @@ +hidden +destructive +needs/target/setup_vault_server +needs/target/setup_tinyproxy_server diff --git a/tests/integration/targets/setup_vault_legacy/defaults/main.yml b/tests/integration/targets/setup_vault_legacy/defaults/main.yml new file mode 100644 index 000000000..ea0a3e9a7 --- /dev/null +++ b/tests/integration/targets/setup_vault_legacy/defaults/main.yml @@ -0,0 +1,2 @@ +--- +vault_integration_legacy: False diff --git a/tests/integration/targets/setup_vault_legacy/tasks/main.yml b/tests/integration/targets/setup_vault_legacy/tasks/main.yml new file mode 100644 index 000000000..c460e8069 --- /dev/null +++ b/tests/integration/targets/setup_vault_legacy/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- include_role: + name: setup_vault_server + when: vault_integration_legacy | bool + +- include_role: + name: setup_tinyproxy_server + when: vault_integration_legacy | bool From 0af2fa2cbcf1970866aa8ba2c4ba59068c1ab399 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 15:54:41 -0400 Subject: [PATCH 108/137] split macos docker tasks --- .github/workflows/ansible-test.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 6c7f912f8..c0b08d4d9 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -334,6 +334,13 @@ jobs: - macos-latest test_container: - default + docker: + - true + - false + exclude: + - runner: ubuntu-latest + docker: false + # vault: # - 1.7.3 # exclude: @@ -477,27 +484,32 @@ jobs: collection: community.general - name: legacy integration - use sample integration_config + if: ${{ matrix.runner != 'macos-latest' || !matrix.docker }} working-directory: ${{ env.COLLECTION_INTEGRATION_PATH }} run: cp "integration_config.yml.sample" "integration_config.yml" - name: legacy integration - venv + if: ${{ matrix.runner != 'macos-latest' || !matrix.docker }} run: ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --venv --requirements --allow-destructive #--coverage working-directory: ${{ env.COLLECTION_PATH }} - name: Install Docker on MacOS - if: ${{ matrix.runner == 'macos-latest' }} + if: ${{ matrix.runner == 'macos-latest' && matrix.docker }} uses: ./.github/actions/macos-docker - name: legacy integration - docker + if: ${{ matrix.docker }} run: | ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage working-directory: ${{ env.COLLECTION_PATH }} - name: legacy integration - remove integration_config + if: ${{ matrix.docker }} working-directory: ${{ env.COLLECTION_INTEGRATION_PATH }} - run: rm "integration_config.yml" + run: rm -f "integration_config.yml" - name: localenv_docker - setup + if: ${{ matrix.docker }} run: | pwd pip install -r files/requirements/requirements.txt -c files/requirements/constraints.txt @@ -505,6 +517,7 @@ jobs: working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_docker - name: localenv_docker - Run integration test (in docker) + if: ${{ matrix.docker }} run: | ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage --docker-network hashi_vault_default working-directory: ${{ env.COLLECTION_PATH }} @@ -512,10 +525,12 @@ jobs: # ansible-test support producing code coverage date - name: Generate coverage report + if: ${{ matrix.docker }} run: ansible-test coverage xml -v --requirements --group-by command --group-by version working-directory: ${{ env.COLLECTION_PATH }} # See the reports at https://codecov.io/gh/ansible-collections/community.hashi_vault - uses: codecov/codecov-action@v1 + if: ${{ matrix.docker }} with: fail_ci_if_error: false From 6276a65525ea4f45517d236a5a1330d886aa15dd Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 16:00:23 -0400 Subject: [PATCH 109/137] fix macos legacy docker --- .github/workflows/ansible-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index c0b08d4d9..3eb6b0f03 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -484,7 +484,6 @@ jobs: collection: community.general - name: legacy integration - use sample integration_config - if: ${{ matrix.runner != 'macos-latest' || !matrix.docker }} working-directory: ${{ env.COLLECTION_INTEGRATION_PATH }} run: cp "integration_config.yml.sample" "integration_config.yml" From 1c7589bc302b7138baf7d1156babeb3f9a424caf Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 16:33:40 -0400 Subject: [PATCH 110/137] better job name? --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 3eb6b0f03..1124ae777 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -318,7 +318,7 @@ jobs: local_test_invocation: runs-on: ${{ matrix.runner }} - name: Local I - ${{ matrix.runner }} (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) + name: LI - ${{ runner.os }} ${{ matrix.docker && ' docker' }} (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) env: # needed to prevent Ansible crashing on MacOS OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES' From b81061f21860f9cfc98e98613435830ee774fb8b Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 16:37:09 -0400 Subject: [PATCH 111/137] correction --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 1124ae777..8f6b55d82 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -318,7 +318,7 @@ jobs: local_test_invocation: runs-on: ${{ matrix.runner }} - name: LI - ${{ runner.os }} ${{ matrix.docker && ' docker' }} (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) + name: LI - ${{ matrix.runner }} ${{ matrix.docker && ' docker' }} (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) env: # needed to prevent Ansible crashing on MacOS OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES' From 5c498995b703a547ce6377d4b371ce781d99523b Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 17:26:46 -0400 Subject: [PATCH 112/137] name tweak --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 8f6b55d82..7534aba8b 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -318,7 +318,7 @@ jobs: local_test_invocation: runs-on: ${{ matrix.runner }} - name: LI - ${{ matrix.runner }} ${{ matrix.docker && ' docker' }} (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) + name: LI - ${{ matrix.runner }} [${{ matrix.docker && '+' || '-' }}d4r] (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }}) env: # needed to prevent Ansible crashing on MacOS OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES' From d06e77f41be8e2bd1ab876d4ed25fe986cb0c881 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 17:56:59 -0400 Subject: [PATCH 113/137] cleanup workflow --- .github/workflows/ansible-test.yml | 218 +---------------------------- 1 file changed, 1 insertion(+), 217 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 7534aba8b..8821abc54 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -1,9 +1,3 @@ -# README FIRST -# 1. If you don't have unit tests remove that section -# 2. If your collection depends on other collections ensure they are installed, see "Install collection dependencies" -# If you need help please ask in the #ansible-community channel on Libera.Chat IRC (https://libera.chat/) -# Refer to https://libera.chat/guides/registration to learn how to register - name: CI on: # Run CI against all pushes (direct commits, also merged PRs), Pull Requests @@ -103,11 +97,6 @@ jobs: - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check - # # OPTIONAL If your unit test requires Python libraries from other collections - # # Install them like this - # - name: Install collection dependencies - # run: ansible-galaxy collection install ansible.netcommon ansible.utils -p . - # Run the unit tests - name: Run unit test run: ansible-test units -v --color --docker ${{ matrix.test_container }} --coverage @@ -128,11 +117,6 @@ jobs: # # https://docs.ansible.com/ansible/latest/dev_guide/testing_integration.html - -# If the application you are testing is available as a docker container and you want to test -# multiple versions see the following for an example: -# https://github.com/ansible-collections/community.zabbix/tree/master/.github/workflows - integration: runs-on: ${{ matrix.runner }} name: I (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }} | Vault ${{ matrix.vault }}) @@ -167,16 +151,6 @@ jobs: python: '3.10' runner: ubuntu-latest test_container: default - # services: - # vault: - # image: vault:${{ matrix.vault }} - # volumes: - # - ${{ github.workspace }}/ansible_collections/community/hashi_vault/tests/integration/targets/setup_localenv_gha/files/.output/vault_config:/vault/config - # env: - # VAULT_DEV_ROOT_TOKEN_ID: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' - # SKIP_CHOWN: 1 - # tinyproxy: - # image: monokal/tinyproxy steps: - name: Initialize env vars @@ -190,19 +164,6 @@ jobs: LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars - # run: | - # COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME} - # # COLLECTION_INTEGRATION_TARGETS=${COLLECTION_PATH}/tests/integration/targets - # # LOOKUP_HASHI_VAULT_PATH=${COLLECTION_INTEGRATION_TARGETS}/lookup_hashi_vault - # # LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/files/bin - # # LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_PATH}/lookup_hashi_vault/vars - - # echo "COLLECTION_PATH=${COLLECTION_PATH}" >> ${GITHUB_ENV} - # # echo "COLLECTION_INTEGRATION_TARGETS=${COLLECTION_INTEGRATION_TARGETS}" >> ${GITHUB_ENV} - # # echo "LOOKUP_HASHI_VAULT_PATH=${LOOKUP_HASHI_VAULT_PATH}" >> ${GITHUB_ENV} - # # echo "LOOKUP_HASHI_VAULT_BIN=${LOOKUP_HASHI_VAULT_BIN}" >> ${GITHUB_ENV} - # # echo "LOOKUP_HASHI_VAULT_VARS=${LOOKUP_HASHI_VAULT_VARS}" >> ${GITHUB_ENV} - - name: Check out code uses: actions/checkout@v2 with: @@ -216,91 +177,13 @@ jobs: with: python-version: 3.8 - # - name: Cache for vault binaries - # id: cache-vault - # uses: actions/cache@v2 - # with: - # path: ${{ env.LOOKUP_HASHI_VAULT_BIN }} - # key: ${{ runner.os }}-vault${{ matrix.vault }} # future: include version/arch when configurable - - # # removing .gitignore lets the files in those dirs be sent to the container via ansible-test - # # the files/bin dir will contain the vault binary downloaded a few steps later (or retrieved from cache) - # # the vars/ dir will be used to write a file overriding role defaults (for Vault version) - # - name: Prepare for Vault version and caching - # run: | - # rm -f "${LOOKUP_HASHI_VAULT_BIN}/.gitignore" - # rm -f "${LOOKUP_HASHI_VAULT_VARS}/.gitignore" - # echo '{ "vault_version": "${{ matrix.vault }}" }' > "${LOOKUP_HASHI_VAULT_VARS}/main.json" - - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check - # #TODO: remove dependency on community.crypto - # - name: Find the latest version of community.crypto & community.docker - # run: | - # # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 - - # cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) - # echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV - - # cdlu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.docker/releases/latest) - # echo "CD_LATEST=$(basename ${cdlu})" >> $GITHUB_ENV - - # - name: Cache for community.crypto - # id: cache-cc - # uses: actions/cache@v2 - # with: - # path: ansible_collections/community/crypto - # key: collection-community.crypto-${{ env.CC_LATEST }} - - # - name: Cache for community.docker - # id: cache-cd - # uses: actions/cache@v2 - # with: - # path: ansible_collections/community/docker - # key: collection-community.docker-${{ env.CD_LATEST }} - - # - name: Install community.crypto - # if: steps.cache-cc.outputs.cache-hit != 'true' - # uses: actions/checkout@v2 - # with: - # repository: ansible-collections/community.crypto - # ref: refs/tags/${{ env.CC_LATEST }} - # path: ansible_collections/community/crypto - - # - name: Install community.docker - # if: steps.cache-cd.outputs.cache-hit != 'true' - # uses: actions/checkout@v2 - # with: - # repository: ansible-collections/community.docker - # ref: refs/tags/${{ env.CD_LATEST }} - # path: ansible_collections/community/docker - - # this will populate files/bin with the selected vault version binary - # - name: Pre-download Vault - # if: steps.cache-vault.outputs.cache-hit != 'true' - # env: - # ANSIBLE_ROLES_PATH: ../ - # working-directory: ${{ env.LOOKUP_HASHI_VAULT_PATH }}/playbooks - # run: ansible-playbook "download_vault.yml" -v - - # Run the integration tests - # env: - # ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_INTEGRATION_TARGETS }} - # ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }} - name: Prepare docker dependencies - run: | - pwd - ./setup.sh + run: ./setup.sh working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_gha - # pip install -r "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/requirements.txt" -c "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/requirements/constraints.txt" - # ansible localhost -m include_role -a "name=setup_localenv_docker" -e vault_version=${{ matrix.vault }} - # ansible localhost -m include_role -a "name=setup_localenv_gha" -e vault_version=${{ matrix.vault }} - # cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_docker/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_PATH}" - # cp "${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_gha/files/.output/integration_config.yml" "${COLLECTION_INTEGRATION_PATH}" - #"${COLLECTION_INTEGRATION_TARGETS}/setup_localenv_gha/files/.output/launch.sh" - - name: Run integration test run: | ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage --docker-network hashi_vault_default @@ -341,16 +224,6 @@ jobs: - runner: ubuntu-latest docker: false - # vault: - # - 1.7.3 - # exclude: - # - ansible: stable-2.9 - # python: 3.9 - # include: - # - ansible: devel - # vault: 1.7.3 - # python: '3.10' - steps: - name: Initialize env vars uses: briantist/ezenv@v1 @@ -376,97 +249,9 @@ jobs: with: python-version: ${{ matrix.python }} - # - name: Fix forks/maxfiles on MacOS - # if: ${{ matrix.runner == 'macos-latest' }} - # env: - # PLIST: /Library/LaunchDaemons/limit.maxfiles.plist - # MAXFILES: 524288 - # run: | - # ulimit -n - # ulimit -Hn - # ulimit -Sn - - # sudo launchctl limit maxfiles $MAXFILES $MAXFILES - - # ulimit -n - # ulimit -Hn - # ulimit -Sn - - # ulimit -S -n $MAXFILES $MAXFILES - - # # cat >${PLIST} < - # # - # # - # # - # # Label - # # limit.maxfiles - # # ProgramArguments - # # - # # launchctl - # # limit - # # maxfiles - # # 524288 - # # 524288 - # # - # # RunAtLoad - # # - # # ServiceIPC - # # - # # - # # - # # EOF - - # # sudo chmod 644 ${PLIST} - - name: Install ansible-base (${{ matrix.ansible }}) run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check - # - name: Find the latest version of community.crypto - # run: | - # # credit to https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c#gistcomment-3294173 - - # cclu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.crypto/releases/latest) - # echo "CC_LATEST=$(basename ${cclu})" >> $GITHUB_ENV - - # cglu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.general/releases/latest) - # echo "CG_LATEST=$(basename ${cglu})" >> $GITHUB_ENV - - # cdlu=$(curl -fs -o/dev/null -w %{redirect_url} https://github.com/ansible-collections/community.docker/releases/latest) - # echo "CD_LATEST=$(basename ${cdlu})" >> $GITHUB_ENV - - # - name: Cache for community.crypto - # id: cache-cc - # uses: actions/cache@v2 - # with: - # path: ansible_collections/community/crypto - # key: collection-community.crypto-${{ env.CC_LATEST }} - - # - name: Install collection dependencies - # if: steps.cache-cc.outputs.cache-hit != 'true' - # uses: actions/checkout@v2 - # with: - # repository: ansible-collections/community.crypto - # ref: refs/tags/${{ env.CC_LATEST }} - # path: ansible_collections/community/crypto - - # - name: Install collection dependencies - # # if: steps.cache-cc.outputs.cache-hit != 'true' - # uses: actions/checkout@v2 - # with: - # repository: ansible-collections/community.general - # ref: refs/tags/${{ env.CG_LATEST }} - # path: ansible_collections/community/general - - # - name: Install collection dependencies - # # if: steps.cache-cc.outputs.cache-hit != 'true' - # uses: actions/checkout@v2 - # with: - # repository: ansible-collections/community.docker - # ref: refs/tags/${{ env.CD_LATEST }} - # path: ansible_collections/community/docker - - name: Install community.crypto uses: ./.github/actions/collection-via-git with: @@ -521,7 +306,6 @@ jobs: ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage --docker-network hashi_vault_default working-directory: ${{ env.COLLECTION_PATH }} - # ansible-test support producing code coverage date - name: Generate coverage report if: ${{ matrix.docker }} From bce66365fa36620ff4a87807df6167293fe1bd83 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 17:58:25 -0400 Subject: [PATCH 114/137] try ubuntu2004 for local integration --- .github/workflows/ansible-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 8821abc54..80ad57c92 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -216,7 +216,7 @@ jobs: - ubuntu-latest - macos-latest test_container: - - default + - ubuntu2004 docker: - true - false @@ -284,7 +284,7 @@ jobs: - name: legacy integration - docker if: ${{ matrix.docker }} run: | - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage + ansible-test integration -v --color --retry-on-error --continue-on-error --docker ${{ matrix.test_container }} --coverage working-directory: ${{ env.COLLECTION_PATH }} - name: legacy integration - remove integration_config @@ -303,7 +303,7 @@ jobs: - name: localenv_docker - Run integration test (in docker) if: ${{ matrix.docker }} run: | - ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage --docker-network hashi_vault_default + ansible-test integration -v --color --retry-on-error --continue-on-error --docker ${{ matrix.test_container }} --coverage --docker-network hashi_vault_default working-directory: ${{ env.COLLECTION_PATH }} # ansible-test support producing code coverage date From b1bda0a68c9eadc39c208bd1658d8dff96a5216b Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 18:07:32 -0400 Subject: [PATCH 115/137] go back to default container --- .github/workflows/ansible-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 80ad57c92..8821abc54 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -216,7 +216,7 @@ jobs: - ubuntu-latest - macos-latest test_container: - - ubuntu2004 + - default docker: - true - false @@ -284,7 +284,7 @@ jobs: - name: legacy integration - docker if: ${{ matrix.docker }} run: | - ansible-test integration -v --color --retry-on-error --continue-on-error --docker ${{ matrix.test_container }} --coverage + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage working-directory: ${{ env.COLLECTION_PATH }} - name: legacy integration - remove integration_config @@ -303,7 +303,7 @@ jobs: - name: localenv_docker - Run integration test (in docker) if: ${{ matrix.docker }} run: | - ansible-test integration -v --color --retry-on-error --continue-on-error --docker ${{ matrix.test_container }} --coverage --docker-network hashi_vault_default + ansible-test integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} --coverage --docker-network hashi_vault_default working-directory: ${{ env.COLLECTION_PATH }} # ansible-test support producing code coverage date From d9959e78bca1fa08f353d66b3e86596f5bd52576 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 18:52:02 -0400 Subject: [PATCH 116/137] more cleanup --- .../templates/integration_config.yml.j2 | 2 -- .../templates/vault_config.hcl.j2 | 4 ---- .../targets/setup_vault_server/defaults/main.yml | 16 ---------------- 3 files changed, 22 deletions(-) diff --git a/tests/integration/targets/setup_localenv_docker/templates/integration_config.yml.j2 b/tests/integration/targets/setup_localenv_docker/templates/integration_config.yml.j2 index e57e3fc03..4add1f4ed 100644 --- a/tests/integration/targets/setup_localenv_docker/templates/integration_config.yml.j2 +++ b/tests/integration/targets/setup_localenv_docker/templates/integration_config.yml.j2 @@ -1,5 +1,3 @@ # {{ ansible_managed }} --- -# can be uncommented once vault configuration has been run once (WIP/broken) -# vault_server_configure: False {{ integration_config | to_nice_yaml }} diff --git a/tests/integration/targets/setup_localenv_docker/templates/vault_config.hcl.j2 b/tests/integration/targets/setup_localenv_docker/templates/vault_config.hcl.j2 index ddc42c4b5..ec89c20e1 100644 --- a/tests/integration/targets/setup_localenv_docker/templates/vault_config.hcl.j2 +++ b/tests/integration/targets/setup_localenv_docker/templates/vault_config.hcl.j2 @@ -1,14 +1,10 @@ # {{ ansible_managed }} -# pid_file = "{{ local_temp_dir | default('/tmp') }}/vault.pid" -# % if vault_run_https_tests | bool % listener "tcp" { tls_key_file = "/vault/config/{{ vault_key_file | basename }}" tls_cert_file = "/vault/config/{{ vault_cert_file | basename }}" tls_disable = false - # address = " vault_test_server_https | regex_replace('^https://([^:]+):(\\d+).*?$', '\\1:\\2') " address = "{{ integration_config.vault_test_server_https | regex_replace('^https://([^:]+):(\\d+).*?$', '\\1:\\2') }}" } -# % endif % diff --git a/tests/integration/targets/setup_vault_server/defaults/main.yml b/tests/integration/targets/setup_vault_server/defaults/main.yml index d13e6e398..f34922096 100644 --- a/tests/integration/targets/setup_vault_server/defaults/main.yml +++ b/tests/integration/targets/setup_vault_server/defaults/main.yml @@ -1,20 +1,4 @@ --- -## vars for vault server - -# vault_ansible_arch_table: -# 'x86_64': 'amd64' # Linux -# 'amd64': 'amd64' # FreeBSD -# 'i386': '386' - -# vault_arch: "{{ vault_ansible_arch_table[ansible_architecture] }}" - -# vault_version: '1.7.3' -# vault_bin: '{{ role_path }}/files/bin/{{ vault_slug }}' -# vault_slug: 'vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}' -# vault_zip: '{{ vault_bin }}/{{ vault_slug }}.zip' -# vault_uri: 'https://releases.hashicorp.com/vault/{{ vault_version }}/{{ vault_slug }}.zip' -# vault_cmd: '{{ vault_bin }}/vault' - vault_dev_root_token_id: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' vault_test_server_https: 'https://localhost:8201' vault_test_server_http: 'http://localhost:8200' From 249b93e921ac42b349d103f35f45f0873350db3f Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 22:36:32 -0400 Subject: [PATCH 117/137] New docs guides --- docs/docsite/extra-docs.yml | 4 +- docs/docsite/rst/contributor_guide.rst | 145 ++++++++++++++++++ docs/docsite/rst/localenv_developer_guide.rst | 63 ++++++++ 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 docs/docsite/rst/contributor_guide.rst create mode 100644 docs/docsite/rst/localenv_developer_guide.rst diff --git a/docs/docsite/extra-docs.yml b/docs/docsite/extra-docs.yml index aa21606c6..5d536e5e2 100644 --- a/docs/docsite/extra-docs.yml +++ b/docs/docsite/extra-docs.yml @@ -1,5 +1,7 @@ --- sections: - - title: User Guide + - title: Guides toctree: - user_guide + - contributor_guide + - localenv_developer_guide diff --git a/docs/docsite/rst/contributor_guide.rst b/docs/docsite/rst/contributor_guide.rst new file mode 100644 index 000000000..b3d97816a --- /dev/null +++ b/docs/docsite/rst/contributor_guide.rst @@ -0,0 +1,145 @@ +.. _ansible_collections.community.hashi_vault.docsite.contributor_guide: + +Contributor Guide +================= + +This guide aims to help PR authors contribute to the ``community.hashi_vault`` collection. + +**NOTE:** this guide is a work-in-progress and should not be considered complete. Check back often as we fill out more details based on experience and feedback, and please let us know how this guide can be improved. + + +.. contents:: + :local: + :depth: 3 + + +Quick Start +----------- + +#. Log into your GitHub account. +#. Fork the `ansible-collections/community.hashi_vault repository `_ by clicking the **Fork** button in the upper right corner. This will create a fork in your own account. +#. Clone the repository locally, following `the example instructions here `_ (but replace ``general`` with ``hashi_vault``). **Pay special attention to the path structure.** +#. As mentioned on that page, commit your changes to a branch, push them to your fork, and create a pull request (GitHub will automatically prompt you to do do when you look at your repository). +#. `See the guidance on Changelogs `_ and include a `changelog fragment `_ if appropriate. + + +Running Tests Locally +--------------------- + +If you're making anything more than very small or one-time changes, you'll want to run the tests locally to avoid having to push a commit for each thing, and waiting for the CI to run tests. + +First, `review the guidance on testing collections `_, as it applies to this collection as well. + +Integration Tests +----------------- + +Unlike other collections, we now require an `integration_config.yml `_ file for properly running integration tests, as the tests require external dependencies (like a Vault server) and need to know where to find them. + +If you have contributed to this collection or to the ``hashi_vault`` lookup plugin in the past, you might remember that the integration tests used to download, extract, and run a Vault server during the course of the tests, by default. This **legacy method** is not recommended but is still available (for now) via opt-in. + +Skip to the next section for a method that's nearly as easy but better off in the long run (docker-compose). + +Legacy Mode +........... + +To get started quickly without having to set anything else, you can use legacy mode by copying the included integration config sample: + +.. code-block:: bash + + $ cp tests/integration/integration_config.yml.sample tests/integration/integration_config.yml + +That file has everything configured to be able to run the integration tests and have them set up the dependencies for you. + +You will also need the following additional Ansible collections: + +* `community.crypto `_ +* `community.general `_ (MacOS local/venv only) + +Running legacy mode tests in docker (recommended): + +.. code-block:: bash + + $ ansible-test integration --docker default -v + +Running legacy mode tests in a controlled python virtual environment (**not recommended**): + +.. code-block:: bash + + $ ansible-test integration --venv --requirements --allow-destructive -v + +Note that your system packages may be manipulated by running locally or in a venv. + +Legacy mode is not recommended because a new Vault server and proxy server will be downloaded, set up, configured, and/or uninstalled, for every *target*. Traditionally, we've only had one target, and so it was a good way to go, but that's no longer going to be the case. This is going to make it slower and slower as you'll incur the overhead on every target, in every run. + +As a result, if you're going to use legacy mode it's best to limit your test run to the specific target needed, for example: + +.. code-block:: bash + + $ ansible-test integration --docker default -v lookup_hashi_vault + +Docker Compose localenv +....................... + +The recommended way to run the tests has Vault and tinyproxy running in their own containers, set up via docker-compose, and the itnegration tests run in their own container separately. + +We have a pre-defined "localenv" setup role for this purpose. + +Usage +~~~~~ + +For ease of typing / length of commands, we'll enter role directory first: + +.. code-block:: bash + + $ cd tests/integration/targets/setup_localenv_docker + +This localenv has both Ansible collection and Python requirements, so let's get those out of the way: + +.. code-block:: bash + + $ pip install -r files/requirements/requirements.txt -c files/requirements/constraints.txt + $ ansible-galaxy collection install -r files/requirements/requirements.yml + +To set up your docker-compose environment with the all defaults: + +.. code-block:: bash + + $ ./setup.sh + +This will do the following: + +#. Template a ``docker-compose.yml`` for the project. +#. Generate a private key and self-signed certificate for Vault. +#. Template a Vault config file. +#. Bring down the existing compose project. +#. Bring up the compose project as defined by the vars (specified or defaults). +#. Template an ``integration_config.yml`` file that has all the right info for integration tests to connect. +#. Will copy the integration config to the correct location *if there isn't already one there* (it won't overwrite, in case you had customized changes). + +With your containers running, you can now run the tests in docker (after returning back to the collection root): + +.. code-block:: bash + + $ cd ../../../../ + $ ansible-test integration --docker default --docker-network hashi_vault_default -v + +The ``--docker-network`` part is important, as it will ensure that the Ansible test container is in the same network as the dependency containers, that way the test container can reach them by their container names. The network name, ``hashi_vault_default`` comes from the default docker-compose project name used by this role (``hashi_vault``). See the next section for more information. + +Running ``setup.sh`` again can be used to re-deploy the containers, or if you prefer you can use the generated ``docker-compose.yml`` in ``files/.output/``. + +If running again, remember to manually copy the contents of newly generated ``files/.output/integration_config.yml`` to the integration root, or delete the file in the root before re-running setup so that it copies the file automatically. + +Customization +~~~~~~~~~~~~~ + +``setup.sh`` will pass any additional params you send it to the ``ansible-playbook`` command it calls, so you can customize variables with the standard ``-e`` option. There are many advanced scenarios possible, but a few things you might want to override: + +* ``docker_compose`` (defaults to ``clean`` but could be set to ``up``, ``down``, or ``none``) + * ``up`` -- similar to running ``docker-compose up`` (no op if the project is running as it should) + * ``down`` -- similar to ``docker-compose down`` (destroys the project) + * ``clean`` -- (default) similar to ``docker-compose down`` followed by ``docker-compose up`` + * ``none`` -- does the other tasks, including templating, but does not bring the project up or down. With this option, the ``community.docker`` collection is not required. +* ``vault_crypto_force`` -- by default this is ``false`` so if the cert and key exist they won't be regenerated. Setting to ``true`` will overwrite them. +* ``vault_port_http``, ``vault_port_https``, ``proxy_port`` -- all of the ports are exposed to the host, so if you already have any of the default ports in use on your host, you may need to override these. +* ``vault_container_name``, ``proxy_container_name`` -- these are the names for their respective containers, which will also be the DNS names used within the container network. In case you have the default names in use you may need to override these. +* ``docker_compose_project_name`` -- unlikely to need to be changed, but it affects the name of the docker network which will be needed for your ``ansible-test`` invocation, so it's worth mentioning. For example, if you set this to ``ansible_hashi_vault`` then the docker network name will be ``ansible_hashi_vault_default``. diff --git a/docs/docsite/rst/localenv_developer_guide.rst b/docs/docsite/rst/localenv_developer_guide.rst new file mode 100644 index 000000000..e6f75d008 --- /dev/null +++ b/docs/docsite/rst/localenv_developer_guide.rst @@ -0,0 +1,63 @@ +.. _ansible_collections.community.hashi_vault.docsite.localenv_developer_guide: + +localenv Developer Guide +======================== + +A "localenv" role in the context of this collection is a role that's used to set up the external dependencies required to run the integration tests. The idea is to provide a pre-packaged way for a contributor to set up their local environment in a certain way. + +**NOTE:** this guide is a work-in-progress and is **very** light on details. For the time being, it's best to open an issue in the repository to discuss it if you're thinking of a new localenv. Looking at ``setup_localenv_docker`` should also be helpful as it's the most complete one to date. + + +.. contents:: + :local: + :depth: 2 + + +Required External Dependencies +------------------------------ + +HashiCorp Vault +............... + +A Vault server is required for the integration tests. Using `Vault Dev Server Mode `_ is recommended as it's the easiest and fastest way to get a server started. + +A unencrypted (plain HTTP) listener is *required* for our purposes as most of the tests will expect to connect that way. + +To run the tests that deal specifically with TLS/HTTPS access, the Vault server will need to be started with a TLS enabled listener as well, and the address/port, as well as the CA cert (or the cert itself if self-signed) will need to be supplied. + +The **root token** of the Vault server is needed, as the integration tests will make changes to Vault's configuration, and will expect to have that token available to do so. It's possible to let Vault generate the token on startup and then retrieve it but it may be easiest to pre-generate one and pass it into Vault, via the ``-dev-root-token-id`` option or ``VAULT_DEV_ROOT_TOKEN_ID`` environment variable (see `Dev Options `_). + +.. csv-table:: Relevant integration_config Variables + :header: "var", "example", "description" + :widths: 15, 20, 65 + + "``vault_test_server_http``", "``http://myvault:8200``", "The full HTTP URL of your Vault test server." + "``vault_test_server_https``", "``https://myvault:8300``", "The full HTTPS URL of your Vault test server." + "``vault_dev_root_token_id``", "``3ee9a1f7-f115-4f7c-90a3-d3c73361bcb5``", "The root token used to authenticate to Vault." + "``vault_version``", "``1.7.3``", "The desired version of Vault to download (only used by legacy setup)." + "``vault_integration_legacy``", "``false``", "When ``true`` legacy integration will be used (see legacy section)." + "``vault_cert_content``", "``-----BEGIN CERTIFICATE-----``", "The public cert of the CA that signed the cert used for Vault's TLS listener (or the cert itself if self-signed)." + + +Proxy Server +............ + +A proxy server is used to test the proxy connectivity options. + +In theory any number of proxy servers could be used for this purpose, but `tinyproxy `_ is recommended for being, well.. tiny, as well as easy to configure and run, and available in package managers and containers. + +.. csv-table:: Relevant integration_config Variables + :header: "var", "example", "description" + :widths: 15, 20, 65 + + "``vault_proxy_server``", "``http://proxy:8080``", "The full HTTP URL of your proxy server." + +localenv Role Conventions +------------------------- + +* Use ``files/.output`` to hold generated artifacts. +* Anything generated should be ``.gitignore``d; conversely anything not in a ``.gitignore`` should not be overwritten or modified by this process. That this, there should be no changes to git status that arise from this. +* Consider providing a ``setup.sh`` to avoid having to manuall run ``ansible-`` commands. It should ideally operate correctly regardless of the current working directory. +* Generate a usable ``integration_config.yml`` that allows for using the result of the localenv. Generate it within the role output, not outside the role. Copy it to the right location, but do not overwrite an existing one. +* If the role has external dependencies, try to codify those in file(s) that can be used by the right tool, like ``requirements.yml`` for ``ansible-galaxy``, etc. +* localenv roles are meant to run **outside** of the ``ansible-test`` environment, but they can make (re)use of other roles; for example a localenv that seeks to run Vault directly on the local machine could reuse the setup roles for downloading and launching Vault that are used (within the test environment) by the legacy integration. From 48b4d022c52bb6850099d2deb1b13647b3f634ce Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 22:55:58 -0400 Subject: [PATCH 118/137] remove unused variables --- tests/integration/integration_config.yml.sample | 2 -- .../targets/setup_localenv_docker/vars/local_client.yml | 2 -- tests/integration/targets/setup_localenv_docker/vars/main.yml | 2 -- .../integration/targets/setup_vault_server/defaults/main.yml | 4 ---- 4 files changed, 10 deletions(-) diff --git a/tests/integration/integration_config.yml.sample b/tests/integration/integration_config.yml.sample index 4179a04a3..0325e2813 100644 --- a/tests/integration/integration_config.yml.sample +++ b/tests/integration/integration_config.yml.sample @@ -8,8 +8,6 @@ vault_version: 1.7.3 vault_dev_root_token_id: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a vault_proxy_server: 'http://127.0.0.1:8001' -vault_proxy_external: false -vault_test_server_external: false vault_test_server_http: http://localhost:8200 vault_test_server_https: https://localhost:8300 diff --git a/tests/integration/targets/setup_localenv_docker/vars/local_client.yml b/tests/integration/targets/setup_localenv_docker/vars/local_client.yml index 0cf3a9ddf..96b718756 100644 --- a/tests/integration/targets/setup_localenv_docker/vars/local_client.yml +++ b/tests/integration/targets/setup_localenv_docker/vars/local_client.yml @@ -10,8 +10,6 @@ integration_config: vault_test_server_https: 'https://{{ vault_target_name }}:{{ vault_port_https}}' vault_dev_root_token_id: '{{ vault_dev_root_token_id }}' vault_proxy_server: 'http://{{ proxy_target_name }}:{{ proxy_port }}' - vault_test_server_external: True - vault_proxy_external: True vault_cert_content: "{{ lookup('file', vault_cert_file) }}" vault_proxy_alt_vault_http: 'http://{{ vault_container_name }}:{{ vault_port_http }}' vault_proxy_alt_vault_https: 'https://{{ vault_container_name }}:{{ vault_port_https }}' diff --git a/tests/integration/targets/setup_localenv_docker/vars/main.yml b/tests/integration/targets/setup_localenv_docker/vars/main.yml index 0d8ef3a48..3da4d0efa 100644 --- a/tests/integration/targets/setup_localenv_docker/vars/main.yml +++ b/tests/integration/targets/setup_localenv_docker/vars/main.yml @@ -5,6 +5,4 @@ integration_config: vault_test_server_https: 'https://{{ vault_target_name }}:{{ vault_port_https}}' vault_dev_root_token_id: '{{ vault_dev_root_token_id }}' vault_proxy_server: 'http://{{ proxy_target_name }}:{{ proxy_port }}' - vault_test_server_external: True - vault_proxy_external: True vault_cert_content: "{{ lookup('file', vault_cert_file) }}" diff --git a/tests/integration/targets/setup_vault_server/defaults/main.yml b/tests/integration/targets/setup_vault_server/defaults/main.yml index f34922096..0688b6009 100644 --- a/tests/integration/targets/setup_vault_server/defaults/main.yml +++ b/tests/integration/targets/setup_vault_server/defaults/main.yml @@ -3,10 +3,6 @@ vault_dev_root_token_id: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' vault_test_server_https: 'https://localhost:8201' vault_test_server_http: 'http://localhost:8200' -# this means "don't download and start a Vault server", instead -# just use the addresses above to connect to one that's already running -vault_test_server_external: False - # WIP vault_test_server_configure: True From 1d94e72ce722ff12b4ca309249967af3d045d1e0 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Sun, 11 Jul 2021 22:59:03 -0400 Subject: [PATCH 119/137] add vault_version detail --- docs/docsite/rst/contributor_guide.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docsite/rst/contributor_guide.rst b/docs/docsite/rst/contributor_guide.rst index b3d97816a..81a6779fa 100644 --- a/docs/docsite/rst/contributor_guide.rst +++ b/docs/docsite/rst/contributor_guide.rst @@ -134,6 +134,7 @@ Customization ``setup.sh`` will pass any additional params you send it to the ``ansible-playbook`` command it calls, so you can customize variables with the standard ``-e`` option. There are many advanced scenarios possible, but a few things you might want to override: +* ``vault_version`` -- can target any version of Vault for which a docker container exists * ``docker_compose`` (defaults to ``clean`` but could be set to ``up``, ``down``, or ``none``) * ``up`` -- similar to running ``docker-compose up`` (no op if the project is running as it should) * ``down`` -- similar to ``docker-compose down`` (destroys the project) From 3c112e7ddfc44864561d39e9d090f27fe93e5335 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Wed, 14 Jul 2021 12:59:19 -0400 Subject: [PATCH 120/137] typos and doc fixups --- docs/docsite/rst/contributor_guide.rst | 10 +++++----- docs/docsite/rst/localenv_developer_guide.rst | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/docsite/rst/contributor_guide.rst b/docs/docsite/rst/contributor_guide.rst index 81a6779fa..4d036e747 100644 --- a/docs/docsite/rst/contributor_guide.rst +++ b/docs/docsite/rst/contributor_guide.rst @@ -19,7 +19,7 @@ Quick Start #. Log into your GitHub account. #. Fork the `ansible-collections/community.hashi_vault repository `_ by clicking the **Fork** button in the upper right corner. This will create a fork in your own account. #. Clone the repository locally, following `the example instructions here `_ (but replace ``general`` with ``hashi_vault``). **Pay special attention to the path structure.** -#. As mentioned on that page, commit your changes to a branch, push them to your fork, and create a pull request (GitHub will automatically prompt you to do do when you look at your repository). +#. As mentioned on that page, commit your changes to a branch, push them to your fork, and create a pull request (GitHub will automatically prompt you to do so when you look at your repository). #. `See the guidance on Changelogs `_ and include a `changelog fragment `_ if appropriate. @@ -33,7 +33,7 @@ First, `review the guidance on testing collections `_ file for properly running integration tests, as the tests require external dependencies (like a Vault server) and need to know where to find them. +Unlike other collections, we now require an `integration_config.yml `_ file for properly running integration tests, as the tests require external dependencies (like a Vault server) and they need to know where to find those dependencies. If you have contributed to this collection or to the ``hashi_vault`` lookup plugin in the past, you might remember that the integration tests used to download, extract, and run a Vault server during the course of the tests, by default. This **legacy method** is not recommended but is still available (for now) via opt-in. @@ -80,7 +80,7 @@ As a result, if you're going to use legacy mode it's best to limit your test run Docker Compose localenv ....................... -The recommended way to run the tests has Vault and tinyproxy running in their own containers, set up via docker-compose, and the itnegration tests run in their own container separately. +The recommended way to run the tests has Vault and tinyproxy running in their own containers, set up via docker-compose, and the integration tests run in their own container separately. We have a pre-defined "localenv" setup role for this purpose. @@ -91,7 +91,7 @@ For ease of typing / length of commands, we'll enter role directory first: .. code-block:: bash - $ cd tests/integration/targets/setup_localenv_docker + $ pushd tests/integration/targets/setup_localenv_docker This localenv has both Ansible collection and Python requirements, so let's get those out of the way: @@ -120,7 +120,7 @@ With your containers running, you can now run the tests in docker (after returni .. code-block:: bash - $ cd ../../../../ + $ popd $ ansible-test integration --docker default --docker-network hashi_vault_default -v The ``--docker-network`` part is important, as it will ensure that the Ansible test container is in the same network as the dependency containers, that way the test container can reach them by their container names. The network name, ``hashi_vault_default`` comes from the default docker-compose project name used by this role (``hashi_vault``). See the next section for more information. diff --git a/docs/docsite/rst/localenv_developer_guide.rst b/docs/docsite/rst/localenv_developer_guide.rst index e6f75d008..eeb0a018b 100644 --- a/docs/docsite/rst/localenv_developer_guide.rst +++ b/docs/docsite/rst/localenv_developer_guide.rst @@ -44,7 +44,7 @@ Proxy Server A proxy server is used to test the proxy connectivity options. -In theory any number of proxy servers could be used for this purpose, but `tinyproxy `_ is recommended for being, well.. tiny, as well as easy to configure and run, and available in package managers and containers. +In theory any proxy supporting http/s targets could be used for this purpose, but `tinyproxy `_ is recommended for being, well.. tiny, as well as easy to configure and run, and available in package managers and containers. .. csv-table:: Relevant integration_config Variables :header: "var", "example", "description" @@ -56,8 +56,8 @@ localenv Role Conventions ------------------------- * Use ``files/.output`` to hold generated artifacts. -* Anything generated should be ``.gitignore``d; conversely anything not in a ``.gitignore`` should not be overwritten or modified by this process. That this, there should be no changes to git status that arise from this. -* Consider providing a ``setup.sh`` to avoid having to manuall run ``ansible-`` commands. It should ideally operate correctly regardless of the current working directory. +* Anything generated should be ``.gitignore``d; conversely anything not in a ``.gitignore`` should not be overwritten or modified by this process. That is, there should be no changes to git status that arise from this. +* Consider providing a ``setup.sh`` to avoid having to manually run ``ansible-`` commands. It should ideally operate correctly regardless of the current working directory. * Generate a usable ``integration_config.yml`` that allows for using the result of the localenv. Generate it within the role output, not outside the role. Copy it to the right location, but do not overwrite an existing one. * If the role has external dependencies, try to codify those in file(s) that can be used by the right tool, like ``requirements.yml`` for ``ansible-galaxy``, etc. * localenv roles are meant to run **outside** of the ``ansible-test`` environment, but they can make (re)use of other roles; for example a localenv that seeks to run Vault directly on the local machine could reuse the setup roles for downloading and launching Vault that are used (within the test environment) by the legacy integration. From a22b16dc7739233375ed65a7ce6c1f94c6a1e762 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 15 Jul 2021 18:15:59 -0400 Subject: [PATCH 121/137] restore Vault 1.7.3 tests --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 8821abc54..e457a7156 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -140,7 +140,7 @@ jobs: - 3.8 - 3.9 vault: - # - 1.7.3 + - 1.7.3 - 1.6.5 exclude: - ansible: stable-2.9 From 068aa0a7a88a6b3ec9f7c9f519914e36493445da Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Thu, 15 Jul 2021 18:32:29 -0400 Subject: [PATCH 122/137] forgot to use vault version from matrix! --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index e457a7156..90940e148 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -181,7 +181,7 @@ jobs: run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check - name: Prepare docker dependencies - run: ./setup.sh + run: ./setup.sh -e vault_version=${{ matrix.vault }} working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_gha - name: Run integration test From 06296b5a866b28c32c22c7935ffe0a0a0b51368e Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Fri, 16 Jul 2021 14:59:14 -0400 Subject: [PATCH 123/137] fix rst formatting --- docs/docsite/rst/localenv_developer_guide.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/docsite/rst/localenv_developer_guide.rst b/docs/docsite/rst/localenv_developer_guide.rst index eeb0a018b..28e319ee3 100644 --- a/docs/docsite/rst/localenv_developer_guide.rst +++ b/docs/docsite/rst/localenv_developer_guide.rst @@ -27,7 +27,10 @@ To run the tests that deal specifically with TLS/HTTPS access, the Vault server The **root token** of the Vault server is needed, as the integration tests will make changes to Vault's configuration, and will expect to have that token available to do so. It's possible to let Vault generate the token on startup and then retrieve it but it may be easiest to pre-generate one and pass it into Vault, via the ``-dev-root-token-id`` option or ``VAULT_DEV_ROOT_TOKEN_ID`` environment variable (see `Dev Options `_). -.. csv-table:: Relevant integration_config Variables +Relevant ``integration_config.yml`` Variables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. csv-table:: :header: "var", "example", "description" :widths: 15, 20, 65 @@ -46,7 +49,10 @@ A proxy server is used to test the proxy connectivity options. In theory any proxy supporting http/s targets could be used for this purpose, but `tinyproxy `_ is recommended for being, well.. tiny, as well as easy to configure and run, and available in package managers and containers. -.. csv-table:: Relevant integration_config Variables +Relevant ``integration_config.yml`` Variables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. csv-table:: :header: "var", "example", "description" :widths: 15, 20, 65 @@ -56,7 +62,7 @@ localenv Role Conventions ------------------------- * Use ``files/.output`` to hold generated artifacts. -* Anything generated should be ``.gitignore``d; conversely anything not in a ``.gitignore`` should not be overwritten or modified by this process. That is, there should be no changes to git status that arise from this. +* Anything generated should be in a ``.gitignore``; conversely anything not in a ``.gitignore`` should not be overwritten or modified by this process. That is, there should be no changes to git status that arise from this. * Consider providing a ``setup.sh`` to avoid having to manually run ``ansible-`` commands. It should ideally operate correctly regardless of the current working directory. * Generate a usable ``integration_config.yml`` that allows for using the result of the localenv. Generate it within the role output, not outside the role. Copy it to the right location, but do not overwrite an existing one. * If the role has external dependencies, try to codify those in file(s) that can be used by the right tool, like ``requirements.yml`` for ``ansible-galaxy``, etc. From e93a2f56f6b576dd5111189433910de2272b446c Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Fri, 16 Jul 2021 18:32:35 -0400 Subject: [PATCH 124/137] Update docs/docsite/rst/contributor_guide.rst Co-authored-by: Alicia Cozine <879121+acozine@users.noreply.github.com> --- docs/docsite/rst/contributor_guide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docsite/rst/contributor_guide.rst b/docs/docsite/rst/contributor_guide.rst index 4d036e747..56c8eb639 100644 --- a/docs/docsite/rst/contributor_guide.rst +++ b/docs/docsite/rst/contributor_guide.rst @@ -71,7 +71,7 @@ Note that your system packages may be manipulated by running locally or in a ven Legacy mode is not recommended because a new Vault server and proxy server will be downloaded, set up, configured, and/or uninstalled, for every *target*. Traditionally, we've only had one target, and so it was a good way to go, but that's no longer going to be the case. This is going to make it slower and slower as you'll incur the overhead on every target, in every run. -As a result, if you're going to use legacy mode it's best to limit your test run to the specific target needed, for example: +If you must use legacy mode testing, you can make it more efficient by limiting your test run to the specific target needed, for example: .. code-block:: bash From 12ff4420aedc5585df09ac0363f45fb59a05d2a8 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Fri, 16 Jul 2021 21:21:17 -0400 Subject: [PATCH 125/137] contributor guide updates --- docs/docsite/rst/contributor_guide.rst | 144 +++++++++++++++++++------ 1 file changed, 110 insertions(+), 34 deletions(-) diff --git a/docs/docsite/rst/contributor_guide.rst b/docs/docsite/rst/contributor_guide.rst index 56c8eb639..f05c0a313 100644 --- a/docs/docsite/rst/contributor_guide.rst +++ b/docs/docsite/rst/contributor_guide.rst @@ -1,11 +1,14 @@ .. _ansible_collections.community.hashi_vault.docsite.contributor_guide: -Contributor Guide -================= +***************** +Contributor guide +***************** This guide aims to help PR authors contribute to the ``community.hashi_vault`` collection. -**NOTE:** this guide is a work-in-progress and should not be considered complete. Check back often as we fill out more details based on experience and feedback, and please let us know how this guide can be improved. +.. note:: + + This guide is a work-in-progress and should not be considered complete. Check back often as we fill out more details based on experience and feedback, and please let us know how this guide can be improved. .. contents:: @@ -13,38 +16,107 @@ This guide aims to help PR authors contribute to the ``community.hashi_vault`` c :depth: 3 -Quick Start ------------ +Quick start +=========== #. Log into your GitHub account. #. Fork the `ansible-collections/community.hashi_vault repository `_ by clicking the **Fork** button in the upper right corner. This will create a fork in your own account. -#. Clone the repository locally, following `the example instructions here `_ (but replace ``general`` with ``hashi_vault``). **Pay special attention to the path structure.** +#. Clone the repository locally, following :ref:`the example instructions here ` (but replace ``general`` with ``hashi_vault``). **Pay special attention to the path structure.** #. As mentioned on that page, commit your changes to a branch, push them to your fork, and create a pull request (GitHub will automatically prompt you to do so when you look at your repository). -#. `See the guidance on Changelogs `_ and include a `changelog fragment `_ if appropriate. +#. :ref:`See the guidance on Changelogs ` and include a :ref:`changelog fragment ` if appropriate. + +Contributing documentation +========================== + +Additions to the collection documentation are very welcome! We have three primary types of documentation, each with their own syntax and rules. + +Module and plugin documentation +------------------------------- + +This type of documentation gets generated from structured YAML, inside of a Python string. It will be included in the same code that it's documenting, or it will be in a separate Python file, such as a doc fragment. Please see the :ref:`module format and documentation guidance ` for more information. + +This type of documentation is tested with ``ansible-test sanity`` and full instructions are available on the :ref:`testing module documentation ` page. + +Although we can't preview how the documentation will look for these, we can be reasonably assured the output is correct because of the highly structured nature and the validation via sanity tests. + +README and other markdown files +------------------------------- + +Markdown files (those with the extension ``.md``) can be found in several directories within the repository. These files are primarily aimed at developers and those browsing the repository, to explain or give context to the other files nearby. + +The main exception to the above is the ``README.md`` in the repository root. This file is more important because it provides introductory information and links for anyone browsing the repository, both on GitHub and on the collection's `Ansible Galaxy page `_. + +Markdown files can be previewed natively in GutHub, so they are easy to validate by reviewers, and there are no specific tests that need to run against them. + +Collection docsite +------------------ + +The collection docsite is a set what you are reading now. It is a set of documentation written in reStructuredText (RST) format that is published on the :ref:`ansible_documentation` site. This is where we have longform documentation that doesn't fit into the other two categories. + +If you are considering adding an entirely new document here it may be best to open an issue first to discuss the idea and how best to organize it. + +Refer to the :ref:`Ansible style guide ` for all submissions to the collection docsite. + +RST files for the docsite are in the ``docs/docsite/rst/`` directory. Some submissions may also require edits to ``docs/docsite/extra-docs.yml``. + +Only the ``extra-docs.yml`` file is validated by the collection's CI, and there is not yet any automated preview for the docsite rendering, but this is an area we are hoping to improve on. +Advanced +^^^^^^^^ + +Docsite pages can be generated locally through a workaround technique. **This is not a supported method** but it may be helpful to get more rapid feedback on docsite if changes, if you're comfortable at a command line. + +The process is: + +#. Clone `ansible/ansible `_ or a fork of it. +#. Copy the ``.rst`` files you want to preview into that repository's ``docs/docsite/`` directory. +#. Install the requirements needed to build the docsite (from the repository root): + + .. code-block:: shell-session + + $ pip install -r requirements.txt + $ pip install -r docs/docsite/requirements.txt + +#. You may also need to remove write permission from group and other: + + .. code-block:: shell-session + + $ chmod -R go-w docs/docsite/rst + +#. Build the docs: + + .. code-block:: shell-session + + $ make coredocs + +The rendered HTML docs should be available in ``docs/docsite/_build/html/`` and can be opened in a browser. Running Tests Locally ---------------------- +===================== If you're making anything more than very small or one-time changes, you'll want to run the tests locally to avoid having to push a commit for each thing, and waiting for the CI to run tests. -First, `review the guidance on testing collections `_, as it applies to this collection as well. +First, :ref:`review the guidance on testing collections `, as it applies to this collection as well. Integration Tests ----------------- Unlike other collections, we now require an `integration_config.yml `_ file for properly running integration tests, as the tests require external dependencies (like a Vault server) and they need to know where to find those dependencies. -If you have contributed to this collection or to the ``hashi_vault`` lookup plugin in the past, you might remember that the integration tests used to download, extract, and run a Vault server during the course of the tests, by default. This **legacy method** is not recommended but is still available (for now) via opt-in. +If you have contributed to this collection or to the ``hashi_vault`` lookup plugin in the past, you might remember that the integration tests used to download, extract, and run a Vault server during the course of the tests, by default. This **legacy mode** is not recommended but is still available (for now) via opt-in. -Skip to the next section for a method that's nearly as easy but better off in the long run (docker-compose). +.. note:: + + Legacy mode is not recommended because a new Vault server and proxy server will be downloaded, set up, configured, and/or uninstalled, for every *target*. Traditionally, we've only had one target, and so it was a good way to go, but that's no longer going to be the case. This will make it slower and slower as you'll incur the overhead on every target, in every run. + + Skip to :ref:`localenv_docker` for a method that's nearly as easy but better off in the long run (docker-compose). Legacy Mode -........... +^^^^^^^^^^^ To get started quickly without having to set anything else, you can use legacy mode by copying the included integration config sample: -.. code-block:: bash +.. code-block:: shell-session $ cp tests/integration/integration_config.yml.sample tests/integration/integration_config.yml @@ -55,54 +127,56 @@ You will also need the following additional Ansible collections: * `community.crypto `_ * `community.general `_ (MacOS local/venv only) -Running legacy mode tests in docker (recommended): +Running legacy mode tests in docker (**recommended**): -.. code-block:: bash +.. code-block:: shell-session $ ansible-test integration --docker default -v Running legacy mode tests in a controlled python virtual environment (**not recommended**): -.. code-block:: bash +.. code-block:: shell-session $ ansible-test integration --venv --requirements --allow-destructive -v -Note that your system packages may be manipulated by running locally or in a venv. +.. warning:: -Legacy mode is not recommended because a new Vault server and proxy server will be downloaded, set up, configured, and/or uninstalled, for every *target*. Traditionally, we've only had one target, and so it was a good way to go, but that's no longer going to be the case. This is going to make it slower and slower as you'll incur the overhead on every target, in every run. + In legacy mode, your system packages may be manipulated by running locally or in a venv. -If you must use legacy mode testing, you can make it more efficient by limiting your test run to the specific target needed, for example: +If you must use legacy mode testing, you can make it more efficient by limiting your test run to the specific target needed, to avoid the overhead of creating and destroying the dependencies for each target. For example: -.. code-block:: bash +.. code-block:: shell-session $ ansible-test integration --docker default -v lookup_hashi_vault +.. _localenv_docker: + Docker Compose localenv -....................... +^^^^^^^^^^^^^^^^^^^^^^^ The recommended way to run the tests has Vault and tinyproxy running in their own containers, set up via docker-compose, and the integration tests run in their own container separately. We have a pre-defined "localenv" setup role for this purpose. Usage -~~~~~ +""""" -For ease of typing / length of commands, we'll enter role directory first: +For ease of typing / length of commands, we'll enter the role directory first: -.. code-block:: bash +.. code-block:: shell-session $ pushd tests/integration/targets/setup_localenv_docker This localenv has both Ansible collection and Python requirements, so let's get those out of the way: -.. code-block:: bash +.. code-block:: shell-session $ pip install -r files/requirements/requirements.txt -c files/requirements/constraints.txt $ ansible-galaxy collection install -r files/requirements/requirements.yml -To set up your docker-compose environment with the all defaults: +To set up your docker-compose environment with all the defaults: -.. code-block:: bash +.. code-block:: shell-session $ ./setup.sh @@ -113,26 +187,28 @@ This will do the following: #. Template a Vault config file. #. Bring down the existing compose project. #. Bring up the compose project as defined by the vars (specified or defaults). -#. Template an ``integration_config.yml`` file that has all the right info for integration tests to connect. -#. Will copy the integration config to the correct location *if there isn't already one there* (it won't overwrite, in case you had customized changes). +#. Template an ``integration_config.yml`` file that has all the right settings for integration tests to connect. +#. Copy the integration config to the correct location *if there isn't already one there* (it won't overwrite, in case you had customized changes). With your containers running, you can now run the tests in docker (after returning back to the collection root): -.. code-block:: bash +.. code-block:: shell-session $ popd $ ansible-test integration --docker default --docker-network hashi_vault_default -v -The ``--docker-network`` part is important, as it will ensure that the Ansible test container is in the same network as the dependency containers, that way the test container can reach them by their container names. The network name, ``hashi_vault_default`` comes from the default docker-compose project name used by this role (``hashi_vault``). See the next section for more information. +The ``--docker-network`` part is important, because it will ensure that the Ansible test container is in the same network as the dependency containers, that way the test container can reach them by their container names. The network name, ``hashi_vault_default`` comes from the default docker-compose project name used by this role (``hashi_vault``). See the :ref:`customization section ` for more information. -Running ``setup.sh`` again can be used to re-deploy the containers, or if you prefer you can use the generated ``docker-compose.yml`` in ``files/.output/``. +Running ``setup.sh`` again can be used to re-deploy the containers, or if you prefer you can use the generated ``files/.output//docker-compose.yml`` directly with local tools. If running again, remember to manually copy the contents of newly generated ``files/.output/integration_config.yml`` to the integration root, or delete the file in the root before re-running setup so that it copies the file automatically. +.. _localenv_docker_customization: + Customization -~~~~~~~~~~~~~ +""""""""""""" -``setup.sh`` will pass any additional params you send it to the ``ansible-playbook`` command it calls, so you can customize variables with the standard ``-e`` option. There are many advanced scenarios possible, but a few things you might want to override: +``setup.sh`` will pass any additional params you send it to the ``ansible-playbook`` command it calls, so you can customize variables with the standard ``--extra-vars`` (or ``-e``) option. There are many advanced scenarios possible, but a few things you might want to override: * ``vault_version`` -- can target any version of Vault for which a docker container exists * ``docker_compose`` (defaults to ``clean`` but could be set to ``up``, ``down``, or ``none``) From c3a0278ccf354bd1eac595b049d9e5e196b2dc97 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Fri, 16 Jul 2021 21:27:12 -0400 Subject: [PATCH 126/137] user guide formatting fixes --- docs/docsite/rst/user_guide.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/docsite/rst/user_guide.rst b/docs/docsite/rst/user_guide.rst index dbc34cdae..cc8af5ef1 100644 --- a/docs/docsite/rst/user_guide.rst +++ b/docs/docsite/rst/user_guide.rst @@ -1,11 +1,14 @@ .. _ansible_collections.community.hashi_vault.docsite.user_guide: +********** User Guide -============ +********** The `community.hashi_vault collection `_ offers Ansible content for working with `HashiCorp Vault `_. -**NOTE:** this guide is a work-in-progress and should not be considered complete. Use it in conjunction with plugin documentation. +.. note:: + + This guide is a work-in-progress and should not be considered complete. Use it in conjunction with plugin documentation. .. contents:: :local: @@ -13,7 +16,7 @@ The `community.hashi_vault collection `_ library. @@ -23,7 +26,7 @@ The content in ``community.hashi_vault`` requires the `hvac `_ and so all of its options are supported. @@ -36,7 +39,7 @@ In ``community.hashi_vault`` you can specify the ``retries`` parameter in two wa About the Collection Defaults -............................. +----------------------------- The collection uses its own set of recommended defaults for retries, including which HTTP status codes to retry, which HTTP methods are subject to retries, and the backoff factor used. **These defaults are subject to change at any time (in any release) and won't be considered breaking changes.** By setting ``retries`` to a number you are opting in to trust the defaults in the collection. To enable retries with full control over its behavior, be sure to specify a dictionary. @@ -56,12 +59,12 @@ Current Defaults (always check the source code to confirm the defaults in your s Any of the ``Retry`` class's parameters that are not specified in the collection defaults or in your custom dictionary, are initialized using the class's defaults, with one exception: the ``raise_on_status`` parameter is always set to ``false`` unless you explicitly added it your custom dictionary. The reason is that this lets our error handling look for the expected ``hvac`` exceptions, instead of the ``Retry``-specfic exceptions. It is recommended that you don't override this as it may cause unexpected error messages on common failures if they are retried. Retry messages -.............. +-------------- By default, if a retry is performed, a warning will be emitted that shows how many retries are remaining. This can be controlled with the ``retry_action`` option which defaults to ``warn``. It is recommended to keep this enabled unless you have other processes that will be thrown off by the warning output. A note about timeouts -..................... +--------------------- Consider setting the ``timeout`` option appropriately when using retries, as a connection timeout doesn't count toward time between retries (backoff). A long timeout can cause very long delays for a connection that isn't going to recover, multiplied by number of retries. From 11d5d322bdb28a9d2ae3fada6ff0eaabfab930fe Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Fri, 16 Jul 2021 21:35:41 -0400 Subject: [PATCH 127/137] more unique anchors --- docs/docsite/rst/contributor_guide.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/docsite/rst/contributor_guide.rst b/docs/docsite/rst/contributor_guide.rst index f05c0a313..a93cdf675 100644 --- a/docs/docsite/rst/contributor_guide.rst +++ b/docs/docsite/rst/contributor_guide.rst @@ -109,7 +109,7 @@ If you have contributed to this collection or to the ``hashi_vault`` lookup plug Legacy mode is not recommended because a new Vault server and proxy server will be downloaded, set up, configured, and/or uninstalled, for every *target*. Traditionally, we've only had one target, and so it was a good way to go, but that's no longer going to be the case. This will make it slower and slower as you'll incur the overhead on every target, in every run. - Skip to :ref:`localenv_docker` for a method that's nearly as easy but better off in the long run (docker-compose). + Skip to :ref:`ansible_collections.community.hashi_vault.docsite.contributor_guide.localenv_docker` for a method that's nearly as easy but better off in the long run (docker-compose). Legacy Mode ^^^^^^^^^^^ @@ -149,7 +149,7 @@ If you must use legacy mode testing, you can make it more efficient by limiting $ ansible-test integration --docker default -v lookup_hashi_vault -.. _localenv_docker: +.. _ansible_collections.community.hashi_vault.docsite.contributor_guide.localenv_docker: Docker Compose localenv ^^^^^^^^^^^^^^^^^^^^^^^ @@ -197,13 +197,13 @@ With your containers running, you can now run the tests in docker (after returni $ popd $ ansible-test integration --docker default --docker-network hashi_vault_default -v -The ``--docker-network`` part is important, because it will ensure that the Ansible test container is in the same network as the dependency containers, that way the test container can reach them by their container names. The network name, ``hashi_vault_default`` comes from the default docker-compose project name used by this role (``hashi_vault``). See the :ref:`customization section ` for more information. +The ``--docker-network`` part is important, because it will ensure that the Ansible test container is in the same network as the dependency containers, that way the test container can reach them by their container names. The network name, ``hashi_vault_default`` comes from the default docker-compose project name used by this role (``hashi_vault``). See the :ref:`customization section ` for more information. Running ``setup.sh`` again can be used to re-deploy the containers, or if you prefer you can use the generated ``files/.output//docker-compose.yml`` directly with local tools. If running again, remember to manually copy the contents of newly generated ``files/.output/integration_config.yml`` to the integration root, or delete the file in the root before re-running setup so that it copies the file automatically. -.. _localenv_docker_customization: +.. _ansible_collections.community.hashi_vault.docsite.contributor_guide.localenv_docker_customization: Customization """"""""""""" From 6aea499e820bf4de1900be009a247cbf8edc98a7 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Fri, 16 Jul 2021 21:35:56 -0400 Subject: [PATCH 128/137] localenv formatting --- docs/docsite/rst/localenv_developer_guide.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/docsite/rst/localenv_developer_guide.rst b/docs/docsite/rst/localenv_developer_guide.rst index 28e319ee3..c6530a923 100644 --- a/docs/docsite/rst/localenv_developer_guide.rst +++ b/docs/docsite/rst/localenv_developer_guide.rst @@ -1,11 +1,14 @@ .. _ansible_collections.community.hashi_vault.docsite.localenv_developer_guide: +************************ localenv Developer Guide -======================== +************************ A "localenv" role in the context of this collection is a role that's used to set up the external dependencies required to run the integration tests. The idea is to provide a pre-packaged way for a contributor to set up their local environment in a certain way. -**NOTE:** this guide is a work-in-progress and is **very** light on details. For the time being, it's best to open an issue in the repository to discuss it if you're thinking of a new localenv. Looking at ``setup_localenv_docker`` should also be helpful as it's the most complete one to date. +.. note:: + + This guide is a work-in-progress and is **very** light on details. For the time being, it's best to open an issue in the repository to discuss it if you're thinking of a new localenv. Looking at ``setup_localenv_docker`` should also be helpful as it's the most complete one to date. .. contents:: @@ -14,10 +17,10 @@ A "localenv" role in the context of this collection is a role that's used to set Required External Dependencies ------------------------------- +============================== HashiCorp Vault -............... +--------------- A Vault server is required for the integration tests. Using `Vault Dev Server Mode `_ is recommended as it's the easiest and fastest way to get a server started. @@ -43,7 +46,7 @@ Relevant ``integration_config.yml`` Variables Proxy Server -............ +------------ A proxy server is used to test the proxy connectivity options. @@ -59,7 +62,7 @@ Relevant ``integration_config.yml`` Variables "``vault_proxy_server``", "``http://proxy:8080``", "The full HTTP URL of your proxy server." localenv Role Conventions -------------------------- +========================= * Use ``files/.output`` to hold generated artifacts. * Anything generated should be in a ``.gitignore``; conversely anything not in a ``.gitignore`` should not be overwritten or modified by this process. That is, there should be no changes to git status that arise from this. From 1440af48f38a2618e65612438869105a18f7423a Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Fri, 16 Jul 2021 21:39:52 -0400 Subject: [PATCH 129/137] Sentence case --- docs/docsite/rst/localenv_developer_guide.rst | 12 ++++++------ docs/docsite/rst/user_guide.rst | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/docsite/rst/localenv_developer_guide.rst b/docs/docsite/rst/localenv_developer_guide.rst index c6530a923..58dbfe0de 100644 --- a/docs/docsite/rst/localenv_developer_guide.rst +++ b/docs/docsite/rst/localenv_developer_guide.rst @@ -1,7 +1,7 @@ .. _ansible_collections.community.hashi_vault.docsite.localenv_developer_guide: ************************ -localenv Developer Guide +localenv developer guide ************************ A "localenv" role in the context of this collection is a role that's used to set up the external dependencies required to run the integration tests. The idea is to provide a pre-packaged way for a contributor to set up their local environment in a certain way. @@ -16,7 +16,7 @@ A "localenv" role in the context of this collection is a role that's used to set :depth: 2 -Required External Dependencies +Required external dependencies ============================== HashiCorp Vault @@ -30,7 +30,7 @@ To run the tests that deal specifically with TLS/HTTPS access, the Vault server The **root token** of the Vault server is needed, as the integration tests will make changes to Vault's configuration, and will expect to have that token available to do so. It's possible to let Vault generate the token on startup and then retrieve it but it may be easiest to pre-generate one and pass it into Vault, via the ``-dev-root-token-id`` option or ``VAULT_DEV_ROOT_TOKEN_ID`` environment variable (see `Dev Options `_). -Relevant ``integration_config.yml`` Variables +Relevant ``integration_config.yml`` variables ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. csv-table:: @@ -45,14 +45,14 @@ Relevant ``integration_config.yml`` Variables "``vault_cert_content``", "``-----BEGIN CERTIFICATE-----``", "The public cert of the CA that signed the cert used for Vault's TLS listener (or the cert itself if self-signed)." -Proxy Server +Proxy server ------------ A proxy server is used to test the proxy connectivity options. In theory any proxy supporting http/s targets could be used for this purpose, but `tinyproxy `_ is recommended for being, well.. tiny, as well as easy to configure and run, and available in package managers and containers. -Relevant ``integration_config.yml`` Variables +Relevant ``integration_config.yml`` variables ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. csv-table:: @@ -61,7 +61,7 @@ Relevant ``integration_config.yml`` Variables "``vault_proxy_server``", "``http://proxy:8080``", "The full HTTP URL of your proxy server." -localenv Role Conventions +localenv role conventions ========================= * Use ``files/.output`` to hold generated artifacts. diff --git a/docs/docsite/rst/user_guide.rst b/docs/docsite/rst/user_guide.rst index cc8af5ef1..c1210f4d0 100644 --- a/docs/docsite/rst/user_guide.rst +++ b/docs/docsite/rst/user_guide.rst @@ -1,7 +1,7 @@ .. _ansible_collections.community.hashi_vault.docsite.user_guide: ********** -User Guide +User guide ********** The `community.hashi_vault collection `_ offers Ansible content for working with `HashiCorp Vault `_. @@ -25,7 +25,7 @@ The content in ``community.hashi_vault`` requires the `hvac `_ and so all of its options are supported. @@ -38,7 +38,7 @@ In ``community.hashi_vault`` you can specify the ``retries`` parameter in two wa * Set a dictionary, where you can set any field that the ``Retry`` class can be initialized with, in order to fully customize your retry experience. -About the Collection Defaults +About the collection defaults ----------------------------- The collection uses its own set of recommended defaults for retries, including which HTTP status codes to retry, which HTTP methods are subject to retries, and the backoff factor used. **These defaults are subject to change at any time (in any release) and won't be considered breaking changes.** By setting ``retries`` to a number you are opting in to trust the defaults in the collection. To enable retries with full control over its behavior, be sure to specify a dictionary. From 6143c4d29e2b0ec48a5f17905c26a42b27fe81c2 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 19 Jul 2021 16:50:14 -0400 Subject: [PATCH 130/137] bash -> shell-session --- docs/docsite/rst/user_guide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docsite/rst/user_guide.rst b/docs/docsite/rst/user_guide.rst index c1210f4d0..321d561d6 100644 --- a/docs/docsite/rst/user_guide.rst +++ b/docs/docsite/rst/user_guide.rst @@ -20,7 +20,7 @@ Requirements The content in ``community.hashi_vault`` requires the `hvac `_ library. -.. code-block:: bash +.. code-block:: shell-session $ pip install hvac From aed642ceea67f5fb024d5bb9147b8da91b156800 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 19 Jul 2021 17:35:13 -0400 Subject: [PATCH 131/137] Apply suggestions from code review Thank you! Co-authored-by: Alicia Cozine <879121+acozine@users.noreply.github.com> --- docs/docsite/rst/contributor_guide.rst | 20 +++++++++---------- docs/docsite/rst/localenv_developer_guide.rst | 2 +- docs/docsite/rst/user_guide.rst | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/docsite/rst/contributor_guide.rst b/docs/docsite/rst/contributor_guide.rst index a93cdf675..982dcf715 100644 --- a/docs/docsite/rst/contributor_guide.rst +++ b/docs/docsite/rst/contributor_guide.rst @@ -33,7 +33,7 @@ Additions to the collection documentation are very welcome! We have three primar Module and plugin documentation ------------------------------- -This type of documentation gets generated from structured YAML, inside of a Python string. It will be included in the same code that it's documenting, or it will be in a separate Python file, such as a doc fragment. Please see the :ref:`module format and documentation guidance ` for more information. +This type of documentation gets generated from structured YAML, inside of a Python string. It is included in the same code that it's documenting, or in a separate Python file, such as a doc fragment. Please see the :ref:`module format and documentation guidance ` for more information. This type of documentation is tested with ``ansible-test sanity`` and full instructions are available on the :ref:`testing module documentation ` page. @@ -51,9 +51,9 @@ Markdown files can be previewed natively in GutHub, so they are easy to validate Collection docsite ------------------ -The collection docsite is a set what you are reading now. It is a set of documentation written in reStructuredText (RST) format that is published on the :ref:`ansible_documentation` site. This is where we have longform documentation that doesn't fit into the other two categories. +The collection docsite is what you are reading now. It is written in reStructuredText (RST) format and published on the :ref:`ansible_documentation` site. This is where we have long-form documentation that doesn't fit into the other two categories. -If you are considering adding an entirely new document here it may be best to open an issue first to discuss the idea and how best to organize it. +If you are considering adding an entirely new document here it may be best to open a GitHub issue first to discuss the idea and how best to organize it. Refer to the :ref:`Ansible style guide ` for all submissions to the collection docsite. @@ -64,7 +64,7 @@ Only the ``extra-docs.yml`` file is validated by the collection's CI, and there Advanced ^^^^^^^^ -Docsite pages can be generated locally through a workaround technique. **This is not a supported method** but it may be helpful to get more rapid feedback on docsite if changes, if you're comfortable at a command line. +Docsite pages can be generated locally through a workaround technique. **This is not a supported method** but it may be helpful to get more rapid feedback on docsite changes, if you're comfortable at a command line. The process is: @@ -91,7 +91,7 @@ The process is: The rendered HTML docs should be available in ``docs/docsite/_build/html/`` and can be opened in a browser. -Running Tests Locally +Running tests locally ===================== If you're making anything more than very small or one-time changes, you'll want to run the tests locally to avoid having to push a commit for each thing, and waiting for the CI to run tests. @@ -109,9 +109,9 @@ If you have contributed to this collection or to the ``hashi_vault`` lookup plug Legacy mode is not recommended because a new Vault server and proxy server will be downloaded, set up, configured, and/or uninstalled, for every *target*. Traditionally, we've only had one target, and so it was a good way to go, but that's no longer going to be the case. This will make it slower and slower as you'll incur the overhead on every target, in every run. - Skip to :ref:`ansible_collections.community.hashi_vault.docsite.contributor_guide.localenv_docker` for a method that's nearly as easy but better off in the long run (docker-compose). + Skip to :ref:`ansible_collections.community.hashi_vault.docsite.contributor_guide.localenv_docker` for a method that's nearly as easy as legacy mode, and far more efficient (docker-compose). -Legacy Mode +Legacy mode ^^^^^^^^^^^ To get started quickly without having to set anything else, you can use legacy mode by copying the included integration config sample: @@ -180,7 +180,7 @@ To set up your docker-compose environment with all the defaults: $ ./setup.sh -This will do the following: +The setup script does the following: #. Template a ``docker-compose.yml`` for the project. #. Generate a private key and self-signed certificate for Vault. @@ -197,7 +197,7 @@ With your containers running, you can now run the tests in docker (after returni $ popd $ ansible-test integration --docker default --docker-network hashi_vault_default -v -The ``--docker-network`` part is important, because it will ensure that the Ansible test container is in the same network as the dependency containers, that way the test container can reach them by their container names. The network name, ``hashi_vault_default`` comes from the default docker-compose project name used by this role (``hashi_vault``). See the :ref:`customization section ` for more information. +The ``--docker-network`` part is important, because it ensures that the Ansible test container is in the same network as the dependency containers, that way the test container can reach them by their container names. The network name, ``hashi_vault_default`` comes from the default docker-compose project name used by this role (``hashi_vault``). See the :ref:`customization section ` for more information. Running ``setup.sh`` again can be used to re-deploy the containers, or if you prefer you can use the generated ``files/.output//docker-compose.yml`` directly with local tools. @@ -208,7 +208,7 @@ If running again, remember to manually copy the contents of newly generated ``fi Customization """"""""""""" -``setup.sh`` will pass any additional params you send it to the ``ansible-playbook`` command it calls, so you can customize variables with the standard ``--extra-vars`` (or ``-e``) option. There are many advanced scenarios possible, but a few things you might want to override: +``setup.sh`` passes any additional params you send it to the ``ansible-playbook`` command it calls, so you can customize variables with the standard ``--extra-vars`` (or ``-e``) option. There are many advanced scenarios possible, but a few things you might want to override: * ``vault_version`` -- can target any version of Vault for which a docker container exists * ``docker_compose`` (defaults to ``clean`` but could be set to ``up``, ``down``, or ``none``) diff --git a/docs/docsite/rst/localenv_developer_guide.rst b/docs/docsite/rst/localenv_developer_guide.rst index 58dbfe0de..e36c47e6c 100644 --- a/docs/docsite/rst/localenv_developer_guide.rst +++ b/docs/docsite/rst/localenv_developer_guide.rst @@ -28,7 +28,7 @@ A unencrypted (plain HTTP) listener is *required* for our purposes as most of th To run the tests that deal specifically with TLS/HTTPS access, the Vault server will need to be started with a TLS enabled listener as well, and the address/port, as well as the CA cert (or the cert itself if self-signed) will need to be supplied. -The **root token** of the Vault server is needed, as the integration tests will make changes to Vault's configuration, and will expect to have that token available to do so. It's possible to let Vault generate the token on startup and then retrieve it but it may be easiest to pre-generate one and pass it into Vault, via the ``-dev-root-token-id`` option or ``VAULT_DEV_ROOT_TOKEN_ID`` environment variable (see `Dev Options `_). +The **root token** of the Vault server is needed, as the integration tests make changes to Vault's configuration, and expect to have that token available to do so. It's possible to let Vault generate the token on startup and then retrieve it but it may be easiest to pre-generate one and pass it into Vault, via the ``-dev-root-token-id`` option or ``VAULT_DEV_ROOT_TOKEN_ID`` environment variable (see `Dev Options `_). Relevant ``integration_config.yml`` variables ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/docsite/rst/user_guide.rst b/docs/docsite/rst/user_guide.rst index 321d561d6..39ded557e 100644 --- a/docs/docsite/rst/user_guide.rst +++ b/docs/docsite/rst/user_guide.rst @@ -25,10 +25,10 @@ The content in ``community.hashi_vault`` requires the `hvac `_ and so all of its options are supported. +Via the ``retries`` parameter, you can control what happens when a request to Vault fails, and automatically retry certain requests. Retries are based on the `urllib3 Retry class `_ and so all of its options are supported. Retries are disabled by default. From 326715b209ce25ac78e04ee08a57855b6ed36552 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 19 Jul 2021 17:50:27 -0400 Subject: [PATCH 132/137] docs fixes from review feedback --- docs/docsite/rst/localenv_developer_guide.rst | 4 ++-- docs/docsite/rst/user_guide.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/docsite/rst/localenv_developer_guide.rst b/docs/docsite/rst/localenv_developer_guide.rst index e36c47e6c..6ddfd2e7f 100644 --- a/docs/docsite/rst/localenv_developer_guide.rst +++ b/docs/docsite/rst/localenv_developer_guide.rst @@ -4,7 +4,7 @@ localenv developer guide ************************ -A "localenv" role in the context of this collection is a role that's used to set up the external dependencies required to run the integration tests. The idea is to provide a pre-packaged way for a contributor to set up their local environment in a certain way. +A "localenv" role in this collection sets up the external dependencies required to run the integration tests. The idea is to provide a pre-packaged way for a contributor to set up their local environment in a consistent, repeatable way. .. note:: @@ -26,7 +26,7 @@ A Vault server is required for the integration tests. Using `Vault Dev Server Mo A unencrypted (plain HTTP) listener is *required* for our purposes as most of the tests will expect to connect that way. -To run the tests that deal specifically with TLS/HTTPS access, the Vault server will need to be started with a TLS enabled listener as well, and the address/port, as well as the CA cert (or the cert itself if self-signed) will need to be supplied. +To run the tests that deal specifically with TLS/HTTPS access, you must start the Vault server with a TLS enabled listener. The TLS address:port, and the CA cert (or the cert itself if self-signed) must be supplied. The **root token** of the Vault server is needed, as the integration tests make changes to Vault's configuration, and expect to have that token available to do so. It's possible to let Vault generate the token on startup and then retrieve it but it may be easiest to pre-generate one and pass it into Vault, via the ``-dev-root-token-id`` option or ``VAULT_DEV_ROOT_TOKEN_ID`` environment variable (see `Dev Options `_). diff --git a/docs/docsite/rst/user_guide.rst b/docs/docsite/rst/user_guide.rst index 39ded557e..5c93ea8d5 100644 --- a/docs/docsite/rst/user_guide.rst +++ b/docs/docsite/rst/user_guide.rst @@ -26,7 +26,7 @@ The content in ``community.hashi_vault`` requires the `hvac `_ and so all of its options are supported. From d6a8cf5d565282604f7628c5ee2c42440bb10c68 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Mon, 19 Jul 2021 17:56:32 -0400 Subject: [PATCH 133/137] missed a spot --- docs/docsite/rst/contributor_guide.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docsite/rst/contributor_guide.rst b/docs/docsite/rst/contributor_guide.rst index 982dcf715..9f1843b44 100644 --- a/docs/docsite/rst/contributor_guide.rst +++ b/docs/docsite/rst/contributor_guide.rst @@ -8,7 +8,7 @@ This guide aims to help PR authors contribute to the ``community.hashi_vault`` c .. note:: - This guide is a work-in-progress and should not be considered complete. Check back often as we fill out more details based on experience and feedback, and please let us know how this guide can be improved. + This guide is a work-in-progress and should not be considered complete. Check back often as we fill out more details based on experience and feedback, and please let us know how this guide can be improved by opening a `GitHub issue in the repository `_. .. contents:: @@ -21,7 +21,7 @@ Quick start #. Log into your GitHub account. #. Fork the `ansible-collections/community.hashi_vault repository `_ by clicking the **Fork** button in the upper right corner. This will create a fork in your own account. -#. Clone the repository locally, following :ref:`the example instructions here ` (but replace ``general`` with ``hashi_vault``). **Pay special attention to the path structure.** +#. Clone the repository locally, following :ref:`the example instructions here ` (but replace ``general`` with ``hashi_vault``). **Pay special attention to the local path structure** of the cloned repository as described in those instructions (for example ``ansible_collections/community/hashi_vault``). #. As mentioned on that page, commit your changes to a branch, push them to your fork, and create a pull request (GitHub will automatically prompt you to do so when you look at your repository). #. :ref:`See the guidance on Changelogs ` and include a :ref:`changelog fragment ` if appropriate. From 0564f3f991b9e1de5776c6e344456fade56b92e2 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 20 Jul 2021 10:39:41 -0400 Subject: [PATCH 134/137] update retry warnings header --- docs/docsite/rst/user_guide.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docsite/rst/user_guide.rst b/docs/docsite/rst/user_guide.rst index 5c93ea8d5..946baee3d 100644 --- a/docs/docsite/rst/user_guide.rst +++ b/docs/docsite/rst/user_guide.rst @@ -58,8 +58,8 @@ Current Defaults (always check the source code to confirm the defaults in your s Any of the ``Retry`` class's parameters that are not specified in the collection defaults or in your custom dictionary, are initialized using the class's defaults, with one exception: the ``raise_on_status`` parameter is always set to ``false`` unless you explicitly added it your custom dictionary. The reason is that this lets our error handling look for the expected ``hvac`` exceptions, instead of the ``Retry``-specfic exceptions. It is recommended that you don't override this as it may cause unexpected error messages on common failures if they are retried. -Retry messages --------------- +Controlling retry warnings +-------------------------- By default, if a retry is performed, a warning will be emitted that shows how many retries are remaining. This can be controlled with the ``retry_action`` option which defaults to ``warn``. It is recommended to keep this enabled unless you have other processes that will be thrown off by the warning output. From 71c5e0f137056d6f1cc3e77fd67c942af9351a16 Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 20 Jul 2021 10:56:59 -0400 Subject: [PATCH 135/137] errant codecov@v1 from main merge conflict --- .github/workflows/ansible-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index fcb0cfce6..b20cd9f1f 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -195,7 +195,7 @@ jobs: working-directory: ${{ env.COLLECTION_PATH }} # See the reports at https://codecov.io/gh/ansible-collections/community.hashi_vault - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v2 with: fail_ci_if_error: false From fc18cd178ab0dd7be4d5c636a70a962cc5c16f1d Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 20 Jul 2021 11:03:45 -0400 Subject: [PATCH 136/137] README update for contributors --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e383add9b..0d9d50b51 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Browsing the [**latest** collection documentation](https://docs.ansible.com/ansi Browsing the [**devel** collection documentation](https://docs.ansible.com/ansible/devel/collections/community/hashi_vault) shows docs for the _latest version released on Galaxy_. -If you use the Ansible package and don't update collections independently, use **latest**, if you install or update this collection directly from Galaxy, use **devel**. +If you use the Ansible package and don't update collections independently, use **latest**, if you install or update this collection directly from Galaxy, use **devel**. If you are looking to contribute, use **devel**. ## Tested with Ansible * 2.9 @@ -71,6 +71,8 @@ See [Ansible Using collections](https://docs.ansible.com/ansible/latest/user_gui ## Contributing to this collection +See the contributor guide in the [**devel** collection documentation](https://docs.ansible.com/ansible/devel/collections/community/hashi_vault). + ## Releasing this collection (for maintainers) From 799d195983bbf8f3f96b0f55c53be49ff7c7e99e Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 20 Jul 2021 11:05:41 -0400 Subject: [PATCH 137/137] Apply suggestions from code review Co-authored-by: Alicia Cozine <879121+acozine@users.noreply.github.com> --- docs/docsite/rst/contributor_guide.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docsite/rst/contributor_guide.rst b/docs/docsite/rst/contributor_guide.rst index 9f1843b44..8ff2f20ee 100644 --- a/docs/docsite/rst/contributor_guide.rst +++ b/docs/docsite/rst/contributor_guide.rst @@ -37,7 +37,7 @@ This type of documentation gets generated from structured YAML, inside of a Pyth This type of documentation is tested with ``ansible-test sanity`` and full instructions are available on the :ref:`testing module documentation ` page. -Although we can't preview how the documentation will look for these, we can be reasonably assured the output is correct because of the highly structured nature and the validation via sanity tests. +Although we can't preview how the documentation will look for these, we can be reasonably sure the output is correct because the documentation is highly structured and validated using sanity tests. README and other markdown files ------------------------------- @@ -94,7 +94,7 @@ The rendered HTML docs should be available in ``docs/docsite/_build/html/`` and Running tests locally ===================== -If you're making anything more than very small or one-time changes, you'll want to run the tests locally to avoid having to push a commit for each thing, and waiting for the CI to run tests. +If you're making anything more than very small or one-time changes, run the tests locally to avoid having to push a commit for each thing, and waiting for the CI to run tests. First, :ref:`review the guidance on testing collections `, as it applies to this collection as well.