From dffb8b48c07076ff7d63872db1f7cc5e6ec11179 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Mon, 22 Apr 2024 17:26:07 -0700 Subject: [PATCH 01/45] buildscripts: move test suite lists to psm_interop_kokoro_lib.sh --- .kokoro/psm_interop_kokoro_lib.sh | 41 +++++++++++++++++++++++++++++ framework/infrastructure/gcp/api.py | 11 ++++---- tests/subsetting_test.py | 7 +++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index c75f4864..40095864 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -30,6 +30,47 @@ readonly GKE_CLUSTER_PSM_LB="psm-lb" readonly GKE_CLUSTER_PSM_SECURITY="psm-security" readonly GKE_CLUSTER_PSM_BASIC="psm-basic" +# TODO(sergiitk): all methods should be using "psm::" package name, +# see https://google.github.io/styleguide/shellguide.html#function-names + +####################################### +# Returns the list of tests in LN test suite. +# Globals: +# TESTING_VERSION: Populated with the version branch under test, +# f.e. master, dev, v1.42.x. +# TESTS: An array of tests LB tests. +# Outputs: +# Sets variable TESTS. +# Prints TESTS to stdout. +####################################### +psm::get_lb_tests() { + # readarray -t myArray < "$1" + + declare -a TESTS + TESTS=( + "affinity_test" + "api_listener_test" + "app_net_test" + "change_backend_service_test" + "custom_lb_test" + "failover_test" + "outlier_detection_test" + "remove_neg_test" + "round_robin_test" + ) + + # master-only tests + if [[ "${TESTING_VERSION}" =~ "master" ]]; then + TESTS+=( + "bootstrap_generator_test" + "subsetting_test" + ) + fi + + echo "Running LB suite tests:" + printf "%s\n" "${TESTS[@]}" +} + ####################################### # Determines the cluster name and zone based on the given cluster identifier. # Globals: diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index cee8af19..a539b1eb 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -126,13 +126,14 @@ def private_api_key(self): return secret.payload.data.decode() @functools.lru_cache(None) - def compute(self, version): + def compute(self, version: str): api_name = "compute" + + if version.startswith("v1") and self.compute_v1_discovery_file: + return self._build_from_file(self.compute_v1_discovery_file) + if version == "v1": - if self.compute_v1_discovery_file: - return self._build_from_file(self.compute_v1_discovery_file) - else: - return self._build_from_discovery_v1(api_name, version) + return self._build_from_discovery_v1(api_name, version) elif version == "v1alpha": return self._build_from_discovery_v1(api_name, "alpha") diff --git a/tests/subsetting_test.py b/tests/subsetting_test.py index 8231b609..8c364033 100644 --- a/tests/subsetting_test.py +++ b/tests/subsetting_test.py @@ -18,6 +18,7 @@ from absl import flags from absl import logging from absl.testing import absltest +from typing_extensions import override from framework import xds_k8s_testcase from framework.helpers import skips @@ -34,6 +35,12 @@ class SubsettingTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): + @classmethod + @override + def setUpClass(cls): + super().setUpClass() + cls.compute_api_version = "v1alpha" + @staticmethod def is_supported(config: skips.TestConfig) -> bool: # Subsetting is an experimental feature where most work is done on the From ac8f5d3d3fae4c25a2824b38aa05b56ed9bad109 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Mon, 22 Apr 2024 17:51:31 -0700 Subject: [PATCH 02/45] Add note on flags.set_default, minor text changes --- .kokoro/psm_interop_kokoro_lib.sh | 2 +- tests/subsetting_test.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 40095864..1c3f9704 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -67,7 +67,7 @@ psm::get_lb_tests() { ) fi - echo "Running LB suite tests:" + echo "LB test suite:" printf "%s\n" "${TESTS[@]}" } diff --git a/tests/subsetting_test.py b/tests/subsetting_test.py index 8c364033..275ec472 100644 --- a/tests/subsetting_test.py +++ b/tests/subsetting_test.py @@ -38,6 +38,8 @@ class SubsettingTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): @classmethod @override def setUpClass(cls): + # TODO(sergiitk): use when absl updated to 1.3.0+, also set + # flags.set_default(xds_flags.COMPUTE_API_VERSION, 'v1alpha') super().setUpClass() cls.compute_api_version = "v1alpha" From 3fcf97d9553079180d3abed23f814a0cc3ac342d Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 10:48:24 -0700 Subject: [PATCH 03/45] Move more common functionality to the shared lib --- .kokoro/psm_interop_kokoro_lib.sh | 120 +++++++++++++++++++++++++++++- 1 file changed, 116 insertions(+), 4 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 1c3f9704..877356c5 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -24,6 +24,9 @@ readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-main}" readonly TEST_DRIVER_PATH="" readonly TEST_DRIVER_PROTOS_PATH="protos/grpc/testing" readonly FORCE_TESTING_VERSION="${FORCE_TESTING_VERSION:-}" +# For compatibility with FORCE_IMAGE_BUILD defined in buildscripts. +# TODO(sergiitk): can be defined as FORCE_IMAGE_BUILD when all buildscripts are updated. +readonly FORCE_IMAGE_BUILD_PSM="${FORCE_IMAGE_BUILD:-0}" # GKE cluster identifiers. readonly GKE_CLUSTER_PSM_LB="psm-lb" @@ -33,6 +36,8 @@ readonly GKE_CLUSTER_PSM_BASIC="psm-basic" # TODO(sergiitk): all methods should be using "psm::" package name, # see https://google.github.io/styleguide/shellguide.html#function-names +# --- LB TESTS --- + ####################################### # Returns the list of tests in LN test suite. # Globals: @@ -43,10 +48,9 @@ readonly GKE_CLUSTER_PSM_BASIC="psm-basic" # Sets variable TESTS. # Prints TESTS to stdout. ####################################### -psm::get_lb_tests() { - # readarray -t myArray < "$1" - - declare -a TESTS +psm::lb::get_tests() { + # TODO(sergiitk): load from env var? + declare -ag TESTS TESTS=( "affinity_test" "api_listener_test" @@ -67,10 +71,118 @@ psm::get_lb_tests() { ) fi + # Prevent buildscripts from appending to the list of tests. + declare -r TESTS echo "LB test suite:" printf "%s\n" "${TESTS[@]}" } +psm::lb::run() { + activate_gke_cluster GKE_CLUSTER_PSM_LB + activate_secondary_gke_cluster GKE_CLUSTER_PSM_LB + psm::setup_test_driver + psm::build_docker_images_if_needed + psm::lb::get_tests + psm::run_tests +} + +# --- Common test run --- + +####################################### +# TBD +# Globals: +# KOKORO_ARTIFACTS_DIR +# GITHUB_REPOSITORY_NAME +# SRC_DIR: Populated with absolute path to the source repo +# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing +# the test driver +# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code +# TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile +# TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report +# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build +# GIT_COMMIT: Populated with the SHA-1 of git commit being built +# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built +# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access +# Arguments: +# None +# Outputs: +# Writes the output of test execution to stdout, stderr +####################################### +psm::run() { + psm::set_docker_images "${GRPC_LANGUAGE}" + case $1 in + lb) + psm::lb::run + ;; + *) + echo "Unknown Test Suite: ${1}" + exit 1 + ;; + esac +} + +psm::set_docker_images() { + case $1 in + java) + SERVER_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-server" + CLIENT_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-client" + ;; + *) + echo "Unknown Language: ${1}" + exit 1 + ;; + esac + declare -r SERVER_IMAGE_NAME + declare -r CLIENT_IMAGE_NAME +} + +psm::run_tests() { + cd "${TEST_DRIVER_FULL_DIR}" + local failed_tests=0 + for test_name in "${TESTS[@]}"; do + run_test "${test_name}" || (( ++failed_tests )) + done + echo "Failed test suites: ${failed_tests}" +} + +psm::setup_test_driver() { + if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then + kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" + else + local_setup_test_driver "$(dirname "$0")" + fi +} + +####################################### +# Builds test app and its docker images unless they already exist +# Globals: +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# FORCE_IMAGE_BUILD +# Arguments: +# None +# Outputs: +# Writes the output to stdout, stderr +####################################### +psm::build_docker_images_if_needed() { + # Check if images already exist + server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" + printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" + echo "${server_tags:-Server image not found}" + + client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" + printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" + echo "${client_tags:-Client image not found}" + + # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 + if [[ "${FORCE_IMAGE_BUILD_PSM}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then + build_test_app_docker_images + else + echo "Skipping ${GRPC_LANGUAGE} test app build" + fi +} + ####################################### # Determines the cluster name and zone based on the given cluster identifier. # Globals: From ee9be1921333b52ae270d7483aec6a14dbecc638 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 11:14:23 -0700 Subject: [PATCH 04/45] Fix array redeclaration; printf tests as a "list" --- .kokoro/psm_interop_kokoro_lib.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 877356c5..4f03fed8 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -71,10 +71,8 @@ psm::lb::get_tests() { ) fi - # Prevent buildscripts from appending to the list of tests. - declare -r TESTS echo "LB test suite:" - printf "%s\n" "${TESTS[@]}" + printf -- "- %s\n" "${TESTS[@]}" } psm::lb::run() { From 52cebf48cde5a7224dddd4f4a2c24c95ef0af2fa Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 12:06:14 -0700 Subject: [PATCH 05/45] Add security suite and move build to psm-interop-build.sh --- .kokoro/psm_interop_kokoro_lib.sh | 46 +++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 4f03fed8..cf7d233f 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -39,7 +39,7 @@ readonly GKE_CLUSTER_PSM_BASIC="psm-basic" # --- LB TESTS --- ####################################### -# Returns the list of tests in LN test suite. +# Returns the list of tests in LB test suite. # Globals: # TESTING_VERSION: Populated with the version branch under test, # f.e. master, dev, v1.42.x. @@ -50,7 +50,6 @@ readonly GKE_CLUSTER_PSM_BASIC="psm-basic" ####################################### psm::lb::get_tests() { # TODO(sergiitk): load from env var? - declare -ag TESTS TESTS=( "affinity_test" "api_listener_test" @@ -62,7 +61,6 @@ psm::lb::get_tests() { "remove_neg_test" "round_robin_test" ) - # master-only tests if [[ "${TESTING_VERSION}" =~ "master" ]]; then TESTS+=( @@ -70,7 +68,6 @@ psm::lb::get_tests() { "subsetting_test" ) fi - echo "LB test suite:" printf -- "- %s\n" "${TESTS[@]}" } @@ -84,6 +81,35 @@ psm::lb::run() { psm::run_tests } +# --- Security TESTS --- + +####################################### +# Returns the list of tests in PSM Security test suite. +# Globals: +# TESTS: An array of tests LB tests. +# Outputs: +# Sets variable TESTS. +# Prints TESTS to stdout. +####################################### +psm::security::get_tests() { + # TODO(sergiitk): load from env var? + TESTS=( + "baseline_test" + "security_test" + "authz_test" + ) + echo "Security test suite:" + printf -- "- %s\n" "${TESTS[@]}" +} + +psm::security::run() { + activate_gke_cluster GKE_CLUSTER_PSM_SECURITY + psm::setup_test_driver + psm::build_docker_images_if_needed + psm::lb::get_tests + psm::run_tests +} + # --- Common test run --- ####################################### @@ -112,6 +138,9 @@ psm::run() { lb) psm::lb::run ;; + security) + psm::security::run + ;; *) echo "Unknown Test Suite: ${1}" exit 1 @@ -144,10 +173,17 @@ psm::run_tests() { } psm::setup_test_driver() { + local script_dir + script_dir="$(dirname "$0")" + if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" else - local_setup_test_driver "$(dirname "$0")" + local_setup_test_driver "${script_dir}" + fi + + if [[ -f "${script_dir}/psm-interop-build.sh" ]]; then + source "${script_dir}/psm-interop-build.sh" fi } From 8f2b47822c1d01f8d01e499a1218714992a63ca8 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 12:20:09 -0700 Subject: [PATCH 06/45] Add URL Map suite --- .kokoro/psm_interop_kokoro_lib.sh | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index cf7d233f..9aa43d39 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# TODO(sergiitk): move to grpc/grpc when implementing support of other languages set -eo pipefail # Constants @@ -24,8 +23,7 @@ readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-main}" readonly TEST_DRIVER_PATH="" readonly TEST_DRIVER_PROTOS_PATH="protos/grpc/testing" readonly FORCE_TESTING_VERSION="${FORCE_TESTING_VERSION:-}" -# For compatibility with FORCE_IMAGE_BUILD defined in buildscripts. -# TODO(sergiitk): can be defined as FORCE_IMAGE_BUILD when all buildscripts are updated. +# TODO(sergiitk): can be defined as readonly FORCE_IMAGE_BUILD when removed from buildscripts. readonly FORCE_IMAGE_BUILD_PSM="${FORCE_IMAGE_BUILD:-0}" # GKE cluster identifiers. @@ -110,6 +108,30 @@ psm::security::run() { psm::run_tests } +# --- URL Map TESTS --- + +####################################### +# Returns the list of tests in URL Map test suite. +# Globals: +# TESTS: An array of tests LB tests. +# Outputs: +# Sets variable TESTS. +# Prints TESTS to stdout. +####################################### +psm::url_map::get_tests() { + TESTS=("url_map") + echo "URL Map test suite:" + printf -- "- %s\n" "${TESTS[@]}" +} + +psm::url_map::run() { + activate_gke_cluster GKE_CLUSTER_PSM_BASIC + psm::setup_test_driver + psm::build_docker_images_if_needed + psm::url_map::get_tests + psm::run_tests +} + # --- Common test run --- ####################################### @@ -141,6 +163,9 @@ psm::run() { security) psm::security::run ;; + url_map) + psm::url_map::run + ;; *) echo "Unknown Test Suite: ${1}" exit 1 From 209e6831762173512963d36c28e35a12b52e4957 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 12:22:45 -0700 Subject: [PATCH 07/45] Minor docblock fixes --- .kokoro/psm_interop_kokoro_lib.sh | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 9aa43d39..5f295091 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -37,13 +37,11 @@ readonly GKE_CLUSTER_PSM_BASIC="psm-basic" # --- LB TESTS --- ####################################### -# Returns the list of tests in LB test suite. +# Prepares the list of tests in PSM LB test suite. # Globals: -# TESTING_VERSION: Populated with the version branch under test, -# f.e. master, dev, v1.42.x. -# TESTS: An array of tests LB tests. +# TESTING_VERSION: The version branch under test, f.e. master, dev, v1.42.x. +# TESTS: Populated with tests in PSM LB test suite. # Outputs: -# Sets variable TESTS. # Prints TESTS to stdout. ####################################### psm::lb::get_tests() { @@ -82,11 +80,10 @@ psm::lb::run() { # --- Security TESTS --- ####################################### -# Returns the list of tests in PSM Security test suite. +# Prepares the list of tests in PSM Security test suite. # Globals: -# TESTS: An array of tests LB tests. +# TESTS: Populated with tests in PSM Security test suite. # Outputs: -# Sets variable TESTS. # Prints TESTS to stdout. ####################################### psm::security::get_tests() { @@ -111,11 +108,10 @@ psm::security::run() { # --- URL Map TESTS --- ####################################### -# Returns the list of tests in URL Map test suite. +# Prepares the list of tests in URL Map test suite. # Globals: -# TESTS: An array of tests LB tests. +# TESTS: Populated with tests in URL Map test suite. # Outputs: -# Sets variable TESTS. # Prints TESTS to stdout. ####################################### psm::url_map::get_tests() { From 10f1fe9a008bce481d2088b7a4a8e5e513199b04 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 12:29:05 -0700 Subject: [PATCH 08/45] debug sourcing psm-interop-build.sh --- .kokoro/psm_interop_kokoro_lib.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 5f295091..1b3795d7 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -196,6 +196,12 @@ psm::run_tests() { psm::setup_test_driver() { local script_dir script_dir="$(dirname "$0")" + set -x + echo "Sourcing ${script_dir}/psm-interop-build.sh" + if [[ -f "${script_dir}/psm-interop-build.sh" ]]; then + source "${script_dir}/psm-interop-build.sh" + fi + set +x if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" @@ -203,9 +209,6 @@ psm::setup_test_driver() { local_setup_test_driver "${script_dir}" fi - if [[ -f "${script_dir}/psm-interop-build.sh" ]]; then - source "${script_dir}/psm-interop-build.sh" - fi } ####################################### From ed4681ccc869fbe881f80db11df0afb3a4845308 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 12:31:12 -0700 Subject: [PATCH 09/45] Rename build_test_app_docker_images --- .kokoro/psm_interop_kokoro_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 1b3795d7..98281f7f 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -235,7 +235,7 @@ psm::build_docker_images_if_needed() { # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 if [[ "${FORCE_IMAGE_BUILD_PSM}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then - build_test_app_docker_images + psm::lang::build_docker_images else echo "Skipping ${GRPC_LANGUAGE} test app build" fi From e9d65f05c4abf90b8e3d9a08eeeedd6e386b509a Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 12:34:14 -0700 Subject: [PATCH 10/45] Use BUILD_SCRIPT_DIR set in the driver --- .kokoro/psm_interop_kokoro_lib.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 98281f7f..13e48a48 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -194,19 +194,17 @@ psm::run_tests() { } psm::setup_test_driver() { - local script_dir - script_dir="$(dirname "$0")" set -x - echo "Sourcing ${script_dir}/psm-interop-build.sh" - if [[ -f "${script_dir}/psm-interop-build.sh" ]]; then - source "${script_dir}/psm-interop-build.sh" + echo "Sourcing ${BUILD_SCRIPT_DIR}/psm-interop-build.sh" + if [[ -f "${BUILD_SCRIPT_DIR}/psm-interop-build.sh" ]]; then + source "${BUILD_SCRIPT_DIR}/psm-interop-build.sh" fi set +x if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" else - local_setup_test_driver "${script_dir}" + local_setup_test_driver "${BUILD_SCRIPT_DIR}" fi } From 686bde9c177aa0d1a11f24e996055d34b68a855d Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 12:39:15 -0700 Subject: [PATCH 11/45] Make psm-interop-build.sh with language suffix --- .kokoro/psm_interop_kokoro_lib.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 13e48a48..967464f9 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -194,12 +194,11 @@ psm::run_tests() { } psm::setup_test_driver() { - set -x - echo "Sourcing ${BUILD_SCRIPT_DIR}/psm-interop-build.sh" - if [[ -f "${BUILD_SCRIPT_DIR}/psm-interop-build.sh" ]]; then - source "${BUILD_SCRIPT_DIR}/psm-interop-build.sh" + local build_docker_script="${BUILD_SCRIPT_DIR}/psm-interop-build-${GRPC_LANGUAGE}.sh" + if [[ -f "${build_docker_script}" ]]; then + echo "Sourcing ${build_docker_script}" + source "${build_docker_script}" fi - set +x if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" From 242b2f5d86bf06787f23a043f582add6c9e5262b Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 12:42:10 -0700 Subject: [PATCH 12/45] Remove psm-interop-build debugging --- .kokoro/psm_interop_kokoro_lib.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 967464f9..e97cc87e 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -194,18 +194,17 @@ psm::run_tests() { } psm::setup_test_driver() { - local build_docker_script="${BUILD_SCRIPT_DIR}/psm-interop-build-${GRPC_LANGUAGE}.sh" - if [[ -f "${build_docker_script}" ]]; then - echo "Sourcing ${build_docker_script}" - source "${build_docker_script}" - fi - if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" else local_setup_test_driver "${BUILD_SCRIPT_DIR}" fi + local build_docker_script="${BUILD_SCRIPT_DIR}/psm-interop-build-${GRPC_LANGUAGE}.sh" + if [[ -f "${build_docker_script}" ]]; then + echo "Sourcing ${build_docker_script}" + source "${build_docker_script}" + fi } ####################################### From 7579354d1a05a8257a6c05c783178c66623c11cd Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 12:50:47 -0700 Subject: [PATCH 13/45] Need more debugging after all --- .kokoro/psm_interop_kokoro_lib.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index e97cc87e..f9205852 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -194,17 +194,18 @@ psm::run_tests() { } psm::setup_test_driver() { + local build_docker_script="${BUILD_SCRIPT_DIR}/psm-interop-build-${GRPC_LANGUAGE}.sh" + echo "Looking for docker image build script ${build_docker_script}" + if [[ -f "${build_docker_script}" ]]; then + echo "Sourcing docker image build script: ${build_docker_script}" + source "${build_docker_script}" + fi + if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" else local_setup_test_driver "${BUILD_SCRIPT_DIR}" fi - - local build_docker_script="${BUILD_SCRIPT_DIR}/psm-interop-build-${GRPC_LANGUAGE}.sh" - if [[ -f "${build_docker_script}" ]]; then - echo "Sourcing ${build_docker_script}" - source "${build_docker_script}" - fi } ####################################### From 751ab60d3685bbd22dc37ac38678acd6f3d6a801 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 13:59:33 -0700 Subject: [PATCH 14/45] try psm::run_generic_test_suite --- .kokoro/psm_interop_kokoro_lib.sh | 39 ++++++++++++------------------- 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index f9205852..20e69c09 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -68,13 +68,9 @@ psm::lb::get_tests() { printf -- "- %s\n" "${TESTS[@]}" } -psm::lb::run() { +psm::lb::setup() { activate_gke_cluster GKE_CLUSTER_PSM_LB activate_secondary_gke_cluster GKE_CLUSTER_PSM_LB - psm::setup_test_driver - psm::build_docker_images_if_needed - psm::lb::get_tests - psm::run_tests } # --- Security TESTS --- @@ -97,12 +93,8 @@ psm::security::get_tests() { printf -- "- %s\n" "${TESTS[@]}" } -psm::security::run() { +psm::security::setup() { activate_gke_cluster GKE_CLUSTER_PSM_SECURITY - psm::setup_test_driver - psm::build_docker_images_if_needed - psm::lb::get_tests - psm::run_tests } # --- URL Map TESTS --- @@ -120,12 +112,8 @@ psm::url_map::get_tests() { printf -- "- %s\n" "${TESTS[@]}" } -psm::url_map::run() { +psm::url_map::setup() { activate_gke_cluster GKE_CLUSTER_PSM_BASIC - psm::setup_test_driver - psm::build_docker_images_if_needed - psm::url_map::get_tests - psm::run_tests } # --- Common test run --- @@ -153,14 +141,8 @@ psm::url_map::run() { psm::run() { psm::set_docker_images "${GRPC_LANGUAGE}" case $1 in - lb) - psm::lb::run - ;; - security) - psm::security::run - ;; - url_map) - psm::url_map::run + lb | security | url_map) + psm::run_generic_test_suite "$1" ;; *) echo "Unknown Test Suite: ${1}" @@ -169,6 +151,15 @@ psm::run() { esac } +psm::run_generic_test_suite() { + local test_suite="$1" + "psm::${test_suite}::setup" + psm::setup_test_driver + psm::build_docker_images_if_needed + "psm::${test_suite}::get_tests" + psm::run_tests +} + psm::set_docker_images() { case $1 in java) @@ -200,7 +191,7 @@ psm::setup_test_driver() { echo "Sourcing docker image build script: ${build_docker_script}" source "${build_docker_script}" fi - + if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" else From 0e070b938a7e9350555a84b20bf59e1b055d130a Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 18:33:31 -0700 Subject: [PATCH 15/45] Start moving python run test logic into the kokoro lib --- .kokoro/psm_interop_kokoro_lib.sh | 181 +++++++++++++++++++++++------- 1 file changed, 143 insertions(+), 38 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 20e69c09..7f1eb81f 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -31,10 +31,17 @@ readonly GKE_CLUSTER_PSM_LB="psm-lb" readonly GKE_CLUSTER_PSM_SECURITY="psm-security" readonly GKE_CLUSTER_PSM_BASIC="psm-basic" -# TODO(sergiitk): all methods should be using "psm::" package name, -# see https://google.github.io/styleguide/shellguide.html#function-names +# --- LB TESTS ------------------------ -# --- LB TESTS --- +####################################### +# LB Test Suite setup. +# Outputs: +# Prints activated cluster names. +####################################### +psm::lb::setup() { + activate_gke_cluster GKE_CLUSTER_PSM_LB + activate_secondary_gke_cluster GKE_CLUSTER_PSM_LB +} ####################################### # Prepares the list of tests in PSM LB test suite. @@ -68,12 +75,39 @@ psm::lb::get_tests() { printf -- "- %s\n" "${TESTS[@]}" } -psm::lb::setup() { - activate_gke_cluster GKE_CLUSTER_PSM_LB - activate_secondary_gke_cluster GKE_CLUSTER_PSM_LB +####################################### +# Executes LB test case +# Globals: +# TBD +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +psm::lb::run_test() { + local test_name="${1:?missing the test name argument}" + PSM_TEST_FLAGS+=( + "--secondary_kube_context=${SECONDARY_KUBE_CONTEXT}" + "--server_image=${SERVER_IMAGE_NAME}:${GIT_COMMIT}" + "--client_image=${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" + ) + echo "PSM test flags:" + printf -- "- %s\n" "${PSM_TEST_FLAGS[@]}" + + psm:tools:run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } -# --- Security TESTS --- +# --- Security TESTS ------------------ + +####################################### +# Security Test Suite setup. +# Outputs: +# Prints activated cluster names. +####################################### +psm::security::setup() { + activate_gke_cluster GKE_CLUSTER_PSM_SECURITY +} ####################################### # Prepares the list of tests in PSM Security test suite. @@ -93,11 +127,17 @@ psm::security::get_tests() { printf -- "- %s\n" "${TESTS[@]}" } -psm::security::setup() { - activate_gke_cluster GKE_CLUSTER_PSM_SECURITY -} -# --- URL Map TESTS --- +# --- URL Map TESTS ------------------ + +####################################### +# URL Map Test Suite setup. +# Outputs: +# Prints activated cluster names. +####################################### +psm::url_map::setup() { + activate_gke_cluster GKE_CLUSTER_PSM_BASIC +} ####################################### # Prepares the list of tests in URL Map test suite. @@ -112,17 +152,14 @@ psm::url_map::get_tests() { printf -- "- %s\n" "${TESTS[@]}" } -psm::url_map::setup() { - activate_gke_cluster GKE_CLUSTER_PSM_BASIC -} - -# --- Common test run --- +# --- Common test run logic ----------- ####################################### # TBD # Globals: # KOKORO_ARTIFACTS_DIR # GITHUB_REPOSITORY_NAME +# GRPC_LANGUAGE # SRC_DIR: Populated with absolute path to the source repo # TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing # the test driver @@ -134,34 +171,95 @@ psm::url_map::setup() { # GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built # KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access # Arguments: -# None +# Test suite name, one of (lb, security, url_map) # Outputs: # Writes the output of test execution to stdout, stderr ####################################### psm::run() { - psm::set_docker_images "${GRPC_LANGUAGE}" - case $1 in + psm::setup::docker_image_names "${GRPC_LANGUAGE}" + local test_suite="${1:?missing the test suite argument}" + case "${test_suite}" in lb | security | url_map) - psm::run_generic_test_suite "$1" + psm::setup::generic_test_suite "${test_suite}" ;; *) - echo "Unknown Test Suite: ${1}" + echo "Unknown Test Suite: ${test_suite}" exit 1 ;; esac + + psm::run:test_suite "${test_suite}" } -psm::run_generic_test_suite() { - local test_suite="$1" +####################################### +# Executes the test suite +# Globals: +# TBD +# Arguments: +# Test suite name +# Outputs: +# TBD +####################################### +psm::run::test_suite() { + local test_suite="${1:?missing the test suite argument}" + cd "${TEST_DRIVER_FULL_DIR}" + local failed_tests=0 + for test_name in "${TESTS[@]}"; do + psm::run::test "${test_suite}" "${test_name}" || (( ++failed_tests )) + done + echo "Failed test suites: ${failed_tests}" +} + +####################################### +# Executes the test case +# Globals: +# TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile +# KUBE_CONTEXT: The name of kubectl context with GKE cluster access +# TEST_XML_OUTPUT_DIR: Output directory for the test xUnit XML report +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# TESTING_VERSION: version branch under test: used by the framework to +# determine the supported PSM features. +# Arguments: +# Test suite name +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +psm::run::test() { + # Test driver usage: https://github.com/grpc/psm-interop#basic-usage + local test_suite="${1:?missing the test suite argument}" + local test_name="${2:?missing the test name argument}" + local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" + mkdir -p "${out_dir}" + + PSM_TEST_FLAGS=( + "--flagfile=${TEST_DRIVER_FLAGFILE}" + "--kube_context=${KUBE_CONTEXT}" + "--force_cleanup" + "--collect_app_logs" + "--log_dir=${out_dir}" + "--testing_version=${TESTING_VERSION}" + "--xml_output_file=${out_dir}/sponge_log.xml" + ) + + # Must be the last line. + "psm::${test_suite}::run_test" "${test_name}" |& tee "${out_dir}/sponge_log.log" +} + +# --- Common test setup logic ----------- + +psm::setup::generic_test_suite() { + local test_suite="${1:?missing the test suite argument}" "psm::${test_suite}::setup" - psm::setup_test_driver - psm::build_docker_images_if_needed + psm::setup::test_driver + psm::build::docker_images_if_needed "psm::${test_suite}::get_tests" - psm::run_tests } -psm::set_docker_images() { - case $1 in +psm::setup::docker_image_names() { + case "$1" in java) SERVER_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-server" CLIENT_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-client" @@ -175,16 +273,7 @@ psm::set_docker_images() { declare -r CLIENT_IMAGE_NAME } -psm::run_tests() { - cd "${TEST_DRIVER_FULL_DIR}" - local failed_tests=0 - for test_name in "${TESTS[@]}"; do - run_test "${test_name}" || (( ++failed_tests )) - done - echo "Failed test suites: ${failed_tests}" -} - -psm::setup_test_driver() { +psm::setup::test_driver() { local build_docker_script="${BUILD_SCRIPT_DIR}/psm-interop-build-${GRPC_LANGUAGE}.sh" echo "Looking for docker image build script ${build_docker_script}" if [[ -f "${build_docker_script}" ]]; then @@ -199,6 +288,8 @@ psm::setup_test_driver() { fi } +# --- Common test build logic ----------- + ####################################### # Builds test app and its docker images unless they already exist # Globals: @@ -229,6 +320,20 @@ psm::build_docker_images_if_needed() { fi } +# --- Common helpers ---------------------- + +psm::tools::run_verbose() { + local exit_code=0 + set -x + "$@" || exit_code=$? + set +x + return $exit_code +} + +# --- "Unsorted" methods -------------- +# TODO(sergiitk): all methods should be using "psm::" package name, +# see https://google.github.io/styleguide/shellguide.html#function-names + ####################################### # Determines the cluster name and zone based on the given cluster identifier. # Globals: From 4f7a66756bbef370b789ed820c8d272c180612e6 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 23 Apr 2024 18:43:48 -0700 Subject: [PATCH 16/45] bugfixign --- .kokoro/psm_interop_kokoro_lib.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 7f1eb81f..9116e05a 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -95,7 +95,7 @@ psm::lb::run_test() { echo "PSM test flags:" printf -- "- %s\n" "${PSM_TEST_FLAGS[@]}" - psm:tools:run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" + psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } # --- Security TESTS ------------------ @@ -188,7 +188,7 @@ psm::run() { ;; esac - psm::run:test_suite "${test_suite}" + psm::run::test_suite "${test_suite}" } ####################################### @@ -302,7 +302,7 @@ psm::setup::test_driver() { # Outputs: # Writes the output to stdout, stderr ####################################### -psm::build_docker_images_if_needed() { +psm::build::docker_images_if_needed() { # Check if images already exist server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" From bbebf799f2dd2d54b3906c5d0ec8c928ead04441 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 24 Apr 2024 11:49:45 -0700 Subject: [PATCH 17/45] Move URL Map and Security --- .kokoro/psm_interop_kokoro_lib.sh | 89 +++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 9116e05a..fc7ab714 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -66,6 +66,7 @@ psm::lb::get_tests() { ) # master-only tests if [[ "${TESTING_VERSION}" =~ "master" ]]; then + echo "Appending master-only tests to the LB suite." TESTS+=( "bootstrap_generator_test" "subsetting_test" @@ -87,14 +88,7 @@ psm::lb::get_tests() { ####################################### psm::lb::run_test() { local test_name="${1:?missing the test name argument}" - PSM_TEST_FLAGS+=( - "--secondary_kube_context=${SECONDARY_KUBE_CONTEXT}" - "--server_image=${SERVER_IMAGE_NAME}:${GIT_COMMIT}" - "--client_image=${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" - ) - echo "PSM test flags:" - printf -- "- %s\n" "${PSM_TEST_FLAGS[@]}" - + psm::tools::print_test_flags psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } @@ -127,6 +121,21 @@ psm::security::get_tests() { printf -- "- %s\n" "${TESTS[@]}" } +####################################### +# Executes Security test case +# Globals: +# TBD +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +psm::security::run_test() { + local test_name="${1:?missing the test name argument}" + psm::tools::print_test_flags + psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" +} # --- URL Map TESTS ------------------ @@ -144,12 +153,29 @@ psm::url_map::setup() { # Globals: # TESTS: Populated with tests in URL Map test suite. # Outputs: -# Prints TESTS to stdout. +# None ####################################### psm::url_map::get_tests() { TESTS=("url_map") - echo "URL Map test suite:" - printf -- "- %s\n" "${TESTS[@]}" +} + +####################################### +# Executes Security test case +# Globals: +# TBD +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +psm::url_map::run_test() { + local test_name="${1:?missing the test name argument}" + PSM_TEST_FLAGS+=( + "--flagfile=config/url-map.cfg" + ) + psm::tools::print_test_flags + psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } # --- Common test run logic ----------- @@ -176,8 +202,9 @@ psm::url_map::get_tests() { # Writes the output of test execution to stdout, stderr ####################################### psm::run() { - psm::setup::docker_image_names "${GRPC_LANGUAGE}" local test_suite="${1:?missing the test suite argument}" + psm::setup::docker_image_names "${GRPC_LANGUAGE}" "${test_suite}" + case "${test_suite}" in lb | security | url_map) psm::setup::generic_test_suite "${test_suite}" @@ -232,6 +259,7 @@ psm::run::test() { local test_suite="${1:?missing the test suite argument}" local test_name="${2:?missing the test name argument}" local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" + local test_log="${out_dir}/sponge_log.log" mkdir -p "${out_dir}" PSM_TEST_FLAGS=( @@ -240,12 +268,24 @@ psm::run::test() { "--force_cleanup" "--collect_app_logs" "--log_dir=${out_dir}" - "--testing_version=${TESTING_VERSION}" "--xml_output_file=${out_dir}/sponge_log.xml" + "--testing_version=${TESTING_VERSION}" + "--client_image=${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" ) + # Some test suites have canonical server image configured in the flagfiles. + if [[ -n "${SERVER_IMAGE_NAME}" ]]; then + PSM_TEST_FLAGS+=("--server_image=${SERVER_IMAGE_NAME}:${GIT_COMMIT}") + fi + + # So far, only LB test uses secondary GKE cluster. + if [[ -n "${SECONDARY_KUBE_CONTEXT}" ]]; then + PSM_TEST_FLAGS+=("--secondary_kube_context=${SECONDARY_KUBE_CONTEXT}") + fi + + echo "Preparing to run ${test_suite} test: ${test_name}" |& tee "${test_log}" # Must be the last line. - "psm::${test_suite}::run_test" "${test_name}" |& tee "${out_dir}/sponge_log.log" + "psm::${test_suite}::run_test" "${test_name}" |& tee -a "${test_log}" } # --- Common test setup logic ----------- @@ -259,7 +299,10 @@ psm::setup::generic_test_suite() { } psm::setup::docker_image_names() { - case "$1" in + local language="${1:?missing the language argument}" + local test_suite="${2:?missing the test suite argument}" + + case "${language}" in java) SERVER_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-server" CLIENT_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-client" @@ -269,6 +312,14 @@ psm::setup::docker_image_names() { exit 1 ;; esac + + case "${test_suite}" in + url_map) + # Uses the canonical server image configured in url-map.cfg + SERVER_IMAGE_NAME="" + ;; + esac + declare -r SERVER_IMAGE_NAME declare -r CLIENT_IMAGE_NAME } @@ -330,6 +381,12 @@ psm::tools::run_verbose() { return $exit_code } +psm::tools::print_test_flags() { + echo "PSM test flags:" + printf -- "%s\n" "${PSM_TEST_FLAGS[@]}" + echo +} + # --- "Unsorted" methods -------------- # TODO(sergiitk): all methods should be using "psm::" package name, # see https://google.github.io/styleguide/shellguide.html#function-names @@ -601,7 +658,7 @@ test_driver_compile_protos() { ####################################### # Installs the test driver and it's requirements. -# https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#installation +# https://github.com/grpc/psm-interop#basic-usage#installation # Globals: # TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing # the test driver From 7135ae4d346398f9f70eaa42a06e5598d5e8a844 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 24 Apr 2024 12:01:29 -0700 Subject: [PATCH 18/45] Handle docker_images_if_needed when server image not needed --- .kokoro/psm_interop_kokoro_lib.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index fc7ab714..e1b657eb 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -355,14 +355,18 @@ psm::setup::test_driver() { ####################################### psm::build::docker_images_if_needed() { # Check if images already exist - server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" - printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" - echo "${server_tags:-Server image not found}" - client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" echo "${client_tags:-Client image not found}" + if [[ -n "${SERVER_IMAGE_NAME}" ]]; then + server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" + printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" + echo "${server_tags:-Server image not found}" + else + server_tags="ignored" + fi + # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 if [[ "${FORCE_IMAGE_BUILD_PSM}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then psm::lang::build_docker_images From cb47650cb28e58426547dfccb77bb3ed47c9a1db Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 24 Apr 2024 13:27:05 -0700 Subject: [PATCH 19/45] fix the unset image name when building --- .kokoro/psm_interop_kokoro_lib.sh | 37 ++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index e1b657eb..39ccc9f4 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -274,7 +274,7 @@ psm::run::test() { ) # Some test suites have canonical server image configured in the flagfiles. - if [[ -n "${SERVER_IMAGE_NAME}" ]]; then + if [[ -z "${SERVER_IMAGE_USE_CANONICAL}" ]]; then PSM_TEST_FLAGS+=("--server_image=${SERVER_IMAGE_NAME}:${GIT_COMMIT}") fi @@ -298,14 +298,28 @@ psm::setup::generic_test_suite() { "psm::${test_suite}::get_tests" } +####################################### +# Executes the test case +# Globals: +# CLIENT_IMAGE_NAME: Populated with xDS test client image name +# SERVER_IMAGE_NAME: Populated with xDS test server image name +# SERVER_IMAGE_USE_CANONICAL: Set to "1" when canonical server image will be used in the tests +# instead of the one configured in SERVER_IMAGE_NAME. +# Arguments: +# gRPC Language +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### psm::setup::docker_image_names() { local language="${1:?missing the language argument}" local test_suite="${2:?missing the test suite argument}" case "${language}" in java) - SERVER_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-server" CLIENT_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-client" + SERVER_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-server" ;; *) echo "Unknown Language: ${1}" @@ -316,12 +330,16 @@ psm::setup::docker_image_names() { case "${test_suite}" in url_map) # Uses the canonical server image configured in url-map.cfg - SERVER_IMAGE_NAME="" + SERVER_IMAGE_USE_CANONICAL="1" + ;; + *) + SERVER_IMAGE_USE_CANONICAL="" ;; esac - declare -r SERVER_IMAGE_NAME declare -r CLIENT_IMAGE_NAME + declare -r SERVER_IMAGE_NAME + declare -r SERVER_IMAGE_USE_CANONICAL } psm::setup::test_driver() { @@ -359,12 +377,12 @@ psm::build::docker_images_if_needed() { printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" echo "${client_tags:-Client image not found}" - if [[ -n "${SERVER_IMAGE_NAME}" ]]; then + if [[ -z "${SERVER_IMAGE_USE_CANONICAL}" ]]; then server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" echo "${server_tags:-Server image not found}" else - server_tags="ignored" + server_tags="ignored-use-canonical" fi # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 @@ -377,6 +395,13 @@ psm::build::docker_images_if_needed() { # --- Common helpers ---------------------- +####################################### +# Print the command with its arguments (aka set xtrace/ set -x) before execution. +# Arguments: +# The command to run +# Returns: +# The exit status of the command executed. +####################################### psm::tools::run_verbose() { local exit_code=0 set -x From d33eb068c0cf3bcff94c12125f7845f83b8252c6 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 24 Apr 2024 17:13:49 -0700 Subject: [PATCH 20/45] Improve logging --- .kokoro/psm_interop_kokoro_lib.sh | 114 ++++++++++++++++++------------ 1 file changed, 70 insertions(+), 44 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 39ccc9f4..ca7b9bd0 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -66,14 +66,12 @@ psm::lb::get_tests() { ) # master-only tests if [[ "${TESTING_VERSION}" =~ "master" ]]; then - echo "Appending master-only tests to the LB suite." + psm::tools::log "Appending master-only tests to the LB suite." TESTS+=( "bootstrap_generator_test" "subsetting_test" ) fi - echo "LB test suite:" - printf -- "- %s\n" "${TESTS[@]}" } ####################################### @@ -88,7 +86,7 @@ psm::lb::get_tests() { ####################################### psm::lb::run_test() { local test_name="${1:?missing the test name argument}" - psm::tools::print_test_flags + psm::tools::print_test_flags "${test_name}" psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } @@ -117,8 +115,6 @@ psm::security::get_tests() { "security_test" "authz_test" ) - echo "Security test suite:" - printf -- "- %s\n" "${TESTS[@]}" } ####################################### @@ -210,7 +206,7 @@ psm::run() { psm::setup::generic_test_suite "${test_suite}" ;; *) - echo "Unknown Test Suite: ${test_suite}" + psm::tools::log "Unknown Test Suite: ${test_suite}" exit 1 ;; esac @@ -234,7 +230,7 @@ psm::run::test_suite() { for test_name in "${TESTS[@]}"; do psm::run::test "${test_suite}" "${test_name}" || (( ++failed_tests )) done - echo "Failed test suites: ${failed_tests}" + psm::tools::log "Failed test suites: ${failed_tests}" } ####################################### @@ -283,7 +279,7 @@ psm::run::test() { PSM_TEST_FLAGS+=("--secondary_kube_context=${SECONDARY_KUBE_CONTEXT}") fi - echo "Preparing to run ${test_suite} test: ${test_name}" |& tee "${test_log}" + psm::tools::log "Running ${test_suite} suite test: ${test_name}" |& tee "${test_log}" # Must be the last line. "psm::${test_suite}::run_test" "${test_name}" |& tee -a "${test_log}" } @@ -295,7 +291,11 @@ psm::setup::generic_test_suite() { "psm::${test_suite}::setup" psm::setup::test_driver psm::build::docker_images_if_needed + "psm::${test_suite}::get_tests" + psm::tools::log "Tests in ${test_suite} test suite:" + printf -- "- %s\n" "${TESTS[@]}" + echo } ####################################### @@ -322,7 +322,7 @@ psm::setup::docker_image_names() { SERVER_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-server" ;; *) - echo "Unknown Language: ${1}" + psm::tools::log "Unknown Language: ${1}" exit 1 ;; esac @@ -344,9 +344,9 @@ psm::setup::docker_image_names() { psm::setup::test_driver() { local build_docker_script="${BUILD_SCRIPT_DIR}/psm-interop-build-${GRPC_LANGUAGE}.sh" - echo "Looking for docker image build script ${build_docker_script}" + psm::tools::log "Looking for docker image build script ${build_docker_script}" if [[ -f "${build_docker_script}" ]]; then - echo "Sourcing docker image build script: ${build_docker_script}" + psm::tools::log "Sourcing docker image build script: ${build_docker_script}" source "${build_docker_script}" fi @@ -375,21 +375,25 @@ psm::build::docker_images_if_needed() { # Check if images already exist client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" - echo "${client_tags:-Client image not found}" + psm::tools::log "${client_tags:-Client image not found}" if [[ -z "${SERVER_IMAGE_USE_CANONICAL}" ]]; then server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" - echo "${server_tags:-Server image not found}" + psm::tools::log "${server_tags:-Server image not found}" else server_tags="ignored-use-canonical" fi # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 if [[ "${FORCE_IMAGE_BUILD_PSM}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then - psm::lang::build_docker_images + { + psm::tools::log "Building xDS interop test app Docker images" + psm::lang::build_docker_images + psm::tools::log "Finished xDS interop test app Docker images" + } | tee -a "${BUILD_LOGS_ROOT}/build-docker.log" else - echo "Skipping ${GRPC_LANGUAGE} test app build" + psm::tools::log "Skipping ${GRPC_LANGUAGE} test app build" fi } @@ -410,8 +414,14 @@ psm::tools::run_verbose() { return $exit_code } +psm::tools::log() { + echo -en "+ $(date "+[%H:%M:%S %Z]")\011 " + echo "$@" +} + psm::tools::print_test_flags() { - echo "PSM test flags:" + local test_name="${1:?missing the test name argument}" + psm::tools::log "${test_name} PSM test flags:" printf -- "%s\n" "${PSM_TEST_FLAGS[@]}" echo } @@ -454,11 +464,11 @@ activate_gke_cluster() { GKE_CLUSTER_ZONE="us-central1-c" ;; *) - echo "Unknown GKE cluster: ${1}" + psm::tools::log "Unknown GKE cluster: ${1}" exit 1 ;; esac - echo -n "Activated GKE cluster: GKE_CLUSTER_NAME=${GKE_CLUSTER_NAME} " + psm::tools::log -n "Activated GKE cluster: GKE_CLUSTER_NAME=${GKE_CLUSTER_NAME} " if [[ -n "${GKE_CLUSTER_REGION}" ]]; then echo "GKE_CLUSTER_REGION=${GKE_CLUSTER_REGION}" else @@ -484,11 +494,11 @@ activate_secondary_gke_cluster() { SECONDARY_GKE_CLUSTER_ZONE="us-west1-b" ;; *) - echo "Unknown secondary GKE cluster: ${1}" + psm::tools::log "Unknown secondary GKE cluster: ${1}" exit 1 ;; esac - echo -n "Activated secondary GKE cluster: SECONDARY_GKE_CLUSTER_NAME=${SECONDARY_GKE_CLUSTER_NAME}" + psm::tools::log -n "Activated secondary GKE cluster: SECONDARY_GKE_CLUSTER_NAME=${SECONDARY_GKE_CLUSTER_NAME}" echo " SECONDARY_GKE_CLUSTER_ZONE=${SECONDARY_GKE_CLUSTER_ZONE}" } @@ -505,7 +515,7 @@ run_ignore_exit_code() { local exit_code=0 "$@" || exit_code=$? if [[ $exit_code != 0 ]]; then - echo "Cmd: '$*', exit code: ${exit_code}" + psm::tools::log "Cmd: '$*', exit code: ${exit_code}" fi } @@ -610,9 +620,9 @@ gcloud_get_cluster_credentials() { ####################################### test_driver_get_source() { if [[ -n "${TEST_DRIVER_REPO_DIR_USE_EXISTING}" && -d "${TEST_DRIVER_REPO_DIR}" ]]; then - echo "Using exiting driver directory: ${TEST_DRIVER_REPO_DIR}." + psm::tools::log "Using exiting driver directory: ${TEST_DRIVER_REPO_DIR}." else - echo "Cloning driver to ${TEST_DRIVER_REPO_URL} branch ${TEST_DRIVER_BRANCH} to ${TEST_DRIVER_REPO_DIR}" + psm::tools::log "Cloning driver to ${TEST_DRIVER_REPO_URL} branch ${TEST_DRIVER_BRANCH} to ${TEST_DRIVER_REPO_DIR}" git clone -b "${TEST_DRIVER_BRANCH}" --depth=1 "${TEST_DRIVER_REPO_URL}" "${TEST_DRIVER_REPO_DIR}" fi } @@ -630,16 +640,16 @@ test_driver_get_source() { # Writes the list of installed modules to stdout ####################################### test_driver_pip_install() { - echo "Install python dependencies" + psm::tools::log "Install python dependencies" cd "${TEST_DRIVER_FULL_DIR}" # Create and activate virtual environment unless already using one if [[ -z "${VIRTUAL_ENV}" ]]; then local venv_dir="${TEST_DRIVER_FULL_DIR}/venv" if [[ -d "${venv_dir}" ]]; then - echo "Found python virtual environment directory: ${venv_dir}" + psm::tools::log "Found python virtual environment directory: ${venv_dir}" else - echo "Creating python virtual environment: ${venv_dir}" + psm::tools::log "Creating python virtual environment: ${venv_dir}" "python${PYTHON_VERSION}" -m venv "${venv_dir}" --upgrade-deps fi # Intentional: No need to check python venv activate script. @@ -647,9 +657,14 @@ test_driver_pip_install() { source "${venv_dir}/bin/activate" fi - python3 -m pip install -r requirements.lock - echo "Installed Python packages:" - python3 -m pip list + psm::tools::log "Installed Python packages with pip, see install-pip.log" + psm::driver::pip_install &>> "${BUILD_LOGS_ROOT}/install-pip.log" +} + +psm::driver::pip_install() { + psm::tools::run_verbose python3 -m pip install -r requirements.lock + echo + psm::tools::run_verbose -m pip list } ####################################### @@ -673,7 +688,7 @@ test_driver_compile_protos() { "${TEST_DRIVER_PROTOS_PATH}/messages.proto" "${TEST_DRIVER_PROTOS_PATH}/empty.proto" ) - echo "Generate python code from grpc.testing protos: ${protos[*]}" + psm::tools::log "Generate python code from grpc.testing protos: ${protos[*]}" cd "${TEST_DRIVER_REPO_DIR}" python3 -m grpc_tools.protoc \ --proto_path=. \ @@ -681,8 +696,8 @@ test_driver_compile_protos() { --grpc_python_out="${TEST_DRIVER_FULL_DIR}" \ "${protos[@]}" local protos_out_dir="${TEST_DRIVER_FULL_DIR}/${TEST_DRIVER_PROTOS_PATH}" - echo "Generated files ${protos_out_dir}:" - ls -Fl "${protos_out_dir}" + psm::tools::log "Generated files ${protos_out_dir}:" + du --time --max-depth=1 --all --bytes "${protos_out_dir}" } ####################################### @@ -713,7 +728,7 @@ test_driver_install() { # Writes the output to stdout ####################################### kokoro_print_version() { - echo "Kokoro Ubuntu version:" + psm::tools::log "Kokoro Ubuntu version:" run_ignore_exit_code lsb_release -a run_ignore_exit_code "python${PYTHON_VERSION}" --version run_ignore_exit_code "python${PYTHON_VERSION}" -m pip --version @@ -744,7 +759,7 @@ TESTGRID_EXCLUDE,${TESTGRID_EXCLUDE:-0} GIT_ORIGIN_URL,${GIT_ORIGIN_URL:?GIT_ORIGIN_URL must be set} GIT_COMMIT_SHORT,${GIT_COMMIT_SHORT:?GIT_COMMIT_SHORT must be set} EOF - echo "Sponge properties:" + psm::tools::log "Sponge properties:" cat "${KOKORO_ARTIFACTS_DIR}/custom_sponge_config.csv" } @@ -831,8 +846,14 @@ kokoro_get_testing_version() { # Writes the output to stdout, stderr, files ####################################### kokoro_setup_test_driver() { - # Unset noisy verbose mode often set in the parent script. + # Unset noisy verbose mode often set in the parent scripts. set +x + + # Prepend verbose mode commands (xtrace) with the date. + PS4='+ $(date "+[%H:%M:%S %Z]")\011 ' + + psm::tools::log "Starting Kokoro provisioning" + local src_repository_name="${1:?Usage kokoro_setup_test_driver GITHUB_REPOSITORY_NAME}" # Capture Kokoro VM version info in the log. kokoro_print_version @@ -843,9 +864,19 @@ kokoro_setup_test_driver() { # Kokoro clones repo to ${KOKORO_ARTIFACTS_DIR}/github/${GITHUB_REPOSITORY} local github_root="${KOKORO_ARTIFACTS_DIR}/github" readonly SRC_DIR="${github_root}/${src_repository_name}" + + # Test artifacts dir: xml reports, logs, etc. + local artifacts_dir="${KOKORO_ARTIFACTS_DIR}/artifacts" + # Folders after $artifacts_dir reported as target name + readonly TEST_XML_OUTPUT_DIR="${artifacts_dir}/${KOKORO_JOB_NAME}" + readonly BUILD_LOGS_ROOT="${TEST_XML_OUTPUT_DIR}" + + mkdir -p "${artifacts_dir}" "${TEST_XML_OUTPUT_DIR}" "${BUILD_LOGS_ROOT}" parse_src_repo_git_info SRC_DIR kokoro_write_sponge_properties - kokoro_install_dependencies + + psm::tools::log "Installing packages with apt, see install-apt.log" + kokoro_install_dependencies &> "${BUILD_LOGS_ROOT}/install-apt.log" # Get kubectl cluster credentials. gcloud_get_cluster_credentials @@ -856,12 +887,6 @@ kokoro_setup_test_driver() { test_driver_install "${test_driver_repo_dir}" # shellcheck disable=SC2034 # Used in the main script readonly TEST_DRIVER_FLAGFILE="config/grpc-testing.cfg" - - # Test artifacts dir: xml reports, logs, etc. - local artifacts_dir="${KOKORO_ARTIFACTS_DIR}/artifacts" - # Folders after $artifacts_dir reported as target name - readonly TEST_XML_OUTPUT_DIR="${artifacts_dir}/${KOKORO_JOB_NAME}" - mkdir -p "${artifacts_dir}" "${TEST_XML_OUTPUT_DIR}" } ####################################### @@ -905,7 +930,8 @@ local_setup_test_driver() { readonly TEST_DRIVER_FLAGFILE="config/local-dev.cfg" # Test out readonly TEST_XML_OUTPUT_DIR="${TEST_DRIVER_FULL_DIR}/out" - mkdir -p "${TEST_XML_OUTPUT_DIR}" + readonly BUILD_LOGS_ROOT="${TEST_XML_OUTPUT_DIR}" + mkdir -p "${TEST_XML_OUTPUT_DIR}" "${BUILD_LOGS_ROOT}" } ####################################### From 0f86579f25e36a57bf90a849d77cdc2794dee98d Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 24 Apr 2024 17:49:45 -0700 Subject: [PATCH 21/45] debugging - tmp use fewer tests in lb --- .kokoro/psm_interop_kokoro_lib.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index ca7b9bd0..eeac79ca 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -52,6 +52,14 @@ psm::lb::setup() { # Prints TESTS to stdout. ####################################### psm::lb::get_tests() { + # TODO(sergiitk): remove after debugging + TESTS=( + "app_net_test" + "baseline_test" + ) + return + + # TODO(sergiitk): load from env var? TESTS=( "affinity_test" From 5d52cd46b103a0bcb84d4bc5df02926caff0597e Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 24 Apr 2024 17:57:30 -0700 Subject: [PATCH 22/45] fix pip list --- .kokoro/psm_interop_kokoro_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index eeac79ca..a6baf4df 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -672,7 +672,7 @@ test_driver_pip_install() { psm::driver::pip_install() { psm::tools::run_verbose python3 -m pip install -r requirements.lock echo - psm::tools::run_verbose -m pip list + psm::tools::run_verbose python3 -m pip list } ####################################### From f3ae087e930131f3f7f1f02f96fa72375b08fd7f Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 24 Apr 2024 18:05:05 -0700 Subject: [PATCH 23/45] Clarify logging --- .kokoro/psm_interop_kokoro_lib.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index a6baf4df..f59d53ae 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -665,7 +665,7 @@ test_driver_pip_install() { source "${venv_dir}/bin/activate" fi - psm::tools::log "Installed Python packages with pip, see install-pip.log" + psm::tools::log "Installing Python packages with pip, see install-pip.log" psm::driver::pip_install &>> "${BUILD_LOGS_ROOT}/install-pip.log" } @@ -887,6 +887,7 @@ kokoro_setup_test_driver() { kokoro_install_dependencies &> "${BUILD_LOGS_ROOT}/install-apt.log" # Get kubectl cluster credentials. + psm::tools::log "Fetching GKE cluster credentials" gcloud_get_cluster_credentials # Install the driver. From fb97b50d78f0f76fcba67cb0842d54576ef9ede4 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 24 Apr 2024 18:11:46 -0700 Subject: [PATCH 24/45] minor log fixes --- .kokoro/psm_interop_kokoro_lib.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index f59d53ae..2acdd168 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -237,6 +237,8 @@ psm::run::test_suite() { local failed_tests=0 for test_name in "${TESTS[@]}"; do psm::run::test "${test_suite}" "${test_name}" || (( ++failed_tests )) + psm::tools::log "Finished ${test_suite} suite test: ${test_name}" + echo done psm::tools::log "Failed test suites: ${failed_tests}" } @@ -302,7 +304,7 @@ psm::setup::generic_test_suite() { "psm::${test_suite}::get_tests" psm::tools::log "Tests in ${test_suite} test suite:" - printf -- "- %s\n" "${TESTS[@]}" + printf -- " - %s\n" "${TESTS[@]}" echo } @@ -382,13 +384,13 @@ psm::setup::test_driver() { psm::build::docker_images_if_needed() { # Check if images already exist client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" - printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" - psm::tools::log "${client_tags:-Client image not found}" + psm::tools::log "Client image: ${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" + echo "${client_tags:-Client image not found}" if [[ -z "${SERVER_IMAGE_USE_CANONICAL}" ]]; then server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" - printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" - psm::tools::log "${server_tags:-Server image not found}" + psm::tools::log "Server image: ${SERVER_IMAGE_NAME}:${GIT_COMMIT}" + echo "${server_tags:-Server image not found}" else server_tags="ignored-use-canonical" fi @@ -429,7 +431,7 @@ psm::tools::log() { psm::tools::print_test_flags() { local test_name="${1:?missing the test name argument}" - psm::tools::log "${test_name} PSM test flags:" + psm::tools::log "Test driver flags for ${test_name}:" printf -- "%s\n" "${PSM_TEST_FLAGS[@]}" echo } From ede49ba72bccebe478088c11f810002d897ae91b Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 24 Apr 2024 18:46:21 -0700 Subject: [PATCH 25/45] Adding cpp --- .kokoro/psm_interop_kokoro_lib.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 2acdd168..36ac2f50 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -26,6 +26,9 @@ readonly FORCE_TESTING_VERSION="${FORCE_TESTING_VERSION:-}" # TODO(sergiitk): can be defined as readonly FORCE_IMAGE_BUILD when removed from buildscripts. readonly FORCE_IMAGE_BUILD_PSM="${FORCE_IMAGE_BUILD:-0}" +# Docker +readonly DOCKER_REGISTRY="us-docker.pkg.dev" + # GKE cluster identifiers. readonly GKE_CLUSTER_PSM_LB="psm-lb" readonly GKE_CLUSTER_PSM_SECURITY="psm-security" @@ -399,6 +402,7 @@ psm::build::docker_images_if_needed() { if [[ "${FORCE_IMAGE_BUILD_PSM}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then { psm::tools::log "Building xDS interop test app Docker images" + gcloud -q auth configure-docker "${DOCKER_REGISTRY}" psm::lang::build_docker_images psm::tools::log "Finished xDS interop test app Docker images" } | tee -a "${BUILD_LOGS_ROOT}/build-docker.log" @@ -959,6 +963,6 @@ tag_and_push_docker_image() { local from_tag="$2" local to_tag="$3" - docker tag "${image_name}:${from_tag}" "${image_name}:${to_tag}" - docker push "${image_name}:${to_tag}" + psm::tools::run_verbose docker tag "${image_name}:${from_tag}" "${image_name}:${to_tag}" + psm::tools::run_verbose push "${image_name}:${to_tag}" } From 40b0f0fbb9cb835ddf015711cb8b87060bcc9a31 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 24 Apr 2024 19:43:01 -0700 Subject: [PATCH 26/45] add generic docker build function --- .kokoro/psm_interop_kokoro_lib.sh | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 36ac2f50..8bb10e1b 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -140,6 +140,12 @@ psm::security::get_tests() { ####################################### psm::security::run_test() { local test_name="${1:?missing the test name argument}" + + # Only java supports extra checks for certificate matches (via channelz socket info). + if [[ "${GRPC_LANGUAGE}" != "java" ]]; then + PSM_TEST_FLAGS+=("--nocheck_local_certs") + fi + psm::tools::print_test_flags psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } @@ -411,6 +417,47 @@ psm::build::docker_images_if_needed() { fi } +####################################### +# Builds test app Docker images and pushes them to GCR +# Globals: +# SRC_DIR: Absolute path to the source repo on Kokoro VM +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# DOCKER_REGISTRY: Docker registry to push to +# Arguments: +# The path to xDS test client Dockerfile. Can be absolute or relative to SRC_DIR +# The path to xDS test server Dockerfile. Optional, server build is skip when omitted. +# Outputs: +# Writes the output of docker image build stdout, stderr +####################################### +psm::build::docker_images_generic() { + local client_dockerfile="${1:?missing the client dockerfile argument}" + local server_dockerfile="${2:-}" + pushd "${SRC_DIR}" + + # Client is required. + psm::tools::log "Building ${GRPC_LANGUAGE} xDS interop test client" + psm::tools::run_verbose docker build -f "${client_dockerfile}" -t "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" . + psm::tools::run_verbose docker push "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" + + # Server is optional + if [[ -n "${server_dockerfile}" ]]; then + psm::tools::log "Building ${GRPC_LANGUAGE} xDS interop test server" + psm::tools::run_verbose docker build -f "${server_dockerfile}" -t "${SERVER_IMAGE_NAME}:${GIT_COMMIT}" . + psm::tools::run_verbose docker push "${SERVER_IMAGE_NAME}:${GIT_COMMIT}" + fi + popd + + if is_version_branch "${TESTING_VERSION}"; then + psm::tools::log "Detected proper version branch ${TESTING_VERSION}, adding version tags." + tag_and_push_docker_image "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" "${TESTING_VERSION}" + if [[ -n "${server_dockerfile}" ]]; then + tag_and_push_docker_image "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" "${TESTING_VERSION}" + fi + fi +} + # --- Common helpers ---------------------- ####################################### From 7186db215cb58eed2589dd60620e4350bb37c5af Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 24 Apr 2024 19:51:09 -0700 Subject: [PATCH 27/45] Support csm --- .kokoro/psm_interop_kokoro_lib.sh | 62 +++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 8bb10e1b..bd9d7783 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -50,9 +50,7 @@ psm::lb::setup() { # Prepares the list of tests in PSM LB test suite. # Globals: # TESTING_VERSION: The version branch under test, f.e. master, dev, v1.42.x. -# TESTS: Populated with tests in PSM LB test suite. -# Outputs: -# Prints TESTS to stdout. +# TESTS: Populated with tests in the test suite. ####################################### psm::lb::get_tests() { # TODO(sergiitk): remove after debugging @@ -86,7 +84,7 @@ psm::lb::get_tests() { } ####################################### -# Executes LB test case +# Executes LB test case. # Globals: # TBD # Arguments: @@ -116,8 +114,6 @@ psm::security::setup() { # Prepares the list of tests in PSM Security test suite. # Globals: # TESTS: Populated with tests in PSM Security test suite. -# Outputs: -# Prints TESTS to stdout. ####################################### psm::security::get_tests() { # TODO(sergiitk): load from env var? @@ -164,16 +160,14 @@ psm::url_map::setup() { ####################################### # Prepares the list of tests in URL Map test suite. # Globals: -# TESTS: Populated with tests in URL Map test suite. -# Outputs: -# None +# TESTS: Populated with tests in the test suite. ####################################### psm::url_map::get_tests() { TESTS=("url_map") } ####################################### -# Executes Security test case +# Executes URL Map test case # Globals: # TBD # Arguments: @@ -191,6 +185,52 @@ psm::url_map::run_test() { psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } +# --- CSM TESTS ------------------ + +####################################### +# CSM Test Suite setup. +# Outputs: +# Prints activated cluster names. +####################################### +psm::csm::setup() { + activate_gke_cluster GKE_CLUSTER_PSM_CSM +} + +####################################### +# Prepares the list of tests in CSM test suite. +# Globals: +# TESTS: Populated with tests in the test suite. +####################################### +psm::csm::get_tests() { + # TODO(sergiitk): load from env var? + TESTS=( + "gamma.gamma_baseline_test" + "gamma.affinity_test" + "gamma.affinity_session_drain_test" + "gamma.csm_observability_test" + "app_net_ssa_test" + ) +} + +####################################### +# Executes CSM test case +# Globals: +# TBD +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +psm::csm::run_test() { + local test_name="${1:?missing the test name argument}" + PSM_TEST_FLAGS+=( + "--flagfile=config/common-csm.cfg" + ) + psm::tools::print_test_flags + psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" +} + # --- Common test run logic ----------- ####################################### @@ -219,7 +259,7 @@ psm::run() { psm::setup::docker_image_names "${GRPC_LANGUAGE}" "${test_suite}" case "${test_suite}" in - lb | security | url_map) + lb | security | url_map | csm) psm::setup::generic_test_suite "${test_suite}" ;; *) From 41b5171bb9a62d20e77308088d33b96fc1fab35f Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Thu, 25 Apr 2024 16:54:26 -0700 Subject: [PATCH 28/45] add cpp am python to psm::setup::docker_image_names --- .kokoro/psm_interop_kokoro_lib.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index bd9d7783..15e8cb2f 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -376,9 +376,9 @@ psm::setup::docker_image_names() { local test_suite="${2:?missing the test suite argument}" case "${language}" in - java) - CLIENT_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-client" - SERVER_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/${GRPC_LANGUAGE}-server" + java | cpp | python) + CLIENT_IMAGE_NAME="${DOCKER_REGISTRY}/grpc-testing/psm-interop/${GRPC_LANGUAGE}-client" + SERVER_IMAGE_NAME="${DOCKER_REGISTRY}/grpc-testing/psm-interop/${GRPC_LANGUAGE}-server" ;; *) psm::tools::log "Unknown Language: ${1}" From e969820b641cca53b791005a9e194ddf65adfd72 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Thu, 25 Apr 2024 17:58:28 -0700 Subject: [PATCH 29/45] fix psm::tools::print_test_flags args and improve missing argument err --- .kokoro/psm_interop_kokoro_lib.sh | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 15e8cb2f..a050b60e 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -94,7 +94,7 @@ psm::lb::get_tests() { # Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml ####################################### psm::lb::run_test() { - local test_name="${1:?missing the test name argument}" + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" psm::tools::print_test_flags "${test_name}" psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } @@ -135,14 +135,14 @@ psm::security::get_tests() { # Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml ####################################### psm::security::run_test() { - local test_name="${1:?missing the test name argument}" + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" # Only java supports extra checks for certificate matches (via channelz socket info). if [[ "${GRPC_LANGUAGE}" != "java" ]]; then PSM_TEST_FLAGS+=("--nocheck_local_certs") fi - psm::tools::print_test_flags + psm::tools::print_test_flags "${test_name}" psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } @@ -177,11 +177,11 @@ psm::url_map::get_tests() { # Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml ####################################### psm::url_map::run_test() { - local test_name="${1:?missing the test name argument}" + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" PSM_TEST_FLAGS+=( "--flagfile=config/url-map.cfg" ) - psm::tools::print_test_flags + psm::tools::print_test_flags "${test_name}" psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } @@ -223,11 +223,11 @@ psm::csm::get_tests() { # Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml ####################################### psm::csm::run_test() { - local test_name="${1:?missing the test name argument}" + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" PSM_TEST_FLAGS+=( "--flagfile=config/common-csm.cfg" ) - psm::tools::print_test_flags + psm::tools::print_test_flags "${test_name}" psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } @@ -255,7 +255,7 @@ psm::csm::run_test() { # Writes the output of test execution to stdout, stderr ####################################### psm::run() { - local test_suite="${1:?missing the test suite argument}" + local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" psm::setup::docker_image_names "${GRPC_LANGUAGE}" "${test_suite}" case "${test_suite}" in @@ -281,7 +281,7 @@ psm::run() { # TBD ####################################### psm::run::test_suite() { - local test_suite="${1:?missing the test suite argument}" + local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" cd "${TEST_DRIVER_FULL_DIR}" local failed_tests=0 for test_name in "${TESTS[@]}"; do @@ -311,8 +311,8 @@ psm::run::test_suite() { ####################################### psm::run::test() { # Test driver usage: https://github.com/grpc/psm-interop#basic-usage - local test_suite="${1:?missing the test suite argument}" - local test_name="${2:?missing the test name argument}" + local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" + local test_name="${2:?${FUNCNAME[0]} missing the test name argument}" local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" local test_log="${out_dir}/sponge_log.log" mkdir -p "${out_dir}" @@ -346,7 +346,7 @@ psm::run::test() { # --- Common test setup logic ----------- psm::setup::generic_test_suite() { - local test_suite="${1:?missing the test suite argument}" + local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" "psm::${test_suite}::setup" psm::setup::test_driver psm::build::docker_images_if_needed @@ -372,8 +372,8 @@ psm::setup::generic_test_suite() { # Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml ####################################### psm::setup::docker_image_names() { - local language="${1:?missing the language argument}" - local test_suite="${2:?missing the test suite argument}" + local language="${1:?${FUNCNAME[0]} missing the language argument}" + local test_suite="${2:?${FUNCNAME[0]} missing the test suite argument}" case "${language}" in java | cpp | python) @@ -472,7 +472,7 @@ psm::build::docker_images_if_needed() { # Writes the output of docker image build stdout, stderr ####################################### psm::build::docker_images_generic() { - local client_dockerfile="${1:?missing the client dockerfile argument}" + local client_dockerfile="${1:?${FUNCNAME[0]} missing the client dockerfile argument}" local server_dockerfile="${2:-}" pushd "${SRC_DIR}" @@ -521,7 +521,7 @@ psm::tools::log() { } psm::tools::print_test_flags() { - local test_name="${1:?missing the test name argument}" + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" psm::tools::log "Test driver flags for ${test_name}:" printf -- "%s\n" "${PSM_TEST_FLAGS[@]}" echo From c55e0b115c897737b6800d7cce82a346b2d5ecef Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 09:20:33 -0700 Subject: [PATCH 30/45] Add go to the list of languages --- .kokoro/psm_interop_kokoro_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index a050b60e..8254c50d 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -376,7 +376,7 @@ psm::setup::docker_image_names() { local test_suite="${2:?${FUNCNAME[0]} missing the test suite argument}" case "${language}" in - java | cpp | python) + java | cpp | python | go) CLIENT_IMAGE_NAME="${DOCKER_REGISTRY}/grpc-testing/psm-interop/${GRPC_LANGUAGE}-client" SERVER_IMAGE_NAME="${DOCKER_REGISTRY}/grpc-testing/psm-interop/${GRPC_LANGUAGE}-server" ;; From 1c0f169f75d96e2ceba397f87b1598e4291a8e10 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 13:17:42 -0700 Subject: [PATCH 31/45] Add node support --- .kokoro/psm_interop_kokoro_lib.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 8254c50d..df96fa1a 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -331,6 +331,9 @@ psm::run::test() { # Some test suites have canonical server image configured in the flagfiles. if [[ -z "${SERVER_IMAGE_USE_CANONICAL}" ]]; then PSM_TEST_FLAGS+=("--server_image=${SERVER_IMAGE_NAME}:${GIT_COMMIT}") + elif [[ "${GRPC_LANGUAGE}" == "node" ]]; then + # TODO(b/261911148): To be replaced with --server_image_use_canonical when implemented. + PSM_TEST_FLAGS+=("--server_image=us-docker.pkg.dev/grpc-testing/psm-interop/java-server:canonical") fi # So far, only LB test uses secondary GKE cluster. @@ -380,6 +383,11 @@ psm::setup::docker_image_names() { CLIENT_IMAGE_NAME="${DOCKER_REGISTRY}/grpc-testing/psm-interop/${GRPC_LANGUAGE}-client" SERVER_IMAGE_NAME="${DOCKER_REGISTRY}/grpc-testing/psm-interop/${GRPC_LANGUAGE}-server" ;; + node) + CLIENT_IMAGE_NAME="${DOCKER_REGISTRY}/grpc-testing/psm-interop/${GRPC_LANGUAGE}-client" + SERVER_IMAGE_NAME="" + SERVER_IMAGE_USE_CANONICAL="1" + ;; *) psm::tools::log "Unknown Language: ${1}" exit 1 From 4ce553e0b056da03ed5d352ec2342e2134cae73c Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 14:17:14 -0700 Subject: [PATCH 32/45] switch from du to md5sum in proto files listing --- .kokoro/psm_interop_kokoro_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index df96fa1a..b8fc8242 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -806,7 +806,7 @@ test_driver_compile_protos() { "${protos[@]}" local protos_out_dir="${TEST_DRIVER_FULL_DIR}/${TEST_DRIVER_PROTOS_PATH}" psm::tools::log "Generated files ${protos_out_dir}:" - du --time --max-depth=1 --all --bytes "${protos_out_dir}" + md5sum "${protos_out_dir}/*" } ####################################### From 65e9d3d0248de8508d90893c8196d6c700d2bc5a Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 14:17:32 -0700 Subject: [PATCH 33/45] Fix wrong SERVER_IMAGE_USE_CANONICAL override --- .kokoro/psm_interop_kokoro_lib.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index b8fc8242..a6705c1e 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -377,6 +377,7 @@ psm::setup::generic_test_suite() { psm::setup::docker_image_names() { local language="${1:?${FUNCNAME[0]} missing the language argument}" local test_suite="${2:?${FUNCNAME[0]} missing the test suite argument}" + SERVER_IMAGE_USE_CANONICAL="" case "${language}" in java | cpp | python | go) @@ -399,9 +400,6 @@ psm::setup::docker_image_names() { # Uses the canonical server image configured in url-map.cfg SERVER_IMAGE_USE_CANONICAL="1" ;; - *) - SERVER_IMAGE_USE_CANONICAL="" - ;; esac declare -r CLIENT_IMAGE_NAME From 9a67d033b9c9d9c390442720ea8f88c5b65c293a Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 14:40:58 -0700 Subject: [PATCH 34/45] Improve psm::tools::run_verbose --- .kokoro/psm_interop_kokoro_lib.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index a6705c1e..3d6c37ea 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -517,8 +517,15 @@ psm::tools::run_verbose() { local exit_code=0 set -x "$@" || exit_code=$? - set +x - return $exit_code + { set +x; } 2>/dev/null + + if (( exit_code == 0 )); then + psm::tools::log "Cmd finished: ${1}" + else + psm::tools::log "Cmd exit code ${exit_code}: '$*'" + fi + + return ${exit_code} } psm::tools::log() { From 1afd454481e0fd35f162c3cef16738ded9aae2ab Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 14:58:48 -0700 Subject: [PATCH 35/45] Set PS4 at the top --- .kokoro/psm_interop_kokoro_lib.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 3d6c37ea..3f07ec45 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -14,6 +14,9 @@ # limitations under the License. set -eo pipefail +# Prepend verbose mode commands (xtrace) with the date. +PS4='+ $(date "+[%H:%M:%S %Z]")\011 ' + # Constants readonly PYTHON_VERSION="${PYTHON_VERSION:-3.10}" # Test driver @@ -962,10 +965,7 @@ kokoro_get_testing_version() { kokoro_setup_test_driver() { # Unset noisy verbose mode often set in the parent scripts. set +x - - # Prepend verbose mode commands (xtrace) with the date. - PS4='+ $(date "+[%H:%M:%S %Z]")\011 ' - + psm::tools::log "Starting Kokoro provisioning" local src_repository_name="${1:?Usage kokoro_setup_test_driver GITHUB_REPOSITORY_NAME}" From ce43c7d678891511426dbbca78ca1cb04fd83e5b Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 16:37:53 -0700 Subject: [PATCH 36/45] md5sum without shell globbing --- .kokoro/psm_interop_kokoro_lib.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 3f07ec45..b737d894 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -814,7 +814,7 @@ test_driver_compile_protos() { "${protos[@]}" local protos_out_dir="${TEST_DRIVER_FULL_DIR}/${TEST_DRIVER_PROTOS_PATH}" psm::tools::log "Generated files ${protos_out_dir}:" - md5sum "${protos_out_dir}/*" + find "${protos_out_dir}" -type f -exec md5sum {} \; } ####################################### @@ -965,7 +965,7 @@ kokoro_get_testing_version() { kokoro_setup_test_driver() { # Unset noisy verbose mode often set in the parent scripts. set +x - + psm::tools::log "Starting Kokoro provisioning" local src_repository_name="${1:?Usage kokoro_setup_test_driver GITHUB_REPOSITORY_NAME}" From 0565352b7f79ec6d32f245c75938afcfc30b9150 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 17:12:48 -0700 Subject: [PATCH 37/45] Minor log format change in psm::tools::run_verbose --- .kokoro/psm_interop_kokoro_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index b737d894..8c71e8b8 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -525,7 +525,7 @@ psm::tools::run_verbose() { if (( exit_code == 0 )); then psm::tools::log "Cmd finished: ${1}" else - psm::tools::log "Cmd exit code ${exit_code}: '$*'" + psm::tools::log "Cmd failed with exit code ${exit_code}: $*" fi return ${exit_code} From 9ab646cc1c0f1e09dc2bc78fd49fcd5ad19245d7 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 17:20:43 -0700 Subject: [PATCH 38/45] Fix compatibility with old-style build scripts --- .kokoro/psm_interop_kokoro_lib.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 8c71e8b8..74c8d2db 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -30,7 +30,10 @@ readonly FORCE_TESTING_VERSION="${FORCE_TESTING_VERSION:-}" readonly FORCE_IMAGE_BUILD_PSM="${FORCE_IMAGE_BUILD:-0}" # Docker -readonly DOCKER_REGISTRY="us-docker.pkg.dev" +# TODO(sergiitk): if can be removed when DOCKER_REGISTRY removed from buildscripts. +if [[ -z "${DOCKER_REGISTRY}" ]] ; then + readonly DOCKER_REGISTRY="us-docker.pkg.dev" +fi # GKE cluster identifiers. readonly GKE_CLUSTER_PSM_LB="psm-lb" From c69c6438506eedbff1c6d27a7d0ab9ffff46de15 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 17:29:47 -0700 Subject: [PATCH 39/45] Deprecate run_ignore_exit_code --- .kokoro/psm_interop_kokoro_lib.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 74c8d2db..32e05bb6 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -534,6 +534,24 @@ psm::tools::run_verbose() { return ${exit_code} } +####################################### +# Run command end report its exit code. Doesn't exit on non-zero exit code. +# Globals: +# None +# Arguments: +# Command to execute +# Outputs: +# Writes the output of given command to stdout, stderr +####################################### +psm::tools::run_ignore_exit_code() { + local exit_code=0 + "$@" || exit_code=$? + + if (( exit_code == 0 )); then + psm::tools::log "Cmd failed with exit code ${exit_code}: $*" + fi +} + psm::tools::log() { echo -en "+ $(date "+[%H:%M:%S %Z]")\011 " echo "$@" @@ -624,6 +642,10 @@ activate_secondary_gke_cluster() { ####################################### # Run command end report its exit code. Doesn't exit on non-zero exit code. +# +# Deprecated. Use psm::tools::run_ignore_exit_code +# TODO(sergiitk): delete this method when no longer used in per-language buildscripts. +# # Globals: # None # Arguments: From 8f2861962f8cfdabad296a3035ba09b62db9d37e Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 18:38:28 -0700 Subject: [PATCH 40/45] Ensure checksum command exists --- .kokoro/psm_interop_kokoro_lib.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 32e05bb6..5a52dbcc 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -838,8 +838,11 @@ test_driver_compile_protos() { --grpc_python_out="${TEST_DRIVER_FULL_DIR}" \ "${protos[@]}" local protos_out_dir="${TEST_DRIVER_FULL_DIR}/${TEST_DRIVER_PROTOS_PATH}" - psm::tools::log "Generated files ${protos_out_dir}:" - find "${protos_out_dir}" -type f -exec md5sum {} \; + + if command -v sha256sum &> /dev/null; then + psm::tools::log "Generated files ${protos_out_dir}:" + find "${protos_out_dir}" -type f -exec sha256sum {} \; + fi } ####################################### From 65aac7b0e3def6cfbf6ec9fe12c52e0db79aab07 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 19:05:01 -0700 Subject: [PATCH 41/45] minor - add start/end log messages --- .kokoro/psm_interop_kokoro_lib.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 5a52dbcc..7fe1e879 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -262,6 +262,8 @@ psm::csm::run_test() { ####################################### psm::run() { local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" + psm::tools::log "Starting PSM Interop tests: ${test_suite}" + psm::setup::docker_image_names "${GRPC_LANGUAGE}" "${test_suite}" case "${test_suite}" in @@ -275,6 +277,7 @@ psm::run() { esac psm::run::test_suite "${test_suite}" + psm::tools::log "PSM Interop tests completed: ${test_suite}" } ####################################### From f3a5a31081b10d85948e8b97468f7c7a08bb04f5 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 19:13:42 -0700 Subject: [PATCH 42/45] resolve temporary todos --- .kokoro/psm_interop_kokoro_lib.sh | 44 +++++++++++++++---------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 7fe1e879..ea0c7a04 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -30,7 +30,7 @@ readonly FORCE_TESTING_VERSION="${FORCE_TESTING_VERSION:-}" readonly FORCE_IMAGE_BUILD_PSM="${FORCE_IMAGE_BUILD:-0}" # Docker -# TODO(sergiitk): if can be removed when DOCKER_REGISTRY removed from buildscripts. +# TODO(sergiitk): 'if' can be removed when DOCKER_REGISTRY removed from buildscripts. if [[ -z "${DOCKER_REGISTRY}" ]] ; then readonly DOCKER_REGISTRY="us-docker.pkg.dev" fi @@ -59,15 +59,6 @@ psm::lb::setup() { # TESTS: Populated with tests in the test suite. ####################################### psm::lb::get_tests() { - # TODO(sergiitk): remove after debugging - TESTS=( - "app_net_test" - "baseline_test" - ) - return - - - # TODO(sergiitk): load from env var? TESTS=( "affinity_test" "api_listener_test" @@ -92,7 +83,7 @@ psm::lb::get_tests() { ####################################### # Executes LB test case. # Globals: -# TBD +# PSM_TEST_FLAGS: The array with flags for the test # Arguments: # Test case name # Outputs: @@ -122,7 +113,6 @@ psm::security::setup() { # TESTS: Populated with tests in PSM Security test suite. ####################################### psm::security::get_tests() { - # TODO(sergiitk): load from env var? TESTS=( "baseline_test" "security_test" @@ -133,7 +123,8 @@ psm::security::get_tests() { ####################################### # Executes Security test case # Globals: -# TBD +# PSM_TEST_FLAGS: The array with flags for the test +# GRPC_LANGUAGE: The name of gRPC languages under test # Arguments: # Test case name # Outputs: @@ -175,7 +166,7 @@ psm::url_map::get_tests() { ####################################### # Executes URL Map test case # Globals: -# TBD +# PSM_TEST_FLAGS: The array with flags for the test # Arguments: # Test case name # Outputs: @@ -208,7 +199,6 @@ psm::csm::setup() { # TESTS: Populated with tests in the test suite. ####################################### psm::csm::get_tests() { - # TODO(sergiitk): load from env var? TESTS=( "gamma.gamma_baseline_test" "gamma.affinity_test" @@ -221,7 +211,7 @@ psm::csm::get_tests() { ####################################### # Executes CSM test case # Globals: -# TBD +# PSM_TEST_FLAGS: The array with flags for the test # Arguments: # Test case name # Outputs: @@ -240,7 +230,11 @@ psm::csm::run_test() { # --- Common test run logic ----------- ####################################### -# TBD +# Prepares and runs the test suite. +# +# Main entry function to be used by the buildscripts. +# Provisions necessary software, configures the test driver, and executes the test suite. +# # Globals: # KOKORO_ARTIFACTS_DIR # GITHUB_REPOSITORY_NAME @@ -281,13 +275,14 @@ psm::run() { } ####################################### -# Executes the test suite +# Executes tests in the test suite # Globals: -# TBD +# TEST_DRIVER_FULL_DIR +# TESTS: the list of the tests to execute # Arguments: -# Test suite name +# Test suite name. # Outputs: -# TBD +# Each test to stdout. ####################################### psm::run::test_suite() { local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" @@ -302,7 +297,7 @@ psm::run::test_suite() { } ####################################### -# Executes the test case +# Executes a single test case of given test suite. # Globals: # TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile # KUBE_CONTEXT: The name of kubectl context with GKE cluster access @@ -362,7 +357,12 @@ psm::setup::generic_test_suite() { "psm::${test_suite}::setup" psm::setup::test_driver psm::build::docker_images_if_needed + psm::setup::get_tests "${test_suite}" +} +psm::setup::get_tests() { + local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" + # TODO(sergiitk): allow to override TESTS from an env var "psm::${test_suite}::get_tests" psm::tools::log "Tests in ${test_suite} test suite:" printf -- " - %s\n" "${TESTS[@]}" From 107a1a026fe192ba2416db8004edc8d0dbed2cc1 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 19:45:44 -0700 Subject: [PATCH 43/45] Override compute_api_version flag default instead of hardcoding --- tests/subsetting_test.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/subsetting_test.py b/tests/subsetting_test.py index 275ec472..ca5421f8 100644 --- a/tests/subsetting_test.py +++ b/tests/subsetting_test.py @@ -18,12 +18,15 @@ from absl import flags from absl import logging from absl.testing import absltest -from typing_extensions import override from framework import xds_k8s_testcase from framework.helpers import skips flags.adopt_module_key_flags(xds_k8s_testcase) +# Change the default value of --compute_api_version to v1alpha. +# Subsetting test requires Compute v1alpha APIs. +# Can be tested with another API version if the flag is passed to the test. +flags.set_default(xds_k8s_testcase.xds_flags.COMPUTE_API_VERSION, "v1alpha") # Type aliases _XdsTestServer = xds_k8s_testcase.XdsTestServer @@ -35,14 +38,6 @@ class SubsettingTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): - @classmethod - @override - def setUpClass(cls): - # TODO(sergiitk): use when absl updated to 1.3.0+, also set - # flags.set_default(xds_flags.COMPUTE_API_VERSION, 'v1alpha') - super().setUpClass() - cls.compute_api_version = "v1alpha" - @staticmethod def is_supported(config: skips.TestConfig) -> bool: # Subsetting is an experimental feature where most work is done on the From 5aeb13f0e1dd104b2e45e56bcfb95967bd7cc547 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 26 Apr 2024 20:40:13 -0700 Subject: [PATCH 44/45] allow to override test list and append extra flags from env vars --- .kokoro/psm_interop_kokoro_lib.sh | 59 +++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index ea0c7a04..28942c6e 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -92,7 +92,7 @@ psm::lb::get_tests() { ####################################### psm::lb::run_test() { local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" - psm::tools::print_test_flags "${test_name}" + psm::run::finalize_test_flags "${test_name}" psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } @@ -139,7 +139,7 @@ psm::security::run_test() { PSM_TEST_FLAGS+=("--nocheck_local_certs") fi - psm::tools::print_test_flags "${test_name}" + psm::run::finalize_test_flags "${test_name}" psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } @@ -178,7 +178,7 @@ psm::url_map::run_test() { PSM_TEST_FLAGS+=( "--flagfile=config/url-map.cfg" ) - psm::tools::print_test_flags "${test_name}" + psm::run::finalize_test_flags "${test_name}" psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } @@ -223,7 +223,7 @@ psm::csm::run_test() { PSM_TEST_FLAGS+=( "--flagfile=config/common-csm.cfg" ) - psm::tools::print_test_flags "${test_name}" + psm::run::finalize_test_flags "${test_name}" psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" } @@ -317,7 +317,7 @@ psm::run::test() { # Test driver usage: https://github.com/grpc/psm-interop#basic-usage local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" local test_name="${2:?${FUNCNAME[0]} missing the test name argument}" - local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" + local out_dir="${TEST_XML_OUTPUT_DIR:-/tmp/psm}/${test_name}" local test_log="${out_dir}/sponge_log.log" mkdir -p "${out_dir}" @@ -350,6 +350,33 @@ psm::run::test() { "psm::${test_suite}::run_test" "${test_name}" |& tee -a "${test_log}" } +####################################### +# Appends extra flags (if any) to the end of PSM_TEST_FLAGS, prints the flag list. +# Globals: +# PSM_TEST_FLAGS: The array with flags for the test +# PSM_EXTRA_FLAGS: Space-separated string with extra flags to append +# Arguments: +# Test case name +# Outputs: +# Prints the list of test flags to the stdout +####################################### +psm::run::finalize_test_flags() { + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" + + # Append extra flags + # TODO(sergiitk): replace BAZEL_FLAGS with PSM_EXTRA_FLAGS when allowed_env_vars configured + if [[ -n "${BAZEL_FLAGS}" ]]; then + declare -a extra_flags + IFS=' ' read -r -a extra_flags <<< "${BAZEL_FLAGS}" + PSM_TEST_FLAGS+=("${extra_flags[@]}") + fi + + psm::tools::log "Test driver flags for ${test_name}:" + printf -- "%s\n" "${PSM_TEST_FLAGS[@]}" + echo +} + + # --- Common test setup logic ----------- psm::setup::generic_test_suite() { @@ -362,8 +389,19 @@ psm::setup::generic_test_suite() { psm::setup::get_tests() { local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" - # TODO(sergiitk): allow to override TESTS from an env var - "psm::${test_suite}::get_tests" + + # TODO(sergiitk): replace BAZEL_TESTS with PSM_TESTS when allowed_env_vars configured + if [[ -n "${BAZEL_TESTS}" ]]; then + # Test list overridden in env var. + IFS=' ' read -r -a TESTS <<< "${BAZEL_TESTS}" + if (( ${#TESTS[@]} == 0 )); then + psm::tools::log "Error: test list overridden, but no tests specified" + exit 1 + fi + else + "psm::${test_suite}::get_tests" + fi + psm::tools::log "Tests in ${test_suite} test suite:" printf -- " - %s\n" "${TESTS[@]}" echo @@ -560,13 +598,6 @@ psm::tools::log() { echo "$@" } -psm::tools::print_test_flags() { - local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" - psm::tools::log "Test driver flags for ${test_name}:" - printf -- "%s\n" "${PSM_TEST_FLAGS[@]}" - echo -} - # --- "Unsorted" methods -------------- # TODO(sergiitk): all methods should be using "psm::" package name, # see https://google.github.io/styleguide/shellguide.html#function-names From 3f078ba6d09a76ae4ab27abee3ab8d963bdc18bc Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 30 Apr 2024 10:31:34 -0700 Subject: [PATCH 45/45] Remove temporary BAZEL_ vars, organize and document constants --- .kokoro/psm_interop_kokoro_lib.sh | 72 ++++++++++++++++++------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index 28942c6e..0ee9fc1b 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -17,28 +17,42 @@ set -eo pipefail # Prepend verbose mode commands (xtrace) with the date. PS4='+ $(date "+[%H:%M:%S %Z]")\011 ' -# Constants -readonly PYTHON_VERSION="${PYTHON_VERSION:-3.10}" +# --- Constants --- +# GKE cluster identifiers. +readonly GKE_CLUSTER_PSM_LB="psm-lb" +readonly GKE_CLUSTER_PSM_SECURITY="psm-security" +readonly GKE_CLUSTER_PSM_BASIC="psm-basic" +# TODO(sergiitk): 'if' can be removed when DOCKER_REGISTRY removed from buildscripts. +if [[ -z "${DOCKER_REGISTRY}" ]] ; then + readonly DOCKER_REGISTRY="us-docker.pkg.dev" +fi # Test driver readonly TEST_DRIVER_REPO_NAME="psm-interop" -readonly TEST_DRIVER_REPO_URL="https://github.com/${TEST_DRIVER_REPO_OWNER:-grpc}/psm-interop.git" -readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-main}" readonly TEST_DRIVER_PATH="" readonly TEST_DRIVER_PROTOS_PATH="protos/grpc/testing" -readonly FORCE_TESTING_VERSION="${FORCE_TESTING_VERSION:-}" -# TODO(sergiitk): can be defined as readonly FORCE_IMAGE_BUILD when removed from buildscripts. -readonly FORCE_IMAGE_BUILD_PSM="${FORCE_IMAGE_BUILD:-0}" -# Docker -# TODO(sergiitk): 'if' can be removed when DOCKER_REGISTRY removed from buildscripts. -if [[ -z "${DOCKER_REGISTRY}" ]] ; then - readonly DOCKER_REGISTRY="us-docker.pkg.dev" -fi +# --- Injectable constants --- +readonly PYTHON_VERSION="${PYTHON_VERSION:-3.10}" + +# Test driver +readonly TEST_DRIVER_REPO_OWNER="${TEST_DRIVER_REPO_OWNER:-grpc}" +readonly TEST_DRIVER_REPO_URL="https://github.com/${TEST_DRIVER_REPO_OWNER}/${TEST_DRIVER_REPO_NAME}.git" +readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-main}" + +# Forces image build, even if a tag with GIT_COMMIT already exists. +readonly PSM_FORCE_IMAGE_BUILD="${PSM_FORCE_IMAGE_BUILD:-0}" +# Overrides the TESTING_VERSION. Can be used to force versioned image tag, see is_version_branch. +readonly PSM_FORCE_TESTING_VERSION="${PSM_FORCE_TESTING_VERSION:-}" +# Overrides the list of test to run. A whitespace-separated string. Example: +# PSM_TESTS="app_net_test baseline_test" +readonly PSM_TESTS="${PSM_TESTS:-}" +# A space-separated string with extra flags to append to the test driver arguments. +# Can be used to execute a run and test a new flag value, f.e.: +# PSM_EXTRA_FLAGS="--noenable_workload_identity --td_bootstrap_image=us-docker.pkg.dev/new-image..." +# In addition, can be used to run a single test in a suite. F.e. to run only test_mtls in security_test: +# PSM_TESTS="security_test" PSM_EXTRA_FLAGS="SecurityTest.test_mtls" +readonly PSM_EXTRA_FLAGS="${PSM_EXTRA_FLAGS:-}" -# GKE cluster identifiers. -readonly GKE_CLUSTER_PSM_LB="psm-lb" -readonly GKE_CLUSTER_PSM_SECURITY="psm-security" -readonly GKE_CLUSTER_PSM_BASIC="psm-basic" # --- LB TESTS ------------------------ @@ -364,10 +378,9 @@ psm::run::finalize_test_flags() { local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" # Append extra flags - # TODO(sergiitk): replace BAZEL_FLAGS with PSM_EXTRA_FLAGS when allowed_env_vars configured - if [[ -n "${BAZEL_FLAGS}" ]]; then + if [[ -n "${PSM_EXTRA_FLAGS}" ]]; then declare -a extra_flags - IFS=' ' read -r -a extra_flags <<< "${BAZEL_FLAGS}" + IFS=' ' read -r -a extra_flags <<< "${PSM_EXTRA_FLAGS}" PSM_TEST_FLAGS+=("${extra_flags[@]}") fi @@ -390,10 +403,9 @@ psm::setup::generic_test_suite() { psm::setup::get_tests() { local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" - # TODO(sergiitk): replace BAZEL_TESTS with PSM_TESTS when allowed_env_vars configured - if [[ -n "${BAZEL_TESTS}" ]]; then + if [[ -n "${PSM_TESTS}" ]]; then # Test list overridden in env var. - IFS=' ' read -r -a TESTS <<< "${BAZEL_TESTS}" + IFS=' ' read -r -a TESTS <<< "${PSM_TESTS}" if (( ${#TESTS[@]} == 0 )); then psm::tools::log "Error: test list overridden, but no tests specified" exit 1 @@ -477,7 +489,7 @@ psm::setup::test_driver() { # SERVER_IMAGE_NAME: Test server Docker image name # CLIENT_IMAGE_NAME: Test client Docker image name # GIT_COMMIT: SHA-1 of git commit being built -# FORCE_IMAGE_BUILD +# PSM_FORCE_IMAGE_BUILD # Arguments: # None # Outputs: @@ -497,8 +509,8 @@ psm::build::docker_images_if_needed() { server_tags="ignored-use-canonical" fi - # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 - if [[ "${FORCE_IMAGE_BUILD_PSM}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then + # Build if any of the images are missing, or PSM_FORCE_IMAGE_BUILD=1 + if [[ "${PSM_FORCE_IMAGE_BUILD}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then { psm::tools::log "Building xDS interop test app Docker images" gcloud -q auth configure-docker "${DOCKER_REGISTRY}" @@ -966,7 +978,7 @@ kokoro_install_dependencies() { # Globals: # KOKORO_JOB_NAME # KOKORO_BUILD_INITIATOR -# FORCE_TESTING_VERSION: Forces the testing version to be something else. +# PSM_FORCE_TESTING_VERSION: Forces the testing version to be something else. # TESTING_VERSION: Populated with the version branch under test, # f.e. master, dev, v1.42.x. # Outputs: @@ -982,10 +994,10 @@ kokoro_get_testing_version() { local version_from_job_name version_from_job_name=$(echo "${KOKORO_JOB_NAME}" | cut -d '/' -f3) - if [[ -n "${FORCE_TESTING_VERSION}" ]]; then + if [[ -n "${PSM_FORCE_TESTING_VERSION}" ]]; then # Allows to override the testing version, and force tagging the built # images, if necessary. - readonly TESTING_VERSION="${FORCE_TESTING_VERSION}" + readonly TESTING_VERSION="${PSM_FORCE_TESTING_VERSION}" elif [[ "${KOKORO_BUILD_INITIATOR:-anonymous}" != "kokoro" ]]; then # If not initiated by Kokoro, it's a dev branch. # This allows to know later down the line that the built image doesn't need @@ -1093,8 +1105,8 @@ local_setup_test_driver() { readonly SECONDARY_KUBE_CONTEXT="${SECONDARY_KUBE_CONTEXT}" # Never override docker image for local runs, unless explicitly forced. - if [[ -n "${FORCE_TESTING_VERSION}" ]]; then - readonly TESTING_VERSION="${FORCE_TESTING_VERSION}" + if [[ -n "${PSM_FORCE_TESTING_VERSION}" ]]; then + readonly TESTING_VERSION="${PSM_FORCE_TESTING_VERSION}" else readonly TESTING_VERSION="dev" fi