diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh index c75f4864..0ee9fc1b 100644 --- a/.kokoro/psm_interop_kokoro_lib.sh +++ b/.kokoro/psm_interop_kokoro_lib.sh @@ -12,23 +12,607 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# TODO(sergiitk): move to grpc/grpc when implementing support of other languages set -eo pipefail -# Constants -readonly PYTHON_VERSION="${PYTHON_VERSION:-3.10}" -# Test driver -readonly TEST_DRIVER_REPO_NAME="psm-interop" -readonly TEST_DRIVER_REPO_URL="https://github.com/${TEST_DRIVER_REPO_OWNER:-grpc}/psm-interop.git" -readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-main}" -readonly TEST_DRIVER_PATH="" -readonly TEST_DRIVER_PROTOS_PATH="protos/grpc/testing" -readonly FORCE_TESTING_VERSION="${FORCE_TESTING_VERSION:-}" +# Prepend verbose mode commands (xtrace) with the date. +PS4='+ $(date "+[%H:%M:%S %Z]")\011 ' +# --- Constants --- # GKE cluster identifiers. readonly GKE_CLUSTER_PSM_LB="psm-lb" readonly GKE_CLUSTER_PSM_SECURITY="psm-security" readonly GKE_CLUSTER_PSM_BASIC="psm-basic" +# TODO(sergiitk): 'if' can be removed when DOCKER_REGISTRY removed from buildscripts. +if [[ -z "${DOCKER_REGISTRY}" ]] ; then + readonly DOCKER_REGISTRY="us-docker.pkg.dev" +fi +# Test driver +readonly TEST_DRIVER_REPO_NAME="psm-interop" +readonly TEST_DRIVER_PATH="" +readonly TEST_DRIVER_PROTOS_PATH="protos/grpc/testing" + +# --- Injectable constants --- +readonly PYTHON_VERSION="${PYTHON_VERSION:-3.10}" + +# Test driver +readonly TEST_DRIVER_REPO_OWNER="${TEST_DRIVER_REPO_OWNER:-grpc}" +readonly TEST_DRIVER_REPO_URL="https://github.com/${TEST_DRIVER_REPO_OWNER}/${TEST_DRIVER_REPO_NAME}.git" +readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-main}" + +# Forces image build, even if a tag with GIT_COMMIT already exists. +readonly PSM_FORCE_IMAGE_BUILD="${PSM_FORCE_IMAGE_BUILD:-0}" +# Overrides the TESTING_VERSION. Can be used to force versioned image tag, see is_version_branch. +readonly PSM_FORCE_TESTING_VERSION="${PSM_FORCE_TESTING_VERSION:-}" +# Overrides the list of test to run. A whitespace-separated string. Example: +# PSM_TESTS="app_net_test baseline_test" +readonly PSM_TESTS="${PSM_TESTS:-}" +# A space-separated string with extra flags to append to the test driver arguments. +# Can be used to execute a run and test a new flag value, f.e.: +# PSM_EXTRA_FLAGS="--noenable_workload_identity --td_bootstrap_image=us-docker.pkg.dev/new-image..." +# In addition, can be used to run a single test in a suite. F.e. to run only test_mtls in security_test: +# PSM_TESTS="security_test" PSM_EXTRA_FLAGS="SecurityTest.test_mtls" +readonly PSM_EXTRA_FLAGS="${PSM_EXTRA_FLAGS:-}" + + +# --- LB TESTS ------------------------ + +####################################### +# LB Test Suite setup. +# Outputs: +# Prints activated cluster names. +####################################### +psm::lb::setup() { + activate_gke_cluster GKE_CLUSTER_PSM_LB + activate_secondary_gke_cluster GKE_CLUSTER_PSM_LB +} + +####################################### +# Prepares the list of tests in PSM LB test suite. +# Globals: +# TESTING_VERSION: The version branch under test, f.e. master, dev, v1.42.x. +# TESTS: Populated with tests in the test suite. +####################################### +psm::lb::get_tests() { + TESTS=( + "affinity_test" + "api_listener_test" + "app_net_test" + "change_backend_service_test" + "custom_lb_test" + "failover_test" + "outlier_detection_test" + "remove_neg_test" + "round_robin_test" + ) + # master-only tests + if [[ "${TESTING_VERSION}" =~ "master" ]]; then + psm::tools::log "Appending master-only tests to the LB suite." + TESTS+=( + "bootstrap_generator_test" + "subsetting_test" + ) + fi +} + +####################################### +# Executes LB test case. +# Globals: +# PSM_TEST_FLAGS: The array with flags for the test +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +psm::lb::run_test() { + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" + psm::run::finalize_test_flags "${test_name}" + psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" +} + +# --- Security TESTS ------------------ + +####################################### +# Security Test Suite setup. +# Outputs: +# Prints activated cluster names. +####################################### +psm::security::setup() { + activate_gke_cluster GKE_CLUSTER_PSM_SECURITY +} + +####################################### +# Prepares the list of tests in PSM Security test suite. +# Globals: +# TESTS: Populated with tests in PSM Security test suite. +####################################### +psm::security::get_tests() { + TESTS=( + "baseline_test" + "security_test" + "authz_test" + ) +} + +####################################### +# Executes Security test case +# Globals: +# PSM_TEST_FLAGS: The array with flags for the test +# GRPC_LANGUAGE: The name of gRPC languages under test +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +psm::security::run_test() { + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" + + # Only java supports extra checks for certificate matches (via channelz socket info). + if [[ "${GRPC_LANGUAGE}" != "java" ]]; then + PSM_TEST_FLAGS+=("--nocheck_local_certs") + fi + + psm::run::finalize_test_flags "${test_name}" + psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" +} + +# --- URL Map TESTS ------------------ + +####################################### +# URL Map Test Suite setup. +# Outputs: +# Prints activated cluster names. +####################################### +psm::url_map::setup() { + activate_gke_cluster GKE_CLUSTER_PSM_BASIC +} + +####################################### +# Prepares the list of tests in URL Map test suite. +# Globals: +# TESTS: Populated with tests in the test suite. +####################################### +psm::url_map::get_tests() { + TESTS=("url_map") +} + +####################################### +# Executes URL Map test case +# Globals: +# PSM_TEST_FLAGS: The array with flags for the test +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +psm::url_map::run_test() { + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" + PSM_TEST_FLAGS+=( + "--flagfile=config/url-map.cfg" + ) + psm::run::finalize_test_flags "${test_name}" + psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" +} + +# --- CSM TESTS ------------------ + +####################################### +# CSM Test Suite setup. +# Outputs: +# Prints activated cluster names. +####################################### +psm::csm::setup() { + activate_gke_cluster GKE_CLUSTER_PSM_CSM +} + +####################################### +# Prepares the list of tests in CSM test suite. +# Globals: +# TESTS: Populated with tests in the test suite. +####################################### +psm::csm::get_tests() { + TESTS=( + "gamma.gamma_baseline_test" + "gamma.affinity_test" + "gamma.affinity_session_drain_test" + "gamma.csm_observability_test" + "app_net_ssa_test" + ) +} + +####################################### +# Executes CSM test case +# Globals: +# PSM_TEST_FLAGS: The array with flags for the test +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +psm::csm::run_test() { + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" + PSM_TEST_FLAGS+=( + "--flagfile=config/common-csm.cfg" + ) + psm::run::finalize_test_flags "${test_name}" + psm::tools::run_verbose python -m "tests.${test_name}" "${PSM_TEST_FLAGS[@]}" +} + +# --- Common test run logic ----------- + +####################################### +# Prepares and runs the test suite. +# +# Main entry function to be used by the buildscripts. +# Provisions necessary software, configures the test driver, and executes the test suite. +# +# Globals: +# KOKORO_ARTIFACTS_DIR +# GITHUB_REPOSITORY_NAME +# GRPC_LANGUAGE +# SRC_DIR: Populated with absolute path to the source repo +# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing +# the test driver +# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code +# TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile +# TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report +# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build +# GIT_COMMIT: Populated with the SHA-1 of git commit being built +# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built +# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access +# Arguments: +# Test suite name, one of (lb, security, url_map) +# Outputs: +# Writes the output of test execution to stdout, stderr +####################################### +psm::run() { + local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" + psm::tools::log "Starting PSM Interop tests: ${test_suite}" + + psm::setup::docker_image_names "${GRPC_LANGUAGE}" "${test_suite}" + + case "${test_suite}" in + lb | security | url_map | csm) + psm::setup::generic_test_suite "${test_suite}" + ;; + *) + psm::tools::log "Unknown Test Suite: ${test_suite}" + exit 1 + ;; + esac + + psm::run::test_suite "${test_suite}" + psm::tools::log "PSM Interop tests completed: ${test_suite}" +} + +####################################### +# Executes tests in the test suite +# Globals: +# TEST_DRIVER_FULL_DIR +# TESTS: the list of the tests to execute +# Arguments: +# Test suite name. +# Outputs: +# Each test to stdout. +####################################### +psm::run::test_suite() { + local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" + cd "${TEST_DRIVER_FULL_DIR}" + local failed_tests=0 + for test_name in "${TESTS[@]}"; do + psm::run::test "${test_suite}" "${test_name}" || (( ++failed_tests )) + psm::tools::log "Finished ${test_suite} suite test: ${test_name}" + echo + done + psm::tools::log "Failed test suites: ${failed_tests}" +} + +####################################### +# Executes a single test case of given test suite. +# Globals: +# TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile +# KUBE_CONTEXT: The name of kubectl context with GKE cluster access +# TEST_XML_OUTPUT_DIR: Output directory for the test xUnit XML report +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# TESTING_VERSION: version branch under test: used by the framework to +# determine the supported PSM features. +# Arguments: +# Test suite name +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +psm::run::test() { + # Test driver usage: https://github.com/grpc/psm-interop#basic-usage + local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" + local test_name="${2:?${FUNCNAME[0]} missing the test name argument}" + local out_dir="${TEST_XML_OUTPUT_DIR:-/tmp/psm}/${test_name}" + local test_log="${out_dir}/sponge_log.log" + mkdir -p "${out_dir}" + + PSM_TEST_FLAGS=( + "--flagfile=${TEST_DRIVER_FLAGFILE}" + "--kube_context=${KUBE_CONTEXT}" + "--force_cleanup" + "--collect_app_logs" + "--log_dir=${out_dir}" + "--xml_output_file=${out_dir}/sponge_log.xml" + "--testing_version=${TESTING_VERSION}" + "--client_image=${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" + ) + + # Some test suites have canonical server image configured in the flagfiles. + if [[ -z "${SERVER_IMAGE_USE_CANONICAL}" ]]; then + PSM_TEST_FLAGS+=("--server_image=${SERVER_IMAGE_NAME}:${GIT_COMMIT}") + elif [[ "${GRPC_LANGUAGE}" == "node" ]]; then + # TODO(b/261911148): To be replaced with --server_image_use_canonical when implemented. + PSM_TEST_FLAGS+=("--server_image=us-docker.pkg.dev/grpc-testing/psm-interop/java-server:canonical") + fi + + # So far, only LB test uses secondary GKE cluster. + if [[ -n "${SECONDARY_KUBE_CONTEXT}" ]]; then + PSM_TEST_FLAGS+=("--secondary_kube_context=${SECONDARY_KUBE_CONTEXT}") + fi + + psm::tools::log "Running ${test_suite} suite test: ${test_name}" |& tee "${test_log}" + # Must be the last line. + "psm::${test_suite}::run_test" "${test_name}" |& tee -a "${test_log}" +} + +####################################### +# Appends extra flags (if any) to the end of PSM_TEST_FLAGS, prints the flag list. +# Globals: +# PSM_TEST_FLAGS: The array with flags for the test +# PSM_EXTRA_FLAGS: Space-separated string with extra flags to append +# Arguments: +# Test case name +# Outputs: +# Prints the list of test flags to the stdout +####################################### +psm::run::finalize_test_flags() { + local test_name="${1:?${FUNCNAME[0]} missing the test name argument}" + + # Append extra flags + if [[ -n "${PSM_EXTRA_FLAGS}" ]]; then + declare -a extra_flags + IFS=' ' read -r -a extra_flags <<< "${PSM_EXTRA_FLAGS}" + PSM_TEST_FLAGS+=("${extra_flags[@]}") + fi + + psm::tools::log "Test driver flags for ${test_name}:" + printf -- "%s\n" "${PSM_TEST_FLAGS[@]}" + echo +} + + +# --- Common test setup logic ----------- + +psm::setup::generic_test_suite() { + local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" + "psm::${test_suite}::setup" + psm::setup::test_driver + psm::build::docker_images_if_needed + psm::setup::get_tests "${test_suite}" +} + +psm::setup::get_tests() { + local test_suite="${1:?${FUNCNAME[0]} missing the test suite argument}" + + if [[ -n "${PSM_TESTS}" ]]; then + # Test list overridden in env var. + IFS=' ' read -r -a TESTS <<< "${PSM_TESTS}" + if (( ${#TESTS[@]} == 0 )); then + psm::tools::log "Error: test list overridden, but no tests specified" + exit 1 + fi + else + "psm::${test_suite}::get_tests" + fi + + psm::tools::log "Tests in ${test_suite} test suite:" + printf -- " - %s\n" "${TESTS[@]}" + echo +} + +####################################### +# Executes the test case +# Globals: +# CLIENT_IMAGE_NAME: Populated with xDS test client image name +# SERVER_IMAGE_NAME: Populated with xDS test server image name +# SERVER_IMAGE_USE_CANONICAL: Set to "1" when canonical server image will be used in the tests +# instead of the one configured in SERVER_IMAGE_NAME. +# Arguments: +# gRPC Language +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +psm::setup::docker_image_names() { + local language="${1:?${FUNCNAME[0]} missing the language argument}" + local test_suite="${2:?${FUNCNAME[0]} missing the test suite argument}" + SERVER_IMAGE_USE_CANONICAL="" + + case "${language}" in + java | cpp | python | go) + CLIENT_IMAGE_NAME="${DOCKER_REGISTRY}/grpc-testing/psm-interop/${GRPC_LANGUAGE}-client" + SERVER_IMAGE_NAME="${DOCKER_REGISTRY}/grpc-testing/psm-interop/${GRPC_LANGUAGE}-server" + ;; + node) + CLIENT_IMAGE_NAME="${DOCKER_REGISTRY}/grpc-testing/psm-interop/${GRPC_LANGUAGE}-client" + SERVER_IMAGE_NAME="" + SERVER_IMAGE_USE_CANONICAL="1" + ;; + *) + psm::tools::log "Unknown Language: ${1}" + exit 1 + ;; + esac + + case "${test_suite}" in + url_map) + # Uses the canonical server image configured in url-map.cfg + SERVER_IMAGE_USE_CANONICAL="1" + ;; + esac + + declare -r CLIENT_IMAGE_NAME + declare -r SERVER_IMAGE_NAME + declare -r SERVER_IMAGE_USE_CANONICAL +} + +psm::setup::test_driver() { + local build_docker_script="${BUILD_SCRIPT_DIR}/psm-interop-build-${GRPC_LANGUAGE}.sh" + psm::tools::log "Looking for docker image build script ${build_docker_script}" + if [[ -f "${build_docker_script}" ]]; then + psm::tools::log "Sourcing docker image build script: ${build_docker_script}" + source "${build_docker_script}" + fi + + if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then + kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" + else + local_setup_test_driver "${BUILD_SCRIPT_DIR}" + fi +} + +# --- Common test build logic ----------- + +####################################### +# Builds test app and its docker images unless they already exist +# Globals: +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# PSM_FORCE_IMAGE_BUILD +# Arguments: +# None +# Outputs: +# Writes the output to stdout, stderr +####################################### +psm::build::docker_images_if_needed() { + # Check if images already exist + client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" + psm::tools::log "Client image: ${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" + echo "${client_tags:-Client image not found}" + + if [[ -z "${SERVER_IMAGE_USE_CANONICAL}" ]]; then + server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" + psm::tools::log "Server image: ${SERVER_IMAGE_NAME}:${GIT_COMMIT}" + echo "${server_tags:-Server image not found}" + else + server_tags="ignored-use-canonical" + fi + + # Build if any of the images are missing, or PSM_FORCE_IMAGE_BUILD=1 + if [[ "${PSM_FORCE_IMAGE_BUILD}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then + { + psm::tools::log "Building xDS interop test app Docker images" + gcloud -q auth configure-docker "${DOCKER_REGISTRY}" + psm::lang::build_docker_images + psm::tools::log "Finished xDS interop test app Docker images" + } | tee -a "${BUILD_LOGS_ROOT}/build-docker.log" + else + psm::tools::log "Skipping ${GRPC_LANGUAGE} test app build" + fi +} + +####################################### +# Builds test app Docker images and pushes them to GCR +# Globals: +# SRC_DIR: Absolute path to the source repo on Kokoro VM +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# DOCKER_REGISTRY: Docker registry to push to +# Arguments: +# The path to xDS test client Dockerfile. Can be absolute or relative to SRC_DIR +# The path to xDS test server Dockerfile. Optional, server build is skip when omitted. +# Outputs: +# Writes the output of docker image build stdout, stderr +####################################### +psm::build::docker_images_generic() { + local client_dockerfile="${1:?${FUNCNAME[0]} missing the client dockerfile argument}" + local server_dockerfile="${2:-}" + pushd "${SRC_DIR}" + + # Client is required. + psm::tools::log "Building ${GRPC_LANGUAGE} xDS interop test client" + psm::tools::run_verbose docker build -f "${client_dockerfile}" -t "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" . + psm::tools::run_verbose docker push "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" + + # Server is optional + if [[ -n "${server_dockerfile}" ]]; then + psm::tools::log "Building ${GRPC_LANGUAGE} xDS interop test server" + psm::tools::run_verbose docker build -f "${server_dockerfile}" -t "${SERVER_IMAGE_NAME}:${GIT_COMMIT}" . + psm::tools::run_verbose docker push "${SERVER_IMAGE_NAME}:${GIT_COMMIT}" + fi + popd + + if is_version_branch "${TESTING_VERSION}"; then + psm::tools::log "Detected proper version branch ${TESTING_VERSION}, adding version tags." + tag_and_push_docker_image "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" "${TESTING_VERSION}" + if [[ -n "${server_dockerfile}" ]]; then + tag_and_push_docker_image "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" "${TESTING_VERSION}" + fi + fi +} + +# --- Common helpers ---------------------- + +####################################### +# Print the command with its arguments (aka set xtrace/ set -x) before execution. +# Arguments: +# The command to run +# Returns: +# The exit status of the command executed. +####################################### +psm::tools::run_verbose() { + local exit_code=0 + set -x + "$@" || exit_code=$? + { set +x; } 2>/dev/null + + if (( exit_code == 0 )); then + psm::tools::log "Cmd finished: ${1}" + else + psm::tools::log "Cmd failed with exit code ${exit_code}: $*" + fi + + return ${exit_code} +} + +####################################### +# Run command end report its exit code. Doesn't exit on non-zero exit code. +# Globals: +# None +# Arguments: +# Command to execute +# Outputs: +# Writes the output of given command to stdout, stderr +####################################### +psm::tools::run_ignore_exit_code() { + local exit_code=0 + "$@" || exit_code=$? + + if (( exit_code == 0 )); then + psm::tools::log "Cmd failed with exit code ${exit_code}: $*" + fi +} + +psm::tools::log() { + echo -en "+ $(date "+[%H:%M:%S %Z]")\011 " + echo "$@" +} + +# --- "Unsorted" methods -------------- +# TODO(sergiitk): all methods should be using "psm::" package name, +# see https://google.github.io/styleguide/shellguide.html#function-names ####################################### # Determines the cluster name and zone based on the given cluster identifier. @@ -64,11 +648,11 @@ activate_gke_cluster() { GKE_CLUSTER_ZONE="us-central1-c" ;; *) - echo "Unknown GKE cluster: ${1}" + psm::tools::log "Unknown GKE cluster: ${1}" exit 1 ;; esac - echo -n "Activated GKE cluster: GKE_CLUSTER_NAME=${GKE_CLUSTER_NAME} " + psm::tools::log -n "Activated GKE cluster: GKE_CLUSTER_NAME=${GKE_CLUSTER_NAME} " if [[ -n "${GKE_CLUSTER_REGION}" ]]; then echo "GKE_CLUSTER_REGION=${GKE_CLUSTER_REGION}" else @@ -94,16 +678,20 @@ activate_secondary_gke_cluster() { SECONDARY_GKE_CLUSTER_ZONE="us-west1-b" ;; *) - echo "Unknown secondary GKE cluster: ${1}" + psm::tools::log "Unknown secondary GKE cluster: ${1}" exit 1 ;; esac - echo -n "Activated secondary GKE cluster: SECONDARY_GKE_CLUSTER_NAME=${SECONDARY_GKE_CLUSTER_NAME}" + psm::tools::log -n "Activated secondary GKE cluster: SECONDARY_GKE_CLUSTER_NAME=${SECONDARY_GKE_CLUSTER_NAME}" echo " SECONDARY_GKE_CLUSTER_ZONE=${SECONDARY_GKE_CLUSTER_ZONE}" } ####################################### # Run command end report its exit code. Doesn't exit on non-zero exit code. +# +# Deprecated. Use psm::tools::run_ignore_exit_code +# TODO(sergiitk): delete this method when no longer used in per-language buildscripts. +# # Globals: # None # Arguments: @@ -115,7 +703,7 @@ run_ignore_exit_code() { local exit_code=0 "$@" || exit_code=$? if [[ $exit_code != 0 ]]; then - echo "Cmd: '$*', exit code: ${exit_code}" + psm::tools::log "Cmd: '$*', exit code: ${exit_code}" fi } @@ -220,9 +808,9 @@ gcloud_get_cluster_credentials() { ####################################### test_driver_get_source() { if [[ -n "${TEST_DRIVER_REPO_DIR_USE_EXISTING}" && -d "${TEST_DRIVER_REPO_DIR}" ]]; then - echo "Using exiting driver directory: ${TEST_DRIVER_REPO_DIR}." + psm::tools::log "Using exiting driver directory: ${TEST_DRIVER_REPO_DIR}." else - echo "Cloning driver to ${TEST_DRIVER_REPO_URL} branch ${TEST_DRIVER_BRANCH} to ${TEST_DRIVER_REPO_DIR}" + psm::tools::log "Cloning driver to ${TEST_DRIVER_REPO_URL} branch ${TEST_DRIVER_BRANCH} to ${TEST_DRIVER_REPO_DIR}" git clone -b "${TEST_DRIVER_BRANCH}" --depth=1 "${TEST_DRIVER_REPO_URL}" "${TEST_DRIVER_REPO_DIR}" fi } @@ -240,16 +828,16 @@ test_driver_get_source() { # Writes the list of installed modules to stdout ####################################### test_driver_pip_install() { - echo "Install python dependencies" + psm::tools::log "Install python dependencies" cd "${TEST_DRIVER_FULL_DIR}" # Create and activate virtual environment unless already using one if [[ -z "${VIRTUAL_ENV}" ]]; then local venv_dir="${TEST_DRIVER_FULL_DIR}/venv" if [[ -d "${venv_dir}" ]]; then - echo "Found python virtual environment directory: ${venv_dir}" + psm::tools::log "Found python virtual environment directory: ${venv_dir}" else - echo "Creating python virtual environment: ${venv_dir}" + psm::tools::log "Creating python virtual environment: ${venv_dir}" "python${PYTHON_VERSION}" -m venv "${venv_dir}" --upgrade-deps fi # Intentional: No need to check python venv activate script. @@ -257,9 +845,14 @@ test_driver_pip_install() { source "${venv_dir}/bin/activate" fi - python3 -m pip install -r requirements.lock - echo "Installed Python packages:" - python3 -m pip list + psm::tools::log "Installing Python packages with pip, see install-pip.log" + psm::driver::pip_install &>> "${BUILD_LOGS_ROOT}/install-pip.log" +} + +psm::driver::pip_install() { + psm::tools::run_verbose python3 -m pip install -r requirements.lock + echo + psm::tools::run_verbose python3 -m pip list } ####################################### @@ -283,7 +876,7 @@ test_driver_compile_protos() { "${TEST_DRIVER_PROTOS_PATH}/messages.proto" "${TEST_DRIVER_PROTOS_PATH}/empty.proto" ) - echo "Generate python code from grpc.testing protos: ${protos[*]}" + psm::tools::log "Generate python code from grpc.testing protos: ${protos[*]}" cd "${TEST_DRIVER_REPO_DIR}" python3 -m grpc_tools.protoc \ --proto_path=. \ @@ -291,13 +884,16 @@ test_driver_compile_protos() { --grpc_python_out="${TEST_DRIVER_FULL_DIR}" \ "${protos[@]}" local protos_out_dir="${TEST_DRIVER_FULL_DIR}/${TEST_DRIVER_PROTOS_PATH}" - echo "Generated files ${protos_out_dir}:" - ls -Fl "${protos_out_dir}" + + if command -v sha256sum &> /dev/null; then + psm::tools::log "Generated files ${protos_out_dir}:" + find "${protos_out_dir}" -type f -exec sha256sum {} \; + fi } ####################################### # Installs the test driver and it's requirements. -# https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#installation +# https://github.com/grpc/psm-interop#basic-usage#installation # Globals: # TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing # the test driver @@ -323,7 +919,7 @@ test_driver_install() { # Writes the output to stdout ####################################### kokoro_print_version() { - echo "Kokoro Ubuntu version:" + psm::tools::log "Kokoro Ubuntu version:" run_ignore_exit_code lsb_release -a run_ignore_exit_code "python${PYTHON_VERSION}" --version run_ignore_exit_code "python${PYTHON_VERSION}" -m pip --version @@ -354,7 +950,7 @@ TESTGRID_EXCLUDE,${TESTGRID_EXCLUDE:-0} GIT_ORIGIN_URL,${GIT_ORIGIN_URL:?GIT_ORIGIN_URL must be set} GIT_COMMIT_SHORT,${GIT_COMMIT_SHORT:?GIT_COMMIT_SHORT must be set} EOF - echo "Sponge properties:" + psm::tools::log "Sponge properties:" cat "${KOKORO_ARTIFACTS_DIR}/custom_sponge_config.csv" } @@ -382,7 +978,7 @@ kokoro_install_dependencies() { # Globals: # KOKORO_JOB_NAME # KOKORO_BUILD_INITIATOR -# FORCE_TESTING_VERSION: Forces the testing version to be something else. +# PSM_FORCE_TESTING_VERSION: Forces the testing version to be something else. # TESTING_VERSION: Populated with the version branch under test, # f.e. master, dev, v1.42.x. # Outputs: @@ -398,10 +994,10 @@ kokoro_get_testing_version() { local version_from_job_name version_from_job_name=$(echo "${KOKORO_JOB_NAME}" | cut -d '/' -f3) - if [[ -n "${FORCE_TESTING_VERSION}" ]]; then + if [[ -n "${PSM_FORCE_TESTING_VERSION}" ]]; then # Allows to override the testing version, and force tagging the built # images, if necessary. - readonly TESTING_VERSION="${FORCE_TESTING_VERSION}" + readonly TESTING_VERSION="${PSM_FORCE_TESTING_VERSION}" elif [[ "${KOKORO_BUILD_INITIATOR:-anonymous}" != "kokoro" ]]; then # If not initiated by Kokoro, it's a dev branch. # This allows to know later down the line that the built image doesn't need @@ -441,8 +1037,11 @@ kokoro_get_testing_version() { # Writes the output to stdout, stderr, files ####################################### kokoro_setup_test_driver() { - # Unset noisy verbose mode often set in the parent script. + # Unset noisy verbose mode often set in the parent scripts. set +x + + psm::tools::log "Starting Kokoro provisioning" + local src_repository_name="${1:?Usage kokoro_setup_test_driver GITHUB_REPOSITORY_NAME}" # Capture Kokoro VM version info in the log. kokoro_print_version @@ -453,11 +1052,22 @@ kokoro_setup_test_driver() { # Kokoro clones repo to ${KOKORO_ARTIFACTS_DIR}/github/${GITHUB_REPOSITORY} local github_root="${KOKORO_ARTIFACTS_DIR}/github" readonly SRC_DIR="${github_root}/${src_repository_name}" + + # Test artifacts dir: xml reports, logs, etc. + local artifacts_dir="${KOKORO_ARTIFACTS_DIR}/artifacts" + # Folders after $artifacts_dir reported as target name + readonly TEST_XML_OUTPUT_DIR="${artifacts_dir}/${KOKORO_JOB_NAME}" + readonly BUILD_LOGS_ROOT="${TEST_XML_OUTPUT_DIR}" + + mkdir -p "${artifacts_dir}" "${TEST_XML_OUTPUT_DIR}" "${BUILD_LOGS_ROOT}" parse_src_repo_git_info SRC_DIR kokoro_write_sponge_properties - kokoro_install_dependencies + + psm::tools::log "Installing packages with apt, see install-apt.log" + kokoro_install_dependencies &> "${BUILD_LOGS_ROOT}/install-apt.log" # Get kubectl cluster credentials. + psm::tools::log "Fetching GKE cluster credentials" gcloud_get_cluster_credentials # Install the driver. @@ -466,12 +1076,6 @@ kokoro_setup_test_driver() { test_driver_install "${test_driver_repo_dir}" # shellcheck disable=SC2034 # Used in the main script readonly TEST_DRIVER_FLAGFILE="config/grpc-testing.cfg" - - # Test artifacts dir: xml reports, logs, etc. - local artifacts_dir="${KOKORO_ARTIFACTS_DIR}/artifacts" - # Folders after $artifacts_dir reported as target name - readonly TEST_XML_OUTPUT_DIR="${artifacts_dir}/${KOKORO_JOB_NAME}" - mkdir -p "${artifacts_dir}" "${TEST_XML_OUTPUT_DIR}" } ####################################### @@ -501,8 +1105,8 @@ local_setup_test_driver() { readonly SECONDARY_KUBE_CONTEXT="${SECONDARY_KUBE_CONTEXT}" # Never override docker image for local runs, unless explicitly forced. - if [[ -n "${FORCE_TESTING_VERSION}" ]]; then - readonly TESTING_VERSION="${FORCE_TESTING_VERSION}" + if [[ -n "${PSM_FORCE_TESTING_VERSION}" ]]; then + readonly TESTING_VERSION="${PSM_FORCE_TESTING_VERSION}" else readonly TESTING_VERSION="dev" fi @@ -515,7 +1119,8 @@ local_setup_test_driver() { readonly TEST_DRIVER_FLAGFILE="config/local-dev.cfg" # Test out readonly TEST_XML_OUTPUT_DIR="${TEST_DRIVER_FULL_DIR}/out" - mkdir -p "${TEST_XML_OUTPUT_DIR}" + readonly BUILD_LOGS_ROOT="${TEST_XML_OUTPUT_DIR}" + mkdir -p "${TEST_XML_OUTPUT_DIR}" "${BUILD_LOGS_ROOT}" } ####################################### @@ -532,6 +1137,6 @@ tag_and_push_docker_image() { local from_tag="$2" local to_tag="$3" - docker tag "${image_name}:${from_tag}" "${image_name}:${to_tag}" - docker push "${image_name}:${to_tag}" + psm::tools::run_verbose docker tag "${image_name}:${from_tag}" "${image_name}:${to_tag}" + psm::tools::run_verbose push "${image_name}:${to_tag}" } diff --git a/framework/infrastructure/gcp/api.py b/framework/infrastructure/gcp/api.py index cee8af19..a539b1eb 100644 --- a/framework/infrastructure/gcp/api.py +++ b/framework/infrastructure/gcp/api.py @@ -126,13 +126,14 @@ def private_api_key(self): return secret.payload.data.decode() @functools.lru_cache(None) - def compute(self, version): + def compute(self, version: str): api_name = "compute" + + if version.startswith("v1") and self.compute_v1_discovery_file: + return self._build_from_file(self.compute_v1_discovery_file) + if version == "v1": - if self.compute_v1_discovery_file: - return self._build_from_file(self.compute_v1_discovery_file) - else: - return self._build_from_discovery_v1(api_name, version) + return self._build_from_discovery_v1(api_name, version) elif version == "v1alpha": return self._build_from_discovery_v1(api_name, "alpha") diff --git a/tests/subsetting_test.py b/tests/subsetting_test.py index 8231b609..ca5421f8 100644 --- a/tests/subsetting_test.py +++ b/tests/subsetting_test.py @@ -23,6 +23,10 @@ from framework.helpers import skips flags.adopt_module_key_flags(xds_k8s_testcase) +# Change the default value of --compute_api_version to v1alpha. +# Subsetting test requires Compute v1alpha APIs. +# Can be tested with another API version if the flag is passed to the test. +flags.set_default(xds_k8s_testcase.xds_flags.COMPUTE_API_VERSION, "v1alpha") # Type aliases _XdsTestServer = xds_k8s_testcase.XdsTestServer