From dc09d23394037acfed47c301a73b5fe3cba0a773 Mon Sep 17 00:00:00 2001 From: Matthew James Briggs Date: Fri, 29 Apr 2022 19:56:00 -0700 Subject: [PATCH] test: Add `cargo make` targets for `testsys` tests --- .gitignore | 1 + Makefile.toml | 107 +++++ TESTING.md | 176 ++++++++ tools/Cargo.lock | 671 +++++++++++++++++++++++++++++ tools/Cargo.toml | 1 + tools/deny.toml | 3 + tools/testsys/Cargo.toml | 26 ++ tools/testsys/src/aws_resources.rs | 229 ++++++++++ tools/testsys/src/delete.rs | 29 ++ tools/testsys/src/install.rs | 47 ++ tools/testsys/src/logs.rs | 46 ++ tools/testsys/src/main.rs | 111 +++++ tools/testsys/src/restart_test.rs | 21 + tools/testsys/src/run.rs | 215 +++++++++ tools/testsys/src/secret.rs | 117 +++++ tools/testsys/src/status.rs | 55 +++ tools/testsys/src/uninstall.rs | 23 + 17 files changed, 1878 insertions(+) create mode 100644 TESTING.md create mode 100644 tools/testsys/Cargo.toml create mode 100644 tools/testsys/src/aws_resources.rs create mode 100644 tools/testsys/src/delete.rs create mode 100644 tools/testsys/src/install.rs create mode 100644 tools/testsys/src/logs.rs create mode 100644 tools/testsys/src/main.rs create mode 100644 tools/testsys/src/restart_test.rs create mode 100644 tools/testsys/src/run.rs create mode 100644 tools/testsys/src/secret.rs create mode 100644 tools/testsys/src/status.rs create mode 100644 tools/testsys/src/uninstall.rs diff --git a/.gitignore b/.gitignore index 9c27e13b8ab..595b687d9df 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /.gomodcache /html /Infra.toml +/testsys.kubeconfig /*.pem /keys /roles diff --git a/Makefile.toml b/Makefile.toml index 5aefe132cdc..17bfc05c82b 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -116,6 +116,13 @@ DOCKER_BUILDKIT = "1" # write AMI information to specifically named files. AMI_DATA_FILE_SUFFIX = "amis.json" +# The type of testsys test that should be run. +# `quick` will run a quick test which usually tests that the instances are reachable. +# `conformance` will run a certified conformance test, these tests may take up to 3 hrs. +TESTSYS_TEST = "quick" +# The path to the testsys cluster's kubeconfig file. This is used for all testsys calls. +TESTSYS_KUBECONFIG_PATH = "${BUILDSYS_ROOT_DIR}/testsys.kubeconfig" + [env.development] # Certain variables are defined here to allow us to override a component value # on the command line. @@ -1431,5 +1438,105 @@ rm -rf ${BUILDSYS_STATE_DIR} ''' ] +[tasks.test-tools] +dependencies = ["setup", "fetch-sources"] +script = [ + ''' + cargo install \ + ${CARGO_MAKE_CARGO_ARGS} \ + --path tools/testsys \ + --root tools \ + --force \ + --quiet + ''' +] + +[tasks.setup-test] +dependencies = ["test-tools"] +script = [ + ''' + set -eu + export PATH="${BUILDSYS_TOOLS_DIR}/bin:${PATH}" + testsys --kubeconfig ${TESTSYS_KUBECONFIG_PATH} install + ''' +] + +# This task is used to test bottlerocket build artifacts. By default the region first listed in Infra.toml +# is used for testing; however, `TESTSYS_REGION` can be used to test in a different region. +[tasks.test] +dependencies = ["test-tools"] +script = [ + ''' + set -eu + ami_input="${BUILDSYS_OUTPUT_DIR}/${BUILDSYS_NAME_FULL}-${AMI_DATA_FILE_SUFFIX}" + if [ ! -s "${ami_input}" ]; then + echo "AMI input file doesn't exist for the current version/commit - ${BUILDSYS_VERSION_FULL} - please run 'cargo make ami'" >&2 + exit 1 + fi + export PATH="${BUILDSYS_TOOLS_DIR}/bin:${PATH}" + # The ami that is selected from `amis.json` is determined by `TESTSYS_REGION` if set; otherwise, + # it is the first region listed in `Infra.toml` (for aws variants). + testsys --kubeconfig ${TESTSYS_KUBECONFIG_PATH} run ${TESTSYS_TEST} \ + --ami-input "${ami_input}" \ + ${TESTSYS_AWS_SECRET_NAME:+--secret ${TESTSYS_AWS_SECRET_NAME}} \ + ''' +] + +# This task will clear all tests and resources from the testsys cluster. +[tasks.clean-test] +dependencies = ["test-tools"] +script = [ + ''' + set -eu + export PATH="${BUILDSYS_TOOLS_DIR}/bin:${PATH}" + testsys --kubeconfig ${TESTSYS_KUBECONFIG_PATH} delete + ''' +] + +# This task will clear all testsys components from the testsys cluster. +[tasks.uninstall-test] +dependencies = ["test-tools"] +script = [ + ''' + set -eu + export PATH="${BUILDSYS_TOOLS_DIR}/bin:${PATH}" + testsys --kubeconfig ${TESTSYS_KUBECONFIG_PATH} uninstall + ''' +] + +# This task will call watch on the `status` testsys command to show the results of a test. +[tasks.watch-test] +dependencies = ["test-tools"] +script = [ + ''' + set -eu + export PATH="${BUILDSYS_TOOLS_DIR}/bin:${PATH}" + watch -- testsys --kubeconfig ${TESTSYS_KUBECONFIG_PATH} status ${@} + ''' +] + +# This task will retrieve testsys logs from a test. You can add `--follow` to continue to recieve +# logs as they come in. +[tasks.log-test] +dependencies = ["test-tools"] +script = [ + ''' + set -eu + export PATH="${BUILDSYS_TOOLS_DIR}/bin:${PATH}" + testsys --kubeconfig ${TESTSYS_KUBECONFIG_PATH} logs --test ${@} + ''' +] + +# This task is useful for using the current tree's testsys without symlinks +[tasks.testsys] +dependencies = ["test-tools"] +script = [ + ''' + set -eu + export PATH="${BUILDSYS_TOOLS_DIR}/bin:${PATH}" + testsys --kubeconfig ${TESTSYS_KUBECONFIG_PATH} ${@} + ''' +] + [tasks.default] alias = "build" diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 00000000000..b3959ba2eef --- /dev/null +++ b/TESTING.md @@ -0,0 +1,176 @@ +# Testing Bottlerocket + +🚧 👷 + +This section is under active development. +We are working on tooling for running Bottlerocket integration tests. +While the work is underway, there will be frequent changes to this document. + +## Unit Tests + +It is easy to execute unit tests, you can run them from the root of the repo with `cargo make unit-tests`. +Note that some code in Bottlerocket is conditionally compiled based on variant thus some tests won't be executed. +Unless you intend to test the default variant, it is best to pass the relevant variant and architecture like this: + +```shell +cargo make \ + -e BUILDSYS_VARIANT="aws-ecs-1" \ + -e BUILDSYS_ARCH="x86_64" \ + unit-tests +``` + +## Integration Tests + +Unit tests will only get us so far. +Ultimately we want to know if Bottlerocket runs correctly as a complete system. +We have created a [command line utility] and [testing system] to help us test Bottlerocket holistically. + +[command line utility]: ./tools/testsys +[testing system]: https://github.com/bottlerocket-os/bottlerocket-test-system + +The test system coordinates: +- the creation of a cluster (or re-use of an existing cluster), +- creation of Bottlerocket instances, +- running tests that target the created cluster and instances, +- terminating the Bottlerocket instances, +- terminating the Kubernetes cluster (if desired) + +Testsys uses a Kubernetes operator to test bottlerocket. +The operator runs in a cluster that is separate from the one where you are testing Bottlerocket nodes. +We call this control cluster the *testsys cluster*. +When you launch a Bottlerocket integration test, pods run in the testsys cluster to perform the steps described above. + +## Setup + +### EKS + +It is possible to run your testsys cluster anywhere so long as it has the necessary authorization and networking. +We have plans to make this easy to do in EKS by providing the instructions and role permissions you need. +However, some work is still needed on the roles, so check back for those instructions in the future! + +### Using a Temporary Kind Cluster + +For developer workflows, the quickest way to run a testsys cluster is using [kind]. + +[kind]: https://kind.sigs.k8s.io/ + +**Important:** only use `kind` for temporary testsys clusters that you will be using yourself. +Do not use `kind` for long-lived clusters or clusters that you will share with other users. + +Here are the steps to set up a testsys cluster using `kind`. + +Create a kind cluster (any name will suffice): + +```shell +kind create cluster --name testsys +``` + +If you want to store the kubeconfig file, set the `KUBECONFIG` variable to some path (there should be no pre-existing file there). +It doesn't really matter where this is, since this is a throwaway cluster and then write the +kubeconfig to that path. +The environment variable `TESTSYS_KUBECONFIG` is used by all testsys +related cargo make tasks. + +```shell +export TESTSYS_KUBECONFIG="${HOME}/testsys-kubeconfig.yaml" +kind get kubeconfig --name testsys > $TESTSYS_KUBECONFIG +``` + +Install the testsys cluster components: + +```shell +cargo make setup-testsys +``` + +Testsys containers will need AWS credentials. + +**Reminder**: this is for your developer workflow only, do not share this cluster with other users. + +```shell +cargo make testsys add secret map \ + --name "creds" \ + "access-key-id=$(aws configure get aws_access_key_id)" \ + "secret-access-key=$(aws configure get aws_secret_access_key)" +``` + +If you have a named profile you can use the following. +```shell +PROFILE= +cargo make testsys add secret map \ + --name "creds" \ + "access-key-id=$(aws configure get aws_access_key_id --profile ${PROFILE})" \ + "secret-access-key=$(aws configure get aws_secret_access_key --profile ${PROFILE})" +``` + +### Conveniences + +All testsys commands can be run using cargo make to eliminate the chance of 2 different versions of +testsys bing used. +Testsys requires the controller and the agent images to be of the same testsys version. + +```shell +cargo make testsys +``` + +The Bottlerocket components are found in the `testsys-bottlerocket-aws` Kubernetes namespace. + +## Run + +Now that you have the testsys cluster set up, it's time to run a Bottlerocket integration test! + +### Variants + +Different Bottlerocket variants require different implementations in the test system. +For example, to ensure that Kubernetes variants are working correctly, we use [Sonobuoy] to run through the K8s E2E conformance test suite. +For ECS, we run a [task] on Bottlerocket to make sure Bottlerocket is working. +We use EC2 and EKS for `aws-k8s` variants and vSphere for `vmware-k8s` variants, and so on. + +[Sonobuoy]: https://sonobuoy.io/ +[task]: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/welcome-features.html + +We have attempted use sensible defaults for these behaviors when calling the `cargo make test` command. + +🚧 👷 **Variant Support** + +We haven't yet enabled `cargo make test` for every variant, though much of the underlying foundation has been laid. +If you run `cargo make test` for a variant that is not yet enabled, it will print an error message. +Check back here and follow the issues relevant to your variant of interest. + +- `aws-k8s` conformance testing is working! +- `aws-ecs`: https://github.com/bottlerocket-os/bottlerocket/issues/2150 +- `vmware-k8s`: https://github.com/bottlerocket-os/bottlerocket/issues/2151 +- `metal-k8s`: https://github.com/bottlerocket-os/bottlerocket/issues/2152 + +### aws-k8s + +You need to [build](BUILDING.md) Bottlerocket and create an AMI before you can run a test. +Change the commands below to the desired `aws-k8s` variant and AWS region: + +**Caution**: An EKS cluster will be created for you. +Because these take a long time to create, the default testsys behavior is to leave this in place so you can re-use it. +You will need to delete the EKS cluster manually when you are done using it. +(EC2 instances are terminated automatically, but it's worth double-checking to make sure they were terminated.) + +```shell +cargo make \ + -e BUILDSYS_VARIANT=aws-k8s-1.21 \ + -e BUILDSYS_ARCH="x86_64" \ + build + +cargo make \ + -e BUILDSYS_VARIANT=aws-k8s-1.21 \ + -e BUILDSYS_ARCH="x86_64" \ + -e PUBLISH_REGIONS="us-west-2" + ami + +cargo make \ + -e BUILDSYS_VARIANT=aws-k8s-1.21 \ + -e BUILDSYS_ARCH="x86_64" \ + test +``` + +(`-r` tells testsys to also show the status of resources like the cluster and instances in addition to tests): + +```shell +cargo make watch-test +``` diff --git a/tools/Cargo.lock b/tools/Cargo.lock index bd559864835..15eee4b0c35 100644 --- a/tools/Cargo.lock +++ b/tools/Cargo.lock @@ -35,6 +35,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "anyhow" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" + [[package]] name = "argh" version = "0.1.7" @@ -74,6 +80,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-recursion" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cda8f4bcc10624c4e85bc66b3f452cca98cfa5ca002dc83a16aad2367641bea" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.56" @@ -147,6 +164,25 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bottlerocket-types" +version = "0.1.0" +source = "git+https://github.com/bottlerocket-os/bottlerocket-test-system?rev=021e8d6#021e8d69b13b7d05e79963a0ff3f1c5c1af10753" +dependencies = [ + "model", + "serde", + "serde_plain", +] + +[[package]] +name = "bottlerocket-variant" +version = "0.1.0" +dependencies = [ + "generate-readme", + "serde", + "snafu", +] + [[package]] name = "bstr" version = "0.2.17" @@ -182,6 +218,12 @@ version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "bytes" version = "1.1.0" @@ -252,13 +294,28 @@ checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" dependencies = [ "atty", "bitflags", + "clap_derive", "clap_lex", "indexmap", + "lazy_static", "strsim 0.10.0", "termcolor", "textwrap 0.15.0", ] +[[package]] +name = "clap_derive" +version = "3.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "clap_lex" version = "0.2.0" @@ -404,6 +461,41 @@ dependencies = [ "subtle", ] +[[package]] +name = "darling" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.9.0" @@ -489,6 +581,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_logger" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "fastrand" version = "1.7.0" @@ -504,6 +609,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -747,6 +867,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + [[package]] name = "httparse" version = "1.7.1" @@ -759,6 +885,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.19" @@ -798,6 +930,37 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.2.3" @@ -886,6 +1049,126 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-patch" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f995a3c8f2bc3dd52a18a583e90f9ec109c047fa1603a853e46bcda14d2e279d" +dependencies = [ + "serde", + "serde_json", + "treediff", +] + +[[package]] +name = "jsonpath_lib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f" +dependencies = [ + "log", + "serde", + "serde_json", +] + +[[package]] +name = "k8s-openapi" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ae2c04fcee6b01b04e3aadd56bb418932c8e0a9d8a93f48bc68c6bdcdb559d" +dependencies = [ + "base64", + "bytes", + "chrono", + "http", + "percent-encoding", + "serde", + "serde-value", + "serde_json", + "url", +] + +[[package]] +name = "kube" +version = "0.73.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f68b954ea9ad888de953fb1488bd8f377c4c78d82d4642efa5925189210b50b7" +dependencies = [ + "k8s-openapi", + "kube-client", + "kube-core", + "kube-derive", +] + +[[package]] +name = "kube-client" +version = "0.73.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150dc7107d9acf4986088f284a0a6dddc5ae37ef1ffdf142f6811dc5998dd58" +dependencies = [ + "base64", + "bytes", + "chrono", + "dirs-next", + "either", + "futures", + "http", + "http-body", + "hyper", + "hyper-timeout", + "hyper-tls", + "jsonpath_lib", + "k8s-openapi", + "kube-core", + "openssl", + "pem", + "pin-project", + "rand", + "secrecy", + "serde", + "serde_json", + "serde_yaml", + "thiserror", + "tokio", + "tokio-native-tls", + "tokio-tungstenite", + "tokio-util", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "kube-core" +version = "0.73.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc8c429676abe6a73b374438d5ca02caaf9ae7a635441253c589b779fa5d0622" +dependencies = [ + "chrono", + "form_urlencoded", + "http", + "json-patch", + "k8s-openapi", + "once_cell", + "schemars", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "kube-derive" +version = "0.73.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfb405f0d39181acbfdc7c79e3fc095330c9b6465ab50aeb662d762e53b662f1" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -923,6 +1206,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matches" version = "0.1.9" @@ -982,6 +1271,54 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "model" +version = "0.1.0" +source = "git+https://github.com/bottlerocket-os/bottlerocket-test-system?rev=021e8d6#021e8d69b13b7d05e79963a0ff3f1c5c1af10753" +dependencies = [ + "async-recursion", + "async-trait", + "base64", + "bytes", + "futures", + "http", + "json-patch", + "k8s-openapi", + "kube", + "lazy_static", + "log", + "maplit", + "regex", + "schemars", + "serde", + "serde_json", + "serde_plain", + "serde_yaml", + "snafu", + "tabled", + "tokio", + "tokio-util", + "topological-sort", +] + +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.24.1" @@ -1076,12 +1413,60 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-float" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +dependencies = [ + "num-traits", +] + [[package]] name = "os_pipe" version = "0.9.2" @@ -1098,6 +1483,15 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" +[[package]] +name = "papergrid" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63709d10e2c2ec58f7bd91d8258d27ce80de090064b0ddf3a4bf38b907b61b8a" +dependencies = [ + "unicode-width", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -1163,6 +1557,26 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pin-project" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1175,6 +1589,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -1704,6 +2124,30 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "schemars" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1847b767a3d62d95cbf3d8a9f0e421cf57a0d8aa4f411d4b16525afb0284d4ed" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4d7e1b012cb3d9129567661a63755ea4b8a7386d339dc945ae187e403c6743" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -1720,6 +2164,16 @@ dependencies = [ "untrusted", ] +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "serde", + "zeroize", +] + [[package]] name = "security-framework" version = "2.6.1" @@ -1761,6 +2215,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.137" @@ -1772,12 +2236,24 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_json" version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ + "indexmap", "itoa", "ryu", "serde", @@ -1816,6 +2292,17 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "sha-1" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.3", +] + [[package]] name = "sha2" version = "0.9.9" @@ -1986,6 +2473,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tabled" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d15827061abcf689257b1841c8e2732b1dfcc3ef825b24ce6c606e1e9e1a7bde" +dependencies = [ + "papergrid", + "tabled_derive", +] + +[[package]] +name = "tabled_derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278ea3921cee8c5a69e0542998a089f7a14fa43c9c4e4f9951295da89bd0c943" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -2019,6 +2527,29 @@ dependencies = [ "winapi", ] +[[package]] +name = "testsys" +version = "0.1.0" +dependencies = [ + "anyhow", + "bottlerocket-types", + "bottlerocket-variant", + "clap 3.1.18", + "env_logger", + "futures", + "k8s-openapi", + "log", + "maplit", + "model", + "pubsys-config", + "serde", + "serde_json", + "serde_plain", + "terminal_size", + "tokio", + "unescape", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -2127,6 +2658,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "1.8.0" @@ -2138,6 +2679,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.23.4" @@ -2160,6 +2711,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06cda1232a49558c46f8a504d5b93101d42c0bf7f911f12a105ba48168f821ae" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.3" @@ -2183,6 +2746,12 @@ dependencies = [ "serde", ] +[[package]] +name = "topological-sort" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa7c7f42dea4b1b99439786f5633aeb9c14c1b53f75e282803c2ec2ad545873c" + [[package]] name = "tough" version = "0.12.2" @@ -2242,6 +2811,49 @@ dependencies = [ "tough", ] +[[package]] +name = "tower" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" +dependencies = [ + "base64", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + [[package]] name = "tower-service" version = "0.3.1" @@ -2255,10 +2867,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" dependencies = [ "cfg-if", + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.26" @@ -2268,18 +2893,52 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "treediff" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff" +dependencies = [ + "serde_json", +] + [[package]] name = "try-lock" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "tungstenite" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96a2dea40e7570482f28eb57afbe42d97551905da6a9400acc5c328d24004f5" +dependencies = [ + "base64", + "byteorder", + "bytes", + "http", + "httparse", + "log", + "rand", + "sha-1", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "typenum" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "unescape" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccb97dac3243214f8d8507998906ca3e2e0b900bf9bf4870477f125b82e68f6e" + [[package]] name = "unicode-bidi" version = "0.3.8" @@ -2347,6 +3006,18 @@ dependencies = [ "serde", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vec_map" version = "0.8.2" diff --git a/tools/Cargo.toml b/tools/Cargo.toml index 58b7f54b1c5..b76b84c55c0 100644 --- a/tools/Cargo.toml +++ b/tools/Cargo.toml @@ -5,4 +5,5 @@ members = [ "pubsys", "pubsys-config", "pubsys-setup", + "testsys", ] diff --git a/tools/deny.toml b/tools/deny.toml index bedc1540492..3a1380beb87 100644 --- a/tools/deny.toml +++ b/tools/deny.toml @@ -68,6 +68,9 @@ skip-tree = [ ] [sources] +allow-git = [ + "https://github.com/bottlerocket-os/bottlerocket-test-system", +] # Deny crates from unknown registries or git repositories. unknown-registry = "deny" unknown-git = "deny" diff --git a/tools/testsys/Cargo.toml b/tools/testsys/Cargo.toml new file mode 100644 index 00000000000..84701cf91a0 --- /dev/null +++ b/tools/testsys/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "testsys" +version = "0.1.0" +authors = ["Ethan Pullen ", "Matt Briggs "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false + +[dependencies] +anyhow = "1.0" +bottlerocket-types = { git = "https://github.com/bottlerocket-os/bottlerocket-test-system", rev = "021e8d6", version = "0.1"} +bottlerocket-variant = { version = "0.1", path = "../../sources/bottlerocket-variant" } +clap = { version = "3", features = ["derive", "env"] } +env_logger = "0.9" +futures = "0.3.8" +k8s-openapi = { version = "0.15", features = ["v1_20", "api"], default-features = false } +log = "0.4" +maplit = "1.0.2" +model = { git = "https://github.com/bottlerocket-os/bottlerocket-test-system", rev = "021e8d6", version = "0.1"} +pubsys-config = { path = "../pubsys-config/", version = "0.1.0" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_plain = "1" +terminal_size = "0.1" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "fs"] } +unescape = "0.1.0" diff --git a/tools/testsys/src/aws_resources.rs b/tools/testsys/src/aws_resources.rs new file mode 100644 index 00000000000..4537cb29698 --- /dev/null +++ b/tools/testsys/src/aws_resources.rs @@ -0,0 +1,229 @@ +use crate::run::{TestType, TestsysImages}; +use anyhow::{anyhow, Context, Result}; +use bottlerocket_types::agent_config::{ + ClusterType, CreationPolicy, Ec2Config, EksClusterConfig, K8sVersion, SonobuoyConfig, + SonobuoyMode, +}; + +use bottlerocket_variant::Variant; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use k8s_openapi::serde_json::Value; +use maplit::btreemap; +use model::constants::NAMESPACE; +use model::{ + Agent, Configuration, Crd, DestructionPolicy, Resource, ResourceSpec, SecretName, Test, + TestSpec, +}; +use std::collections::BTreeMap; + +pub(crate) struct AwsK8s { + pub(crate) arch: String, + pub(crate) variant: String, + pub(crate) region: String, + pub(crate) assume_role: Option, + pub(crate) instance_type: Option, + pub(crate) ami: String, + pub(crate) secrets: Option>, + pub(crate) kube_conformance_image: Option, + pub(crate) target_cluster_name: Option, +} + +impl AwsK8s { + /// Create the necessary test and resource crds for the specified test type. + pub(crate) fn create_crds( + &self, + test: TestType, + testsys_images: &TestsysImages, + ) -> Result> { + match test { + TestType::Conformance => { + self.sonobuoy_test_crds(testsys_images, SonobuoyMode::CertifiedConformance) + } + TestType::Quick => self.sonobuoy_test_crds(testsys_images, SonobuoyMode::Quick), + } + } + + fn sonobuoy_test_crds( + &self, + testsys_images: &TestsysImages, + sonobuoy_mode: SonobuoyMode, + ) -> Result> { + let crds = vec![ + self.eks_crd("", testsys_images)?, + self.ec2_crd("", testsys_images)?, + self.sonobuoy_crd("", "-test", sonobuoy_mode, None, testsys_images)?, + ]; + Ok(crds) + } + + /// Labels help filter test results with `testsys status`. + fn labels(&self) -> BTreeMap { + btreemap! { + "testsys/arch".to_string() => self.arch.to_string(), + "testsys/variant".to_string() => self.variant.to_string(), + } + } + + fn kube_arch(&self) -> String { + self.arch.replace('_', "-") + } + + fn kube_variant(&self) -> String { + self.variant.replace('.', "") + } + + /// Bottlerocket cluster naming convention. + fn cluster_name(&self, suffix: &str) -> String { + self.target_cluster_name + .clone() + .unwrap_or_else(|| format!("{}-{}{}", self.kube_arch(), self.kube_variant(), suffix)) + } + + fn eks_crd(&self, cluster_suffix: &str, testsys_images: &TestsysImages) -> Result { + let cluster_version = K8sVersion::parse( + Variant::new(&self.variant) + .context("The provided variant cannot be interpreted.")? + .version() + .context("aws-k8s variant is missing k8s version")?, + ) + .map_err(|e| anyhow!(e))?; + let cluster_name = self.cluster_name(cluster_suffix); + let eks_crd = Resource { + metadata: ObjectMeta { + name: Some(cluster_name.clone()), + namespace: Some(NAMESPACE.into()), + labels: Some(self.labels()), + ..Default::default() + }, + spec: ResourceSpec { + depends_on: None, + agent: Agent { + name: "eks-provider".to_string(), + image: testsys_images.eks_resource.clone(), + pull_secret: testsys_images.secret.clone(), + keep_running: false, + timeout: None, + configuration: Some( + EksClusterConfig { + cluster_name, + creation_policy: Some(CreationPolicy::IfNotExists), + region: Some(self.region.clone()), + zones: None, + version: Some(cluster_version), + assume_role: self.assume_role.clone(), + } + .into_map() + .context("Unable to convert eks config to map")?, + ), + secrets: self.secrets.clone(), + capabilities: None, + }, + destruction_policy: DestructionPolicy::Never, + }, + status: None, + }; + Ok(Crd::Resource(eks_crd)) + } + + fn ec2_crd(&self, cluster_suffix: &str, testsys_images: &TestsysImages) -> Result { + let cluster_name = self.cluster_name(cluster_suffix); + let mut ec2_config = Ec2Config { + node_ami: self.ami.clone(), + instance_count: Some(2), + instance_type: self.instance_type.clone(), + cluster_name: format!("${{{}.clusterName}}", cluster_name), + region: format!("${{{}.region}}", cluster_name), + instance_profile_arn: format!("${{{}.iamInstanceProfileArn}}", cluster_name), + subnet_id: format!("${{{}.privateSubnetId}}", cluster_name), + cluster_type: ClusterType::Eks, + endpoint: Some(format!("${{{}.endpoint}}", cluster_name)), + certificate: Some(format!("${{{}.certificate}}", cluster_name)), + cluster_dns_ip: Some(format!("${{{}.clusterDnsIp}}", cluster_name)), + security_groups: vec![], + assume_role: self.assume_role.clone(), + } + .into_map() + .context("Unable to create ec2 config")?; + + // TODO - we have change the raw map to reference/template a non string field. + ec2_config.insert( + "securityGroups".to_owned(), + Value::String(format!("${{{}.securityGroups}}", cluster_name)), + ); + + let ec2_resource = Resource { + metadata: ObjectMeta { + name: Some(format!("{}-instances", cluster_name)), + namespace: Some(NAMESPACE.into()), + labels: Some(self.labels()), + ..Default::default() + }, + spec: ResourceSpec { + depends_on: Some(vec![cluster_name]), + agent: Agent { + name: "ec2-provider".to_string(), + image: testsys_images.ec2_resource.clone(), + pull_secret: testsys_images.secret.clone(), + keep_running: false, + timeout: None, + configuration: Some(ec2_config), + secrets: self.secrets.clone(), + capabilities: None, + }, + destruction_policy: DestructionPolicy::OnDeletion, + }, + status: None, + }; + Ok(Crd::Resource(ec2_resource)) + } + + fn sonobuoy_crd( + &self, + cluster_suffix: &str, + test_name_suffix: &str, + sonobuoy_mode: SonobuoyMode, + depends_on: Option>, + testsys_images: &TestsysImages, + ) -> Result { + let cluster_name = self.cluster_name(cluster_suffix); + let ec2_resource_name = format!("{}-instances", cluster_name); + let test_name = format!("{}{}", cluster_name, test_name_suffix); + let sonobuoy = Test { + metadata: ObjectMeta { + name: Some(test_name), + namespace: Some(NAMESPACE.into()), + labels: Some(self.labels()), + ..Default::default() + }, + spec: TestSpec { + resources: vec![ec2_resource_name, cluster_name.to_string()], + depends_on, + retries: Some(5), + agent: Agent { + name: "sonobuoy-test-agent".to_string(), + image: testsys_images.sonobuoy_test.clone(), + pull_secret: testsys_images.secret.clone(), + keep_running: true, + timeout: None, + configuration: Some( + SonobuoyConfig { + kubeconfig_base64: format!("${{{}.encodedKubeconfig}}", cluster_name), + plugin: "e2e".to_string(), + mode: sonobuoy_mode, + kubernetes_version: None, + kube_conformance_image: self.kube_conformance_image.clone(), + assume_role: self.assume_role.clone(), + } + .into_map() + .context("Unable to convert sonobuoy config to `Map`")?, + ), + secrets: self.secrets.clone(), + capabilities: None, + }, + }, + status: None, + }; + + Ok(Crd::Test(sonobuoy)) + } +} diff --git a/tools/testsys/src/delete.rs b/tools/testsys/src/delete.rs new file mode 100644 index 00000000000..09d87921b2b --- /dev/null +++ b/tools/testsys/src/delete.rs @@ -0,0 +1,29 @@ +use anyhow::{Context, Result}; +use clap::Parser; +use futures::TryStreamExt; +use log::info; +use model::test_manager::{DeleteEvent, TestManager}; + +/// Delete all tests and resources from a testsys cluster. +#[derive(Debug, Parser)] +pub(crate) struct Delete {} + +impl Delete { + pub(crate) async fn run(self, client: TestManager) -> Result<()> { + let mut stream = client.delete_all().await.context("Unable to delete all")?; + + while let Some(delete) = stream + .try_next() + .await + .context("A deletion error occured")? + { + match delete { + DeleteEvent::Starting(crd) => println!("Starting delete for {}", crd.name()), + DeleteEvent::Deleted(crd) => println!("Delete finished for {}", crd.name()), + DeleteEvent::Failed(crd) => println!("Delete failed for {}", crd.name()), + } + } + info!("Delete finished"); + Ok(()) + } +} diff --git a/tools/testsys/src/install.rs b/tools/testsys/src/install.rs new file mode 100644 index 00000000000..11e7de59c0f --- /dev/null +++ b/tools/testsys/src/install.rs @@ -0,0 +1,47 @@ +use anyhow::{Context, Result}; +use clap::Parser; +use log::{info, trace}; +use model::test_manager::{ImageConfig, TestManager}; + +/// The install subcommand is responsible for putting all of the necessary components for testsys in +/// a k8s cluster. +#[derive(Debug, Parser)] +pub(crate) struct Install { + /// Controller image pull secret. This is the name of a Kubernetes secret that will be used to + /// pull the container image from a private registry. For example, if you created a pull secret + /// with `kubectl create secret docker-registry regcred` then you would pass + /// `--controller-pull-secret regcred`. + #[clap( + long = "controller-pull-secret", + env = "TESTSYS_CONTROLLER_PULL_SECRET" + )] + secret: Option, + + /// Controller image uri. If not provided the latest released controller image will be used. + #[clap( + long = "controller-uri", + env = "TESTSYS_CONTROLLER_IMAGE", + default_value = "public.ecr.aws/bottlerocket-test-system/controller:v0.0.1" + )] + controller_uri: String, +} + +impl Install { + pub(crate) async fn run(self, client: TestManager) -> Result<()> { + trace!( + "Installing testsys using controller image '{}'", + &self.controller_uri + ); + let controller_image = match (self.secret, self.controller_uri) { + (Some(secret), image) => ImageConfig::WithCreds { secret, image }, + (None, image) => ImageConfig::Image(image), + }; + client.install(controller_image).await.context( + "Unable to install testsys to the cluster. (Some artifacts may be left behind)", + )?; + + info!("testsys components were successfully installed."); + + Ok(()) + } +} diff --git a/tools/testsys/src/logs.rs b/tools/testsys/src/logs.rs new file mode 100644 index 00000000000..07e5b2cb517 --- /dev/null +++ b/tools/testsys/src/logs.rs @@ -0,0 +1,46 @@ +use anyhow::{Context, Error, Result}; +use clap::Parser; +use futures::TryStreamExt; +use model::test_manager::{ResourceState, TestManager}; +use unescape::unescape; + +/// Stream the logs of an object from a testsys cluster. +#[derive(Debug, Parser)] +pub(crate) struct Logs { + /// The name of the test we want logs from. + #[clap(long, conflicts_with = "resource")] + test: Option, + + /// The name of the resource we want logs from. + #[clap(long, conflicts_with = "test", requires = "state")] + resource: Option, + + /// The resource state we want logs for (Creation, Destruction). + #[clap(long = "state", conflicts_with = "test")] + resource_state: Option, + + /// Follow logs + #[clap(long, short)] + follow: bool, +} + +impl Logs { + pub(crate) async fn run(self, client: TestManager) -> Result<()> { + match (self.test, self.resource, self.resource_state) { + (Some(test), None, None) => { + let mut logs = client.test_logs(test, self.follow).await.context("Unable to get logs.")?; + while let Some(line) = logs.try_next().await? { + println!("{}", unescape(&String::from_utf8_lossy(&line)).context("Unable to unescape log string")?); + } + } + (None, Some(resource), Some(state)) => { + let mut logs = client.resource_logs(resource, state, self.follow).await.context("Unable to get logs.")?; + while let Some(line) = logs.try_next().await? { + println!("{}", unescape(&String::from_utf8_lossy(&line)).context("Unable to unescape log string")?); + } + } + _ => return Err(Error::msg("Invalid arguments were provided. Exactly one of `--test` or `--resource` must be given.")), + }; + Ok(()) + } +} diff --git a/tools/testsys/src/main.rs b/tools/testsys/src/main.rs new file mode 100644 index 00000000000..bc86a991f30 --- /dev/null +++ b/tools/testsys/src/main.rs @@ -0,0 +1,111 @@ +use anyhow::{Context, Result}; +use clap::{Parser, Subcommand}; +use delete::Delete; +use env_logger::Builder; +use install::Install; +use log::{debug, error, LevelFilter}; +use logs::Logs; +use model::test_manager::TestManager; +use restart_test::RestartTest; +use run::Run; +use secret::Add; +use status::Status; +use std::path::PathBuf; +use uninstall::Uninstall; + +mod aws_resources; +mod delete; +mod install; +mod logs; +mod restart_test; +mod run; +mod secret; +mod status; +mod uninstall; + +/// A program for running and controlling Bottlerocket tests in a Kubernetes cluster using +/// bottlerocket-test-system +#[derive(Parser, Debug)] +#[clap(about, long_about = None)] +struct TestsysArgs { + #[structopt(global = true, long, default_value = "INFO")] + /// How much detail to log; from least to most: ERROR, WARN, INFO, DEBUG, TRACE + log_level: LevelFilter, + + /// Path to the kubeconfig file for the testsys cluster. Can also be passed with the KUBECONFIG + /// environment variable. + #[clap(long)] + kubeconfig: Option, + + #[clap(subcommand)] + command: Command, +} + +impl TestsysArgs { + async fn run(self) -> Result<()> { + let client = match self.kubeconfig { + Some(path) => TestManager::new_from_kubeconfig_path(&path) + .await + .context(format!( + "Unable to create testsys client using kubeconfig '{}'", + path.display() + ))?, + None => TestManager::new().await.context( + "Unable to create testsys client using KUBECONFIG variable or default kubeconfig", + )?, + }; + match self.command { + Command::Run(run) => run.run(client).await?, + Command::Install(install) => install.run(client).await?, + Command::Delete(delete) => delete.run(client).await?, + Command::Status(status) => status.run(client).await?, + Command::Logs(logs) => logs.run(client).await?, + Command::RestartTest(restart_test) => restart_test.run(client).await?, + Command::Add(add) => add.run(client).await?, + Command::Uninstall(uninstall) => uninstall.run(client).await?, + }; + Ok(()) + } +} + +#[derive(Subcommand, Debug)] +enum Command { + Install(Install), + // We need to box run because it requires significantly more arguments than the other commands. + Run(Box), + Delete(Delete), + Status(Status), + Logs(Logs), + RestartTest(RestartTest), + Add(Add), + Uninstall(Uninstall), +} + +#[tokio::main] +async fn main() { + let args = TestsysArgs::parse(); + init_logger(args.log_level); + debug!("{:?}", args); + if let Err(e) = args.run().await { + error!("{}", e); + std::process::exit(1); + } +} + +/// Initialize the logger with the value passed by `--log-level` (or its default) when the +/// `RUST_LOG` environment variable is not present. If present, the `RUST_LOG` environment variable +/// overrides `--log-level`/`level`. +fn init_logger(level: LevelFilter) { + match std::env::var(env_logger::DEFAULT_FILTER_ENV).ok() { + Some(_) => { + // RUST_LOG exists; env_logger will use it. + Builder::from_default_env().init(); + } + None => { + // RUST_LOG does not exist; use default log level for this crate only. + Builder::new() + .filter(Some(env!("CARGO_CRATE_NAME")), level) + .init(); + } + } +} diff --git a/tools/testsys/src/restart_test.rs b/tools/testsys/src/restart_test.rs new file mode 100644 index 00000000000..cbd4264cd23 --- /dev/null +++ b/tools/testsys/src/restart_test.rs @@ -0,0 +1,21 @@ +use anyhow::{Context, Result}; +use clap::Parser; +use model::test_manager::TestManager; + +/// Restart a test. This will delete the test object from the testsys cluster and replace it with +/// a new, identical test object with a clean state. +#[derive(Debug, Parser)] +pub(crate) struct RestartTest { + /// The name of the test to be restarted. + #[clap()] + test_name: String, +} + +impl RestartTest { + pub(crate) async fn run(self, client: TestManager) -> Result<()> { + client + .restart_test(&self.test_name) + .await + .context(format!("Unable to restart test '{}'", self.test_name)) + } +} diff --git a/tools/testsys/src/run.rs b/tools/testsys/src/run.rs new file mode 100644 index 00000000000..cc881670316 --- /dev/null +++ b/tools/testsys/src/run.rs @@ -0,0 +1,215 @@ +use crate::aws_resources::AwsK8s; +use anyhow::{anyhow, ensure, Context, Result}; +use bottlerocket_variant::Variant; +use clap::Parser; +use log::{debug, info}; +use model::test_manager::TestManager; +use model::SecretName; +use pubsys_config::InfraConfig; +use serde::Deserialize; +use serde_plain::derive_fromstr_from_deserialize; +use std::collections::HashMap; +use std::fs::File; +use std::path::PathBuf; + +/// Run a set of tests for a given arch and variant +#[derive(Debug, Parser)] +pub(crate) struct Run { + /// The type of test to run. Options are `quick` and `conformance`. + test_flavor: TestType, + + /// The architecture to test. Either x86_64 or aarch64. + #[clap(long, env = "BUILDSYS_ARCH")] + arch: String, + + /// The variant to test + #[clap(long, env = "BUILDSYS_VARIANT")] + variant: String, + + /// The path to `Infra.toml` + #[clap(long, env = "PUBLISH_INFRA_CONFIG_PATH", parse(from_os_str))] + infra_config_path: PathBuf, + + /// The path to `amis.json` + #[clap(long, env = "AMI_INPUT")] + ami_input: String, + + /// Override for the region the tests should be run in. If none is provided the first region in + /// Infra.toml will be used. This is the region that the aws client is created with for testing + /// and resource agents. + #[clap(long, env = "TESTSYS_TARGET_REGION")] + target_region: Option, + + /// The name of the cluster for resource agents (eks resource agent, ecs resource agent). Note: + /// This is not the name of the `testsys cluster` this is the name of the cluster that tests + /// should be run on. If no cluster name is provided, the bottlerocket cluster + /// naming convention `-` will be used. + #[clap(long, env = "TESTSYS_TARGET_CLUSTER_NAME")] + target_cluster_name: Option, + + /// The custom kube conformance image that should be used by sonobuoy. This is only applicable + /// for k8s variants. It can be omitted for non-k8s variants and it can be omitted to use the + /// default sonobuoy conformance image. + #[clap(long)] + kube_conformance_image: Option, + + /// The role that should be assumed by the agents + #[clap(long, env = "TESTSYS_ASSUME_ROLE")] + assume_role: Option, + + /// Specify the instance type that should be used. This is only applicable for aws-* variants. + /// It can be omitted for non-aws variants and can be omitted to use default instance types. + #[clap(long)] + instance_type: Option, + + /// Add secrets to the testsys agents (`--secret aws-credentials=my-secret`) + #[clap(long, short, parse(try_from_str = parse_key_val), number_of_values = 1)] + secret: Vec<(String, SecretName)>, + + #[clap(flatten)] + agent_images: TestsysImages, +} + +impl Run { + pub(crate) async fn run(self, client: TestManager) -> Result<()> { + let variant = + Variant::new(&self.variant).context("The provided variant cannot be interpreted.")?; + debug!("Using variant '{}'", variant); + let secrets = if self.secret.is_empty() { + None + } else { + Some(self.secret.into_iter().collect()) + }; + // If a lock file exists, use that, otherwise use Infra.toml or default + let infra_config = InfraConfig::from_path_or_lock(&self.infra_config_path, true) + .context("Unable to read infra config")?; + + let aws = infra_config.aws.unwrap_or_default(); + + // If the user gave an override region, use that, otherwise use the first region from the + // config. + let region = if let Some(region) = self.target_region { + debug!("Using provided region for testing"); + region + } else { + debug!("No region was provided, determining region from `Infra.toml`"); + aws.regions + .clone() + .pop_front() + .context("No region was provided and no regions found in infra config")? + }; + + match variant.family() { + "aws-k8s" => { + debug!("Variant is in 'aws-k8s' family"); + let bottlerocket_ami = ami(&self.ami_input, ®ion)?; + debug!("Using ami '{}'", bottlerocket_ami); + let aws_k8s = AwsK8s { + arch: self.arch, + variant: self.variant, + region, + assume_role: self.assume_role, + instance_type: self.instance_type, + ami: bottlerocket_ami.to_string(), + secrets, + kube_conformance_image: self.kube_conformance_image, + target_cluster_name: self.target_cluster_name, + }; + debug!("Creating crds for aws-k8s testing"); + let crds = aws_k8s.create_crds(self.test_flavor, &self.agent_images)?; + debug!("Adding crds to testsys cluster"); + for crd in crds { + let crd = client + .create_object(crd) + .await + .context("Unable to create object")?; + if let Some(name) = crd.name() { + info!("Successfully added '{}'", name) + }; + } + } + other => { + return Err(anyhow!( + "testsys has not yet added support for the '{}' variant family", + other + )) + } + }; + + Ok(()) + } +} + +fn ami(ami_input: &str, region: &str) -> Result { + let file = File::open(ami_input).context("Unable to open amis.json")?; + let ami_input: HashMap = + serde_json::from_reader(file).context(format!("Unable to deserialize '{}'", ami_input))?; + ensure!(!ami_input.is_empty(), "amis.json is empty"); + Ok(ami_input + .get(region) + .context(format!("ami not found for region '{}'", region))? + .id + .clone()) +} + +fn parse_key_val(s: &str) -> Result<(String, SecretName)> { + let mut iter = s.splitn(2, '='); + let key = iter.next().context("Key is missing")?; + let value = iter.next().context("Value is missing")?; + Ok((key.to_string(), SecretName::new(value)?)) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "lowercase")] +pub(crate) enum TestType { + /// Conformance testing is a full integration test that asserts that Bottlerocket is working for + /// customer workloads. For k8s variants, for example, this will run the full suite of sonobuoy + /// conformance tests. + Conformance, + /// Run a quick test that ensures a basic workload can run on Bottlerocket. For example, on k8s + /// variance this will run sonobuoy in "quick" mode. For ECS variants, this will run a simple + /// ECS task. + Quick, +} + +derive_fromstr_from_deserialize!(TestType); + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct Image { + pub(crate) id: String, + // This is used to deserialize amis.json +} + +#[derive(Debug, Parser)] +pub(crate) struct TestsysImages { + /// Eks resource agent uri. If not provided the latest released resource agent will be used. + #[clap( + long = "eks-resource-agent-image", + env = "TESTSYS_EKS_RESOURCE_AGENT_IMAGE", + default_value = "public.ecr.aws/bottlerocket-test-system/eks-resource-agent:v0.0.1" + )] + pub(crate) eks_resource: String, + + /// Ec2 resource agent uri. If not provided the latest released resource agent will be used. + #[clap( + long = "ec2-resource-agent-image", + env = "TESTSYS_EC2_RESOURCE_AGENT_IMAGE", + default_value = "public.ecr.aws/bottlerocket-test-system/ec2-resource-agent:v0.0.1" + )] + pub(crate) ec2_resource: String, + + /// Sonobuoy test agent uri. If not provided the latest released test agent will be used. + #[clap( + long = "sonobuoy-test-agent-image", + env = "TESTSYS_SONOBUOY_TEST_AGENT_IMAGE", + default_value = "public.ecr.aws/bottlerocket-test-system/sonobuoy-test-agent:v0.0.1" + )] + pub(crate) sonobuoy_test: String, + + /// Images pull secret. This is the name of a Kubernetes secret that will be used to + /// pull the container image from a private registry. For example, if you created a pull secret + /// with `kubectl create secret docker-registry regcred` then you would pass + /// `--images-pull-secret regcred`. + #[clap(long = "images-pull-secret", env = "TESTSYS_IMAGES_PULL_SECRET")] + pub(crate) secret: Option, +} diff --git a/tools/testsys/src/secret.rs b/tools/testsys/src/secret.rs new file mode 100644 index 00000000000..a9d2faa3532 --- /dev/null +++ b/tools/testsys/src/secret.rs @@ -0,0 +1,117 @@ +use anyhow::{Context, Result}; +use clap::Parser; +use model::test_manager::TestManager; +use model::SecretName; + +/// Add a testsys object to the testsys cluster. +#[derive(Debug, Parser)] +pub(crate) struct Add { + #[clap(subcommand)] + command: AddCommand, +} + +#[derive(Debug, Parser)] +enum AddCommand { + /// Add a secret to the testsys cluster. + Secret(AddSecret), +} + +impl Add { + pub(crate) async fn run(self, client: TestManager) -> Result<()> { + match self.command { + AddCommand::Secret(add_secret) => add_secret.run(client).await, + } + } +} + +/// Add a secret to the cluster. +#[derive(Debug, Parser)] +pub(crate) struct AddSecret { + #[clap(subcommand)] + command: Command, +} + +#[derive(Debug, Parser)] +enum Command { + /// Create a secret for image pulls. + Image(AddSecretImage), + /// Create a secret from key value pairs. + Map(AddSecretMap), +} + +impl AddSecret { + pub(crate) async fn run(self, client: TestManager) -> Result<()> { + match self.command { + Command::Image(add_secret_image) => add_secret_image.run(client).await, + Command::Map(add_secret_map) => add_secret_map.run(client).await, + } + } +} + +/// Add a `Secret` with key value pairs. +#[derive(Debug, Parser)] +pub(crate) struct AddSecretMap { + /// Name of the secret + #[clap(short, long)] + name: SecretName, + + /// Key value pairs for secrets. (Key=value) + #[clap(parse(try_from_str = parse_key_val))] + args: Vec<(String, String)>, +} + +impl AddSecretMap { + pub(crate) async fn run(self, client: TestManager) -> Result<()> { + client + .create_secret(&self.name, self.args) + .await + .context("Unable to create secret")?; + println!("Successfully added '{}' to secrets.", self.name); + Ok(()) + } +} + +fn parse_key_val(s: &str) -> Result<(String, String)> { + let mut iter = s.splitn(2, '='); + let key = iter.next().context("Key is missing")?; + let value = iter.next().context("Value is missing")?; + Ok((key.to_string(), value.to_string())) +} + +/// Add a secret to the testsys cluster for image pulls. +#[derive(Debug, Parser)] +pub(crate) struct AddSecretImage { + /// Controller image pull username + #[clap(long, short = 'u')] + pull_username: String, + + /// Controller image pull password + #[clap(long, short = 'p')] + pull_password: String, + + /// Image uri + #[clap(long = "image-uri", short)] + image_uri: String, + + /// Controller image uri + #[clap(long, short = 'n')] + secret_name: String, +} + +impl AddSecretImage { + pub(crate) async fn run(self, client: TestManager) -> Result<()> { + client + .create_image_pull_secret( + &self.secret_name, + &self.pull_username, + &self.pull_password, + &self.image_uri, + ) + .await + .context("Unable to create pull secret")?; + + println!("The secret was added."); + + Ok(()) + } +} diff --git a/tools/testsys/src/status.rs b/tools/testsys/src/status.rs new file mode 100644 index 00000000000..8f6df68477b --- /dev/null +++ b/tools/testsys/src/status.rs @@ -0,0 +1,55 @@ +use anyhow::{Context, Result}; +use clap::Parser; +use log::{debug, info}; +use model::test_manager::{SelectionParams, TestManager}; +use terminal_size::{Height, Width}; + +/// Check the status of testsys objects. +#[derive(Debug, Parser)] +pub(crate) struct Status { + /// Output the results in JSON format. + #[clap(long = "json")] + json: bool, + + /// Check the status of the testsys controller + #[clap(long, short = 'c')] + controller: bool, + + /// Focus status on a particular arch + #[clap(long)] + arch: Option, + + /// Focus status on a particular variant + #[clap(long)] + variant: Option, +} + +impl Status { + pub(crate) async fn run(self, client: TestManager) -> Result<()> { + let mut labels = Vec::new(); + if let Some(arch) = self.arch { + labels.push(format!("testsys/arch={}", arch)) + }; + if let Some(variant) = self.variant { + labels.push(format!("testsys/variant={}", variant)) + }; + let status = client + .status(&SelectionParams::Label(labels.join(",")), self.controller) + .await + .context("Unable to get status")?; + + if self.json { + info!( + "{}", + serde_json::to_string_pretty(&status) + .context("Could not create string from status.")? + ); + } else { + let (terminal_size::Width(width), _) = + terminal_size::terminal_size().unwrap_or((Width(80), Height(0))); + debug!("Window width '{}'", width); + println!("{}", status.to_string(width as usize)); + } + Ok(()) + } +} diff --git a/tools/testsys/src/uninstall.rs b/tools/testsys/src/uninstall.rs new file mode 100644 index 00000000000..aa4b8961afa --- /dev/null +++ b/tools/testsys/src/uninstall.rs @@ -0,0 +1,23 @@ +use anyhow::{Context, Result}; +use clap::Parser; +use log::{info, trace}; +use model::test_manager::TestManager; + +/// The uninstall subcommand is responsible for removing all of the components for testsys in +/// a k8s cluster. This is completed by removing the `testsys-bottlerocket-aws` namespace. +#[derive(Debug, Parser)] +pub(crate) struct Uninstall {} + +impl Uninstall { + pub(crate) async fn run(self, client: TestManager) -> Result<()> { + trace!("Uninstalling testsys"); + + client.uninstall().await.context( + "Unable to uninstall testsys from the cluster. (Some artifacts may be left behind)", + )?; + + info!("testsys components were successfully uninstalled."); + + Ok(()) + } +}