Cleanup: remove yaml tests and TEST_SUITE (#263)

Author: swallez

.buildkite/DockerFile

@@ -25,7 +25,6 @@ COPY api_generator ./api_generator
 COPY elasticsearch/Cargo.toml ./elasticsearch/Cargo.toml
 COPY elasticsearch/src ./elasticsearch/src
 COPY elasticsearch/build.rs ./elasticsearch/build.rs
-COPY yaml_test_runner ./yaml_test_runner
 COPY xtask ./xtask
 
 RUN cargo build --tests

.buildkite/functions/imports.sh

@@ -6,6 +6,9 @@
 # - Initial version after refactor
 # - Validate STACK_VERSION asap
 
+# Hardcoded, there's no more a distinction between "free" and "platinum" and security is on by default
+export TEST_SUITE="platinum"
+
 function require_stack_version() {
   if [[ -z $STACK_VERSION ]]; then
     echo -e "\033[31;1mERROR:\033[0m Required environment variable [STACK_VERSION] not set\033[0m"

.buildkite/pipeline.yml

@@ -4,14 +4,11 @@ steps:
       provider: "gcp"
     env:
       STACK_VERSION: "{{ matrix.stack_version }}"
-      TEST_SUITE: "{{ matrix.test_suite }}"
       RUST_TOOLCHAIN: "{{ matrix.toolchain }}"
     matrix:
       setup:
         stack_version:
           - "9.0.0"
-        test_suite:
-          - "platinum"
         toolchain:
           - "latest"
     command: ./.buildkite/run-tests

.dockerignore

@@ -9,6 +9,3 @@ docs
 *.iml
 
 api_generator/src/bin
-
-yaml_test_runner/tests
-!yaml_test_runner/tests/common

.gitignore

@@ -6,9 +6,5 @@ pkg
 *.iml
 .vscode/
 *.log
-yaml_test_runner/tests/free
-yaml_test_runner/tests/xpack
-yaml_test_runner/tests/platinum
-yaml_test_runner/tests/mod.rs
 test_results/
 checkout/

CONTRIBUTING.md

@@ -76,18 +76,6 @@ The `Elasticsearch` category of steps are specifically defined for this project
   cargo make test --env STACK_VERSION=7.9.0
   ```
 
-- Run YAML tests
-
-  Optionally pass
-
-  - `STACK_VERSION`: Elasticsearch version like `7.9.0` or can be
-  a snapshot release like `7.x-SNAPSHOT`
-  - `TEST_SUITE`: Elasticsearch distribution of `free` or `platinum`
-
-  ```sh
-  cargo make test-yaml --env STACK_VERSION=7.9.0 --env TEST_SUITE=free
-  ```
-
 ### Packages
 
 The workspace contains the following packages:
@@ -109,15 +97,6 @@ The `quote!` macro is particularly useful as it accepts Rust code that can inclu
 that will be interpolated during expansion. Unlike procedural macros, the token stream returned by the `quote!` macro
 can be `to_string()`'ed and written to disk, and this is used to create much of the client scaffolding.
 
-- #### `yaml_test_runner`
-
-  A small executable that downloads YAML tests from GitHub and generates client tests from the YAML tests. The
-  version of YAML tests to download are determined from the commit hash of a running Elasticsearch instance.
-
-  The `yaml_test_runner` package can be run with `cargo make test-yaml` to run the generated client tests,
-  passing environment variables `TEST_SUITE` and `STACK_VERSION` to control the distribution and version,
-  respectively.
-
 ### Design principles
 
 1. Generate as much of the client as feasible from the REST API specs

Cargo.lock

@@ -2,7 +2,6 @@
 members = [
     "api_generator",
     "elasticsearch",
-    "yaml_test_runner",
     "xtask"
 ]
 

Cargo.toml

@@ -1,29 +1,21 @@
 [config]
 default_to_workspace = false
+init_task = "init"
 
 [env]
-# Determines the version of Elasticsearch docker container used
-STACK_VERSION = { value = "9.0.0-SNAPSHOT", condition = { env_not_set = ["STACK_VERSION"] }}
-# Determines the distribution of docker container used. Either platinum or free
-TEST_SUITE = { value = "free", condition = { env_not_set = ["TEST_SUITE"] }}
 # Set publish flags to dry-run by default, to force user to explicitly define for publishing
 CARGO_MAKE_CARGO_PUBLISH_FLAGS = "--dry-run"
 # RUST_BACKTRACE is set to "full" in cargo make's builtin makefiles/stable.toml
 RUST_BACKTRACE = { value = "0", condition = { env_not_set = ["RUST_BACKTRACE"]}}
+ELASTICSEARCH_URL = "https://elastic:changeme@localhost:9200"
 
-[tasks.set-free-env]
-category = "Elasticsearch"
-description = "Sets ELASTICSEARCH_URL environment variable if not already set for later tasks when free test suite used"
-private = true
-condition = { env = { "TEST_SUITE" = "free" }, env_not_set = ["ELASTICSEARCH_URL"] }
-env = { "ELASTICSEARCH_URL" = "http://elastic:changeme@localhost:9200" }
-
-[tasks.set-platinum-env]
-category = "Elasticsearch"
-description = "Sets ELASTICSEARCH_URL environment variable if not already set for later tasks when platinum test suite used"
+[tasks.init]
 private = true
-condition = { env = { "TEST_SUITE" = "platinum" }, env_not_set = ["ELASTICSEARCH_URL"] }
-env = { "ELASTICSEARCH_URL" = "https://elastic:changeme@localhost:9200" }
+condition = { env_not_set = ["STACK_VERSION"] }
+script = [
+    'echo "Environment variable STACK_VERSION is required but not set."',
+    'exit 1',
+]
 
 [tasks.download-specs]
 category = "Elasticsearch"
@@ -35,36 +27,9 @@ command = "cargo"
 args = ["run", "-q", "-p", "xtask", "--", "download-specs", "--url", "${ELASTICSEARCH_URL}"]
 dependencies = ["start-elasticsearch"]
 
-[tasks.run-yaml-test-runner]
-category = "Elasticsearch"
-description = '''
-Runs yaml_test_runner package to generate tests from yaml files for a given Elasticsearch commit.
-The commit to use is retrieved from the running Elasticsearch instance
-'''
-private = true
-command = "cargo"
-args = ["run", "-p", "yaml_test_runner", "--", "-u", "${ELASTICSEARCH_URL}"]
-dependencies = ["download-specs"]
-
-[tasks.test-yaml-test-runner]
-category = "Elasticsearch"
-private = true
-condition = { env_set = [ "ELASTICSEARCH_URL" ], env_false = ["CARGO_MAKE_CI"] }
-command = "cargo"
-args = ["test", "-p", "yaml_test_runner", "--", "--test-threads", "1"]
-dependencies = ["generate-yaml-tests"]
-
-[tasks.test-yaml-test-runner-ci]
-category = "Elasticsearch"
-private = true
-condition = { env_set = [ "ELASTICSEARCH_URL" ], env_true = ["CARGO_MAKE_CI"] }
-script = ["cargo test -p yaml_test_runner -- --test-threads 1 | tee test_results/results.txt"]
-dependencies = ["generate-yaml-tests"]
-
 [tasks.test-elasticsearch]
 category = "Elasticsearch"
 private = true
-condition = { env_set = [ "ELASTICSEARCH_URL" ], env = { "TEST_SUITE" = "platinum" } }
 command = "cargo"
 args = ["test", "-p", "elasticsearch"]
 dependencies = ["start-elasticsearch"]
@@ -97,8 +62,7 @@ script = ["[ -d test_results ] || mkdir -p test_results"]
 [tasks.run-elasticsearch]
 category = "Elasticsearch"
 private = true
-condition = { env_set = [ "STACK_VERSION", "TEST_SUITE" ], env_false = ["CARGO_MAKE_CI"] }
-dependencies = ["set-free-env", "set-platinum-env"]
+condition = { env_set = [ "STACK_VERSION" ], env_false = ["CARGO_MAKE_CI"] }
 
 [tasks.run-elasticsearch.linux]
 command = "./.buildkite/run-elasticsearch.sh"
@@ -110,7 +74,7 @@ command = "./.buildkite/run-elasticsearch.sh"
 script_runner = "cmd"
 script = [
 '''
-bash -c "STACK_VERSION=%STACK_VERSION% TEST_SUITE=%TEST_SUITE% DETACH=%DETACH% CLEANUP=%CLEANUP% bash .buildkite/run-elasticsearch.sh"
+bash -c "STACK_VERSION=%STACK_VERSION% DETACH=%DETACH% CLEANUP=%CLEANUP% bash .buildkite/run-elasticsearch.sh"
 '''
 ]
 
@@ -147,13 +111,6 @@ private = false
 description = "Stops Elasticsearch docker container, if running"
 env = { "CLEANUP" = true, "DETACH" = false }
 
-[tasks.test-yaml]
-category = "Elasticsearch"
-description = "Generates and runs yaml_test_runner package platinum/free tests against a given Elasticsearch version"
-condition = { env_set = [ "STACK_VERSION", "TEST_SUITE" ] }
-dependencies = ["generate-yaml-tests", "create-test-results-dir", "test-yaml-test-runner", "test-yaml-test-runner-ci"]
-run_task = "stop-elasticsearch"
-
 [tasks.test-generator]
 category = "Elasticsearch"
 clear = true
@@ -165,16 +122,9 @@ args = ["test", "-p", "api_generator"]
 category = "Elasticsearch"
 clear = true
 description = "Runs elasticsearch package tests against a given Elasticsearch version"
-env = { "TEST_SUITE" = { value = "platinum", condition = { env_set = ["TEST_SUITE"] } } }
 dependencies = ["test-elasticsearch"]
 run_task = "stop-elasticsearch"
 
-[tasks.generate-yaml-tests]
-category = "Elasticsearch"
-description = "Generates Elasticsearch client tests from YAML tests"
-dependencies = ["run-yaml-test-runner"]
-run_task = "format"
-
 [tasks.generate-api]
 category = "Elasticsearch"
 description = "Generates Elasticsearch client from REST API specs"
@@ -343,7 +293,6 @@ script = ['''
     echo - start-elasticsearch: Starts Elasticsearch docker container with the given version and distribution
     echo - stop-elasticsearch: Stops Elasticsearch docker container, if running
     echo
-    echo - test-yaml: Generates and runs yaml_test_runner package platinum/free tests against a given Elasticsearch version
     echo - test-generator: Generates and runs api_generator package tests
     echo - test: Runs elasticsearch package tests against a given Elasticsearch version
     echo
@@ -358,7 +307,6 @@ script = ['''
     echo
     echo Most tasks use these environment variables:
     echo - STACK_VERSION (default '${STACK_VERSION}'): the version of Elasticsearch
-    echo - TEST_SUITE ('free' or 'platinum', default '${TEST_SUITE}'): the distribution of Elasticsearch
     echo - CI (default not set): set when running on CI to determine whether to start Elasticsearch and format test output as JSON
     echo
     echo Run 'cargo make --list-all-steps' for a complete list of available tasks.