From 1ea0103ee17a1a057e793634e12261f639731f32 Mon Sep 17 00:00:00 2001 From: Guillermo Sanchez Gavier Date: Wed, 3 Aug 2022 18:33:53 +0200 Subject: [PATCH 1/4] add e2e test based on local environment --- .github/workflows/push_pr.yaml | 17 ++++++++++++----- README.md | 4 ++++ Tiltfile | 9 +++++++-- test/e2e/metrics.yml | 24 ++++++++++++++++++++++++ test/e2e/test-specs.yml | 21 +++++++++++++++++++++ values-dev.yaml | 10 ++++++++-- 6 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 test/e2e/metrics.yml create mode 100644 test/e2e/test-specs.yml diff --git a/.github/workflows/push_pr.yaml b/.github/workflows/push_pr.yaml index e1d7d324..bcd90e01 100644 --- a/.github/workflows/push_pr.yaml +++ b/.github/workflows/push_pr.yaml @@ -70,8 +70,8 @@ jobs: # if: steps.list-changed.outputs.changed == 'true' # run: ct install --namespace ct --config .github/ct.yaml --debug --upgrade - tilt: - name: Check testing environment + e2e-test: + name: E2e Tests runs-on: ubuntu-latest # If the environment is broken this job could timeout since the default timeout for tilt ci is 30m. timeout-minutes: 10 @@ -91,9 +91,16 @@ jobs: - name: Install Tilt run: | curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash - - name: Run Tilt ci - run: | - make tilt-ci + - name: Run e2e-test + uses: newrelic/newrelic-integration-e2e-action@v1 + with: + retry_seconds: 60 + retry_attempts: 5 + agent_enabled: false + spec_path: test/e2e/test-specs.yml + account_id: ${{ secrets.COREINT_E2E_ACCOUNT_ID }} + api_key: ${{ secrets.COREINT_E2E_API_KEY }} + license_key: ${{ secrets.COREINT_E2E_LICENSE_KEY }} test: name: Run tests diff --git a/README.md b/README.md index 74c72085..55f30829 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,12 @@ Make sure you have these tools or install them: - [Install Tilt](https://docs.tilt.dev/install.html) - [Install Helm](https://helm.sh/docs/intro/install/) +A license key and cluster name are required to run the environment. Configure them by the environment variables `NR_PROM_CLUSTER` and `NR_PROM_LICENSE_KEY`. + Start the local environment: ```shell +export NR_PROM_CLUSTER= +export NR_PROM_LICENSE_KEY= make start-local-cluster make tilt-up ``` diff --git a/Tiltfile b/Tiltfile index 97506214..3ea611b9 100644 --- a/Tiltfile +++ b/Tiltfile @@ -9,7 +9,6 @@ allow_k8s_contexts(cluster_context) local_resource('Configurator binary', 'make compile-multiarch', deps=[ './cmd', - './internal', './configurator', ]) @@ -18,7 +17,13 @@ docker_build('prometheus-configurator', '.') docker_build('openmetrics-fake-exporter', './test/openmetrics-fake-exporter/.') # Deploying Kubernetes resources. -k8s_yaml(helm('./charts/%s' % project_name, name=project_name, values=['values-dev.yaml'])) +k8s_yaml( + helm( + './charts/%s' % project_name, + name=project_name, + values=['values-dev.yaml'], + set=['licenseKey=%s' % os.getenv('NR_PROM_LICENSE_KEY'), 'cluster=%s' % os.getenv('NR_PROM_CLUSTER')], + )) k8s_yaml(helm('./charts/internal/test-resources', name='test-resources')) # Tracking the deployment. diff --git a/test/e2e/metrics.yml b/test/e2e/metrics.yml new file mode 100644 index 00000000..eeceb5ee --- /dev/null +++ b/test/e2e/metrics.yml @@ -0,0 +1,24 @@ +entities: + - entityType: FakeSelfMetrics + metrics: + - name: prometheus_agent_active_series + - name: prometheus_target_interval_length_seconds + - name: prometheus_target_scrape_pool_targets + - name: prometheus_remote_storage_samples_pending + - name: prometheus_remote_storage_samples_in_total + - name: prometheus_remote_storage_samples_retried_total + - name: prometheus_agent_corruptions_total + - name: prometheus_remote_storage_shards + - name: prometheus_sd_kubernetes_events_total + - name: prometheus_agent_checkpoint_creations_failed_total + - name: prometheus_agent_checkpoint_deletions_failed_total + - name: prometheus_remote_storage_dropped_samples_total + - name: prometheus_remote_storage_failed_samples_total + - name: prometheus_sd_kubernetes_http_request_total + - name: prometheus_agent_truncate_duration_seconds_sum + - name: prometheus_build_info + - name: process_resident_memory_bytes + - name: process_virtual_memory_bytes + - name: process_cpu_seconds_total + - name: scrape_pool_targets + - name: scrape_duration_seconds diff --git a/test/e2e/test-specs.yml b/test/e2e/test-specs.yml new file mode 100644 index 00000000..c0a7bdd5 --- /dev/null +++ b/test/e2e/test-specs.yml @@ -0,0 +1,21 @@ +description: Newrelic Prometheus Configurator E2E Test + +custom_test_key: cluster_name + +scenarios: + - description: Scrape test environment metrics + before: + - cd ../../ && NR_PROM_LICENSE_KEY=${LICENSE_KEY} NR_PROM_CLUSTER=${SCENARIO_TAG} make tilt-ci + after: + - tilt down + tests: + metrics: + - source: "metrics.yml" + except_metrics: + # TODO check why this ones don't appear + - prometheus_remote_storage_dropped_samples_total + - prometheus_remote_storage_failed_samples_total + - scrape_pool_targets + + # Need to activate kubernetes_sd for this metric to appear + - prometheus_sd_kubernetes_http_request_total diff --git a/values-dev.yaml b/values-dev.yaml index eac98a5f..e8619d02 100644 --- a/values-dev.yaml +++ b/values-dev.yaml @@ -4,5 +4,11 @@ images: pullPolicy: Never tag: "latest" -cluster: tilt-local -licenseKey: fake-license +# Configure your and by the env vars NR_PROM_CLUSTER and NR_PROM_LICENSE_KEY +# cluster: +# licenseKey: +# nrStaging: true + +config: + common: + scrape_interval: 15s From 67a9ae4a493bdfd1e5f5d1c53dda34ad433e2d55 Mon Sep 17 00:00:00 2001 From: Guillermo Sanchez Gavier Date: Thu, 4 Aug 2022 13:56:50 +0200 Subject: [PATCH 2/4] clean environment before running tests --- configurator/parser_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/configurator/parser_test.go b/configurator/parser_test.go index deeb5cdd..bd27d415 100644 --- a/configurator/parser_test.go +++ b/configurator/parser_test.go @@ -20,8 +20,10 @@ import ( _ "github.com/prometheus/prometheus/discovery/kubernetes" ) +// nolint: paralleltest,tparallel func TestParser(t *testing.T) { - t.Parallel() + t.Setenv(configurator.LicenseKeyEnvKey, "") + t.Setenv(configurator.DataSourceNameEnvKey, "") // it relies on testdata/.yaml and testdata/.expected.yaml testCases := []string{ @@ -51,6 +53,9 @@ func TestParser(t *testing.T) { //nolint: tparallel func TestDataSourceName(t *testing.T) { + t.Setenv(configurator.LicenseKeyEnvKey, "") + t.Setenv(configurator.DataSourceNameEnvKey, "") + configWithDataSourceName := ` data_source_name: %s newrelic_remote_write: @@ -79,6 +84,9 @@ newrelic_remote_write: //nolint: tparallel func TestLicenseKey(t *testing.T) { + t.Setenv(configurator.LicenseKeyEnvKey, "") + t.Setenv(configurator.DataSourceNameEnvKey, "") + configWithLicense := ` newrelic_remote_write: license_key: %s From b4af3dd7dac4df47bef61dcf5749b8319a72a51e Mon Sep 17 00:00:00 2001 From: Guillermo Sanchez Gavier Date: Fri, 5 Aug 2022 09:55:04 +0200 Subject: [PATCH 3/4] docs --- Makefile | 13 +++++++++++++ README.md | 22 +++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 68a96682..892751e8 100644 --- a/Makefile +++ b/Makefile @@ -74,3 +74,16 @@ chart-unit-test: ct --config .github/ct.yaml lint --debug helm dependency update ./charts/newrelic-prometheus helm unittest ./charts/newrelic-prometheus -3 + +.PHONY: e2e-test +e2e-test: + newrelic-integration-e2e \ + --commit_sha=test-string \ + --retry_attempts=5 \ + --retry_seconds=60 \ + --account_id=${ACCOUNT_ID} \ + --api_key=${API_REST_KEY} \ + --license_key=${LICENSE_KEY} \ + --spec_path=./test/e2e/test-specs.yml \ + --verbose_mode=true \ + --agent_enabled="false" diff --git a/README.md b/README.md index 55f30829..7dc9c1f0 100644 --- a/README.md +++ b/README.md @@ -45,13 +45,33 @@ A license key and cluster name are required to run the environment. Configure th Start the local environment: ```shell export NR_PROM_CLUSTER= -export NR_PROM_LICENSE_KEY= +export NR_PROM_LICENSE_KEY= make start-local-cluster make tilt-up ``` Notice that local images are build and pushed to docker running inside the minikube cluster since we are running `eval $(minikube docker-env)` before launching Tilt. +### Running e2e tests + +This test are based on the `newrelic-integration-e2e`. This tool will start the local environment and check if the expected metrics has been reach the NewRelic platform. + +To run it locally you should already be able to run the local cluster and apart from that you must install the `newrelic-integration-e2e` binary: +```bash +git clone https://github.com/newrelic/newrelic-integration-e2e-action +cd newrelic-integration-e2e-action/newrelic-integration-e2e +go build -o $GOPATH/bin/newrelic-integration-e2e ./cmd/... +``` + +Then run: +```bash +export ACCOUNT_ID= +export API_REST_KEY= +export LICENSE_KEY= +make start-local-cluster +make e2e-test +``` + ## Support New Relic hosts and moderates an online forum where customers can interact with New Relic employees as well as other customers to get help and share best practices. Like all official New Relic open source projects, there's a related Community topic in the New Relic Explorers Hub. You can find this project's topic/threads here: From a91741f67143fc92735bbcd5da71012c748721ec Mon Sep 17 00:00:00 2001 From: Guillermo Sanchez Gavier Date: Fri, 5 Aug 2022 10:29:44 +0200 Subject: [PATCH 4/4] fix lint issue --- configurator/parser_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configurator/parser_test.go b/configurator/parser_test.go index bd27d415..abf32fea 100644 --- a/configurator/parser_test.go +++ b/configurator/parser_test.go @@ -20,7 +20,7 @@ import ( _ "github.com/prometheus/prometheus/discovery/kubernetes" ) -// nolint: paralleltest,tparallel +//nolint: paralleltest,tparallel func TestParser(t *testing.T) { t.Setenv(configurator.LicenseKeyEnvKey, "") t.Setenv(configurator.DataSourceNameEnvKey, "")