Allow to run Elastic stack with other licenses

.buildkite/pipeline.trigger.integration.tests.sh

@@ -22,6 +22,8 @@ STACK_COMMAND_TESTS=(
     test-stack-command-8x
     test-stack-command-9x
     test-stack-command-with-apm-server
+    test-stack-command-with-basic-subscription
+    test-stack-command-with-self-monitor
 )
 
 for test in "${STACK_COMMAND_TESTS[@]}"; do

Makefile

@@ -84,6 +84,9 @@ test-stack-command-with-apm-server:
 test-stack-command-with-self-monitor:
 	SELF_MONITOR_ENABLED=true ./scripts/test-stack-command.sh
 
+test-stack-command-with-basic-subscription:
+	ELASTIC_SUBSCRIPTION=basic ./scripts/test-stack-command.sh
+
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x test-stack-command-9x test-stack-command-with-apm-server
 
 test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-logstash

README.md

@@ -652,6 +652,9 @@ The following settings are available per profile:
   the serverless stack provider.
 * `stack.serverless.region` can be used to select the region to use when starting
   serverless projects.
+* `stack.elastic_subscription` allows to select the Elastic subscription type to be used in the stack.
+  Currently, it is supported "basic" and "[trial](https://www.elastic.co/guide/en/elasticsearch/reference/current/start-trial.html)",
+  which enables all subscription features for 30 days.  Defaults to "trial".
 
 ## Useful environment variables
 

internal/profile/_static/config.yml.example

@@ -27,3 +27,5 @@
 # stack.agent.ports:
 # - 127.0.0.1:1514:1514/udp
 
+## Set license subscription
+# stack.elastic_subscription: "basic"

internal/profile/testdata/config.yml

@@ -2,6 +2,7 @@
 stack.geoip_dir: "/home/foo/Documents/ingest-geoip"
 stack.apm_enabled: true
 stack.logstash_enabled: true
+stack.elastic_subscription: basic
 
 # An empty string, should exist, but return empty.
 other.empty: ""

internal/stack/_static/elasticsearch.yml.tmpl

@@ -4,7 +4,8 @@ http.host: "0.0.0.0"
 
 indices.id_field_data.enabled: true
 
-xpack.license.self_generated.type: "trial"
+{{ $elastic_subscription := fact "elastic_subscription" }}
+xpack.license.self_generated.type: "{{ $elastic_subscription }}"
 xpack.security.enabled: true
 xpack.security.authc.api_key.enabled: true
 xpack.security.http.ssl.enabled: true

internal/stack/_static/kibana.yml.tmpl

@@ -161,3 +161,7 @@ xpack.fleet.outputs:
 {{- if eq $version "9.0.0-SNAPSHOT" }}
 xpack.fleet.internal.registry.kibanaVersionCheckEnabled: false
 {{- end }}
+
+logging.loggers:
+  - name: plugins.fleet
+    level: debug

internal/stack/resources.go

@@ -11,6 +11,7 @@ import (
 	"html/template"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"github.com/Masterminds/semver/v3"
@@ -57,12 +58,13 @@ const (
 	elasticsearchUsername = "elastic"
 	elasticsearchPassword = "changeme"
 
-	configAPMEnabled         = "stack.apm_enabled"
-	configGeoIPDir           = "stack.geoip_dir"
-	configKibanaHTTP2Enabled = "stack.kibana_http2_enabled"
-	configLogsDBEnabled      = "stack.logsdb_enabled"
-	configLogstashEnabled    = "stack.logstash_enabled"
-	configSelfMonitorEnabled = "stack.self_monitor_enabled"
+	configAPMEnabled          = "stack.apm_enabled"
+	configGeoIPDir            = "stack.geoip_dir"
+	configKibanaHTTP2Enabled  = "stack.kibana_http2_enabled"
+	configLogsDBEnabled       = "stack.logsdb_enabled"
+	configLogstashEnabled     = "stack.logstash_enabled"
+	configSelfMonitorEnabled  = "stack.self_monitor_enabled"
+	configElasticSubscription = "stack.elastic_subscription"
 )
 
 var (
@@ -135,6 +137,11 @@ var (
 			Content: staticSource.File("_static/Dockerfile.logstash"),
 		},
 	}
+
+	elasticSubscriptionsSupported = []string{
+		"basic",
+		"trial",
+	}
 )
 
 func applyResources(profile *profile.Profile, stackVersion string) error {
@@ -145,6 +152,11 @@ func applyResources(profile *profile.Profile, stackVersion string) error {
 		return fmt.Errorf("failed to unmarshal stack.agent.ports: %w", err)
 	}
 
+	elasticSubscriptionProfile := profile.Config(configElasticSubscription, "trial")
+	if !slices.Contains(elasticSubscriptionsSupported, elasticSubscriptionProfile) {
+		return fmt.Errorf("unsupported Elastic subscription %q: supported subscriptions: %s", elasticSubscriptionProfile, strings.Join(elasticSubscriptionsSupported, ", "))
+	}
+
 	resourceManager := resource.NewManager()
 	resourceManager.AddFacter(resource.StaticFacter{
 		"registry_base_image":   PackageRegistryBaseImage,
@@ -168,6 +180,7 @@ func applyResources(profile *profile.Profile, stackVersion string) error {
 		"logsdb_enabled":       profile.Config(configLogsDBEnabled, "false"),
 		"logstash_enabled":     profile.Config(configLogstashEnabled, "false"),
 		"self_monitor_enabled": profile.Config(configSelfMonitorEnabled, "false"),
+		"elastic_subscription": elasticSubscriptionProfile,
 	})
 
 	if err := os.MkdirAll(stackDir, 0755); err != nil {

scripts/test-stack-command.sh

@@ -5,6 +5,7 @@ set -euxo pipefail
 VERSION=${1:-default}
 APM_SERVER_ENABLED=${APM_SERVER_ENABLED:-false}
 SELF_MONITOR_ENABLED=${SELF_MONITOR_ENABLED:-false}
+ELASTIC_SUBSCRIPTION=${ELASTIC_SUBSCRIPTION:-""}
 
 cleanup() {
   r=$?
@@ -23,6 +24,10 @@ cleanup() {
     elastic-package profiles delete with-self-monitor
   fi
 
+  if [[ "${ELASTIC_SUBSCRIPTION}" != "" ]]; then
+    elastic-package profiles delete with-elastic-subscription
+  fi
+
   exit $r
 }
 
@@ -71,6 +76,16 @@ stack.self_monitor_enabled: true
 EOF
 fi
 
+if [[ "${ELASTIC_SUBSCRIPTION}" != "" ]]; then
+  profile=with-elastic-subscription
+  elastic-package profiles create -v ${profile}
+  elastic-package profiles use ${profile}
+
+  cat ~/.elastic-package/profiles/${profile}/config.yml.example - <<EOF > ~/.elastic-package/profiles/${profile}/config.yml
+stack.elastic_subscription: ${ELASTIC_SUBSCRIPTION}
+EOF
+fi
+
 mkdir -p "${OUTPUT_PATH_STATUS}"
 
 # Initial status empty
@@ -115,6 +130,8 @@ elastic-package stack status -v 2> "${OUTPUT_PATH_STATUS}/running.txt"
 clean_status_output "${OUTPUT_PATH_STATUS}/expected_running.txt" > "${OUTPUT_PATH_STATUS}/expected_no_spaces.txt"
 clean_status_output "${OUTPUT_PATH_STATUS}/running.txt" > "${OUTPUT_PATH_STATUS}/running_no_spaces.txt"
 
+diff -q "${OUTPUT_PATH_STATUS}/running_no_spaces.txt" "${OUTPUT_PATH_STATUS}/expected_no_spaces.txt"
+
 if [ "${APM_SERVER_ENABLED}" = true ]; then
   curl http://localhost:8200/
 fi
@@ -127,4 +144,17 @@ if [ "${SELF_MONITOR_ENABLED}" = true ]; then
     -f "${ELASTIC_PACKAGE_ELASTICSEARCH_HOST}/metrics-system.*/_search?allow_no_indices=false&size=0"
 fi
 
-diff -q "${OUTPUT_PATH_STATUS}/running_no_spaces.txt" "${OUTPUT_PATH_STATUS}/expected_no_spaces.txt"
+subscription=$(curl -s -S \
+  -u "${ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME}:${ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD}" \
+  --cacert "${ELASTIC_PACKAGE_CA_CERT}" \
+  -f "${ELASTIC_PACKAGE_ELASTICSEARCH_HOST}/_license" |jq -r '.license.type')
+
+expected_subscription="trial"
+if [[ "${ELASTIC_SUBSCRIPTION}" != "" ]]; then
+    expected_subscription="${ELASTIC_SUBSCRIPTION}"
+fi
+
+if [[ "${subscription}" != "${expected_subscription}" ]]; then
+    echo "Unexpected \"${subscription}\" subscription found, but expected \"${expected_subscription}\""
+    exit 1
+fi

test/packages/parallel/apache_basic_license.stack_provider_settings

@@ -0,0 +1 @@
+stack.elastic_subscription=basic

test/packages/parallel/apache_basic_license/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@8.1

test/packages/parallel/apache_basic_license/_dev/build/docs/README.md

@@ -0,0 +1,34 @@
+# Apache Integration
+
+This integration periodically fetches metrics from [Apache](https://httpd.apache.org/) servers. It can parse access and error
+logs created by the Apache server.
+
+## Compatibility
+
+The Apache datasets were tested with Apache 2.4.12 and 2.4.46 and are expected to work with
+all versions >= 2.2.31 and >= 2.4.16 (independent from operating system).
+
+## Logs
+
+### Access Logs
+
+Access logs collects the Apache access logs.
+
+{{fields "access"}}
+
+### Error Logs
+
+Error logs collects the Apache error logs.
+
+{{fields "error"}}
+
+## Metrics
+
+### Status Metrics
+
+The server status stream collects data from the Apache Status module. It scrapes the status data from the web page
+generated by the `mod_status` module.
+
+{{event "status"}}
+
+{{fields "status"}}

test/packages/parallel/apache_basic_license/_dev/deploy/docker/Dockerfile

@@ -0,0 +1,6 @@
+ARG SERVICE_VERSION=${SERVICE_VERSION:-2.4.46}
+FROM httpd:$SERVICE_VERSION
+RUN sed -i "/jessie-updates/d" /etc/apt/sources.list
+RUN apt-get update && apt-get install -y curl
+HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost
+COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf

test/packages/parallel/apache_basic_license/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,11 @@
+version: '2.3'
+services:
+  apache_basic_license:
+    # Commented out `image:` below until we have a process to refresh the hosted images from
+    # Dockerfiles in this repo. Until then, we build the image locally using `build:` below.
+    # image: docker.elastic.co/integrations-ci/beats-apache:${SERVICE_VERSION:-2.4.20}-1
+    build: .
+    ports:
+      - 80
+    volumes:
+      - ${SERVICE_LOGS_DIR}:/usr/local/apache2/logs