diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 9ec2bffbe1..396b7f4d07 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -22,6 +22,8 @@ STACK_COMMAND_TESTS=( test-stack-command-8x test-stack-command-9x test-stack-command-with-apm-server + test-stack-command-with-basic-subscription + test-stack-command-with-self-monitor ) for test in "${STACK_COMMAND_TESTS[@]}"; do diff --git a/Makefile b/Makefile index 4aff454c13..4d9a6bb30d 100644 --- a/Makefile +++ b/Makefile @@ -84,6 +84,9 @@ test-stack-command-with-apm-server: test-stack-command-with-self-monitor: SELF_MONITOR_ENABLED=true ./scripts/test-stack-command.sh +test-stack-command-with-basic-subscription: + ELASTIC_SUBSCRIPTION=basic ./scripts/test-stack-command.sh + test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x test-stack-command-9x test-stack-command-with-apm-server test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-logstash diff --git a/README.md b/README.md index 1a6c103a0c..00fe61b1b8 100644 --- a/README.md +++ b/README.md @@ -652,6 +652,9 @@ The following settings are available per profile: the serverless stack provider. * `stack.serverless.region` can be used to select the region to use when starting serverless projects. +* `stack.elastic_subscription` allows to select the Elastic subscription type to be used in the stack. + Currently, it is supported "basic" and "[trial](https://www.elastic.co/guide/en/elasticsearch/reference/current/start-trial.html)", + which enables all subscription features for 30 days. Defaults to "trial". ## Useful environment variables diff --git a/internal/profile/_static/config.yml.example b/internal/profile/_static/config.yml.example index 2d8d21893c..d9f00f00a8 100644 --- a/internal/profile/_static/config.yml.example +++ b/internal/profile/_static/config.yml.example @@ -27,3 +27,5 @@ # stack.agent.ports: # - 127.0.0.1:1514:1514/udp +## Set license subscription +# stack.elastic_subscription: "basic" diff --git a/internal/profile/testdata/config.yml b/internal/profile/testdata/config.yml index d116928f4d..8ddc703cda 100644 --- a/internal/profile/testdata/config.yml +++ b/internal/profile/testdata/config.yml @@ -2,6 +2,7 @@ stack.geoip_dir: "/home/foo/Documents/ingest-geoip" stack.apm_enabled: true stack.logstash_enabled: true +stack.elastic_subscription: basic # An empty string, should exist, but return empty. other.empty: "" diff --git a/internal/stack/_static/elasticsearch.yml.tmpl b/internal/stack/_static/elasticsearch.yml.tmpl index d796e98247..ce1c1cca24 100644 --- a/internal/stack/_static/elasticsearch.yml.tmpl +++ b/internal/stack/_static/elasticsearch.yml.tmpl @@ -4,7 +4,8 @@ http.host: "0.0.0.0" indices.id_field_data.enabled: true -xpack.license.self_generated.type: "trial" +{{ $elastic_subscription := fact "elastic_subscription" }} +xpack.license.self_generated.type: "{{ $elastic_subscription }}" xpack.security.enabled: true xpack.security.authc.api_key.enabled: true xpack.security.http.ssl.enabled: true diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index 6020c25925..a86be3d45c 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -161,3 +161,7 @@ xpack.fleet.outputs: {{- if eq $version "9.0.0-SNAPSHOT" }} xpack.fleet.internal.registry.kibanaVersionCheckEnabled: false {{- end }} + +logging.loggers: + - name: plugins.fleet + level: debug diff --git a/internal/stack/resources.go b/internal/stack/resources.go index a7d23f5df1..a16de5a4d9 100644 --- a/internal/stack/resources.go +++ b/internal/stack/resources.go @@ -11,6 +11,7 @@ import ( "html/template" "os" "path/filepath" + "slices" "strings" "github.com/Masterminds/semver/v3" @@ -57,12 +58,13 @@ const ( elasticsearchUsername = "elastic" elasticsearchPassword = "changeme" - configAPMEnabled = "stack.apm_enabled" - configGeoIPDir = "stack.geoip_dir" - configKibanaHTTP2Enabled = "stack.kibana_http2_enabled" - configLogsDBEnabled = "stack.logsdb_enabled" - configLogstashEnabled = "stack.logstash_enabled" - configSelfMonitorEnabled = "stack.self_monitor_enabled" + configAPMEnabled = "stack.apm_enabled" + configGeoIPDir = "stack.geoip_dir" + configKibanaHTTP2Enabled = "stack.kibana_http2_enabled" + configLogsDBEnabled = "stack.logsdb_enabled" + configLogstashEnabled = "stack.logstash_enabled" + configSelfMonitorEnabled = "stack.self_monitor_enabled" + configElasticSubscription = "stack.elastic_subscription" ) var ( @@ -135,6 +137,11 @@ var ( Content: staticSource.File("_static/Dockerfile.logstash"), }, } + + elasticSubscriptionsSupported = []string{ + "basic", + "trial", + } ) func applyResources(profile *profile.Profile, stackVersion string) error { @@ -145,6 +152,11 @@ func applyResources(profile *profile.Profile, stackVersion string) error { return fmt.Errorf("failed to unmarshal stack.agent.ports: %w", err) } + elasticSubscriptionProfile := profile.Config(configElasticSubscription, "trial") + if !slices.Contains(elasticSubscriptionsSupported, elasticSubscriptionProfile) { + return fmt.Errorf("unsupported Elastic subscription %q: supported subscriptions: %s", elasticSubscriptionProfile, strings.Join(elasticSubscriptionsSupported, ", ")) + } + resourceManager := resource.NewManager() resourceManager.AddFacter(resource.StaticFacter{ "registry_base_image": PackageRegistryBaseImage, @@ -168,6 +180,7 @@ func applyResources(profile *profile.Profile, stackVersion string) error { "logsdb_enabled": profile.Config(configLogsDBEnabled, "false"), "logstash_enabled": profile.Config(configLogstashEnabled, "false"), "self_monitor_enabled": profile.Config(configSelfMonitorEnabled, "false"), + "elastic_subscription": elasticSubscriptionProfile, }) if err := os.MkdirAll(stackDir, 0755); err != nil { diff --git a/scripts/test-stack-command.sh b/scripts/test-stack-command.sh index c644b0d819..1f4aebdf04 100755 --- a/scripts/test-stack-command.sh +++ b/scripts/test-stack-command.sh @@ -5,6 +5,7 @@ set -euxo pipefail VERSION=${1:-default} APM_SERVER_ENABLED=${APM_SERVER_ENABLED:-false} SELF_MONITOR_ENABLED=${SELF_MONITOR_ENABLED:-false} +ELASTIC_SUBSCRIPTION=${ELASTIC_SUBSCRIPTION:-""} cleanup() { r=$? @@ -23,6 +24,10 @@ cleanup() { elastic-package profiles delete with-self-monitor fi + if [[ "${ELASTIC_SUBSCRIPTION}" != "" ]]; then + elastic-package profiles delete with-elastic-subscription + fi + exit $r } @@ -71,6 +76,16 @@ stack.self_monitor_enabled: true EOF fi +if [[ "${ELASTIC_SUBSCRIPTION}" != "" ]]; then + profile=with-elastic-subscription + elastic-package profiles create -v ${profile} + elastic-package profiles use ${profile} + + cat ~/.elastic-package/profiles/${profile}/config.yml.example - < ~/.elastic-package/profiles/${profile}/config.yml +stack.elastic_subscription: ${ELASTIC_SUBSCRIPTION} +EOF +fi + mkdir -p "${OUTPUT_PATH_STATUS}" # Initial status empty @@ -115,6 +130,8 @@ elastic-package stack status -v 2> "${OUTPUT_PATH_STATUS}/running.txt" clean_status_output "${OUTPUT_PATH_STATUS}/expected_running.txt" > "${OUTPUT_PATH_STATUS}/expected_no_spaces.txt" clean_status_output "${OUTPUT_PATH_STATUS}/running.txt" > "${OUTPUT_PATH_STATUS}/running_no_spaces.txt" +diff -q "${OUTPUT_PATH_STATUS}/running_no_spaces.txt" "${OUTPUT_PATH_STATUS}/expected_no_spaces.txt" + if [ "${APM_SERVER_ENABLED}" = true ]; then curl http://localhost:8200/ fi @@ -127,4 +144,17 @@ if [ "${SELF_MONITOR_ENABLED}" = true ]; then -f "${ELASTIC_PACKAGE_ELASTICSEARCH_HOST}/metrics-system.*/_search?allow_no_indices=false&size=0" fi -diff -q "${OUTPUT_PATH_STATUS}/running_no_spaces.txt" "${OUTPUT_PATH_STATUS}/expected_no_spaces.txt" +subscription=$(curl -s -S \ + -u "${ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME}:${ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD}" \ + --cacert "${ELASTIC_PACKAGE_CA_CERT}" \ + -f "${ELASTIC_PACKAGE_ELASTICSEARCH_HOST}/_license" |jq -r '.license.type') + +expected_subscription="trial" +if [[ "${ELASTIC_SUBSCRIPTION}" != "" ]]; then + expected_subscription="${ELASTIC_SUBSCRIPTION}" +fi + +if [[ "${subscription}" != "${expected_subscription}" ]]; then + echo "Unexpected \"${subscription}\" subscription found, but expected \"${expected_subscription}\"" + exit 1 +fi diff --git a/test/packages/parallel/apache_basic_license.stack_provider_settings b/test/packages/parallel/apache_basic_license.stack_provider_settings new file mode 100644 index 0000000000..4a31f52113 --- /dev/null +++ b/test/packages/parallel/apache_basic_license.stack_provider_settings @@ -0,0 +1 @@ +stack.elastic_subscription=basic diff --git a/test/packages/parallel/apache_basic_license/_dev/build/build.yml b/test/packages/parallel/apache_basic_license/_dev/build/build.yml new file mode 100644 index 0000000000..6b5cc3fbcc --- /dev/null +++ b/test/packages/parallel/apache_basic_license/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@8.1 diff --git a/test/packages/parallel/apache_basic_license/_dev/build/docs/README.md b/test/packages/parallel/apache_basic_license/_dev/build/docs/README.md new file mode 100644 index 0000000000..f57e928c13 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/_dev/build/docs/README.md @@ -0,0 +1,34 @@ +# Apache Integration + +This integration periodically fetches metrics from [Apache](https://httpd.apache.org/) servers. It can parse access and error +logs created by the Apache server. + +## Compatibility + +The Apache datasets were tested with Apache 2.4.12 and 2.4.46 and are expected to work with +all versions >= 2.2.31 and >= 2.4.16 (independent from operating system). + +## Logs + +### Access Logs + +Access logs collects the Apache access logs. + +{{fields "access"}} + +### Error Logs + +Error logs collects the Apache error logs. + +{{fields "error"}} + +## Metrics + +### Status Metrics + +The server status stream collects data from the Apache Status module. It scrapes the status data from the web page +generated by the `mod_status` module. + +{{event "status"}} + +{{fields "status"}} diff --git a/test/packages/parallel/apache_basic_license/_dev/deploy/docker/Dockerfile b/test/packages/parallel/apache_basic_license/_dev/deploy/docker/Dockerfile new file mode 100644 index 0000000000..d6a2916a3c --- /dev/null +++ b/test/packages/parallel/apache_basic_license/_dev/deploy/docker/Dockerfile @@ -0,0 +1,6 @@ +ARG SERVICE_VERSION=${SERVICE_VERSION:-2.4.46} +FROM httpd:$SERVICE_VERSION +RUN sed -i "/jessie-updates/d" /etc/apt/sources.list +RUN apt-get update && apt-get install -y curl +HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost +COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/_dev/deploy/docker/docker-compose.yml b/test/packages/parallel/apache_basic_license/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 0000000000..40e9e18081 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,11 @@ +version: '2.3' +services: + apache_basic_license: + # Commented out `image:` below until we have a process to refresh the hosted images from + # Dockerfiles in this repo. Until then, we build the image locally using `build:` below. + # image: docker.elastic.co/integrations-ci/beats-apache:${SERVICE_VERSION:-2.4.20}-1 + build: . + ports: + - 80 + volumes: + - ${SERVICE_LOGS_DIR}:/usr/local/apache2/logs diff --git a/test/packages/parallel/apache_basic_license/_dev/deploy/docker/httpd.conf b/test/packages/parallel/apache_basic_license/_dev/deploy/docker/httpd.conf new file mode 100644 index 0000000000..f402947317 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/_dev/deploy/docker/httpd.conf @@ -0,0 +1,511 @@ +# +# This is the main Apache HTTP server configuration file. It contains the +# configuration directives that give the server its instructions. +# See for detailed information. +# In particular, see +# +# for a discussion of each configuration directive. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so "logs/access_log" +# with ServerRoot set to "/usr/local/apache2" will be interpreted by the +# server as "/usr/local/apache2/logs/access_log", whereas "/logs/access_log" +# will be interpreted as '/logs/access_log'. + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# Do not add a slash at the end of the directory path. If you point +# ServerRoot at a non-local disk, be sure to specify a local disk on the +# Mutex directive, if file-based mutexes are used. If you wish to share the +# same ServerRoot for multiple httpd daemons, you will need to change at +# least PidFile. +# +ServerRoot "/usr/local/apache2" + +# +# Mutex: Allows you to set the mutex mechanism and mutex file directory +# for individual mutexes, or change the global defaults +# +# Uncomment and change the directory if mutexes are file-based and the default +# mutex file directory is not on a local disk or is not appropriate for some +# other reason. +# +# Mutex default:logs + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses. +# +#Listen 12.34.56.78:80 +Listen 80 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +LoadModule authn_file_module modules/mod_authn_file.so +#LoadModule authn_dbm_module modules/mod_authn_dbm.so +#LoadModule authn_anon_module modules/mod_authn_anon.so +#LoadModule authn_dbd_module modules/mod_authn_dbd.so +#LoadModule authn_socache_module modules/mod_authn_socache.so +LoadModule authn_core_module modules/mod_authn_core.so +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule authz_groupfile_module modules/mod_authz_groupfile.so +LoadModule authz_user_module modules/mod_authz_user.so +#LoadModule authz_dbm_module modules/mod_authz_dbm.so +#LoadModule authz_owner_module modules/mod_authz_owner.so +#LoadModule authz_dbd_module modules/mod_authz_dbd.so +LoadModule authz_core_module modules/mod_authz_core.so +#LoadModule authnz_ldap_module modules/mod_authnz_ldap.so +LoadModule access_compat_module modules/mod_access_compat.so +LoadModule auth_basic_module modules/mod_auth_basic.so +#LoadModule auth_form_module modules/mod_auth_form.so +#LoadModule auth_digest_module modules/mod_auth_digest.so +#LoadModule allowmethods_module modules/mod_allowmethods.so +#LoadModule file_cache_module modules/mod_file_cache.so +#LoadModule cache_module modules/mod_cache.so +#LoadModule cache_disk_module modules/mod_cache_disk.so +#LoadModule cache_socache_module modules/mod_cache_socache.so +#LoadModule socache_shmcb_module modules/mod_socache_shmcb.so +#LoadModule socache_dbm_module modules/mod_socache_dbm.so +#LoadModule socache_memcache_module modules/mod_socache_memcache.so +#LoadModule macro_module modules/mod_macro.so +#LoadModule dbd_module modules/mod_dbd.so +#LoadModule dumpio_module modules/mod_dumpio.so +#LoadModule buffer_module modules/mod_buffer.so +#LoadModule ratelimit_module modules/mod_ratelimit.so +LoadModule reqtimeout_module modules/mod_reqtimeout.so +#LoadModule ext_filter_module modules/mod_ext_filter.so +#LoadModule request_module modules/mod_request.so +#LoadModule include_module modules/mod_include.so +LoadModule filter_module modules/mod_filter.so +#LoadModule substitute_module modules/mod_substitute.so +#LoadModule sed_module modules/mod_sed.so +#LoadModule deflate_module modules/mod_deflate.so +LoadModule mime_module modules/mod_mime.so +#LoadModule ldap_module modules/mod_ldap.so +LoadModule log_config_module modules/mod_log_config.so +#LoadModule log_debug_module modules/mod_log_debug.so +#LoadModule logio_module modules/mod_logio.so +LoadModule env_module modules/mod_env.so +#LoadModule expires_module modules/mod_expires.so +LoadModule headers_module modules/mod_headers.so +#LoadModule unique_id_module modules/mod_unique_id.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule version_module modules/mod_version.so +#LoadModule remoteip_module modules/mod_remoteip.so +#LoadModule proxy_module modules/mod_proxy.so +#LoadModule proxy_connect_module modules/mod_proxy_connect.so +#LoadModule proxy_ftp_module modules/mod_proxy_ftp.so +#LoadModule proxy_http_module modules/mod_proxy_http.so +#LoadModule proxy_fcgi_module modules/mod_proxy_fcgi.so +#LoadModule proxy_scgi_module modules/mod_proxy_scgi.so +#LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so +#LoadModule proxy_ajp_module modules/mod_proxy_ajp.so +#LoadModule proxy_balancer_module modules/mod_proxy_balancer.so +#LoadModule proxy_express_module modules/mod_proxy_express.so +#LoadModule session_module modules/mod_session.so +#LoadModule session_cookie_module modules/mod_session_cookie.so +#LoadModule session_crypto_module modules/mod_session_crypto.so +#LoadModule session_dbd_module modules/mod_session_dbd.so +#LoadModule slotmem_shm_module modules/mod_slotmem_shm.so +#LoadModule ssl_module modules/mod_ssl.so +#LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so +#LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so +#LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so +#LoadModule lbmethod_heartbeat_module modules/mod_lbmethod_heartbeat.so +LoadModule unixd_module modules/mod_unixd.so +#LoadModule dav_module modules/mod_dav.so +LoadModule status_module modules/mod_status.so +LoadModule autoindex_module modules/mod_autoindex.so +#LoadModule info_module modules/mod_info.so +#LoadModule cgid_module modules/mod_cgid.so +#LoadModule dav_fs_module modules/mod_dav_fs.so +#LoadModule vhost_alias_module modules/mod_vhost_alias.so +#LoadModule negotiation_module modules/mod_negotiation.so +LoadModule dir_module modules/mod_dir.so +#LoadModule actions_module modules/mod_actions.so +#LoadModule speling_module modules/mod_speling.so +#LoadModule userdir_module modules/mod_userdir.so +LoadModule alias_module modules/mod_alias.so +#LoadModule rewrite_module modules/mod_rewrite.so +LoadModule mpm_event_module modules/mod_mpm_event.so + + +# +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# It is usually good practice to create a dedicated user and group for +# running httpd, as with most system services. +# +User daemon +Group daemon + + + +# 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# definition. These values also provide defaults for +# any containers you may define later in the file. +# +# All of these directives may appear inside containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +# +ServerAdmin you@example.com + +# +# ServerName gives the name and port that the server uses to identify itself. +# This can often be determined automatically, but we recommend you specify +# it explicitly to prevent problems during startup. +# +# If your host doesn't have a registered DNS name, enter its IP address here. +# +ServerName localhost + +# +# Deny access to the entirety of your server's filesystem. You must +# explicitly permit access to web content directories in other +# blocks below. +# + + AllowOverride none + Require all denied + + +# +# Note that from this point forward you must specifically allow +# particular features to be enabled - so if something's not working as +# you might expect, make sure that you have specifically enabled it +# below. +# + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +DocumentRoot "/usr/local/apache2/htdocs" + + # + # Possible values for the Options directive are "None", "All", + # or any combination of: + # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews + # + # Note that "MultiViews" must be named *explicitly* --- "Options All" + # doesn't give it to you. + # + # The Options directive is both complicated and important. Please see + # http://httpd.apache.org/docs/2.4/mod/core.html#options + # for more information. + # + Options Indexes FollowSymLinks + + # + # AllowOverride controls what directives may be placed in .htaccess files. + # It can be "All", "None", or any combination of the keywords: + # AllowOverride FileInfo AuthConfig Limit + # + AllowOverride None + + # + # Controls who can get stuff from this server. + # + Require all granted + + +# +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# + + DirectoryIndex index.html + + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Require all denied + + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog "/usr/local/apache2/logs/error.log" + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + + + # + # The following directives define some format nicknames for use with + # a CustomLog directive (see below). + # + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + # You need to enable mod_logio.c to use %I and %O + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + # + # The location and format of the access logfile (Common Logfile Format). + # If you do not define any access logfiles within a + # container, they will be logged here. Contrariwise, if you *do* + # define per- access logfiles, transactions will be + # logged therein and *not* in this file. + # + CustomLog "/usr/local/apache2/logs/access.log" common + + # + # If you prefer a logfile with access, agent, and referer information + # (Combined Logfile Format) you can use the following directive. + # + #CustomLog "logs/access_log" combined + + + + # + # Redirect: Allows you to tell clients about documents that used to + # exist in your server's namespace, but do not anymore. The client + # will make a new request for the document at its new location. + # Example: + # Redirect permanent /foo http://www.example.com/bar + + # + # Alias: Maps web paths into filesystem paths and is used to + # access content that does not live under the DocumentRoot. + # Example: + # Alias /webpath /full/filesystem/path + # + # If you include a trailing / on /webpath then the server will + # require it to be present in the URL. You will also likely + # need to provide a section to allow access to + # the filesystem path. + + # + # ScriptAlias: This controls which directories contain server scripts. + # ScriptAliases are essentially the same as Aliases, except that + # documents in the target directory are treated as applications and + # run by the server when requested rather than as documents sent to the + # client. The same rules about trailing "/" apply to ScriptAlias + # directives as to Alias. + # + ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/" + + + + + # + # ScriptSock: On threaded servers, designate the path to the UNIX + # socket used to communicate with the CGI daemon of mod_cgid. + # + #Scriptsock cgisock + + +# +# "/usr/local/apache2/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. +# + + AllowOverride None + Options None + Require all granted + + + + # + # TypesConfig points to the file containing the list of mappings from + # filename extension to MIME-type. + # + TypesConfig conf/mime.types + + # + # AddType allows you to add to or override the MIME configuration + # file specified in TypesConfig for specific file types. + # + #AddType application/x-gzip .tgz + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # + #AddEncoding x-compress .Z + #AddEncoding x-gzip .gz .tgz + # + # If the AddEncoding directives above are commented-out, then you + # probably should define those extensions to indicate media types: + # + AddType application/x-compress .Z + AddType application/x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers": + # actions unrelated to filetype. These can be either built into the server + # or added with the Action directive (see below) + # + # To use CGI scripts outside of ScriptAliased directories: + # (You will also need to add "ExecCGI" to the "Options" directive.) + # + #AddHandler cgi-script .cgi + + # For type maps (negotiated resources): + #AddHandler type-map var + + # + # Filters allow you to process content before it is sent to the client. + # + # To parse .shtml files for server-side includes (SSI): + # (You will also need to add "Includes" to the "Options" directive.) + # + #AddType text/html .shtml + #AddOutputFilter INCLUDES .shtml + + +# +# The mod_mime_magic module allows the server to use various hints from the +# contents of the file itself to determine its type. The MIMEMagicFile +# directive tells the module where the hint definitions are located. +# +#MIMEMagicFile conf/magic + +# +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# MaxRanges: Maximum number of Ranges in a request before +# returning the entire resource, or one of the special +# values 'default', 'none' or 'unlimited'. +# Default setting is to accept 200 Ranges. +#MaxRanges unlimited + +# +# EnableMMAP and EnableSendfile: On systems that support it, +# memory-mapping or the sendfile syscall may be used to deliver +# files. This usually improves server performance, but must +# be turned off when serving from networked-mounted +# filesystems or if support for these functions is otherwise +# broken on your system. +# Defaults: EnableMMAP On, EnableSendfile Off +# +#EnableMMAP off +#EnableSendfile on + +# Supplemental configuration +# +# The configuration files in the conf/extra/ directory can be +# included to add extra features or to modify the default configuration of +# the server, or you may simply copy their contents here and change as +# necessary. + +# Server-pool management (MPM specific) +#Include conf/extra/httpd-mpm.conf + +# Multi-language error messages +#Include conf/extra/httpd-multilang-errordoc.conf + +# Fancy directory listings +#Include conf/extra/httpd-autoindex.conf + +# Language settings +#Include conf/extra/httpd-languages.conf + +# User home directories +#Include conf/extra/httpd-userdir.conf + +# Real-time info on requests and configuration +#Include conf/extra/httpd-info.conf + +# Virtual hosts +#Include conf/extra/httpd-vhosts.conf + +# Local access to the Apache HTTP Server Manual +#Include conf/extra/httpd-manual.conf + +# Distributed authoring and versioning (WebDAV) +#Include conf/extra/httpd-dav.conf + +# Various default settings +#Include conf/extra/httpd-default.conf + +# Configure mod_proxy_html to understand HTML4/XHTML1 + +Include conf/extra/proxy-html.conf + + +# Secure (SSL/TLS) connections +#Include conf/extra/httpd-ssl.conf +# +# Note: The following must must be present to support +# starting without SSL on platforms with no /dev/random equivalent +# but a statically compiled-in mod_ssl. +# + +SSLRandomSeed startup builtin +SSLRandomSeed connect builtin + + + + + + SetHandler server-status + + + # Keep track of extended status information for each request + ExtendedStatus On + + + # Show Proxy LoadBalancer status in mod_status + ProxyStatus On + + + + diff --git a/test/packages/parallel/apache_basic_license/_dev/deploy/variants.yml b/test/packages/parallel/apache_basic_license/_dev/deploy/variants.yml new file mode 100644 index 0000000000..a4f638620c --- /dev/null +++ b/test/packages/parallel/apache_basic_license/_dev/deploy/variants.yml @@ -0,0 +1,4 @@ +variants: + v2: + SERVICE_VERSION: 2.4.46 +default: v2 diff --git a/test/packages/parallel/apache_basic_license/_dev/test/config.yml b/test/packages/parallel/apache_basic_license/_dev/test/config.yml new file mode 100644 index 0000000000..a57750f85e --- /dev/null +++ b/test/packages/parallel/apache_basic_license/_dev/test/config.yml @@ -0,0 +1,2 @@ +system: + parallel: true diff --git a/test/packages/parallel/apache_basic_license/changelog.yml b/test/packages/parallel/apache_basic_license/changelog.yml new file mode 100644 index 0000000000..89cff330bc --- /dev/null +++ b/test/packages/parallel/apache_basic_license/changelog.yml @@ -0,0 +1,111 @@ +# newer versions go on top +- version: "999.999.999" + changes: + - description: Change test public IPs to the supported subset + type: bugfix + link: https://github.com/elastic/integrations/pull/2327 +- version: "1.3.2" + changes: + - description: Fix ML module manifest query to ignore frozen and cold tiers + type: bugfix + link: https://github.com/elastic/integrations/pull/2217 +- version: "1.3.1" + changes: + - description: Fix parsing of trace log levels + type: bugfix + link: https://github.com/elastic/integrations/pull/2064 +- version: "1.3.0" + changes: + - description: Support Kibana 8.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/2122 +- version: "1.2.0" + changes: + - description: Uniform with guidelines + type: enhancement + link: https://github.com/elastic/integrations/pull/2001 +- version: "1.1.1" + changes: + - description: Fix logic that checks for the 'forwarded' tag + type: bugfix + link: https://github.com/elastic/integrations/pull/1794 +- version: "1.1.0" + changes: + - description: Update to ECS 1.12.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/1686 +- version: "1.0.0" + changes: + - description: Release Apache as GA + type: enhancement + link: https://github.com/elastic/integrations/pull/1607 +- version: "0.9.2" + changes: + - description: Convert to generated ECS fields + type: enhancement + link: https://github.com/elastic/integrations/pull/1463 +- version: '0.9.1' + changes: + - description: update to ECS 1.11.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/1369 +- version: "0.9.0" + changes: + - description: Update integration description + type: enhancement + link: https://github.com/elastic/integrations/pull/1364 +- version: "0.8.1" + changes: + - description: Add support for Splunk authorization tokens + type: enhancement + link: https://github.com/elastic/integrations/pull/1147 +- version: "0.8.0" + changes: + - description: Set event.module and event.dataset + type: bugfix + link: https://github.com/elastic/integrations/pull/1230 +- version: "0.7.1" + changes: + - description: Fix bug in Third Party REST API ingest pipeline + type: bugfix + link: https://github.com/elastic/integrations/pull/1201 +- version: "0.7.0" + changes: + - description: Update to ECS 1.10.0 and adding items that all packages should have + type: enhancement + link: https://github.com/elastic/integrations/pull/1068 +- version: "0.6.0" + changes: + - description: Render units and metric types in exported fields table + type: enhancement + link: https://github.com/elastic/integrations/pull/1028 +- version: "0.5.1" + changes: + - description: Move ecs.version to the ingest pipeline and make event.original optional + type: enhancement + link: https://github.com/elastic/integrations/pull/1025 +- version: "0.5.0" + changes: + - description: Adds ML jobs for finding unusual activity in HTTP access logs + type: enhancement + link: https://github.com/elastic/integrations/pull/910 +- version: "0.4.1" + changes: + - description: update to ECS 1.9.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/832 +- version: "0.3.5" + changes: + - description: Updating package owner + type: enhancement + link: https://github.com/elastic/integrations/pull/766 +- version: "0.3.4" + changes: + - description: Use correct types for `source.port` and `source.ip` + type: bugfix + link: https://github.com/elastic/integrations/pull/737 +- version: "0.1.0" + changes: + - description: initial release + type: enhancement # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/98 diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-basic.log b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-basic.log new file mode 100644 index 0000000000..4e2cbbe7e5 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-basic.log @@ -0,0 +1,7 @@ +::1 - - [26/Dec/2016:16:16:29 +0200] "GET /favicon.ico HTTP/1.1" 404 209 +192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] "GET /hello HTTP/1.1" 404 499 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +::1 - - [26/Dec/2016:16:16:48 +0200] "-" 408 - +172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] "-" 408 152 "-" "-" +monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json new file mode 100644 index 0000000000..9040732867 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -0,0 +1,389 @@ +{ + "expected": [ + { + "@timestamp": "2016-12-26T14:16:29.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.863598691Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "body": { + "bytes": 209 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "::1", + "ip": "::1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" + }, + "user": { + "name": "-" + } + }, + { + "@timestamp": "2016-12-26T16:22:13.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.863610730Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 499 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hello", + "path": "/hello" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "full": "Mac OS X 10.12", + "name": "Mac OS X", + "version": "10.12" + }, + "version": "50.0." + } + }, + { + "@timestamp": "2016-12-26T14:16:48.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.863613299Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "response": { + "status_code": 408 + } + }, + "source": { + "address": "::1", + "ip": "::1" + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "name": "-" + } + }, + { + "@timestamp": "2017-05-29T19:02:48.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.863614844Z", + "kind": "event", + "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 612 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "172.17.0.1", + "ip": "172.17.0.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/stringpatch", + "path": "/stringpatch" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "full": "Windows 7", + "name": "Windows", + "version": "7" + }, + "version": "15.0.a2" + } + }, + { + "@timestamp": "2017-05-29T19:02:48.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.863616333Z", + "kind": "event", + "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "success", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 612 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "source": { + "address": "monitoring-server", + "domain": "monitoring-server" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/status", + "path": "/status" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "full": "Windows 7", + "name": "Windows", + "version": "7" + }, + "version": "15.0.a2" + } + }, + { + "@timestamp": "2019-02-02T04:38:45.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.863617808Z", + "kind": "event", + "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "referrer": "-" + }, + "response": { + "body": { + "bytes": 152 + }, + "status_code": 408 + } + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "-" + } + }, + { + "@timestamp": "2017-05-29T19:02:48.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.863619268Z", + "kind": "event", + "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "success", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 612 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "source": { + "address": "monitoring-server", + "domain": "monitoring-server" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "mp4", + "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "full": "Windows 7", + "name": "Windows", + "version": "7" + }, + "version": "15.0.a2" + } + } + ] +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-darwin.log b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-darwin.log new file mode 100644 index 0000000000..7966578f57 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-darwin.log @@ -0,0 +1,7 @@ +::1 - - [26/Dec/2016:16:16:28 +0200] "GET / HTTP/1.1" 200 45 +::1 - - [26/Dec/2016:16:16:29 +0200] "GET /favicon.ico HTTP/1.1" 404 209 +::1 - - [26/Dec/2016:16:16:48 +0200] "-" 408 - +89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] "GET / HTTP/1.1" 200 45 +89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] "GET /notfound HTTP/1.1" 404 206 +89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] "GET /hmm HTTP/1.1" 404 201 +192.0.2.100 - - [26/Dec/2016:18:23:45 +0200] "GET /hmm HTTP/1.1" 404 201 diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json new file mode 100644 index 0000000000..fb2b090ffc --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -0,0 +1,401 @@ +{ + "expected": [ + { + "@timestamp": "2016-12-26T14:16:28.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.950483828Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", + "outcome": "success", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "body": { + "bytes": 45 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "source": { + "address": "::1", + "ip": "::1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" + }, + "user": { + "name": "-" + } + }, + { + "@timestamp": "2016-12-26T14:16:29.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.950496473Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "body": { + "bytes": 209 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "::1", + "ip": "::1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" + }, + "user": { + "name": "-" + } + }, + { + "@timestamp": "2016-12-26T14:16:48.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.950497950Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "response": { + "status_code": 408 + } + }, + "source": { + "address": "::1", + "ip": "::1" + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "name": "-" + } + }, + { + "@timestamp": "2016-12-26T16:23:35.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.950499298Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", + "outcome": "success", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "body": { + "bytes": 45 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "source": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" + }, + "user": { + "name": "-" + } + }, + { + "@timestamp": "2016-12-26T16:23:41.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.950500622Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "body": { + "bytes": 206 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/notfound", + "path": "/notfound" + }, + "user": { + "name": "-" + } + }, + { + "@timestamp": "2016-12-26T16:23:45.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:05.950501935Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "body": { + "bytes": 201 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hmm", + "path": "/hmm" + }, + "user": { + "name": "-" + } + }, + { + "@timestamp": "2016-12-26T16:23:45.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "original": "192.0.2.100 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "body": { + "bytes": 201 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "192.0.2.100", + "as": { + "number": 64500, + "organization": { + "name": "Documentation ASN" + } + }, + "geo": { + "city_name": "Las Vegas", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 36.17497, + "lon": -115.13722 + }, + "region_iso_code": "US-NV", + "region_name": "Nevada" + }, + "ip": "192.0.2.100" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hmm", + "path": "/hmm" + }, + "user": { + "name": "-" + } + } + ] +} diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log new file mode 100644 index 0000000000..0a59aed766 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log @@ -0,0 +1,2 @@ +[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 "GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1" 1375 +[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 "GET /appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1" - diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json new file mode 100644 index 0000000000..d2a6f20aff --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -0,0 +1,127 @@ +{ + "expected": [ + { + "@timestamp": "2018-08-10T07:45:56.000Z", + "apache": { + "access": { + "ssl": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "protocol": "TLSv1.2" + } + } + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.029928962Z", + "kind": "event", + "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "body": { + "bytes": 1375 + } + }, + "version": "1.1" + }, + "source": { + "address": "172.30.0.119", + "ip": "172.30.0.119" + }, + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" + }, + "url": { + "extension": "php", + "original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21", + "path": "/nagiosxi/ajaxhelper.php", + "query": "cmd=getxicoreajax\u0026amp;opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" + } + }, + { + "@timestamp": "2019-10-16T09:53:47.000Z", + "apache": { + "access": { + "ssl": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "protocol": "TLSv1.2" + } + } + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.029934826Z", + "kind": "event", + "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1" + }, + "source": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" + }, + "url": { + "extension": "php", + "original": "/appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d", + "path": "/appl/ajaxhelper.php", + "query": "cmd=getxicoreajax\u0026opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" + } + } + ] +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log new file mode 100644 index 0000000000..92d1bdd85a --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log @@ -0,0 +1,9 @@ +127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] "GET / HTTP/1.1" 200 491 "-" "Wget/1.13.4 (linux-gnu)" +192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] "GET / HTTP/1.1" 200 484 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] "GET /favicon.ico HTTP/1.1" 404 504 "http://192.168.33.72/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] "GET / HTTP/1.1" 200 484 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] "GET /favicon.ico HTTP/1.1" 404 504 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] "GET /favicon.ico HTTP/1.1" 404 504 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] "GET /test HTTP/1.1" 404 498 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] "GET /hello HTTP/1.1" 404 499 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] "GET /crap HTTP/1.1" 404 499 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json new file mode 100644 index 0000000000..0aab255bcd --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -0,0 +1,563 @@ +{ + "expected": [ + { + "@timestamp": "2016-12-26T16:18:09.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.078187403Z", + "kind": "event", + "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", + "outcome": "success", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 491 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Wget", + "original": "Wget/1.13.4 (linux-gnu)", + "os": { + "name": "Linux" + }, + "version": "1.13.4" + } + }, + { + "@timestamp": "2016-12-26T16:22:00.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.078193659Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "outcome": "success", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 484 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", + "os": { + "full": "Mac OS X 10.12.0", + "name": "Mac OS X", + "version": "10.12.0" + }, + "version": "54.0.2840.98" + } + }, + { + "@timestamp": "2016-12-26T16:22:00.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.078195340Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "http://192.168.33.72/" + }, + "response": { + "body": { + "bytes": 504 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", + "os": { + "full": "Mac OS X 10.12.0", + "name": "Mac OS X", + "version": "10.12.0" + }, + "version": "54.0.2840.98" + } + }, + { + "@timestamp": "2016-12-26T16:22:08.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.078196730Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "success", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 484 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "full": "Mac OS X 10.12", + "name": "Mac OS X", + "version": "10.12" + }, + "version": "50.0." + } + }, + { + "@timestamp": "2016-12-26T16:22:08.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.078198115Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 504 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "full": "Mac OS X 10.12", + "name": "Mac OS X", + "version": "10.12" + }, + "version": "50.0." + } + }, + { + "@timestamp": "2016-12-26T16:22:08.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.078199480Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 504 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "full": "Mac OS X 10.12", + "name": "Mac OS X", + "version": "10.12" + }, + "version": "50.0." + } + }, + { + "@timestamp": "2016-12-26T16:22:10.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.078200843Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 498 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/test", + "path": "/test" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "full": "Mac OS X 10.12", + "name": "Mac OS X", + "version": "10.12" + }, + "version": "50.0." + } + }, + { + "@timestamp": "2016-12-26T16:22:13.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.078202210Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 499 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hello", + "path": "/hello" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "full": "Mac OS X 10.12", + "name": "Mac OS X", + "version": "10.12" + }, + "version": "50.0." + } + }, + { + "@timestamp": "2016-12-26T16:22:17.000Z", + "apache": { + "access": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.078203552Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 499 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/crap", + "path": "/crap" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "full": "Mac OS X 10.12", + "name": "Mac OS X", + "version": "10.12" + }, + "version": "50.0." + } + } + ] +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-vhost.log b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-vhost.log new file mode 100644 index 0000000000..64a432e4a8 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-vhost.log @@ -0,0 +1 @@ +vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] "GET /hello HTTP/1.1" 404 499 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json new file mode 100644 index 0000000000..0e6ac6978e --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -0,0 +1,69 @@ +{ + "expected": [ + { + "@timestamp": "2016-12-26T16:22:14.000Z", + "apache": { + "access": {} + }, + "destination": { + "domain": "vhost1.domaine.fr" + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-08T10:05:06.135584519Z", + "kind": "event", + "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure", + "type": [ + "access" + ] + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 499 + }, + "status_code": 404 + }, + "version": "1.1" + }, + "source": { + "ip": "192.168.33.2" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "domain": "vhost1.domaine.fr", + "original": "/hello", + "path": "/hello" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "full": "Mac OS X 10.12", + "name": "Mac OS X", + "version": "10.12" + }, + "version": "50.0." + } + } + ] +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-common-config.yml b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 0000000000..fba77e700c --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,10 @@ +dynamic_fields: + event.ingested: ".*" + # This can be removed after ES 8.16.2 is set as the minimum version supported in the manifest. + # Once removed, it requires to update pipeline tests to remove the trailing dot where required. + # Relates: https://github.com/elastic/elasticsearch/pull/117213 + "user_agent.version": '^\d+\.\d+(\.|\..*)?$' +fields: + "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/policy/test-default.expected b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/policy/test-default.expected new file mode 100644 index 0000000000..92ecfec6c3 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/policy/test-default.expected @@ -0,0 +1,34 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: apache + name: test-default-apache_basic_license + streams: + - data_stream: + dataset: apache.access + type: logs + exclude_files: + - .gz$ + paths: + - /var/logs/apache/access.log* + tags: + - apache-access + type: logfile + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-apache.access-ep + privileges: + - auto_configure + - create_doc +secret_references: [] diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/policy/test-default.yml b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/policy/test-default.yml new file mode 100644 index 0000000000..0d208e2e95 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/policy/test-default.yml @@ -0,0 +1,4 @@ +data_stream: + vars: + paths: + - "/var/logs/apache/access.log*" diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/system/test-default-config.yml b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f73c5dd9e --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: ~ +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/access.log*" diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/agent/stream/httpjson.yml.hbs b/test/packages/parallel/apache_basic_license/data_stream/access/agent/stream/httpjson.yml.hbs new file mode 100644 index 0000000000..96e6d31467 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/agent/stream/httpjson.yml.hbs @@ -0,0 +1,64 @@ +config_version: "2" +interval: {{interval}} +{{#unless token}} +{{#if username}} +{{#if password}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +{{/if}} +{{/if}} +{{/unless}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: |- + {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +{{#unless username}} +{{#unless password}} +{{#if token}} + - set: + target: header.Authorization + value: {{token}} +{{/if}} +{{/unless}} +{{/unless}} +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/agent/stream/log.yml.hbs b/test/packages/parallel/apache_basic_license/data_stream/access/agent/stream/log.yml.hbs new file mode 100644 index 0000000000..c6e5ed4c73 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/agent/stream/log.yml.hbs @@ -0,0 +1,19 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +exclude_files: [".gz$"] +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/test/packages/parallel/apache_basic_license/data_stream/access/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 0000000000..48763c70d8 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,155 @@ +--- +description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins." +processors: + - pipeline: + if: ctx.message.startsWith('{') + name: '{{ IngestPipeline "third-party" }}' + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - rename: + field: message + target_field: event.original + - grok: + field: event.original + patterns: + - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" + %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( + "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' + - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" + %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( + "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' + - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + "-" %{NUMBER:http.response.status_code:long} -' + - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} + %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:_tmp.url_orig} + HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long}) + ignore_missing: true + - uri_parts: + field: _tmp.url_orig + ignore_failure: true + - remove: + field: + - _tmp + ignore_missing: true + - set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null + - set: + field: event.kind + value: event + - set: + field: event.category + value: ["web"] + - set: + field: event.type + value: ["access"] + - set: + field: event.outcome + value: success + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + - set: + field: event.outcome + value: failure + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399" + - grok: + field: source.address + ignore_missing: true + patterns: + - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ + - remove: + field: event.created + ignore_missing: true + ignore_failure: true + - rename: + field: '@timestamp' + target_field: event.created + - date: + field: apache.access.time + target_field: '@timestamp' + formats: + - dd/MMM/yyyy:H:m:s Z + ignore_failure: true + - remove: + field: apache.access.time + ignore_failure: true + - user_agent: + field: user_agent.original + ignore_failure: true + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - set: + field: tls.cipher + value: '{{apache.access.ssl.cipher}}' + if: ctx?.apache?.access?.ssl?.cipher != null + - script: + lang: painless + if: ctx?.apache?.access?.ssl?.protocol != null + source: >- + def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v"); + if (parts.length != 2) { + return; + } + if (parts[1].contains(".")) { + ctx.tls.version = parts[1]; + } else { + ctx.tls.version = parts[1] + ".0"; + } + ctx.tls.version_protocol = parts[0]; + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/elasticsearch/ingest_pipeline/third-party.yml b/test/packages/parallel/apache_basic_license/data_stream/access/elasticsearch/ingest_pipeline/third-party.yml new file mode 100644 index 0000000000..42a2ca83ff --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/elasticsearch/ingest_pipeline/third-party.yml @@ -0,0 +1,42 @@ +--- +description: Pipeline for parsing Apache HTTP Server logs from third party api +processors: + - json: + field: message + target_field: json + - drop: + if: ctx.json?.result == null + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: '_id' + ignore_missing: true + - set: + copy_from: json.result._raw + field: message + ignore_empty_value: true + - set: + copy_from: json.result.host + field: host.name + ignore_empty_value: true + - set: + copy_from: json.result.source + field: file.path + ignore_empty_value: true + - remove: + field: + - json + ignore_missing: true +on_failure: + - append: + field: error.message + value: >- + error in third-party pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/fields/agent.yml b/test/packages/parallel/apache_basic_license/data_stream/access/fields/agent.yml new file mode 100644 index 0000000000..3c8ad89f03 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/fields/agent.yml @@ -0,0 +1,200 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/fields/base-fields.yml b/test/packages/parallel/apache_basic_license/data_stream/access/fields/base-fields.yml new file mode 100644 index 0000000000..7a1b27e3da --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: apache +- name: event.dataset + type: constant_keyword + description: Event dataset + value: apache.access diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/fields/ecs.yml b/test/packages/parallel/apache_basic_license/data_stream/access/fields/ecs.yml new file mode 100644 index 0000000000..db3a3a47cd --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/fields/ecs.yml @@ -0,0 +1,102 @@ +- external: ecs + name: destination.domain +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: event.category +- external: ecs + name: event.created +- external: ecs + name: event.kind +- external: ecs + name: event.original +- external: ecs + name: event.outcome +- external: ecs + name: file.path +- external: ecs + name: http.request.method +- external: ecs + name: http.request.referrer +- external: ecs + name: http.response.body.bytes +- external: ecs + name: http.response.status_code +- external: ecs + name: http.version +- external: ecs + name: log.file.path +- external: ecs + name: log.level +- external: ecs + name: message +- external: ecs + name: process.command_line +- external: ecs + name: process.pid +- external: ecs + name: process.thread.id +- external: ecs + name: source.address +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- external: ecs + name: source.domain +- external: ecs + name: source.geo.city_name +- external: ecs + name: source.geo.continent_name +- external: ecs + name: source.geo.country_iso_code +- external: ecs + name: source.geo.country_name +- description: Longitude and latitude. + level: core + name: source.geo.location + type: geo_point +- external: ecs + name: source.geo.region_iso_code +- external: ecs + name: source.geo.region_name +- external: ecs + name: source.ip +- external: ecs + name: tags +- external: ecs + name: tls.cipher +- external: ecs + name: tls.version +- external: ecs + name: tls.version_protocol +- external: ecs + name: url.domain +- external: ecs + name: url.extension +- external: ecs + name: url.original +- external: ecs + name: url.path +- external: ecs + name: url.query +- external: ecs + name: user.name +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.os.full +- external: ecs + name: user_agent.os.name +- external: ecs + name: user_agent.os.version +- external: ecs + name: user_agent.version +- external: ecs + name: host.ip diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/fields/fields.yml b/test/packages/parallel/apache_basic_license/data_stream/access/fields/fields.yml new file mode 100644 index 0000000000..402fb490e4 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/fields/fields.yml @@ -0,0 +1,11 @@ +- name: apache.access + type: group + fields: + - name: ssl.protocol + type: keyword + description: | + SSL protocol version. + - name: ssl.cipher + type: keyword + description: | + SSL cipher name. diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/manifest.yml b/test/packages/parallel/apache_basic_license/data_stream/access/manifest.yml new file mode 100644 index 0000000000..dc93238a33 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/manifest.yml @@ -0,0 +1,87 @@ +title: Apache access logs +type: logs +streams: + - input: logfile + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/apache2/access.log* + - /var/log/apache2/other_vhosts_access.log* + - /var/log/httpd/access_log* + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - apache-access + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + template_path: log.yml.hbs + title: Apache access logs + description: Collect Apache access logs + - input: httpjson + title: Apache access logs via Splunk Enterprise REST API + description: Collect apache access logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"access*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - apache-access + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + diff --git a/test/packages/parallel/apache_basic_license/data_stream/access/sample_event.json b/test/packages/parallel/apache_basic_license/data_stream/access/sample_event.json new file mode 100644 index 0000000000..167963b4de --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/access/sample_event.json @@ -0,0 +1,96 @@ +{ + "@timestamp": "2022-09-08T10:05:46.000Z", + "agent": { + "ephemeral_id": "62fff331-1942-4ca6-b1f4-7b09d72e4e6b", + "id": "488aa68b-25f3-4a6c-aeaf-bbdacf341968", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.3.3" + }, + "apache": { + "access": {} + }, + "data_stream": { + "dataset": "apache.access", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "488aa68b-25f3-4a6c-aeaf-bbdacf341968", + "snapshot": false, + "version": "8.3.3" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "created": "2022-09-08T10:06:04.881Z", + "dataset": "apache.access", + "ingested": "2022-09-08T10:06:05Z", + "kind": "event", + "outcome": "success", + "type": [ + "access" + ] + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "ip": [ + "192.168.128.4" + ], + "mac": [ + "02:42:c0:a8:80:04" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.0-43-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "body": { + "bytes": 45 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/access.log" + }, + "offset": 0 + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "apache-access" + ], + "url": { + "original": "/", + "path": "/" + }, + "user": { + "name": "-" + } +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-common-config.yml b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 0000000000..91c9eaa210 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,6 @@ +dynamic_fields: + event.ingested: ".*" +fields: + event.timezone: "GMT+2" + tags: + - preserve_original_event diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-basic.log b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-basic.log new file mode 100644 index 0000000000..b8120aacfd --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-basic.log @@ -0,0 +1,4 @@ +[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico +[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd' +[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico +[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json new file mode 100644 index 0000000000..3672b5ad22 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -0,0 +1,189 @@ +{ + "expected": [ + { + "@timestamp": "2016-12-26T16:22:08.000+02:00", + "apache": { + "error": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.154511517Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/favicon.ico" + }, + "log": { + "level": "error" + }, + "message": "File does not exist: /var/www/favicon.ico", + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2016-12-26T16:15:55.103+02:00", + "apache": { + "error": { + "module": "core" + } + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.154513602Z", + "kind": "event", + "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "timezone": "GMT+2", + "type": [ + "info" + ] + }, + "log": { + "level": "notice" + }, + "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "process": { + "pid": 11379 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2011-09-09T10:42:29.902+02:00", + "apache": { + "error": { + "module": "core" + } + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.154514592Z", + "kind": "event", + "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/usr/local/apache2/htdocs/favicon.ico" + }, + "log": { + "level": "error" + }, + "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "process": { + "pid": 35708, + "thread": { + "id": 4328636416 + } + }, + "source": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2019-06-27T06:58:09.169+02:00", + "apache": { + "error": { + "module": "include" + } + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.154515567Z", + "kind": "event", + "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "log": { + "level": "warn" + }, + "message": "AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", + "process": { + "pid": 15934 + }, + "source": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156", + "port": 12345 + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-darwin.log b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-darwin.log new file mode 100644 index 0000000000..428ba85a7f --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-darwin.log @@ -0,0 +1,2 @@ +[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations +[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd' diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json new file mode 100644 index 0000000000..732c38fc71 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -0,0 +1,70 @@ +{ + "expected": [ + { + "@timestamp": "2016-12-26T16:15:55.103+02:00", + "apache": { + "error": { + "module": "mpm_prefork" + } + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.194317231Z", + "kind": "event", + "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", + "timezone": "GMT+2", + "type": [ + "info" + ] + }, + "log": { + "level": "notice" + }, + "message": "AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", + "process": { + "pid": 11379 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2016-12-26T16:15:55.103+02:00", + "apache": { + "error": { + "module": "core" + } + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.194320072Z", + "kind": "event", + "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "timezone": "GMT+2", + "type": [ + "info" + ] + }, + "log": { + "level": "notice" + }, + "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "process": { + "pid": 11379 + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-trace.log b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-trace.log new file mode 100644 index 0000000000..64a89473fb --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-trace.log @@ -0,0 +1 @@ +[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/' diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json new file mode 100644 index 0000000000..ea82b6718a --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -0,0 +1,40 @@ +{ + "expected": [ + { + "@timestamp": "2021-10-20T19:20:59.121+02:00", + "apache": { + "error": { + "module": "rewrite" + } + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.227400182Z", + "kind": "event", + "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", + "timezone": "GMT+2", + "type": [ + "info" + ] + }, + "log": { + "level": "trace3" + }, + "message": "mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", + "process": { + "pid": 121591, + "thread": { + "id": 140413273032448 + } + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log new file mode 100644 index 0000000000..e457f0bc78 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log @@ -0,0 +1,7 @@ +[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations +[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/ +[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico +[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico +[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test +[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello +[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json new file mode 100644 index 0000000000..d3274d559c --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -0,0 +1,247 @@ +{ + "expected": [ + { + "@timestamp": "2016-12-26T16:17:53.000+02:00", + "apache": { + "error": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259657304Z", + "kind": "event", + "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", + "timezone": "GMT+2", + "type": [ + "info" + ] + }, + "log": { + "level": "notice" + }, + "message": "Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2016-12-26T16:22:00.000+02:00", + "apache": { + "error": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259659782Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/favicon.ico" + }, + "http": { + "request": { + "referrer": "http://192.168.33.72/" + } + }, + "log": { + "level": "error" + }, + "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2016-12-26T16:22:08.000+02:00", + "apache": { + "error": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259660793Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/favicon.ico" + }, + "log": { + "level": "error" + }, + "message": "File does not exist: /var/www/favicon.ico", + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2016-12-26T16:22:08.000+02:00", + "apache": { + "error": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259661763Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/favicon.ico" + }, + "log": { + "level": "error" + }, + "message": "File does not exist: /var/www/favicon.ico", + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2016-12-26T16:22:10.000+02:00", + "apache": { + "error": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259662714Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/test" + }, + "log": { + "level": "error" + }, + "message": "File does not exist: /var/www/test", + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2016-12-26T16:22:13.000+02:00", + "apache": { + "error": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259663668Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/hello" + }, + "log": { + "level": "error" + }, + "message": "File does not exist: /var/www/hello", + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2016-12-26T16:22:17.000+02:00", + "apache": { + "error": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259664611Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/crap" + }, + "log": { + "level": "error" + }, + "message": "File does not exist: /var/www/crap", + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/policy/test-default.expected b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/policy/test-default.expected new file mode 100644 index 0000000000..e2573ed97e --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/policy/test-default.expected @@ -0,0 +1,36 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: apache + name: test-default-apache_basic_license + streams: + - data_stream: + dataset: apache.error + type: logs + exclude_files: + - .gz$ + paths: + - /var/logs/apache/error.log* + processors: + - add_locale: null + tags: + - apache-error + type: logfile + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-apache.error-ep + privileges: + - auto_configure + - create_doc +secret_references: [] diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/policy/test-default.yml b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/policy/test-default.yml new file mode 100644 index 0000000000..4bfc94f7d2 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/policy/test-default.yml @@ -0,0 +1,4 @@ +data_stream: + vars: + paths: + - "/var/logs/apache/error.log*" diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/system/test-default-config.yml b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..ec7356ee93 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: ~ +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/error.log*" diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/agent/stream/httpjson.yml.hbs b/test/packages/parallel/apache_basic_license/data_stream/error/agent/stream/httpjson.yml.hbs new file mode 100644 index 0000000000..d96b2f6048 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/agent/stream/httpjson.yml.hbs @@ -0,0 +1,64 @@ +config_version: 2 +interval: {{interval}} +{{#unless token}} +{{#if username}} +{{#if password}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +{{/if}} +{{/if}} +{{/unless}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: |- + {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +{{#unless username}} +{{#unless password}} +{{#if token}} + - set: + target: header.Authorization + value: {{token}} +{{/if}} +{{/unless}} +{{/unless}} +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/agent/stream/log.yml.hbs b/test/packages/parallel/apache_basic_license/data_stream/error/agent/stream/log.yml.hbs new file mode 100644 index 0000000000..781c90981a --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/agent/stream/log.yml.hbs @@ -0,0 +1,20 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +processors: +- add_locale: ~ +{{#if processors}} +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/elasticsearch/ingest_pipeline/default.yml b/test/packages/parallel/apache_basic_license/data_stream/error/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 0000000000..92037c49d0 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,140 @@ +--- +description: Pipeline for parsing apache error logs +processors: + - pipeline: + if: ctx.message.startsWith('{') + name: '{{ IngestPipeline "third-party" }}' + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - rename: + field: message + target_field: event.original + - grok: + field: event.original + patterns: + - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{LOGLEVEL:log.level}\]( \[client + %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message} + - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{DATA:apache.error.module}:%{APACHE_LOGLEVEL:log.level}\] + \[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\]( + \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message} + pattern_definitions: + # Apache log level can have numeric sub-levels such as trace1. + APACHE_LOGLEVEL: '%{LOGLEVEL}[0-9]*' + APACHE_TIME: '%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}' + ignore_missing: true + - grok: + field: message + patterns: + - 'File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}' + - 'File does not exist: %{URIPATH:file.path}' + ignore_missing: true + ignore_failure: true + - date: + if: ctx.event.timezone == null + field: apache.error.timestamp + target_field: '@timestamp' + formats: + - EEE MMM dd H:m:s yyyy + - EEE MMM dd H:m:s.SSSSSS yyyy + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' + - date: + if: ctx.event.timezone != null + field: apache.error.timestamp + target_field: '@timestamp' + formats: + - EEE MMM dd H:m:s yyyy + - EEE MMM dd H:m:s.SSSSSS yyyy + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' + - remove: + field: apache.error.timestamp + ignore_failure: true + - set: + field: event.kind + value: event + - set: + field: event.category + value: ["web"] + - script: + if: "ctx?.log?.level != null" + lang: painless + source: >- + def err_levels = ["emerg", "alert", "crit", "error", "warn"]; + if (err_levels.contains(ctx.log.level)) { + ctx.event.type = ["error"]; + } else { + ctx.event.type = ["info"]; + } + + - grok: + field: source.address + ignore_missing: true + patterns: + - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - convert: + field: source.port + type: long + ignore_missing: true + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/elasticsearch/ingest_pipeline/third-party.yml b/test/packages/parallel/apache_basic_license/data_stream/error/elasticsearch/ingest_pipeline/third-party.yml new file mode 100644 index 0000000000..42a2ca83ff --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/elasticsearch/ingest_pipeline/third-party.yml @@ -0,0 +1,42 @@ +--- +description: Pipeline for parsing Apache HTTP Server logs from third party api +processors: + - json: + field: message + target_field: json + - drop: + if: ctx.json?.result == null + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: '_id' + ignore_missing: true + - set: + copy_from: json.result._raw + field: message + ignore_empty_value: true + - set: + copy_from: json.result.host + field: host.name + ignore_empty_value: true + - set: + copy_from: json.result.source + field: file.path + ignore_empty_value: true + - remove: + field: + - json + ignore_missing: true +on_failure: + - append: + field: error.message + value: >- + error in third-party pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/fields/agent.yml b/test/packages/parallel/apache_basic_license/data_stream/error/fields/agent.yml new file mode 100644 index 0000000000..e313ec8287 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/fields/base-fields.yml b/test/packages/parallel/apache_basic_license/data_stream/error/fields/base-fields.yml new file mode 100644 index 0000000000..15365c71bd --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: apache +- name: event.dataset + type: constant_keyword + description: Event dataset + value: apache.error diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/fields/ecs.yml b/test/packages/parallel/apache_basic_license/data_stream/error/fields/ecs.yml new file mode 100644 index 0000000000..0a88a11039 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/fields/ecs.yml @@ -0,0 +1,82 @@ +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: event.category +- external: ecs + name: event.kind +- external: ecs + name: event.timezone +- external: ecs + name: event.type +- external: ecs + name: file.path +- external: ecs + name: http.request.method +- external: ecs + name: http.request.referrer +- external: ecs + name: http.response.body.bytes +- external: ecs + name: http.response.status_code +- external: ecs + name: http.version +- external: ecs + name: log.file.path +- external: ecs + name: log.level +- external: ecs + name: message +- external: ecs + name: process.pid +- external: ecs + name: process.thread.id +- external: ecs + name: source.address +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- external: ecs + name: source.geo.city_name +- external: ecs + name: source.geo.continent_name +- external: ecs + name: source.geo.country_iso_code +- external: ecs + name: source.geo.country_name +- description: Longitude and latitude. + level: core + name: source.geo.location + type: geo_point +- external: ecs + name: source.geo.region_iso_code +- external: ecs + name: source.geo.region_name +- external: ecs + name: source.ip +- external: ecs + name: source.port +- external: ecs + name: tags +- external: ecs + name: url.domain +- external: ecs + name: url.extension +- external: ecs + name: url.original +- external: ecs + name: url.path +- external: ecs + name: url.query +- external: ecs + name: user.name +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.os.name diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/fields/fields.yml b/test/packages/parallel/apache_basic_license/data_stream/error/fields/fields.yml new file mode 100644 index 0000000000..a86664d968 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/fields/fields.yml @@ -0,0 +1,7 @@ +- name: apache.error + type: group + fields: + - name: module + type: keyword + description: | + The module producing the logged message. diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/manifest.yml b/test/packages/parallel/apache_basic_license/data_stream/error/manifest.yml new file mode 100644 index 0000000000..e0ccdffd68 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/manifest.yml @@ -0,0 +1,86 @@ +title: Apache error logs +type: logs +streams: + - input: logfile + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/apache2/error.log* + - /var/log/httpd/error_log* + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - apache-error + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + template_path: log.yml.hbs + title: Apache error logs + description: Collect Apache error logs + - input: httpjson + title: Apache error logs via Splunk Enterprise REST API + description: Collect apache error logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: search sourcetype=apache:error OR sourcetype=apache_error + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - apache-error + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + diff --git a/test/packages/parallel/apache_basic_license/data_stream/error/sample_event.json b/test/packages/parallel/apache_basic_license/data_stream/error/sample_event.json new file mode 100644 index 0000000000..afe50f5260 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/error/sample_event.json @@ -0,0 +1,82 @@ +{ + "@timestamp": "2022-09-06T07:48:48.030Z", + "agent": { + "ephemeral_id": "7d905f37-6778-4228-95af-98b28a0fcb12", + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.3.3" + }, + "apache": { + "error": { + "module": "mpm_event" + } + }, + "data_stream": { + "dataset": "apache.error", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "snapshot": false, + "version": "8.3.3" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "dataset": "apache.error", + "ingested": "2022-09-06T07:49:06Z", + "kind": "event", + "timezone": "+00:00", + "type": [ + "info" + ] + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "ip": [ + "172.21.0.7" + ], + "mac": [ + "02:42:ac:15:00:07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.0-43-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/error.log" + }, + "level": "notice", + "offset": 0 + }, + "message": "AH00489: Apache/2.4.46 (Unix) configured -- resuming normal operations", + "process": { + "pid": 1, + "thread": { + "id": 140493350368384 + } + }, + "tags": [ + "apache-error" + ] +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/data_stream/status/_dev/test/policy/test-default.expected b/test/packages/parallel/apache_basic_license/data_stream/status/_dev/test/policy/test-default.expected new file mode 100644 index 0000000000..fef828730b --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/status/_dev/test/policy/test-default.expected @@ -0,0 +1,34 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: apache + name: test-default-apache_basic_license + streams: + - data_stream: + dataset: apache.status + type: metrics + hosts: + - http://localhost:8080 + metricsets: + - status + period: 30s + server_status_path: /server-status + type: apache/metrics + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - metrics-apache.status-ep + privileges: + - auto_configure + - create_doc +secret_references: [] diff --git a/test/packages/parallel/apache_basic_license/data_stream/status/_dev/test/policy/test-default.yml b/test/packages/parallel/apache_basic_license/data_stream/status/_dev/test/policy/test-default.yml new file mode 100644 index 0000000000..c7678ce910 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/status/_dev/test/policy/test-default.yml @@ -0,0 +1,3 @@ +vars: + hosts: + - http://localhost:8080 diff --git a/test/packages/parallel/apache_basic_license/data_stream/status/_dev/test/system/test-default-config.yml b/test/packages/parallel/apache_basic_license/data_stream/status/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..83fc84f62e --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/status/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: + hosts: + - http://{{Hostname}}:{{Port}} +data_stream: + vars: ~ diff --git a/test/packages/parallel/apache_basic_license/data_stream/status/agent/stream/stream.yml.hbs b/test/packages/parallel/apache_basic_license/data_stream/status/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..9c7975244b --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/status/agent/stream/stream.yml.hbs @@ -0,0 +1,9 @@ +metricsets: ["status"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if server_status_path}} +server_status_path: {{server_status_path}} +{{/if}} diff --git a/test/packages/parallel/apache_basic_license/data_stream/status/fields/agent.yml b/test/packages/parallel/apache_basic_license/data_stream/status/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/status/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/parallel/apache_basic_license/data_stream/status/fields/base-fields.yml b/test/packages/parallel/apache_basic_license/data_stream/status/fields/base-fields.yml new file mode 100644 index 0000000000..2cf8da79f7 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/status/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: apache +- name: event.dataset + type: constant_keyword + description: Event dataset + value: apache.status diff --git a/test/packages/parallel/apache_basic_license/data_stream/status/fields/ecs.yml b/test/packages/parallel/apache_basic_license/data_stream/status/fields/ecs.yml new file mode 100644 index 0000000000..28044872f5 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/status/fields/ecs.yml @@ -0,0 +1,8 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: error.message diff --git a/test/packages/parallel/apache_basic_license/data_stream/status/fields/fields.yml b/test/packages/parallel/apache_basic_license/data_stream/status/fields/fields.yml new file mode 100644 index 0000000000..5ed7663e2e --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/status/fields/fields.yml @@ -0,0 +1,184 @@ +- name: apache.status + type: group + fields: + - name: total_accesses + type: long + description: | + Total number of access requests. + metric_type: counter + - name: total_bytes + type: long + description: | + Total number of bytes served. + metric_type: counter + unit: byte + - name: requests_per_sec + type: scaled_float + description: | + Requests per second. + metric_type: gauge + - name: bytes_per_sec + type: scaled_float + description: | + Bytes per second. + metric_type: gauge + - name: bytes_per_request + type: scaled_float + description: | + Bytes per request. + metric_type: gauge + - name: workers.busy + type: long + description: | + Number of busy workers. + metric_type: gauge + - name: workers.idle + type: long + description: | + Number of idle workers. + metric_type: gauge + - name: uptime + type: group + fields: + - name: server_uptime + type: long + description: | + Server uptime in seconds. + metric_type: counter + - name: uptime + type: long + description: | + Server uptime. + metric_type: counter + - name: cpu + type: group + fields: + - name: load + type: scaled_float + description: | + CPU Load. + metric_type: gauge + - name: user + type: scaled_float + description: | + CPU user load. + metric_type: gauge + - name: system + type: scaled_float + description: | + System cpu. + metric_type: gauge + - name: children_user + type: scaled_float + description: | + CPU of children user. + metric_type: gauge + - name: children_system + type: scaled_float + description: | + CPU of children system. + metric_type: gauge + - name: connections + type: group + fields: + - name: total + type: long + description: | + Total connections. + metric_type: counter + - name: async.writing + type: long + description: | + Async connection writing. + metric_type: gauge + - name: async.keep_alive + type: long + description: | + Async keeped alive connections. + metric_type: gauge + - name: async.closing + type: long + description: | + Async closed connections. + metric_type: gauge + - name: load + type: group + fields: + - name: "1" + type: scaled_float + description: | + Load average for the last minute. + metric_type: gauge + - name: "5" + type: scaled_float + description: | + Load average for the last 5 minutes. + metric_type: gauge + - name: "15" + type: scaled_float + description: | + Load average for the last 15 minutes. + metric_type: gauge + - name: scoreboard + type: group + fields: + - name: starting_up + type: long + description: | + Starting up. + metric_type: gauge + - name: reading_request + type: long + description: | + Reading requests. + metric_type: gauge + - name: sending_reply + type: long + description: | + Sending Reply. + metric_type: gauge + - name: keepalive + type: long + description: | + Keep alive. + metric_type: gauge + - name: dns_lookup + type: long + description: | + Dns Lookups. + metric_type: gauge + - name: closing_connection + type: long + description: | + Closing connections. + metric_type: gauge + - name: logging + type: long + description: | + Logging + metric_type: gauge + - name: gracefully_finishing + type: long + description: | + Gracefully finishing. + metric_type: gauge + - name: idle_cleanup + type: long + description: | + Idle cleanups. + metric_type: gauge + - name: open_slot + type: long + description: | + Open slots. + metric_type: gauge + - name: waiting_for_connection + type: long + description: | + Waiting for connections. + metric_type: gauge + - name: total + type: long + description: | + Total. + metric_type: gauge diff --git a/test/packages/parallel/apache_basic_license/data_stream/status/manifest.yml b/test/packages/parallel/apache_basic_license/data_stream/status/manifest.yml new file mode 100644 index 0000000000..f4b63ba327 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/status/manifest.yml @@ -0,0 +1,21 @@ +title: Apache status metrics +type: metrics +streams: + - input: apache/metrics + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 30s + - name: server_status_path + type: text + title: Server Status Path + multi: false + required: true + show_user: false + default: /server-status + title: Apache status metrics + description: Collect Apache status metrics diff --git a/test/packages/parallel/apache_basic_license/data_stream/status/sample_event.json b/test/packages/parallel/apache_basic_license/data_stream/status/sample_event.json new file mode 100644 index 0000000000..fe58078259 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/data_stream/status/sample_event.json @@ -0,0 +1,110 @@ +{ + "@timestamp": "2022-09-06T07:49:38.359Z", + "agent": { + "ephemeral_id": "8d98054a-3077-4bb5-81b8-89c4da73f566", + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.3.3" + }, + "apache": { + "status": { + "bytes_per_request": 0, + "bytes_per_sec": 0, + "connections": { + "async": { + "closing": 0, + "keep_alive": 0, + "writing": 0 + }, + "total": 0 + }, + "cpu": { + "children_system": 0, + "children_user": 0, + "load": 0.230769, + "system": 0.02, + "user": 0.01 + }, + "load": { + "1": 2.68, + "15": 2.79, + "5": 3.48 + }, + "requests_per_sec": 0.923077, + "scoreboard": { + "closing_connection": 0, + "dns_lookup": 0, + "gracefully_finishing": 0, + "idle_cleanup": 0, + "keepalive": 0, + "logging": 0, + "open_slot": 325, + "reading_request": 0, + "sending_reply": 1, + "starting_up": 0, + "total": 400, + "waiting_for_connection": 74 + }, + "total_accesses": 12, + "total_bytes": 0, + "uptime": { + "server_uptime": 13, + "uptime": 13 + }, + "workers": { + "busy": 1, + "idle": 74 + } + } + }, + "data_stream": { + "dataset": "apache.status", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "snapshot": false, + "version": "8.3.3" + }, + "event": { + "agent_id_status": "verified", + "dataset": "apache.status", + "duration": 1049700, + "ingested": "2022-09-06T07:49:38Z", + "module": "apache" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "ip": [ + "172.21.0.7" + ], + "mac": [ + "02:42:ac:15:00:07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.0-43-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "status", + "period": 30000 + }, + "service": { + "address": "http://elastic-package-service-apache-1:80/server-status?auto=", + "type": "apache" + } +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/docs/README.md b/test/packages/parallel/apache_basic_license/docs/README.md new file mode 100644 index 0000000000..411b02c509 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/docs/README.md @@ -0,0 +1,414 @@ +# Apache Integration + +This integration periodically fetches metrics from [Apache](https://httpd.apache.org/) servers. It can parse access and error +logs created by the Apache server. + +## Compatibility + +The Apache datasets were tested with Apache 2.4.12 and 2.4.46 and are expected to work with +all versions >= 2.2.31 and >= 2.4.16 (independent from operating system). + +## Logs + +### Access Logs + +Access logs collects the Apache access logs. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| apache.access.ssl.cipher | SSL cipher name. | keyword | +| apache.access.ssl.protocol | SSL protocol version. | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | +| event.dataset | Event dataset | constant_keyword | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | +| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | +| file.path.text | Multi-field of `file.path`. | match_only_text | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.name.text | Multi-field of `host.os.name`. | text | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | +| http.request.referrer | Referrer for this HTTP request. | keyword | +| http.response.body.bytes | Size in bytes of the response body. | long | +| http.response.status_code | HTTP response status code. | long | +| http.version | HTTP version. | keyword | +| input.type | Input type | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | +| log.offset | Log offset | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | +| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | +| process.pid | Process id. | long | +| process.thread.id | Thread ID. | long | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | +| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | +| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| tags | List of keywords used to tag each event. | keyword | +| tls.cipher | String indicating the cipher used during the current connection. | keyword | +| tls.version | Numeric part of the version parsed from the original string. | keyword | +| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | +| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.original.text | Multi-field of `url.original`. | match_only_text | +| url.path | Path of the request, such as "/search". | wildcard | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | +| user.name | Short name or login of the user. | keyword | +| user.name.text | Multi-field of `user.name`. | match_only_text | +| user_agent.device.name | Name of the device. | keyword | +| user_agent.name | Name of the user agent. | keyword | +| user_agent.original | Unparsed user_agent string. | keyword | +| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | +| user_agent.os.full | Operating system name, including the version or code name. | keyword | +| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | +| user_agent.os.name | Operating system name, without the version. | keyword | +| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | +| user_agent.os.version | Operating system version as a raw string. | keyword | +| user_agent.version | Version of the user agent. | keyword | + + +### Error Logs + +Error logs collects the Apache error logs. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| apache.error.module | The module producing the logged message. | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | +| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | +| file.path.text | Multi-field of `file.path`. | match_only_text | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.name.text | Multi-field of `host.os.name`. | text | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | +| http.request.referrer | Referrer for this HTTP request. | keyword | +| http.response.body.bytes | Size in bytes of the response body. | long | +| http.response.status_code | HTTP response status code. | long | +| http.version | HTTP version. | keyword | +| input.type | Input type | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | +| log.offset | Log offset | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| process.pid | Process id. | long | +| process.thread.id | Thread ID. | long | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | +| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | +| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.original.text | Multi-field of `url.original`. | match_only_text | +| url.path | Path of the request, such as "/search". | wildcard | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | +| user.name | Short name or login of the user. | keyword | +| user.name.text | Multi-field of `user.name`. | match_only_text | +| user_agent.device.name | Name of the device. | keyword | +| user_agent.name | Name of the user agent. | keyword | +| user_agent.original | Unparsed user_agent string. | keyword | +| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | +| user_agent.os.name | Operating system name, without the version. | keyword | +| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | + + +## Metrics + +### Status Metrics + +The server status stream collects data from the Apache Status module. It scrapes the status data from the web page +generated by the `mod_status` module. + +An example event for `status` looks as following: + +```json +{ + "@timestamp": "2022-09-06T07:49:38.359Z", + "agent": { + "ephemeral_id": "8d98054a-3077-4bb5-81b8-89c4da73f566", + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.3.3" + }, + "apache": { + "status": { + "bytes_per_request": 0, + "bytes_per_sec": 0, + "connections": { + "async": { + "closing": 0, + "keep_alive": 0, + "writing": 0 + }, + "total": 0 + }, + "cpu": { + "children_system": 0, + "children_user": 0, + "load": 0.230769, + "system": 0.02, + "user": 0.01 + }, + "load": { + "1": 2.68, + "15": 2.79, + "5": 3.48 + }, + "requests_per_sec": 0.923077, + "scoreboard": { + "closing_connection": 0, + "dns_lookup": 0, + "gracefully_finishing": 0, + "idle_cleanup": 0, + "keepalive": 0, + "logging": 0, + "open_slot": 325, + "reading_request": 0, + "sending_reply": 1, + "starting_up": 0, + "total": 400, + "waiting_for_connection": 74 + }, + "total_accesses": 12, + "total_bytes": 0, + "uptime": { + "server_uptime": 13, + "uptime": 13 + }, + "workers": { + "busy": 1, + "idle": 74 + } + } + }, + "data_stream": { + "dataset": "apache.status", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "snapshot": false, + "version": "8.3.3" + }, + "event": { + "agent_id_status": "verified", + "dataset": "apache.status", + "duration": 1049700, + "ingested": "2022-09-06T07:49:38Z", + "module": "apache" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "ip": [ + "172.21.0.7" + ], + "mac": [ + "02:42:ac:15:00:07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.0-43-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "status", + "period": 30000 + }, + "service": { + "address": "http://elastic-package-service-apache-1:80/server-status?auto=", + "type": "apache" + } +} +``` + +**Exported fields** + +| Field | Description | Type | Unit | Metric Type | +|---|---|---|---|---| +| @timestamp | Event timestamp. | date | | | +| apache.status.bytes_per_request | Bytes per request. | scaled_float | | gauge | +| apache.status.bytes_per_sec | Bytes per second. | scaled_float | | gauge | +| apache.status.connections.async.closing | Async closed connections. | long | | gauge | +| apache.status.connections.async.keep_alive | Async keeped alive connections. | long | | gauge | +| apache.status.connections.async.writing | Async connection writing. | long | | gauge | +| apache.status.connections.total | Total connections. | long | | counter | +| apache.status.cpu.children_system | CPU of children system. | scaled_float | | gauge | +| apache.status.cpu.children_user | CPU of children user. | scaled_float | | gauge | +| apache.status.cpu.load | CPU Load. | scaled_float | | gauge | +| apache.status.cpu.system | System cpu. | scaled_float | | gauge | +| apache.status.cpu.user | CPU user load. | scaled_float | | gauge | +| apache.status.load.1 | Load average for the last minute. | scaled_float | | gauge | +| apache.status.load.15 | Load average for the last 15 minutes. | scaled_float | | gauge | +| apache.status.load.5 | Load average for the last 5 minutes. | scaled_float | | gauge | +| apache.status.requests_per_sec | Requests per second. | scaled_float | | gauge | +| apache.status.scoreboard.closing_connection | Closing connections. | long | | gauge | +| apache.status.scoreboard.dns_lookup | Dns Lookups. | long | | gauge | +| apache.status.scoreboard.gracefully_finishing | Gracefully finishing. | long | | gauge | +| apache.status.scoreboard.idle_cleanup | Idle cleanups. | long | | gauge | +| apache.status.scoreboard.keepalive | Keep alive. | long | | gauge | +| apache.status.scoreboard.logging | Logging | long | | gauge | +| apache.status.scoreboard.open_slot | Open slots. | long | | gauge | +| apache.status.scoreboard.reading_request | Reading requests. | long | | gauge | +| apache.status.scoreboard.sending_reply | Sending Reply. | long | | gauge | +| apache.status.scoreboard.starting_up | Starting up. | long | | gauge | +| apache.status.scoreboard.total | Total. | long | | gauge | +| apache.status.scoreboard.waiting_for_connection | Waiting for connections. | long | | gauge | +| apache.status.total_accesses | Total number of access requests. | long | | counter | +| apache.status.total_bytes | Total number of bytes served. | long | byte | counter | +| apache.status.uptime.server_uptime | Server uptime in seconds. | long | | counter | +| apache.status.uptime.uptime | Server uptime. | long | | counter | +| apache.status.workers.busy | Number of busy workers. | long | | gauge | +| apache.status.workers.idle | Number of idle workers. | long | | gauge | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | | | +| cloud.image.id | Image ID for the cloud instance. | keyword | | | +| cloud.instance.id | Instance ID of the host machine. | keyword | | | +| cloud.instance.name | Instance name of the host machine. | keyword | | | +| cloud.machine.type | Machine type of the host machine. | keyword | | | +| cloud.project.id | Name of the project in Google Cloud. | keyword | | | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | +| cloud.region | Region in which this host is running. | keyword | | | +| container.id | Unique container id. | keyword | | | +| container.image.name | Name of the image the container was built on. | keyword | | | +| container.labels | Image labels. | object | | | +| container.name | Container name. | keyword | | | +| data_stream.dataset | Data stream dataset. | constant_keyword | | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | | +| data_stream.type | Data stream type. | constant_keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | +| error.message | Error message. | match_only_text | | | +| event.dataset | Event dataset | constant_keyword | | | +| event.module | Event module | constant_keyword | | | +| host.architecture | Operating system architecture. | keyword | | | +| host.containerized | If the host is a container. | boolean | | | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | +| host.ip | Host ip addresses. | ip | | | +| host.mac | Host mac addresses. | keyword | | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | +| host.os.build | OS build information. | keyword | | | +| host.os.codename | OS codename, if any. | keyword | | | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | +| host.os.name | Operating system name, without the version. | keyword | | | +| host.os.name.text | Multi-field of `host.os.name`. | text | | | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | +| host.os.version | Operating system version as a raw string. | keyword | | | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | + diff --git a/test/packages/parallel/apache_basic_license/img/apache-logs-overview.png b/test/packages/parallel/apache_basic_license/img/apache-logs-overview.png new file mode 100644 index 0000000000..5597f61a27 Binary files /dev/null and b/test/packages/parallel/apache_basic_license/img/apache-logs-overview.png differ diff --git a/test/packages/parallel/apache_basic_license/img/apache-metrics-overview.png b/test/packages/parallel/apache_basic_license/img/apache-metrics-overview.png new file mode 100644 index 0000000000..ec24030bbd Binary files /dev/null and b/test/packages/parallel/apache_basic_license/img/apache-metrics-overview.png differ diff --git a/test/packages/parallel/apache_basic_license/img/logo_apache.svg b/test/packages/parallel/apache_basic_license/img/logo_apache.svg new file mode 100644 index 0000000000..384761f641 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/img/logo_apache.svg @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/packages/parallel/apache_basic_license/kibana/dashboard/apache-Logs-Apache-Dashboard.json b/test/packages/parallel/apache_basic_license/kibana/dashboard/apache-Logs-Apache-Dashboard.json new file mode 100644 index 0000000000..7332f02b4d --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/dashboard/apache-Logs-Apache-Dashboard.json @@ -0,0 +1,229 @@ +{ + "attributes": { + "description": "Logs Apache integration dashboard", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "mapBounds": { + "bottom_right": { + "lat": -3.864254615721396, + "lon": 205.3125 + }, + "top_left": { + "lat": 67.7427590666639, + "lon": -205.6640625 + } + }, + "mapCenter": [ + 40.713955826286046, + -0.17578125 + ], + "mapCollar": { + "bottom_right": { + "lat": -39.667755, + "lon": 180 + }, + "top_left": { + "lat": 90, + "lon": -180 + }, + "zoom": 2 + }, + "mapZoom": 2 + }, + "gridData": { + "h": 12, + "i": "1", + "w": 35, + "x": 13, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "2", + "w": 32, + "x": 0, + "y": 20 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "3", + "w": 16, + "x": 32, + "y": 20 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "4", + "w": 8, + "x": 40, + "y": 12 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "5", + "w": 48, + "x": 0, + "y": 32 + }, + "panelIndex": "5", + "panelRefName": "panel_4", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "6", + "w": 40, + "x": 0, + "y": 12 + }, + "panelIndex": "6", + "panelRefName": "panel_5", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "columns": [ + "source.address", + "log.level", + "apache2.error.integration", + "message" + ], + "enhancements": {}, + "sort": [ + "@timestamp", + "desc" + ] + }, + "gridData": { + "h": 12, + "i": "7", + "w": 48, + "x": 0, + "y": 40 + }, + "panelIndex": "7", + "panelRefName": "panel_6", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "f5d1286d-411a-4759-a2e2-0b3227b93cfa", + "w": 13, + "x": 0, + "y": 0 + }, + "panelIndex": "f5d1286d-411a-4759-a2e2-0b3227b93cfa", + "panelRefName": "panel_7", + "version": "7.9.3" + } + ], + "timeRestore": false, + "title": "[Logs Apache] Access and error logs", + "version": 1 + }, + "id": "apache-Logs-Apache-Dashboard", + "migrationVersion": { + "dashboard": "7.11.0" + }, + "references": [ + { + "id": "apache-access-unique-IPs-map", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "apache-response-codes-of-top-URLs", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "apache-browsers", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "apache-operating-systems", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "apache-error-logs-over-time", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "apache-response-codes-over-time", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "apache-errors-log", + "name": "panel_6", + "type": "search" + }, + { + "id": "apache-ed44f820-3a10-11eb-8946-296aab7b13db", + "name": "panel_7", + "type": "visualization" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/dashboard/apache-Metrics-Apache-HTTPD-server-status.json b/test/packages/parallel/apache_basic_license/kibana/dashboard/apache-Metrics-Apache-HTTPD-server-status.json new file mode 100644 index 0000000000..35a2998fd1 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/dashboard/apache-Metrics-Apache-HTTPD-server-status.json @@ -0,0 +1,273 @@ +{ + "attributes": { + "description": "Overview of Apache server status", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "1", + "w": 24, + "x": 24, + "y": 50 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "3", + "w": 24, + "x": 0, + "y": 50 + }, + "panelIndex": "3", + "panelRefName": "panel_1", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "4", + "w": 24, + "x": 24, + "y": 5 + }, + "panelIndex": "4", + "panelRefName": "panel_2", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "4cc4755f-49a7-43c1-8a21-0a78291f0b3f", + "w": 13, + "x": 0, + "y": 0 + }, + "panelIndex": "4cc4755f-49a7-43c1-8a21-0a78291f0b3f", + "panelRefName": "panel_3", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "7b7a1f18-e274-4f4e-a3b3-3760e7896897", + "w": 11, + "x": 13, + "y": 0 + }, + "panelIndex": "7b7a1f18-e274-4f4e-a3b3-3760e7896897", + "panelRefName": "panel_4", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "01794c9e-0ce3-4e1e-bc87-6c15b6434ba8", + "w": 12, + "x": 24, + "y": 0 + }, + "panelIndex": "01794c9e-0ce3-4e1e-bc87-6c15b6434ba8", + "panelRefName": "panel_5", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "e2b2dd17-dcda-4b17-b250-dd30c596f7f6", + "w": 12, + "x": 36, + "y": 0 + }, + "panelIndex": "e2b2dd17-dcda-4b17-b250-dd30c596f7f6", + "panelRefName": "panel_6", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "f90d54d8-034c-4cfd-8640-0e8f10c2ca99", + "w": 24, + "x": 0, + "y": 5 + }, + "panelIndex": "f90d54d8-034c-4cfd-8640-0e8f10c2ca99", + "panelRefName": "panel_7", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "3b23e577-34f6-4cd0-b636-01581b8ce1c0", + "w": 24, + "x": 0, + "y": 20 + }, + "panelIndex": "3b23e577-34f6-4cd0-b636-01581b8ce1c0", + "panelRefName": "panel_8", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "100b598b-4223-4a6b-95d9-ee94147fa5ac", + "w": 24, + "x": 24, + "y": 20 + }, + "panelIndex": "100b598b-4223-4a6b-95d9-ee94147fa5ac", + "panelRefName": "panel_9", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "a7dc6253-4f39-4aae-984f-3108d1cf3cfb", + "w": 24, + "x": 0, + "y": 35 + }, + "panelIndex": "a7dc6253-4f39-4aae-984f-3108d1cf3cfb", + "panelRefName": "panel_10", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "4204480f-8c9f-426f-b3f7-0714a70d418b", + "w": 24, + "x": 24, + "y": 35 + }, + "panelIndex": "4204480f-8c9f-426f-b3f7-0714a70d418b", + "panelRefName": "panel_11", + "version": "7.9.3" + } + ], + "timeRestore": false, + "title": "[Metrics Apache] Overview", + "version": 1 + }, + "id": "apache-Metrics-Apache-HTTPD-server-status", + "migrationVersion": { + "dashboard": "7.11.0" + }, + "references": [ + { + "id": "apache-HTTPD-CPU", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "apache-HTTPD-Load1-slash-5-slash-15", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "apache-HTTPD-Scoreboard", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "apache-805d7bb0-3a10-11eb-8946-296aab7b13db", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "apache-22057f20-3a12-11eb-8946-296aab7b13db", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "apache-47820ce0-3a1d-11eb-8946-296aab7b13db", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "apache-99666080-3a20-11eb-8946-296aab7b13db", + "name": "panel_6", + "type": "visualization" + }, + { + "id": "apache-f4ffec70-3a36-11eb-8946-296aab7b13db", + "name": "panel_7", + "type": "visualization" + }, + { + "id": "apache-320cd980-3a36-11eb-8946-296aab7b13db", + "name": "panel_8", + "type": "visualization" + }, + { + "id": "apache-a45311f0-3a34-11eb-8946-296aab7b13db", + "name": "panel_9", + "type": "visualization" + }, + { + "id": "apache-7d68f730-3a39-11eb-8946-296aab7b13db", + "name": "panel_10", + "type": "visualization" + }, + { + "id": "apache-7724cf20-3a39-11eb-8946-296aab7b13db", + "name": "panel_11", + "type": "visualization" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/ml_module/apache-Logs-ml.json b/test/packages/parallel/apache_basic_license/kibana/ml_module/apache-Logs-ml.json new file mode 100644 index 0000000000..cccc306d3f --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/ml_module/apache-Logs-ml.json @@ -0,0 +1,419 @@ +{ + "attributes": { + "id": "apache_data_stream", + "title": "Apache access logs", + "description": "Find unusual activity in HTTP access logs.", + "type": "Web Access Logs", + "logo": { + "icon": "logoApache" + }, + "defaultIndexPattern": "logs-*", + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + }, + { + "exists": { + "field": "source.address" + } + }, + { + "exists": { + "field": "url.original" + } + }, + { + "exists": { + "field": "http.response.status_code" + } + } + ], + "must_not": { + "terms": { + "_tier": [ + "data_frozen", + "data_cold" + ] + } + } + } + }, + "jobs": [ + { + "id": "visitor_rate_apache", + "config": { + "groups": [ + "apache" + ], + "description": "HTTP Access Logs: Detect unusual visitor rates", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "dc_source_address", + "detectors": [ + { + "detector_description": "Apache access visitor rate", + "function": "non_zero_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-apache-access-data-stream", + "custom_urls": [ + { + "url_name": "Apache logs overview", + "url_value": "dashboards#/view/apache-Logs-Apache-Dashboard?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "status_code_rate_apache", + "config": { + "groups": [ + "apache" + ], + "description": "HTTP Access Logs: Detect unusual status code rates", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "Apache access status code rate", + "function": "count", + "partition_field_name": "http.response.status_code" + } + ], + "influencers": [ + "http.response.status_code", + "source.address" + ] + }, + "analysis_limits": { + "model_memory_limit": "100mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-apache-access-data-stream", + "custom_urls": [ + { + "url_name": "Apache logs overview", + "url_value": "dashboards#/view/apache-Logs-Apache-Dashboard?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "source_ip_url_count_apache", + "config": { + "groups": [ + "apache" + ], + "description": "HTTP Access Logs: Detect unusual source IPs - high distinct count of URLs", + "analysis_config": { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "Apache access source IP high dc URL", + "function": "high_distinct_count", + "field_name": "url.original", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-apache-access-data-stream", + "custom_urls": [ + { + "url_name": "Apache logs overview", + "url_value": "dashboards#/view/apache-Logs-Apache-Dashboard?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "source_ip_request_rate_apache", + "config": { + "groups": [ + "apache" + ], + "description": "HTTP Access Logs: Detect unusual source IPs - high request rates", + "analysis_config": { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "Apache access source IP high count", + "function": "high_count", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-apache-access-data-stream", + "custom_urls": [ + { + "url_name": "Apache logs overview", + "url_value": "dashboards#/view/apache-Logs-Apache-Dashboard?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "low_request_rate_apache", + "config": { + "groups": [ + "apache" + ], + "description": "HTTP Access Logs: Detect low request rates", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "doc_count", + "detectors": [ + { + "detector_description": "Apache access low request rate", + "function": "low_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-apache-access-data-stream", + "custom_urls": [ + { + "url_name": "Apache logs overview", + "url_value": "dashboards#/view/apache-Logs-Apache-Dashboard?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + } + ], + "datafeeds": [ + { + "id": "datafeed-visitor_rate_apache", + "job_id": "visitor_rate_apache", + "config": { + "job_id": "visitor_rate_apache", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "15m", + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "dc_source_address": { + "cardinality": { + "field": "source.address" + } + } + } + } + } + } + }, + { + "id": "datafeed-status_code_rate_apache", + "job_id": "status_code_rate_apache", + "config": { + "job_id": "status_code_rate_apache", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-source_ip_url_count_apache", + "job_id": "source_ip_url_count_apache", + "config": { + "job_id": "source_ip_url_count_apache", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-source_ip_request_rate_apache", + "job_id": "source_ip_request_rate_apache", + "config": { + "job_id": "source_ip_request_rate_apache", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-low_request_rate_apache", + "job_id": "low_request_rate_apache", + "config": { + "job_id": "low_request_rate_apache", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "15m", + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + } + } + } + ] + }, + "id": "apache-Logs-ml", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [], + "type": "ml-module" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/search/apache-HTTPD.json b/test/packages/parallel/apache_basic_license/kibana/search/apache-HTTPD.json new file mode 100644 index 0000000000..5afdb95ba7 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/search/apache-HTTPD.json @@ -0,0 +1,52 @@ +{ + "attributes": { + "columns": [ + "_source" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlight": { + "fields": { + "*": {} + }, + "fragment_size": 2147483647, + "post_tags": [ + "@/kibana-highlighted-field@" + ], + "pre_tags": [ + "@kibana-highlighted-field@" + ], + "require_field_match": false + }, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "(data_stream.dataset:apache.status)" + } + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Apache HTTPD", + "version": 1 + }, + "id": "apache-HTTPD", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/search/apache-access-logs.json b/test/packages/parallel/apache_basic_license/kibana/search/apache-access-logs.json new file mode 100644 index 0000000000..5d2d2e31b5 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/search/apache-access-logs.json @@ -0,0 +1,55 @@ +{ + "attributes": { + "columns": [ + "source.address", + "http.request.method", + "url.original", + "http.response.status_code" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlight": { + "fields": { + "*": {} + }, + "fragment_size": 2147483647, + "post_tags": [ + "@/kibana-highlighted-field@" + ], + "pre_tags": [ + "@kibana-highlighted-field@" + ], + "require_field_match": false + }, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "data_stream.dataset:apache.access" + } + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Apache access logs [Logs Apache]", + "version": 1 + }, + "id": "apache-access-logs", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/search/apache-errors-log.json b/test/packages/parallel/apache_basic_license/kibana/search/apache-errors-log.json new file mode 100644 index 0000000000..ff5742a373 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/search/apache-errors-log.json @@ -0,0 +1,55 @@ +{ + "attributes": { + "columns": [ + "source.address", + "log.level", + "apache2.error.integration", + "message" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlight": { + "fields": { + "*": {} + }, + "fragment_size": 2147483647, + "post_tags": [ + "@/kibana-highlighted-field@" + ], + "pre_tags": [ + "@kibana-highlighted-field@" + ], + "require_field_match": false + }, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "data_stream.dataset:apache.error" + } + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Apache errors log [Logs Apache]", + "version": 1 + }, + "id": "apache-errors-log", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-22057f20-3a12-11eb-8946-296aab7b13db.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-22057f20-3a12-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..c7d5744883 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-22057f20-3a12-11eb-8946-296aab7b13db.json @@ -0,0 +1,78 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Uptime [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "c3c89690-3a11-11eb-8a27-5ff1727df0e0" + } + ], + "bar_color_rules": [ + { + "id": "c2fc9400-3a11-11eb-8a27-5ff1727df0e0" + } + ], + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "s,humanize,", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Uptime", + "line_width": 1, + "metrics": [ + { + "field": "apache.status.uptime.uptime", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "max" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "kibana", + "split_mode": "everything", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "tooltip_mode": "show_all", + "type": "metric" + }, + "title": "Uptime [Metrics Apache]", + "type": "metrics" + } + }, + "id": "apache-22057f20-3a12-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-320cd980-3a36-11eb-8946-296aab7b13db.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-320cd980-3a36-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..66fe71c6f4 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-320cd980-3a36-11eb-8946-296aab7b13db.json @@ -0,0 +1,152 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "savedSearchRefName": "search_0", + "title": "Requests per sec [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Requests per sec", + "field": "apache.status.requests_per_sec" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, + "interpolate": "linear", + "labels": {}, + "legendPosition": "right", + "radiusRatio": 9, + "row": true, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Requests per sec" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "shareYAxis": true, + "showCircles": true, + "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Requests per sec" + }, + "type": "value" + } + ], + "yAxis": {} + }, + "title": "Requests per sec [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-320cd980-3a36-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-HTTPD", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-47820ce0-3a1d-11eb-8946-296aab7b13db.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-47820ce0-3a1d-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..232a587a63 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-47820ce0-3a1d-11eb-8946-296aab7b13db.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total accesses [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "5414c4a0-3a1a-11eb-8b9a-851db9ca6ca8" + } + ], + "bar_color_rules": [ + { + "id": "c532ace0-3a1c-11eb-8b9a-851db9ca6ca8" + } + ], + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", + "gauge_color_rules": [ + { + "id": "586a5890-3a19-11eb-8b9a-851db9ca6ca8" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": { + "language": "kuery", + "query": "" + }, + "formatter": "0a", + "id": "6ccbc140-3a1c-11eb-8b9a-851db9ca6ca8", + "label": "Total accesses", + "line_width": 1, + "metrics": [ + { + "field": "apache.status.total_accesses", + "id": "6ccbc141-3a1c-11eb-8b9a-851db9ca6ca8", + "type": "max" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "tooltip_mode": "show_all", + "type": "metric" + }, + "title": "Total accesses [Metrics Apache]", + "type": "metrics" + } + }, + "id": "apache-47820ce0-3a1d-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-7724cf20-3a39-11eb-8946-296aab7b13db.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-7724cf20-3a39-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..6c7e554b74 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-7724cf20-3a39-11eb-8946-296aab7b13db.json @@ -0,0 +1,189 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Connections [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Writing", + "field": "apache.status.connections.async.writing" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "30s", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Keep alive", + "field": "apache.status.connections.async.keep_alive" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Closing", + "field": "apache.status.connections.async.closing" + }, + "schema": "metric", + "type": "max" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false + }, + "labels": {}, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Writing" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Keep alive" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "4", + "label": "Closing" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Connections" + }, + "type": "value" + } + ] + }, + "title": "Connections [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-7724cf20-3a39-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-7d68f730-3a39-11eb-8946-296aab7b13db.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-7d68f730-3a39-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..0f925bc270 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-7d68f730-3a39-11eb-8946-296aab7b13db.json @@ -0,0 +1,141 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total connections [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "30s", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Total", + "field": "apache.status.connections.total" + }, + "schema": "metric", + "type": "max" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false + }, + "labels": {}, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "4", + "label": "Total" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Connections" + }, + "type": "value" + } + ] + }, + "title": "Total connections [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-7d68f730-3a39-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-805d7bb0-3a10-11eb-8946-296aab7b13db.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-805d7bb0-3a10-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..c663484193 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-805d7bb0-3a10-11eb-8946-296aab7b13db.json @@ -0,0 +1,56 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Apache Hostname [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "host.hostname", + "id": "1607512709833", + "indexPatternRefName": "control_0_index_pattern", + "label": "Hostname", + "options": { + "dynamicOptions": true, + "multiselect": false, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "Apache Hostname [Metrics Apache]", + "type": "input_control_vis" + } + }, + "id": "apache-805d7bb0-3a10-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "metrics-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-99666080-3a20-11eb-8946-296aab7b13db.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-99666080-3a20-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..fcf0f34c90 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-99666080-3a20-11eb-8946-296aab7b13db.json @@ -0,0 +1,90 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total egress [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "5414c4a0-3a1a-11eb-8b9a-851db9ca6ca8" + } + ], + "bar_color_rules": [ + { + "id": "c532ace0-3a1c-11eb-8b9a-851db9ca6ca8" + } + ], + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", + "gauge_color_rules": [ + { + "id": "586a5890-3a19-11eb-8b9a-851db9ca6ca8" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": { + "language": "kuery", + "query": "" + }, + "formatter": "bytes", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Total egress", + "line_width": 1, + "metrics": [ + { + "field": "apache.status.total_bytes", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "max" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "kibana", + "split_mode": "everything", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "tooltip_mode": "show_all", + "type": "metric" + }, + "title": "Total egress [Metrics Apache]", + "type": "metrics" + } + }, + "id": "apache-99666080-3a20-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-HTTPD-CPU.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-HTTPD-CPU.json new file mode 100644 index 0000000000..dc0b8c8496 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-HTTPD-CPU.json @@ -0,0 +1,264 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "savedSearchRefName": "search_0", + "title": "CPU usage [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "CPU load", + "field": "apache.status.cpu.load" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "host.hostname", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "split", + "type": "terms" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "CPU user", + "field": "apache.status.cpu.user" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "CPU system", + "field": "apache.status.cpu.system" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "CPU children user", + "field": "apache.status.cpu.children_user" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "7", + "params": { + "customLabel": "CPU children system", + "field": "apache.status.cpu.children_system" + }, + "schema": "metric", + "type": "avg" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, + "interpolate": "linear", + "labels": {}, + "legendPosition": "right", + "radiusRatio": 9, + "row": true, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "CPU load" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "4", + "label": "CPU user" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "5", + "label": "CPU system" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "6", + "label": "CPU children user" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "7", + "label": "CPU children system" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "shareYAxis": true, + "showCircles": true, + "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ], + "yAxis": {} + }, + "title": "CPU usage [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-HTTPD-CPU", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-HTTPD", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-HTTPD-Load1-slash-5-slash-15.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-HTTPD-Load1-slash-5-slash-15.json new file mode 100644 index 0000000000..8d0ba87a59 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-HTTPD-Load1-slash-5-slash-15.json @@ -0,0 +1,217 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "savedSearchRefName": "search_0", + "title": "Load1/5/15 [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Load 5", + "field": "apache.status.load.5" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Load 1", + "field": "apache.status.load.1" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Load 15", + "field": "apache.status.load.15" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Hostname", + "field": "host.hostname", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "split", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, + "interpolate": "linear", + "labels": {}, + "legendPosition": "right", + "radiusRatio": 9, + "row": true, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Load 5" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Load 1" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "4", + "label": "Load 15" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "shareYAxis": true, + "showCircles": true, + "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ], + "yAxis": {} + }, + "title": "Load1/5/15 [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-HTTPD-Load1-slash-5-slash-15", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-HTTPD", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-HTTPD-Scoreboard.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-HTTPD-Scoreboard.json new file mode 100644 index 0000000000..1167e1092a --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-HTTPD-Scoreboard.json @@ -0,0 +1,433 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "savedSearchRefName": "search_0", + "title": "Scoreboard [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Closing connection", + "field": "apache.status.scoreboard.closing_connection" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Hostname", + "field": "host.hostname", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "split", + "type": "terms" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "DNS lookup", + "field": "apache.status.scoreboard.dns_lookup" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Gracefully finishing", + "field": "apache.status.scoreboard.gracefully_finishing" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "Idle cleanup", + "field": "apache.status.scoreboard.idle_cleanup" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "7", + "params": { + "customLabel": "Keepalive", + "field": "apache.status.scoreboard.keepalive" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "8", + "params": { + "customLabel": "Logging", + "field": "apache.status.scoreboard.logging" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "9", + "params": { + "customLabel": "Open slot", + "field": "apache.status.scoreboard.open_slot" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "10", + "params": { + "customLabel": "Reading request", + "field": "apache.status.scoreboard.reading_request" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "11", + "params": { + "customLabel": "Sending reply", + "field": "apache.status.scoreboard.sending_reply" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "12", + "params": { + "customLabel": "Starting up", + "field": "apache.status.scoreboard.starting_up" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "13", + "params": { + "customLabel": "Total", + "field": "apache.status.scoreboard.total" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "14", + "params": { + "customLabel": "Waiting for connection", + "field": "apache.status.scoreboard.waiting_for_connection" + }, + "schema": "metric", + "type": "avg" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, + "interpolate": "linear", + "labels": {}, + "legendPosition": "right", + "radiusRatio": 9, + "row": true, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Closing connection" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "4", + "label": "DNS lookup" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "5", + "label": "Gracefully finishing" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "6", + "label": "Idle cleanup" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "7", + "label": "Keepalive" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "8", + "label": "Logging" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "9", + "label": "Open slot" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "10", + "label": "Reading request" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "11", + "label": "Sending reply" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "12", + "label": "Starting up" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "13", + "label": "Total" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "14", + "label": "Waiting for connection" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "shareYAxis": true, + "showCircles": true, + "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ], + "yAxis": {} + }, + "title": "Scoreboard [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-HTTPD-Scoreboard", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-HTTPD", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-a45311f0-3a34-11eb-8946-296aab7b13db.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-a45311f0-3a34-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..ec859d0cc0 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-a45311f0-3a34-11eb-8946-296aab7b13db.json @@ -0,0 +1,152 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "savedSearchRefName": "search_0", + "title": "Bytes per sec [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Bytes per sec", + "field": "apache.status.bytes_per_sec" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, + "interpolate": "linear", + "labels": {}, + "legendPosition": "right", + "radiusRatio": 9, + "row": true, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Bytes per sec" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "shareYAxis": true, + "showCircles": true, + "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Bytes per sec" + }, + "type": "value" + } + ], + "yAxis": {} + }, + "title": "Bytes per sec [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-a45311f0-3a34-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-HTTPD", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-access-unique-IPs-map.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-access-unique-IPs-map.json new file mode 100644 index 0000000000..4504b84a68 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-access-unique-IPs-map.json @@ -0,0 +1,85 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Unique IPs map [Logs Apache]", + "uiStateJSON": { + "mapCenter": [ + 14.944784875088372, + 5.09765625 + ] + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "field": "source.address" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "2", + "params": { + "autoPrecision": true, + "field": "source.geo.location" + }, + "schema": "segment", + "type": "geohash_grid" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "heatBlur": 15, + "heatMaxZoom": 16, + "heatMinOpacity": 0.1, + "heatNormalizeData": true, + "heatRadius": 25, + "isDesaturated": true, + "legendPosition": "bottomright", + "mapCenter": [ + 15, + 5 + ], + "mapType": "Scaled Circle Markers", + "mapZoom": 2, + "wms": { + "enabled": false, + "options": { + "attribution": "Maps provided by USGS", + "format": "image/png", + "layers": "0", + "styles": "", + "transparent": true, + "version": "1.3.0" + }, + "url": "https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer" + } + }, + "title": "Apache access unique IPs map", + "type": "tile_map" + } + }, + "id": "apache-access-unique-IPs-map", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-access-logs", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-browsers.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-browsers.json new file mode 100644 index 0000000000..5cc8e7bc52 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-browsers.json @@ -0,0 +1,73 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Browsers breakdown [Logs Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "field": "source.address" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "user_agent.name", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "user_agent.version", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "legendPosition": "bottom", + "shareYAxis": true + }, + "title": "Apache browsers", + "type": "pie" + } + }, + "id": "apache-browsers", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-access-logs", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-ed44f820-3a10-11eb-8946-296aab7b13db.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-ed44f820-3a10-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..bc84a0c66d --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-ed44f820-3a10-11eb-8946-296aab7b13db.json @@ -0,0 +1,56 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Apache Hostname [Logs Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "host.hostname", + "id": "1607512709833", + "indexPatternRefName": "control_0_index_pattern", + "label": "Hostname", + "options": { + "dynamicOptions": true, + "multiselect": false, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "Apache Hostname [Logs Apache]", + "type": "input_control_vis" + } + }, + "id": "apache-ed44f820-3a10-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "logs-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-error-logs-over-time.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-error-logs-over-time.json new file mode 100644 index 0000000000..bb97fac6a7 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-error-logs-over-time.json @@ -0,0 +1,77 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Error logs over time [Logs Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "log.level", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "legendPosition": "right", + "mode": "stacked", + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "times": [], + "yAxis": {} + }, + "title": "Apache error logs over time", + "type": "histogram" + } + }, + "id": "apache-error-logs-over-time", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-errors-log", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-f4ffec70-3a36-11eb-8946-296aab7b13db.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-f4ffec70-3a36-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..fad15583fb --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-f4ffec70-3a36-11eb-8946-296aab7b13db.json @@ -0,0 +1,180 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "savedSearchRefName": "search_0", + "title": "Workers [Metrics Apache]", + "uiStateJSON": { + "vis": { + "legendOpen": true + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Busy workers", + "field": "apache.status.workers.busy" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Idle workers", + "field": "apache.status.workers.idle" + }, + "schema": "metric", + "type": "avg" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, + "interpolate": "linear", + "labels": {}, + "legendPosition": "right", + "radiusRatio": 9, + "row": true, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Busy workers" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Idle workers" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "shareYAxis": true, + "showCircles": true, + "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": false, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Workers" + }, + "type": "value" + } + ], + "yAxis": {} + }, + "title": "Workers [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-f4ffec70-3a36-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-HTTPD", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-operating-systems.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-operating-systems.json new file mode 100644 index 0000000000..196f983cd6 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-operating-systems.json @@ -0,0 +1,73 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Operating systems breakdown [Logs Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "field": "source.address" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "user_agent.os.name", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "user_agent.os.version", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "legendPosition": "bottom", + "shareYAxis": true + }, + "title": "Apache operating systems", + "type": "pie" + } + }, + "id": "apache-operating-systems", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-access-logs", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-response-codes-of-top-URLs.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-response-codes-of-top-URLs.json new file mode 100644 index 0000000000..6b8eb93302 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-response-codes-of-top-URLs.json @@ -0,0 +1,80 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top URLs by response code [Logs Apache]", + "uiStateJSON": { + "vis": { + "colors": { + "200": "#7EB26D", + "404": "#EF843C" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "URL", + "field": "url.original", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "split", + "type": "terms" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "http.response.status_code", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": false, + "legendPosition": "right", + "row": false, + "shareYAxis": true + }, + "title": "Apache response codes of top URLs", + "type": "pie" + } + }, + "id": "apache-response-codes-of-top-URLs", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-access-logs", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/kibana/visualization/apache-response-codes-over-time.json b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-response-codes-over-time.json new file mode 100644 index 0000000000..8f36ed2601 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/kibana/visualization/apache-response-codes-over-time.json @@ -0,0 +1,84 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Response codes over time [Logs Apache]", + "uiStateJSON": { + "vis": { + "colors": { + "200": "#629E51", + "404": "#EF843C" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "http.response.status_code", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "legendPosition": "right", + "mode": "stacked", + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "times": [], + "yAxis": {} + }, + "title": "Apache response codes over time", + "type": "histogram" + } + }, + "id": "apache-response-codes-over-time", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-access-logs", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/parallel/apache_basic_license/manifest.yml b/test/packages/parallel/apache_basic_license/manifest.yml new file mode 100644 index 0000000000..01ddfe7e29 --- /dev/null +++ b/test/packages/parallel/apache_basic_license/manifest.yml @@ -0,0 +1,110 @@ +format_version: 2.0.0 +name: apache +title: Apache HTTP Server +# version is set to something very large to so this test package can +# be installed in the package registry regardless of the version of +# the actual apache package in the registry at any given time. +version: 999.999.999 +description: Collect logs and metrics from Apache servers with Elastic Agent. +type: integration +categories: + - web +release: ga +conditions: + kibana.version: "^7.14.0 || ^8.0.0" + elastic.subscription: basic +screenshots: + - src: /img/apache-metrics-overview.png + title: Apache metrics overview + size: 3360x3064 + type: image/png + - src: /img/apache-logs-overview.png + title: Apache logs overview + size: 3342x1384 + type: image/png +icons: + - src: /img/logo_apache.svg + title: Apache Logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: apache + title: Apache logs and metrics + description: Collect logs and metrics from Apache instances + inputs: + - type: logfile + title: Collect logs from Apache instances + description: Collecting Apache access and error logs + - type: httpjson + title: Collect logs from third-party REST API (experimental) + description: Collect logs from third-party REST API (experimental) + vars: + - name: url + type: text + title: URL of Splunk Enterprise Server + description: i.e. scheme://host:port, path is automatic + show_user: true + required: true + default: https://server.example.com:8089 + - name: username + type: text + title: Splunk REST API Username + show_user: true + required: false + - name: password + type: password + title: Splunk REST API Password + show_user: true + required: false + - name: token + type: password + title: Splunk Authorization Token + description: | + Bearer Token or Session Key, e.g. "Bearer eyJFd3e46..." + or "Splunk 192fd3e...". Cannot be used with username + and password. + show_user: true + required: false + - name: ssl + type: yaml + title: SSL Configuration + description: i.e. certificate_authorities, supported_protocols, verification_mode etc. + multi: false + required: false + show_user: false + default: | + #certificate_authorities: + # - | + # -----BEGIN CERTIFICATE----- + # MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF + # ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2 + # MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB + # BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n + # fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl + # 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t + # /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP + # PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41 + # CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O + # BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux + # 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D + # 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw + # 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA + # H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu + # 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0 + # yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk + # sxSmbIUfc2SGJGCJD4I= + # -----END CERTIFICATE----- + - type: apache/metrics + title: Collect metrics from Apache instances + description: Collecting Apache status metrics + vars: + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - http://127.0.0.1 +owner: + github: elastic/integrations diff --git a/tools/readme/readme.md.tmpl b/tools/readme/readme.md.tmpl index c32f612f75..49bb18bdf5 100644 --- a/tools/readme/readme.md.tmpl +++ b/tools/readme/readme.md.tmpl @@ -206,6 +206,9 @@ The following settings are available per profile: the serverless stack provider. * `stack.serverless.region` can be used to select the region to use when starting serverless projects. +* `stack.elastic_subscription` allows to select the Elastic subscription type to be used in the stack. + Currently, it is supported "basic" and "[trial](https://www.elastic.co/guide/en/elasticsearch/reference/current/start-trial.html)", + which enables all subscription features for 30 days. Defaults to "trial". ## Useful environment variables