diff --git a/.kitchen.yml b/.kitchen.yml index f59d428b..8a11f204 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -132,3 +132,7 @@ suites: provisioner: playbook: test/integration/issue-test.yml idempotency_test: false + - name: xpack-upgrade-trial + provisioner: + playbook: test/integration/xpack-upgrade-trial.yml + idempotency_test: false diff --git a/README.md b/README.md index f68ccba6..12fa889f 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ The latest Elasticsearch versions of 7.x & 6.x are actively tested. * For multi-instances use cases, we are now recommending Docker containers using our official images (https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html). ## Dependency + This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine. ## Usage @@ -69,6 +70,10 @@ This playbook uses [Kitchen](https://kitchen.ci/) for CI and local testing. ### Running the tests +* Ensure you have checked out this repository to `elaticsearch`, not `ansible-elasticsearch`. +* If you don't have a Gold or Platinum license to test with you can run the trial versions of the `xpack-upgrade` and `issue-test` suites by appending `-trial` to the `PATTERN` variable. +* You may need to explicity specify `VERSION=7.x` if some suites are failing. + Install the ruby dependencies with bundler ```sh @@ -151,12 +156,13 @@ Whilst the role installs Elasticsearch with the default configuration parameters * ```es_config['http.port']``` - the http port for the node * ```es_config['transport.port']``` - the transport port for the node * ```es_config['discovery.seed_hosts']``` - the unicast discovery list, in the comma separated format ```":,:"``` (typically the clusters dedicated masters) +* ```es_config['cluster.initial_master_nodes']``` - for 7.x and above the list of master-eligible nodes to boostrap the cluster, in the comma separated format ```":,:"``` (typically the node names of the clusters dedicated masters) * ```es_config['network.host']``` - sets both network.bind_host and network.publish_host to the same host value. The network.bind_host setting allows to control the host different network components will bind on. -The network.publish_host setting allows to control the host the node will publish itself within the cluster so other nodes will be able to connect to it. +The `network.publish_host` setting allows to control the host the node will publish itself within the cluster so other nodes will be able to connect to it. See https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html for further details on default binding behaviour and available options. -The role makes no attempt to enforce the setting of these are requires users to specify them appropriately. IT is recommended master nodes are listed and thus deployed first where possible. +The role makes no attempt to enforce the setting of these are requires users to specify them appropriately. It is recommended master nodes are listed and thus deployed first where possible. A more complex example: @@ -264,7 +270,7 @@ ansible-playbook -i hosts ./your-playbook.yml X-Pack features, such as Security, are supported. The parameter `es_xpack_features` allows to list xpack features to install (example: `["alerting","monitoring","graph","security","ml"]`). -When the list is empty, it install all features available with the current licence. +When the list is empty, it installs all features available with the current licence. * ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/x-pack/current/mapping-roles.html) @@ -349,6 +355,8 @@ es_roles: es_xpack_license: "{{ lookup('file', playbook_dir + '/files/' + es_cluster_name + '/license.json') }}" ``` +If you don't have a license you can enable the 30-day trial by setting `es_xpack_trial` to `true`. + X-Pack configuration parameters can be added to the elasticsearch.yml file using the normal `es_config` parameter. For a full example see [here](https://github.com/elastic/ansible-elasticsearch/blob/master/test/integration/xpack-upgrade.yml) @@ -362,26 +370,35 @@ In order for native users and roles to be configured, the role calls the Elastic These can either be set to a user declared in the file based realm, with admin permissions, or the default "elastic" superuser (default password is changeme). +#### X-Pack Security SSL/TLS + +* To configure your cluster with SSL/TLS for HTTP and/or transport communications follow the [SSL/TLS setup procedure](./docs/ssl-tls-setup.md) + ### Additional Configuration In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behaviour. Options include: * ```es_enable_xpack``` Default `true`. Setting this to `false` will install the oss release of elasticsearch +* `es_xpack_trial` Default `false`. Setting this to `true` will start the 30-day trail once the cluster starts. * ```es_version``` (e.g. "7.4.1"). * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200** * ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Security is enabled. Ensure this user is admin. * ```es_api_basic_auth_password``` The password associated with the user declared in `es_api_basic_auth_username` +* `es_delete_unmanaged_file` Default `true`. Set to false to keep file realm users that have been added outside of ansible. +* `es_delete_unmanaged_native` Default `true`. Set to false to keep native realm users that have been added outside of ansible. * ```es_start_service``` (true (default) or false) * ```es_plugins_reinstall``` (true or false (default) ) * ```es_plugins``` an array of plugin definitions e.g.: + ```yaml es_plugins: - plugin: ingest-attachment ``` + * ```es_path_repo``` Sets the whitelist for allowing local back-up repositories -* ```es_action_auto_create_index ``` Sets the value for auto index creation, use the syntax below for specifying indexes (else true/false): +* ```es_action_auto_create_index``` Sets the value for auto index creation, use the syntax below for specifying indexes (else true/false): es_action_auto_create_index: '[".watches", ".triggered_watches", ".watcher-history-*"]' * ```es_allow_downgrades``` For development purposes only. (true or false (default) ) * ```es_java_install``` If set to true, Java will be installed. (false (default for 7.x) or true (default for 6.x)) @@ -399,6 +416,7 @@ Earlier examples illustrate the installation of plugins using `es_plugins`. For If installing Monitoring or Alerting, ensure the license plugin is also specified. Security configuration currently has limited support, but more support is planned for later versions. To configure X-pack to send mail, the following configuration can be added to the role. When require_auth is true, you will also need to provide the user and password. If not these can be removed: + ```yaml es_mail_config: account: @@ -447,7 +465,7 @@ To define proxy only for a particular plugin during its installation: * The playbook relies on the inventory_name of each host to ensure its directories are unique * KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test the latest version of 7.x and 6.x on all supported platforms. * The role aims to be idempotent. Running the role multiple times, with no changes, should result in no state change on the server. If the configuration is changed, these will be applied and Elasticsearch restarted where required. -* In order to run x-pack tests a license file with security enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests. +* In order to run x-pack tests a license file with security enabled is required. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests. A trial license is appropriate and can be used by setting `es_xpack_trial` to `true` ## IMPORTANT NOTES RE PLUGIN MANAGEMENT diff --git a/ansible.cfg b/ansible.cfg index d9a8c501..0440d489 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1 +1 @@ -[defaults] \ No newline at end of file +[defaults] diff --git a/defaults/main.yml b/defaults/main.yml index dd021a6a..32e0b730 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -33,11 +33,35 @@ es_max_threads: 8192 es_max_map_count: 262144 es_allow_downgrades: false es_xpack_features: [] +es_xpack_trial: false #These are used for internal operations performed by ansible. #They do not affect the current configuration +es_api_scheme: "http" es_api_host: "localhost" es_api_port: 9200 +es_api_uri: "{{ es_api_scheme }}://{{ es_api_host }}:{{ es_api_port }}" +es_security_api: "{{ '_security' if es_version is version_compare('7.0.0', '>=') else '_xpack/security' }}" +es_license_api: "{{ '_license' if es_version is version_compare('7.0.0', '>=') else '_xpack/license' }}" +es_api_sleep: 15 es_debian_startup_timeout: 10 # JVM custom parameters es_jvm_custom_parameters: '' + +# SSL/TLS parameters +es_enable_auto_ssl_configuration: true +es_enable_http_ssl: false +es_enable_transport_ssl: false +es_ssl_keystore: "" +es_ssl_keystore_password: "" +es_ssl_truststore: "" +es_ssl_truststore_password: "" +es_ssl_key: "" +es_ssl_key_password: "" +es_ssl_certificate: "" +es_ssl_certificate_authority: "" +es_ssl_certificate_path: "{{ es_conf_dir }}/certs" +es_ssl_verification_mode: "certificate" +es_validate_certs: "yes" +es_delete_unmanaged_file: true +es_delete_unmanaged_native: true diff --git a/docs/ssl-tls-setup.md b/docs/ssl-tls-setup.md new file mode 100644 index 00000000..a887e653 --- /dev/null +++ b/docs/ssl-tls-setup.md @@ -0,0 +1,78 @@ +# X-Pack Security SSL/TLS + +The role allows configuring HTTP and transport layer SSL/TLS for the cluster. You will need to generate and provide your own PKCS12 or PEM encoded certificates as described in [Encrypting communications in Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/7.4/configuring-tls.html#configuring-tls). + +If you don't want this role to add autogenerated SSL configuration to elasticsearch.yml set `es_enable_auto_ssl_configuration` to `false` (default: `true`). + +The following should be configured to ensure a security-enabled cluster successfully forms: + +* `es_enable_http_ssl` Default `false`. Setting this to `true` will enable HTTP client SSL/TLS +* `es_enable_transport_ssl` - Default `false`. Setting this to `true` will enable transport layer SSL/TLS + +When using a [PKCS12](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#security-http-pkcs12-files) keystore and truststore: + +* `es_ssl_keystore` path to your PKCS12 keystore (can be the same as `es_ssl_truststore`) +* `es_ssl_keystore_password` set this if your keystore is protected with a password +* `es_ssl_truststore` path to your PKCS12 keystore (can be the same as `es_ssl_keystore`) +* `es_ssl_truststore_password` set this if your truststore is protected with a password + +When using [PEM encoded](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#_pem_encoded_files_3) certificates: + +* `es_ssl_key` path to your SSL key +* `es_ssl_key_password` set this if your SSL key is protected with a password +* `es_ssl_certificate` the path to your SSL certificate + +## Generating an SSL keystore + +With a password: + +```shell +$ bin/elasticsearch-certutil ca --out ./my-ca.p12 --pass "ca_password" +$ bin/elasticsearch-certutil cert --ca ./my-ca.p12 --ca-pass "ca_password" --out ./my-keystore.p12 --pass "keystore_password" +``` + +Without a password: + +```shell +$ bin/elasticsearch-certutil ca --out ./my-ca.p12 --pass "" +$ bin/elasticsearch-certutil cert --ca ./my-ca.p12 --out ./my-keystore.p12 --pass "" +``` + +## Additional optional SSL/TLS configuration + +* `es_enable_auto_ssl_configuration` Default `true`. Whether this role should add automatically generated SSL config to elasticsearch.yml. +* `es_ssl_certificate_path` Default `{{ es_conf_dir }}/certs`. The location where certificates should be stored on the ES node. +* `es_ssl_verification_mode` Default `certificate`. See [SSL verification_mode](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#ssl-tls-settings) for options. +* `es_ssl_certificate_authority` PEM encoded certificate file that should be trusted. +* `es_validate_certs` Default `yes`. Determines if ansible should validate SSL certificates when performing actions over HTTPS. e.g. installing templates and managing native users. + +## Example SSL/TLS configuration + +```yaml +- name: Elasticsearch with SSL/TLS enabled + hosts: localhost + roles: + - role: elastic.elasticsearch + vars: + es_config: + node.name: "node1" + cluster.name: "custom-cluster" + discovery.seed_hosts: "localhost:9301" + http.port: 9201 + transport.port: 9301 + node.data: false + node.master: true + bootstrap.memory_lock: true + xpack.security.authc.realms.file.file1.order: 0 + xpack.security.authc.realms.native.native1.order: 1 + es_heap_size: 1g + es_api_basic_auth_username: elastic + es_api_basic_auth_password: changeme + es_enable_http_ssl: true + es_enable_transport_ssl: true + es_ssl_keystore: "files/certs/my-keystore.p12" + es_ssl_truststore: "files/certs/my-truststore.p12" + es_ssl_keystore_password: "keystore_password" + es_ssl_truststore_password: "truststore_password" + es_validate_certs: no +``` diff --git a/tasks/compatibility-variables.yml b/tasks/compatibility-variables.yml index a0b6dfc8..f624d145 100644 --- a/tasks/compatibility-variables.yml +++ b/tasks/compatibility-variables.yml @@ -23,3 +23,8 @@ es_other_package_name: "elasticsearch" when: - not es_enable_xpack + +- name: Set the URL scheme to https if SSL/TLS is enabled + set_fact: + es_api_scheme: "https" + when: es_enable_http_ssl | bool diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 36c3fe13..b25e4f34 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -17,6 +17,13 @@ - es_api_basic_auth_username is not defined - es_api_basic_auth_password is not defined +- name: fail when ssl enabled without defining a key and certificate + fail: msg="Enabling SSL/TLS (es_enable_http_ssl or es_enable_transport_ssl) requires es_ssl_keystore and es_ssl_truststore or es_ssl_key and es_ssl_certificate to be provided" + when: + - es_enable_http_ssl or es_enable_transport_ssl + - (es_ssl_key == "" or es_ssl_certificate == "") + - (es_ssl_keystore == "" or es_ssl_truststore == "") + - name: set fact file_reserved_users set_fact: file_reserved_users={{ es_users.file.keys() | list | intersect (reserved_xpack_users) }} when: es_users is defined and es_users.file is defined and (es_users.file.keys() | list | length > 0) and (es_users.file.keys() | list | intersect (reserved_xpack_users) | length > 0) diff --git a/tasks/elasticsearch-ssl.yml b/tasks/elasticsearch-ssl.yml new file mode 100644 index 00000000..b4d2212d --- /dev/null +++ b/tasks/elasticsearch-ssl.yml @@ -0,0 +1,112 @@ +--- + +- name: set fact es_same_keystore + set_fact: es_same_keystore=false + +- name: set fact es_same_keystore if stores match + set_fact: es_same_keystore=true + when: es_ssl_keystore == es_ssl_truststore + +- name: ensure certificate directory exists + file: + dest: "{{ es_ssl_certificate_path }}" + state: directory + owner: root + group: "{{ es_group }}" + mode: 0750 + +- name: Upload SSL/TLS keystore + copy: + src: "{{ es_ssl_keystore }}" + dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_keystore | basename }}" + owner: "{{ es_user }}" + group: "{{ es_group }}" + mode: 0640 + when: es_ssl_keystore and es_ssl_truststore + notify: restart elasticsearch + register: copy_keystore + +- name: Upload SSL/TLS truststore + copy: + src: "{{ es_ssl_truststore }}" + dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_truststore | basename }}" + owner: "{{ es_user }}" + group: "{{ es_group }}" + mode: 0640 + when: es_ssl_keystore and es_ssl_truststore + notify: restart elasticsearch + register: copy_truststore + +- name: Upload SSL/TLS key and certificate + copy: + src: "{{ item }}" + dest: "{{ es_ssl_certificate_path }}/{{ item | basename }}" + owner: "{{ es_user }}" + group: "{{ es_group }}" + mode: 0640 + with_items: + - "{{ es_ssl_key }}" + - "{{ es_ssl_certificate }}" + when: es_ssl_key and es_ssl_certificate + #Restart if these change + notify: restart elasticsearch + register: copy_certificates + +- name: Upload SSL Certificate Authority + copy: + src: "{{ es_ssl_certificate_authority }}" + dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate_authority | basename }}" + owner: "{{ es_user }}" + group: "{{ es_group }}" + mode: 0640 + #Restart if this changes + notify: restart elasticsearch + when: es_ssl_certificate_authority | bool + +- name: Set keystore password + shell: echo "{{ es_ssl_keystore_password }}" | {{ es_home }}/bin/elasticsearch-keystore add -x -f 'xpack.security.{{ item }}.ssl.keystore.secure_password' + no_log: True + when: es_ssl_keystore_password and (copy_keystore.changed or (es_same_keystore and copy_truststore.changed)) + with_items: + - http + - transport + +- name: Set truststore password + shell: echo "{{ es_ssl_truststore_password }}" | {{ es_home }}/bin/elasticsearch-keystore add -x -f 'xpack.security.{{ item }}.ssl.truststore.secure_password' + no_log: True + when: es_ssl_truststore_password and (copy_truststore.changed or (es_same_keystore and copy_keystore.changed)) + with_items: + - http + - transport + +- name: Remove keystore password + shell: "{{ es_home }}/bin/elasticsearch-keystore remove 'xpack.security.{{ item }}.ssl.keystore.secure_password'" + when: es_ssl_keystore_password == "" and (copy_keystore.changed or (es_same_keystore and copy_truststore.changed)) + ignore_errors: yes + with_items: + - http + - transport + +- name: Remove truststore password + shell: "{{ es_home }}/bin/elasticsearch-keystore remove 'xpack.security.{{ item }}.ssl.truststore.secure_password'" + when: es_ssl_truststore_password == "" and (copy_truststore.changed or (es_same_keystore and copy_keystore.changed)) + ignore_errors: yes + with_items: + - http + - transport + +- name: Set key password + shell: echo "{{ es_ssl_key_password }}" | {{ es_home }}/bin/elasticsearch-keystore add -x -f 'xpack.security.{{ item }}.ssl.secure_key_passphrase' + no_log: True + when: es_ssl_key_password and copy_certificates.changed + with_items: + - http + - transport + +- name: Remove key password + shell: "{{ es_home }}/bin/elasticsearch-keystore remove 'xpack.security.{{ item }}.ssl.secure_key_passphrase'" + when: es_ssl_key_password == "" and copy_certificates.changed + ignore_errors: yes + with_items: + - http + - transport diff --git a/tasks/elasticsearch-template.yml b/tasks/elasticsearch-template.yml index 41f5f41d..a04097f6 100644 --- a/tasks/elasticsearch-template.yml +++ b/tasks/elasticsearch-template.yml @@ -16,7 +16,7 @@ - name: Install templates uri: - url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}" + url: "{{ es_api_uri }}/_template/{{item | filename}}" method: PUT status_code: 200 user: "{{es_api_basic_auth_username | default(omit)}}" @@ -24,6 +24,7 @@ force_basic_auth: yes body_format: json body: "{{ lookup('file', item) }}" + validate_certs: "{{ es_validate_certs }}" when: load_templates.changed and es_start_service with_fileglob: - "{{ es_templates_fileglob | default('') }}" diff --git a/tasks/main.yml b/tasks/main.yml index 0c7cb83d..8cd4bf57 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -53,6 +53,12 @@ tags: - xpack +- name: include elasticsearch-ssl.yml + include: elasticsearch-ssl.yml + when: es_enable_http_ssl or es_enable_transport_ssl + tags: + - xpack + - name: flush handlers meta: flush_handlers @@ -62,7 +68,7 @@ when: es_start_service - name: Wait for elasticsearch to startup - wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1 + wait_for: host={{ es_api_host }} port={{ es_api_port }} delay=5 connect_timeout=1 when: es_restarted is defined and es_restarted.changed and es_start_service - name: set fact manage_native_realm to false @@ -76,18 +82,22 @@ - (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) # If playbook runs too fast, Native commands could fail as the Native Realm is not yet up -- name: Wait 15 seconds for the Native Relm to come up - command: sleep 15 - when: manage_native_realm +- name: Wait {{ es_api_sleep }} seconds for the Native Realm to come up + command: "sleep {{ es_api_sleep }}" + when: manage_native_realm | bool - name: activate-license include: ./xpack/security/elasticsearch-xpack-activation.yml when: es_start_service and es_enable_xpack and es_xpack_license is defined and es_xpack_license != '' +- name: activate-trial + include: ./xpack/security/elasticsearch-xpack-trial-activation.yml + when: es_start_service and es_enable_xpack and es_xpack_trial + #perform security actions here now elasticsearch is started - name: include xpack/security/elasticsearch-security-native.yml include: ./xpack/security/elasticsearch-security-native.yml - when: manage_native_realm + when: manage_native_realm | bool #Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed if we don't wait for the node to restart. #We also do after the native realm to ensure any changes are applied here first and its denf up. diff --git a/tasks/xpack/security/elasticsearch-security-file.yml b/tasks/xpack/security/elasticsearch-security-file.yml index f81117a5..b3ed0274 100644 --- a/tasks/xpack/security/elasticsearch-security-file.yml +++ b/tasks/xpack/security/elasticsearch-security-file.yml @@ -11,7 +11,7 @@ register: old_users_file check_mode: no -- name: Copy the old users file from the old depreacted location +- name: Copy the old users file from the old deprecated location copy: remote_src: yes force: no # only copy it if the new path doesn't exist yet @@ -33,7 +33,7 @@ - name: set fact users_to_remove set_fact: users_to_remove={{ current_file_users.stdout_lines | difference (es_users.file.keys() | list) }} - when: manage_file_users + when: manage_file_users and es_delete_unmanaged_file #Remove users - name: Remove Users @@ -49,7 +49,7 @@ - name: set fact users_to_add set_fact: users_to_add={{ es_users.file.keys() | list | difference (current_file_users.stdout_lines) }} - when: manage_file_users + when: manage_file_users and es_delete_unmanaged_file #Add users - name: Add Users diff --git a/tasks/xpack/security/elasticsearch-security-native.yml b/tasks/xpack/security/elasticsearch-security-native.yml index 6235952e..7a27fd4a 100644 --- a/tasks/xpack/security/elasticsearch-security-native.yml +++ b/tasks/xpack/security/elasticsearch-security-native.yml @@ -21,12 +21,13 @@ #List current users - name: List Native Users uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user + url: "{{ es_api_uri }}/{{ es_security_api }}/user" method: GET user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes status_code: 200 + validate_certs: "{{ es_validate_certs }}" register: user_list_response when: manage_native_users check_mode: no @@ -51,7 +52,7 @@ - name: Update API User Password uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{es_api_basic_auth_username}}/_password + url: "{{ es_api_uri }}/{{ es_security_api }}/user/{{es_api_basic_auth_username}}/_password" method: POST body_format: json body: "{ \"password\":\"{{native_users[es_api_basic_auth_username].password}}\" }" @@ -59,6 +60,7 @@ user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes + validate_certs: "{{ es_validate_certs }}" when: change_api_password - name: set fact es_api_basic_auth_password @@ -73,13 +75,14 @@ #Delete all non required users NOT inc. reserved - name: Delete Native Users uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}} + url: "{{ es_api_uri }}/{{ es_security_api }}/user/{{item}}" method: DELETE status_code: 200 user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes - when: manage_native_users + validate_certs: "{{ es_validate_certs }}" + when: manage_native_users and es_delete_unmanaged_native with_items: "{{ users_to_remove | default([]) }}" - name: set fact users_to_ignore @@ -94,7 +97,7 @@ #Update password on all reserved users - name: Update Reserved User Passwords uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}/_password + url: "{{ es_api_uri }}/{{ es_security_api }}/user/{{ item | urlencode }}/_password" method: POST body_format: json body: "{ \"password\":\"{{native_users[item].password}}\" }" @@ -102,6 +105,7 @@ user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes + validate_certs: "{{ es_validate_certs }}" when: native_users[item].password is defined no_log: True with_items: "{{ users_to_ignore | default([]) }}" @@ -113,7 +117,7 @@ #Overwrite all other users NOT inc. those reserved - name: Update Non-Reserved Native User Details uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}} + url: "{{ es_api_uri }}/{{ es_security_api }}/user/{{ item | urlencode }}" method: POST body_format: json body: "{{ native_users[item] | to_json }}" @@ -121,6 +125,7 @@ user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes + validate_certs: "{{ es_validate_certs }}" when: manage_native_users no_log: True with_items: "{{ users_to_modify | default([]) }}" @@ -130,13 +135,14 @@ #List current roles not. inc those reserved - name: List Native Roles uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role + url: "{{ es_api_uri }}/{{ es_security_api }}/role" method: GET body_format: json user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes status_code: 200 + validate_certs: "{{ es_validate_certs }}" register: role_list_response when: manage_native_roles check_mode: no @@ -165,13 +171,14 @@ #Delete all non required roles NOT inc. reserved - name: Delete Native Roles uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}} + url: "{{ es_api_uri }}/{{ es_security_api }}/role/{{ item | urlencode }}" method: DELETE status_code: 200 user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes - when: manage_native_roles + validate_certs: "{{ es_validate_certs }}" + when: manage_native_roles and es_delete_unmanaged_native with_items: "{{roles_to_remove | default([]) }}" - name: set fact roles_to_modify @@ -181,7 +188,7 @@ #Update other roles - NOT inc. reserved roles - name: Update Native Roles uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}} + url: "{{ es_api_uri }}/{{ es_security_api }}/role/{{ item | urlencode }}" method: POST body_format: json body: "{{ es_roles.native[item] | to_json}}" @@ -189,5 +196,6 @@ user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes + validate_certs: "{{ es_validate_certs }}" when: manage_native_roles with_items: "{{ roles_to_modify | default([]) }}" diff --git a/tasks/xpack/security/elasticsearch-xpack-activation.yml b/tasks/xpack/security/elasticsearch-xpack-activation.yml index 7da085e5..5bf08cb6 100644 --- a/tasks/xpack/security/elasticsearch-xpack-activation.yml +++ b/tasks/xpack/security/elasticsearch-xpack-activation.yml @@ -2,13 +2,14 @@ - name: Activate ES license (with security authentication) uri: method: PUT - url: "http://{{es_api_host}}:{{es_api_port}}/_xpack/license?acknowledge=true" + url: "{{ es_api_uri }}/{{ es_license_api }}?acknowledge=true" user: "{{es_api_basic_auth_username | default(omit)}}" password: "{{es_api_basic_auth_password | default(omit)}}" body_format: json body: "{{ es_xpack_license }}" return_content: yes force_basic_auth: yes + validate_certs: "{{ es_validate_certs }}" register: license_activated no_log: True failed_when: > @@ -16,5 +17,5 @@ license_activated.json.license_status is not defined or license_activated.json.license_status != 'valid' -- debug: - msg: "License: {{ license_activated }}" +- name: License + debug: msg={{ license_activated }} diff --git a/tasks/xpack/security/elasticsearch-xpack-trial-activation.yml b/tasks/xpack/security/elasticsearch-xpack-trial-activation.yml new file mode 100644 index 00000000..e0cc73e0 --- /dev/null +++ b/tasks/xpack/security/elasticsearch-xpack-trial-activation.yml @@ -0,0 +1,18 @@ +--- +- name: Activate ES trial license (with security authentication) + uri: + method: POST + url: "{{ es_api_uri }}/{{ es_license_api }}/start_trial?acknowledge=true" + user: "{{es_api_basic_auth_username | default(omit)}}" + password: "{{es_api_basic_auth_password | default(omit)}}" + return_content: yes + force_basic_auth: yes + status_code: + - 200 + - 403 + validate_certs: "{{ es_validate_certs }}" + register: trial_license_activated + when: es_xpack_trial + +- name: Trial license + debug: msg={{ trial_license_activated }} \ No newline at end of file diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index 9ceac83d..6adaa0dd 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -33,6 +33,35 @@ action.auto_create_index: {{ es_action_auto_create_index }} {% if es_enable_xpack and es_api_basic_auth_username is defined and es_api_basic_auth_password is defined %} xpack.security.enabled: true + +{% if es_enable_transport_ssl and es_enable_auto_ssl_configuration %} +xpack.security.transport.ssl.enabled: true +xpack.security.transport.ssl.verification_mode: "{{ es_ssl_verification_mode }}" +{% if es_ssl_keystore and es_ssl_truststore %} +xpack.security.transport.ssl.keystore.path: "{{ es_ssl_certificate_path }}/{{ es_ssl_keystore | basename }}" +xpack.security.transport.ssl.truststore.path: "{{ es_ssl_certificate_path }}/{{ es_ssl_truststore | basename }}" +{% elif es_ssl_key and es_ssl_certificate %} +xpack.security.transport.ssl.key: "{{ es_ssl_certificate_path }}/{{ es_ssl_key | basename }}" +xpack.security.transport.ssl.certificate: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate | basename }}" +{% if es_ssl_certificate_authority %} +xpack.security.transport.ssl.certificate_authorities: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate_authority | basename }}" +{% endif %} +{% endif %} +{% endif %} + +{% if es_enable_http_ssl and es_enable_auto_ssl_configuration %} +xpack.security.http.ssl.enabled: true +{% if es_ssl_keystore and es_ssl_truststore %} +xpack.security.http.ssl.keystore.path: "{{ es_ssl_certificate_path }}/{{ es_ssl_keystore | basename }}" +xpack.security.http.ssl.truststore.path: "{{ es_ssl_certificate_path }}/{{ es_ssl_truststore | basename }}" +{% elif es_ssl_key and es_ssl_certificate %} +xpack.security.http.ssl.key: "{{ es_ssl_certificate_path }}/{{ es_ssl_key | basename }}" +xpack.security.http.ssl.certificate: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate | basename }}" +{% if es_ssl_certificate_authority %} +xpack.security.http.ssl.certificate_authorities: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate_authority | basename }}" +{% endif %} +{% endif %} +{% endif %} {% endif %} {% if es_mail_config is defined %} diff --git a/test/integration/files/certs/keystore-password-ca.p12 b/test/integration/files/certs/keystore-password-ca.p12 new file mode 100644 index 00000000..e8331cf9 Binary files /dev/null and b/test/integration/files/certs/keystore-password-ca.p12 differ diff --git a/test/integration/files/certs/keystore-password.p12 b/test/integration/files/certs/keystore-password.p12 new file mode 100644 index 00000000..6a58eaac Binary files /dev/null and b/test/integration/files/certs/keystore-password.p12 differ diff --git a/test/integration/files/certs/shared-store-no-password-ca.p12 b/test/integration/files/certs/shared-store-no-password-ca.p12 new file mode 100644 index 00000000..9b76934b Binary files /dev/null and b/test/integration/files/certs/shared-store-no-password-ca.p12 differ diff --git a/test/integration/files/certs/shared-store-no-password.p12 b/test/integration/files/certs/shared-store-no-password.p12 new file mode 100644 index 00000000..c675ae21 Binary files /dev/null and b/test/integration/files/certs/shared-store-no-password.p12 differ diff --git a/test/integration/files/certs/truststore-password-ca.p12 b/test/integration/files/certs/truststore-password-ca.p12 new file mode 100644 index 00000000..a1662a83 Binary files /dev/null and b/test/integration/files/certs/truststore-password-ca.p12 differ diff --git a/test/integration/files/certs/truststore-password.p12 b/test/integration/files/certs/truststore-password.p12 new file mode 100644 index 00000000..25d9aa4c Binary files /dev/null and b/test/integration/files/certs/truststore-password.p12 differ diff --git a/files/templates-6.x/basic.json b/test/integration/files/templates-6.x/basic.json similarity index 100% rename from files/templates-6.x/basic.json rename to test/integration/files/templates-6.x/basic.json diff --git a/files/templates-7.x/basic.json b/test/integration/files/templates-7.x/basic.json similarity index 100% rename from files/templates-7.x/basic.json rename to test/integration/files/templates-7.x/basic.json diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb index 93d30254..0522fe49 100644 --- a/test/integration/helpers/serverspec/shared_spec.rb +++ b/test/integration/helpers/serverspec/shared_spec.rb @@ -17,7 +17,7 @@ family = families[vars['ansible_os_family']] -es_api_url = "http://localhost:#{vars['es_api_port']}" +es_api_url = "#{vars['es_api_scheme']}://localhost:#{vars['es_api_port']}" username = vars['es_api_basic_auth_username'] password = vars['es_api_basic_auth_password'] diff --git a/test/integration/helpers/serverspec/spec_helper.rb b/test/integration/helpers/serverspec/spec_helper.rb index 20ca46b3..2f18ef3d 100644 --- a/test/integration/helpers/serverspec/spec_helper.rb +++ b/test/integration/helpers/serverspec/spec_helper.rb @@ -19,7 +19,12 @@ def curl_json(uri, username=nil, password=nil) if username && password req.basic_auth username, password end - res = Net::HTTP.start(uri.hostname, uri.port) {|http| + res = Net::HTTP.start( + uri.hostname, + uri.port, + :use_ssl => uri.scheme == 'https', + :verify_mode => OpenSSL::SSL::VERIFY_NONE + ) {|http| http.request(req) } return JSON.parse(res.body) diff --git a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb index 62c95282..237b1de8 100644 --- a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb +++ b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb @@ -1,7 +1,20 @@ require 'spec_helper' require 'json' +require 'pathname' vars = JSON.parse(File.read('/tmp/vars.json')) +es_api_url = "#{vars['es_api_scheme']}://localhost:#{vars['es_api_port']}" +username = vars['es_api_basic_auth_username'] +password = vars['es_api_basic_auth_password'] +es_keystore = Pathname.new(vars['es_ssl_keystore']).basename.to_s +es_truststore = Pathname.new(vars['es_ssl_truststore']).basename.to_s + +if vars['es_major_version'] == '7.x' + es_security_api = "_security" +else + es_security_api = "_xpack/security" +end + shared_examples 'xpack_upgrade::init' do |vars| #Test users file, users_roles and roles.yml describe file("/etc/elasticsearch/users_roles") do @@ -18,7 +31,7 @@ describe 'security roles' do it 'should list the security roles' do - roles = curl_json('http://localhost:9200/_xpack/security/role', username='es_admin', password='changeMeAgain') + roles = curl_json("#{es_api_url}/#{es_security_api}/role", username='es_admin', password='changeMeAgain') expect(roles.key?('superuser')) end end @@ -33,6 +46,10 @@ it { should contain 'security.authc.realms.native1.order: 1' } it { should contain 'security.authc.realms.native1.type: native' } end + it { should contain 'xpack.security.transport.ssl.enabled: true' } + it { should contain 'xpack.security.http.ssl.enabled: true' } + it { should contain es_keystore } + it { should contain es_truststore } end #Test contents of role_mapping.yml @@ -47,14 +64,12 @@ #check accounts are correct i.e. we can auth and they have the correct roles describe 'kibana4_server access check' do it 'should be reported as version '+vars['es_version'] do - command = command('curl -s localhost:9200/ -u kibana4_server:changeMe | grep number') - expect(command.stdout).to match(vars['es_version']) - expect(command.exit_status).to eq(0) + expect(curl_json(es_api_url, username='kibana4_server', password='changeMe')['version']['number']).to eq(vars['es_version']) end end describe 'security users' do - result = curl_json('http://localhost:9200/_xpack/security/user', username='elastic', password='elasticChanged') + result = curl_json("#{es_api_url}/#{es_security_api}/user", username=username, password=password) it 'should have the elastic user' do expect(result['elastic']['username']).to eq('elastic') expect(result['elastic']['roles']).to eq(['superuser']) @@ -79,9 +94,17 @@ describe 'logstash_system access check' do it 'should be reported as version '+vars['es_version'] do - command = command('curl -s localhost:9200/ -u logstash_system:aNewLogstashPassword | grep number') - expect(command.stdout).to match(vars['es_version']) - expect(command.exit_status).to eq(0) + expect(curl_json(es_api_url, username='logstash_system', password='aNewLogstashPassword')['version']['number']).to eq(vars['es_version']) + end + end + + describe 'SSL certificate check' do + certificates = curl_json("#{es_api_url}/_ssl/certificates", username=username, password=password) + it 'should list the keystore file' do + expect(certificates.any? { |cert| cert['path'].include? es_keystore }).to be true + end + it 'should list the truststore file' do + expect(certificates.any? { |cert| cert['path'].include? es_truststore }).to be true end end end diff --git a/test/integration/xpack-upgrade-trial b/test/integration/xpack-upgrade-trial new file mode 120000 index 00000000..3021ce03 --- /dev/null +++ b/test/integration/xpack-upgrade-trial @@ -0,0 +1 @@ +xpack-upgrade \ No newline at end of file diff --git a/test/integration/xpack-upgrade-trial.yml b/test/integration/xpack-upgrade-trial.yml new file mode 100644 index 00000000..e2500d2d --- /dev/null +++ b/test/integration/xpack-upgrade-trial.yml @@ -0,0 +1,181 @@ +--- +- name: Elasticsearch Xpack HTTP different keystore and truststore with password + hosts: localhost + post_tasks: + - include: elasticsearch/test/integration/debug.yml + roles: + - elasticsearch + vars: + es_config_6x: + xpack.security.authc.realms.file1.order: 0 + xpack.security.authc.realms.file1.type: file + xpack.security.authc.realms.native1.order: 1 + xpack.security.authc.realms.native1.type: native + es_config_7x: + xpack.security.authc.realms.file.file1.order: 0 + xpack.security.authc.realms.native.native1.order: 1 + es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" + es_heap_size: "1g" + es_templates: true + es_templates_fileglob: "test/integration/files/templates-{{ es_major_version }}/*.json" + es_major_version: "7.x" + es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade + es_xpack_license: "" + es_xpack_trial: true + es_plugins: + - plugin: ingest-attachment + es_xpack_features: + - security + - alerting + es_api_basic_auth_username: elastic + es_api_basic_auth_password: changeme + es_api_sleep: 5 + es_enable_http_ssl: false + es_enable_transport_ssl: true + es_ssl_keystore: "test/integration/files/certs/keystore-password.p12" + es_ssl_truststore: "test/integration/files/certs/truststore-password.p12" + es_ssl_keystore_password: password1 + es_ssl_truststore_password: password2 + es_validate_certs: no + es_role_mapping: + power_user: + - "cn=admins,dc=example,dc=com" + user: + - "cn=users,dc=example,dc=com" + - "cn=admins,dc=example,dc=com" + es_users: + native: + kibana4_server: + password: changeMe + roles: + - kibana4_server + logstash_system: + #this should be successfully modified + password: aNewLogstashPassword + #this will be ignored + roles: + - kibana4_server + elastic: + password: elasticChanged + file: + es_admin: + password: changeMe + roles: + - admin + testUser: + password: changeMeAlso! + roles: + - power_user + - user + es_roles: + file: + admin: + cluster: + - all + indices: + - names: '*' + privileges: + - all + power_user: + cluster: + - monitor + indices: + - names: '*' + privileges: + - all + user: + indices: + - names: '*' + privileges: + - read + kibana4_server: + cluster: + - monitor + indices: + - names: '.kibana' + privileges: + - all + native: + logstash: + cluster: + - manage_index_templates + indices: + - names: 'logstash-*' + privileges: + - write + - delete + - create_index + #this will be ignored - its reserved + logstash_system: + cluster: + - manage_index_templates + indices: + - names: 'logstash-*' + privileges: + - write + - delete + - create_index + +#modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed. +- name: Elasticsearch Xpack HTTP SSL and shared keystore without password + hosts: localhost + post_tasks: + - include: elasticsearch/test/integration/debug.yml + roles: + - elasticsearch + vars: + es_config_6x: + xpack.security.authc.realms.file1.order: 0 + xpack.security.authc.realms.file1.type: file + xpack.security.authc.realms.native1.order: 1 + xpack.security.authc.realms.native1.type: native + es_config_7x: + xpack.security.authc.realms.file.file1.order: 0 + xpack.security.authc.realms.native.native1.order: 1 + es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" + es_heap_size: "1g" + es_templates: true + es_templates_fileglob: "test/integration/files/templates-{{ es_major_version }}/*.json" + es_xpack_license: "" + es_xpack_trial: false + es_plugins: + - plugin: ingest-attachment + es_xpack_features: + - security + - alerting + es_api_basic_auth_username: elastic + es_api_basic_auth_password: elasticChanged + es_api_sleep: 5 + es_enable_http_ssl: true + es_enable_transport_ssl: true + es_ssl_keystore: "test/integration/files/certs/shared-store-no-password.p12" + es_ssl_truststore: "test/integration/files/certs/shared-store-no-password.p12" + es_ssl_keystore_password: "" + es_ssl_truststore_password: "" + es_validate_certs: no + es_role_mapping: + power_user: + - "cn=admins,dc=example,dc=com" + user: + - "cn=users,dc=example,dc=com" + - "cn=admins,dc=example,dc=com" + es_users: + native: + kibana4_server: + password: changeMe + roles: + - kibana4_server + logstash_system: + #this will be ignored + roles: + - kibana4_server + file: + es_admin: + password: changeMeAgain + roles: + - admin + testUser: + password: changeMeAlso! + roles: + - power_user + - user diff --git a/test/integration/xpack-upgrade.yml b/test/integration/xpack-upgrade.yml index 12700076..abfe8ab5 100644 --- a/test/integration/xpack-upgrade.yml +++ b/test/integration/xpack-upgrade.yml @@ -1,5 +1,5 @@ --- -- name: Elasticsearch Xpack tests initial +- name: Elasticsearch Xpack HTTP different keystore and truststore with password hosts: localhost post_tasks: - include: elasticsearch/test/integration/debug.yml @@ -17,6 +17,7 @@ es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" es_heap_size: "1g" es_templates: true + es_templates_fileglob: "test/integration/files/templates-{{ es_major_version }}/*.json" es_major_version: "7.x" es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" @@ -27,6 +28,14 @@ - alerting es_api_basic_auth_username: elastic es_api_basic_auth_password: changeme + es_api_sleep: 5 + es_enable_http_ssl: false + es_enable_transport_ssl: true + es_ssl_keystore: "test/integration/files/certs/keystore-password.p12" + es_ssl_truststore: "test/integration/files/certs/truststore-password.p12" + es_ssl_keystore_password: password1 + es_ssl_truststore_password: password2 + es_validate_certs: no es_role_mapping: power_user: - "cn=admins,dc=example,dc=com" @@ -107,7 +116,7 @@ - create_index #modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed. -- name: Elasticsearch Xpack modify +- name: Elasticsearch Xpack HTTP SSL and shared keystore without password hosts: localhost post_tasks: - include: elasticsearch/test/integration/debug.yml @@ -125,6 +134,7 @@ es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" es_heap_size: "1g" es_templates: true + es_templates_fileglob: "test/integration/files/templates-{{ es_major_version }}/*.json" es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" es_plugins: - plugin: ingest-attachment @@ -133,6 +143,14 @@ - alerting es_api_basic_auth_username: elastic es_api_basic_auth_password: elasticChanged + es_api_sleep: 5 + es_enable_http_ssl: true + es_enable_transport_ssl: true + es_ssl_keystore: "test/integration/files/certs/shared-store-no-password.p12" + es_ssl_truststore: "test/integration/files/certs/shared-store-no-password.p12" + es_ssl_keystore_password: "" + es_ssl_truststore_password: "" + es_validate_certs: no es_role_mapping: power_user: - "cn=admins,dc=example,dc=com"