From e0ae86a58c1f13d406d5afabcb7a13e18574508c Mon Sep 17 00:00:00 2001 From: Xavier Naveira Date: Thu, 25 Jan 2024 16:39:15 +0100 Subject: [PATCH 1/8] Add downsample to ILMs There is a parameter for the hot, warm and cold phases called downsample. According to the description: "Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity." As of 0.11.0 there is no support for this parameter in the proivider, this PR adds it. --- .../elasticsearch_index_lifecycle.md | 30 +++++++++++++++++++ internal/elasticsearch/index/ilm.go | 28 +++++++++++++++-- internal/elasticsearch/index/ilm_test.go | 7 +++++ 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/docs/resources/elasticsearch_index_lifecycle.md b/docs/resources/elasticsearch_index_lifecycle.md index 3b8e2ccc6..21a1ea662 100644 --- a/docs/resources/elasticsearch_index_lifecycle.md +++ b/docs/resources/elasticsearch_index_lifecycle.md @@ -81,6 +81,7 @@ resource "elasticstack_elasticsearch_index_lifecycle" "my_ilm" { Optional: - `allocate` (Block List, Max: 1) Updates the index settings to change which nodes are allowed to host the index shards and change the number of replicas. (see [below for nested schema](#nestedblock--cold--allocate)) +- `downsample` (Block List, Max: 2) Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity. (see [below for nested schema](#nestedblock--cold--downsample)) - `freeze` (Block List, Max: 1) Freeze the index to minimize its memory footprint. (see [below for nested schema](#nestedblock--cold--freeze)) - `migrate` (Block List, Max: 1) Moves the index to the data tier that corresponds to the current phase by updating the "index.routing.allocation.include._tier_preference" index setting. (see [below for nested schema](#nestedblock--cold--migrate)) - `min_age` (String) ILM moves indices through the lifecycle according to their age. To control the timing of these transitions, you set a minimum age for each phase. @@ -101,6 +102,15 @@ Optional: - `total_shards_per_node` (Number) The maximum number of shards for the index on a single Elasticsearch node. Defaults to `-1` (unlimited). Supported from Elasticsearch version **7.16** + +### Nested Schema for `cold.downsample` + +Required: + +- `fixed_interval` (String) Downsampling interval +- `wait_timeout` (String) Downsampling interval + + ### Nested Schema for `cold.freeze` @@ -226,6 +236,7 @@ Optional: Optional: +- `downsample` (Block List, Max: 2) Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity. (see [below for nested schema](#nestedblock--hot--downsample)) - `forcemerge` (Block List, Max: 1) Force merges the index into the specified maximum number of segments. This action makes the index read-only. (see [below for nested schema](#nestedblock--hot--forcemerge)) - `min_age` (String) ILM moves indices through the lifecycle according to their age. To control the timing of these transitions, you set a minimum age for each phase. - `readonly` (Block List, Max: 1) Makes the index read-only. (see [below for nested schema](#nestedblock--hot--readonly)) @@ -235,6 +246,15 @@ Optional: - `shrink` (Block List, Max: 1) Sets a source index to read-only and shrinks it into a new index with fewer primary shards. (see [below for nested schema](#nestedblock--hot--shrink)) - `unfollow` (Block List, Max: 1) Convert a follower index to a regular index. Performed automatically before a rollover, shrink, or searchable snapshot action. (see [below for nested schema](#nestedblock--hot--unfollow)) + +### Nested Schema for `hot.downsample` + +Required: + +- `fixed_interval` (String) Downsampling interval +- `wait_timeout` (String) Downsampling interval + + ### Nested Schema for `hot.forcemerge` @@ -315,6 +335,7 @@ Optional: Optional: - `allocate` (Block List, Max: 1) Updates the index settings to change which nodes are allowed to host the index shards and change the number of replicas. (see [below for nested schema](#nestedblock--warm--allocate)) +- `downsample` (Block List, Max: 2) Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity. (see [below for nested schema](#nestedblock--warm--downsample)) - `forcemerge` (Block List, Max: 1) Force merges the index into the specified maximum number of segments. This action makes the index read-only. (see [below for nested schema](#nestedblock--warm--forcemerge)) - `migrate` (Block List, Max: 1) Moves the index to the data tier that corresponds to the current phase by updating the "index.routing.allocation.include._tier_preference" index setting. (see [below for nested schema](#nestedblock--warm--migrate)) - `min_age` (String) ILM moves indices through the lifecycle according to their age. To control the timing of these transitions, you set a minimum age for each phase. @@ -335,6 +356,15 @@ Optional: - `total_shards_per_node` (Number) The maximum number of shards for the index on a single Elasticsearch node. Defaults to `-1` (unlimited). Supported from Elasticsearch version **7.16** + +### Nested Schema for `warm.downsample` + +Required: + +- `fixed_interval` (String) Downsampling interval +- `wait_timeout` (String) Downsampling interval + + ### Nested Schema for `warm.forcemerge` diff --git a/internal/elasticsearch/index/ilm.go b/internal/elasticsearch/index/ilm.go index 5a8fc9650..16ec576b4 100644 --- a/internal/elasticsearch/index/ilm.go +++ b/internal/elasticsearch/index/ilm.go @@ -46,7 +46,7 @@ func ResourceIlm() *schema.Resource { MaxItems: 1, AtLeastOneOf: []string{"hot", "warm", "cold", "frozen", "delete"}, Elem: &schema.Resource{ - Schema: getSchema("set_priority", "unfollow", "rollover", "readonly", "shrink", "forcemerge", "searchable_snapshot"), + Schema: getSchema("set_priority", "unfollow", "rollover", "readonly", "shrink", "forcemerge", "searchable_snapshot", "downsample"), }, }, "warm": { @@ -56,7 +56,7 @@ func ResourceIlm() *schema.Resource { MaxItems: 1, AtLeastOneOf: []string{"hot", "warm", "cold", "frozen", "delete"}, Elem: &schema.Resource{ - Schema: getSchema("set_priority", "unfollow", "readonly", "allocate", "migrate", "shrink", "forcemerge"), + Schema: getSchema("set_priority", "unfollow", "readonly", "allocate", "migrate", "shrink", "forcemerge", "downsample"), }, }, "cold": { @@ -66,7 +66,7 @@ func ResourceIlm() *schema.Resource { MaxItems: 1, AtLeastOneOf: []string{"hot", "warm", "cold", "frozen", "delete"}, Elem: &schema.Resource{ - Schema: getSchema("set_priority", "unfollow", "readonly", "searchable_snapshot", "allocate", "migrate", "freeze"), + Schema: getSchema("set_priority", "unfollow", "readonly", "searchable_snapshot", "allocate", "migrate", "freeze", "downsample"), }, }, "frozen": { @@ -389,6 +389,26 @@ var supportedActions = map[string]*schema.Schema{ }, }, }, + "downsample": { + Description: "Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity.", + Type: schema.TypeList, + Optional: true, + MaxItems: 2, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "fixed_interval": { + Description: "Downsampling interval", + Type: schema.TypeString, + Required: true, + }, + "wait_timeout": { + Description: "Downsampling interval", + Type: schema.TypeString, + Required: true, + }, + }, + }, + }, } func getSchema(actions ...string) map[string]*schema.Schema { @@ -519,6 +539,8 @@ func expandPhase(p map[string]interface{}, serverVersion *version.Version) (*mod } case "wait_for_snapshot": actions[actionName], diags = expandAction(a, serverVersion, "policy") + case "downsample": + actions[actionName], diags = expandAction(a, serverVersion, "fixed_interval", "wait_timeout") default: diags = append(diags, diag.Diagnostic{ Severity: diag.Error, diff --git a/internal/elasticsearch/index/ilm_test.go b/internal/elasticsearch/index/ilm_test.go index d30e0a5c0..985ab6486 100644 --- a/internal/elasticsearch/index/ilm_test.go +++ b/internal/elasticsearch/index/ilm_test.go @@ -31,6 +31,8 @@ func TestAccResourceILM(t *testing.T) { resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "name", policyName), resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.min_age", "1h"), resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.set_priority.0.priority", "10"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.downsample.0.fixed_interval", "1d"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.downsample.0.wait_timeout", "1d"), resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.rollover.0.max_age", "1d"), resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.readonly.0.enabled", "true"), resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "delete.0.min_age", "0ms"), @@ -128,6 +130,11 @@ resource "elasticstack_elasticsearch_index_lifecycle" "test" { max_age = "1d" } + downsample { + fixed_interval = "1d" + wait_timeout = "1d" + } + readonly {} } From 75b63167b7751e0d6c048085f7e291db7ec29caa Mon Sep 17 00:00:00 2001 From: Xavier Naveira <122811510+xaviervolvo@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:38:37 +0100 Subject: [PATCH 2/8] Change MaxItems for downsample from 2 to 1 Only one downsample section per resource should be allowed. --- internal/elasticsearch/index/ilm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/elasticsearch/index/ilm.go b/internal/elasticsearch/index/ilm.go index 16ec576b4..d69c93036 100644 --- a/internal/elasticsearch/index/ilm.go +++ b/internal/elasticsearch/index/ilm.go @@ -393,7 +393,7 @@ var supportedActions = map[string]*schema.Schema{ Description: "Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity.", Type: schema.TypeList, Optional: true, - MaxItems: 2, + MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "fixed_interval": { From bcfb27c1e95f19c76e7fc5a88440f20ab9eddeb6 Mon Sep 17 00:00:00 2001 From: Xavier Naveira <122811510+xaviervolvo@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:40:30 +0100 Subject: [PATCH 3/8] Fix indentation --- internal/elasticsearch/index/ilm_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/elasticsearch/index/ilm_test.go b/internal/elasticsearch/index/ilm_test.go index 985ab6486..123c49383 100644 --- a/internal/elasticsearch/index/ilm_test.go +++ b/internal/elasticsearch/index/ilm_test.go @@ -130,10 +130,10 @@ resource "elasticstack_elasticsearch_index_lifecycle" "test" { max_age = "1d" } - downsample { - fixed_interval = "1d" - wait_timeout = "1d" - } + downsample { + fixed_interval = "1d" + wait_timeout = "1d" + } readonly {} } From 476f3545a53e8445970f5bacab77da2a5df46063 Mon Sep 17 00:00:00 2001 From: Xavier Naveira <122811510+xaviervolvo@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:43:45 +0100 Subject: [PATCH 4/8] Update CHANGELOG.md --- CHANGELOG.md | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a64f47b0..45bcfea47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,19 @@ ## [Unreleased] ### Added + +- Add downsample section to ILMs [#538](https://github.com/elastic/terraform-provider-elasticstack/pull/538) - Add new optional `ca_certs` attribute for Kibana ([#507](https://github.com/elastic/terraform-provider-elasticstack/pull/507)) ### Fixed + - Handle nil LastExecutionDate's in Kibana alerting rules. ([#508](https://github.com/elastic/terraform-provider-elasticstack/pull/508)) - Import all relevant attributes during `elasticstack_fleet_output` import ([#522](https://github.com/elastic/terraform-provider-elasticstack/pull/522)) ## [0.11.0] - 2023-12-12 ### Added + - Switch to Terraform [protocol version 6](https://developer.hashicorp.com/terraform/plugin/terraform-plugin-protocol#protocol-version-6) that is compatible with Terraform CLI version 1.0 and later. - Add 'elasticstack_fleet_package' data source ([#469](https://github.com/elastic/terraform-provider-elasticstack/pull/469)) - Add `tags` option to Kibana's SLOs ([#495](https://github.com/elastic/terraform-provider-elasticstack/pull/495)) @@ -19,6 +23,7 @@ - Support for Kibana API Key authentication ([#372](https://github.com/elastic/terraform-provider-elasticstack/pull/372)) ### Fixed + - Rename fleet package objects to `elasticstack_fleet_integration` and `elasticstack_fleet_integration_policy` ([#476](https://github.com/elastic/terraform-provider-elasticstack/pull/476)) - Fix a provider crash when managing SLOs outside of the default Kibana space. ([#485](https://github.com/elastic/terraform-provider-elasticstack/pull/485)) - Make input optional for `elasticstack_fleet_integration_policy` ([#493](https://github.com/elastic/terraform-provider-elasticstack/pull/493)) @@ -28,6 +33,7 @@ ## [0.10.0] - 2023-11-02 ### Added + - Add support for Kibana security role ([#435](https://github.com/elastic/terraform-provider-elasticstack/pull/435)) - Introduce `elasticstack_kibana_import_saved_objects` resource as an additive only way to manage Kibana saved objects ([#343](https://github.com/elastic/terraform-provider-elasticstack/pull/343)). - Add support for Terraform Plugin Framework ([#343](https://github.com/elastic/terraform-provider-elasticstack/pull/343)). @@ -39,45 +45,53 @@ ## [0.9.0] - 2023-10-09 ### Added + - Update `elasticstack_fleet_output` to use new API schema format ([#434](https://github.com/elastic/terraform-provider-elasticstack/pull/434)) ### Fixed + - Fix mapping of webhook connectors that is stored in tfstate ([#433](https://github.com/elastic/terraform-provider-elasticstack/pull/433)) ## [0.8.0] - 2023-09-26 ### Added + - Add support for the `.slack_api` connector type for Kibana action connectors ([#419](https://github.com/elastic/terraform-provider-elasticstack/pull/419)) - resource `elasticstack_kibana_slo`: Update `histogram_custom_indicator` `from` and `to` fields to float ([#430](https://github.com/elastic/terraform-provider-elasticstack/pull/430)) ## [0.7.0] - 2023-08-22 ### Added + - Add support for Kibana SLOs ([#385](https://github.com/elastic/terraform-provider-elasticstack/pull/385)) - Document all available environment variables ([#405](https://github.com/elastic/terraform-provider-elasticstack/pull/405)) ## [0.6.2] - 2023-06-19 ### Added + - Logging of Kibana action connectors HTTP requests and responses when [Terraform logs are enabled](https://developer.hashicorp.com/terraform/internals/debugging). - Add `skip_destroy` flag to `elasticstack_fleet_agent_policy` resource ([#357](https://github.com/elastic/terraform-provider-elasticstack/pull/357)) ## [0.6.1] - 2023-05-30 ### Added + - Add `path_style_access` setting to `elasticstack_elasticsearch_snapshot_repository` on s3 repositories to enable path style access pattern ([#331](https://github.com/elastic/terraform-provider-elasticstack/pull/331)) - Add `transform` field to `elasticstack_elasticsearch_watch` to allow for payload transforms to be defined ([#340](https://github.com/elastic/terraform-provider-elasticstack/pull/340)) ### Fixed + - Fix error presented by incorrect handling of `disabled_features` field in `elasticstack_kibana_space` resource ([#340](https://github.com/elastic/terraform-provider-elasticstack/pull/340)) ## [0.6.0] - 2023-05-24 ### Added + - New resource `elasticstack_elasticsearch_enrich_policy` to manage enrich policies ([#286](https://github.com/elastic/terraform-provider-elasticstack/pull/286)) ([Enrich API](https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-apis.html)) - New data source `elasticstack_elasticsearch_enrich_policy` to read enrich policies ([#293](https://github.com/elastic/terraform-provider-elasticstack/pull/293)) ([Enrich API](https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-apis.html)) - Add 'mapping_coerce' field to index resource ([#229](https://github.com/elastic/terraform-provider-elasticstack/pull/229)) -- Add 'min_*' conditions to ILM rollover ([#250](https://github.com/elastic/terraform-provider-elasticstack/pull/250)) +- Add 'min\_\*' conditions to ILM rollover ([#250](https://github.com/elastic/terraform-provider-elasticstack/pull/250)) - Add support for Kibana connections ([#226](https://github.com/elastic/terraform-provider-elasticstack/pull/226)) - **[Breaking Change] Add 'deletion_protection' field to index resource** to avoid unintentional deletion. ([#167](https://github.com/elastic/terraform-provider-elasticstack/pull/167)) - To delete index resource, you'll need to explicitly set `deletion_protection = false` as follows. @@ -102,11 +116,12 @@ - Add `elasticstack_kibana_action_connector` for managing Kibana action connectors ([#306](https://github.com/elastic/terraform-provider-elasticstack/pull/306)) ### Fixed + - Updated unsupported queue_max_bytes_number and queue_max_bytes_units with queue.max_bytes ([#266](https://github.com/elastic/terraform-provider-elasticstack/issues/266)) - Respect `ignore_unavailable` and `include_global_state` values when configuring SLM policies ([#224](https://github.com/elastic/terraform-provider-elasticstack/pull/224)) - Refactor API client functions and return diagnostics ([#220](https://github.com/elastic/terraform-provider-elasticstack/pull/220)) - Fix not to recreate index when field is removed from mapping ([#232](https://github.com/elastic/terraform-provider-elasticstack/pull/232)) -- Add query params fields to index resource ([#244](https://github.com/elastic/terraform-provider-elasticstack/pull/244)) +- Add query params fields to index resource ([#244](https://github.com/elastic/terraform-provider-elasticstack/pull/244)) - Properly handle errors which occur during provider execution ([#262](https://github.com/elastic/terraform-provider-elasticstack/pull/262)) - Correctly handle empty logstash pipeline metadata in plan diffs ([#256](https://github.com/elastic/terraform-provider-elasticstack/pull/256)) - Fix error when logging API requests in debug mode ([#259](https://github.com/elastic/terraform-provider-elasticstack/pull/259)) @@ -139,6 +154,7 @@ ## [0.5.0] - 2022-12-07 ### Added + - New resource `elasticstack_elasticsearch_logstash_pipeline` to manage Logstash pipelines ([Centralized Pipeline Management](https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html)) ([#151](https://github.com/elastic/terraform-provider-elasticstack/pull/151)) - Add `elasticstack_elasticsearch_script` resource ([#173](https://github.com/elastic/terraform-provider-elasticstack/pull/173)) - Add `elasticstack_elasticsearch_security_role` data source ([#177](https://github.com/elastic/terraform-provider-elasticstack/pull/177)) @@ -151,12 +167,14 @@ - Deprecate `elasticsearch_connection` blocks on individual resources/data sources. Connection configuration should be configured directly on the provider with multiple provider instances used to connect to different clusters. ([#218](https://github.com/elastic/terraform-provider-elasticstack/pull/218)) ### Fixed + - Remove unnecessary unsetting id on delete ([#174](https://github.com/elastic/terraform-provider-elasticstack/pull/174)) - Fix not found handling for snapshot repository ([#175](https://github.com/elastic/terraform-provider-elasticstack/pull/175)) - Add warn log when to remove resource from state ([#185](https://github.com/elastic/terraform-provider-elasticstack/pull/185)) - Import snapshot repository name when importing ([#187](https://github.com/elastic/terraform-provider-elasticstack/pull/187)) ## [0.4.0] - 2022-10-07 + ### Added - Add ca_data field to provider schema ([#145](https://github.com/elastic/terraform-provider-elasticstack/pull/145)) @@ -168,6 +186,7 @@ - Add elasticstack_elasticsearch_security_role_mapping resource ([148](https://github.com/elastic/terraform-provider-elasticstack/pull/148)) ### Fixed + - Refactor main function not to use deprecated debug method ([#149](https://github.com/elastic/terraform-provider-elasticstack/pull/149)) - Expose provider package ([#142](https://github.com/elastic/terraform-provider-elasticstack/pull/142)) - Upgrade Go version to 1.19 and sdk version to v2.22.0 ([#139](https://github.com/elastic/terraform-provider-elasticstack/pull/139)) @@ -176,50 +195,65 @@ - Support **7.x** Elasticsearch < **7.15** by removing the default `media_type` attribute in the Append processor ([#118](https://github.com/elastic/terraform-provider-elasticstack/pull/118)) ## [0.3.3] - 2022-03-22 + ### Fixed + - Make sure it is possible to set priority to `0` in ILM template ([#88](https://github.com/elastic/terraform-provider-elasticstack/issues/88)) - Set the ILM name on read operation ([#87](https://github.com/elastic/terraform-provider-elasticstack/issues/87)) - Always use data_stream setting if it's present ([#91](https://github.com/elastic/terraform-provider-elasticstack/issues/91)) ## [0.3.2] - 2022-02-28 + ### Fixed + - Properly apply `number_of_replicas` setting in `allocate` action in ILM ([#80](https://github.com/elastic/terraform-provider-elasticstack/issues/80)) ## [0.3.1] - 2022-02-24 + ### Fixed + - Add new field `allow_custom_routing` in `data_stream` section of [`index_template`](https://www.elastic.co/guide/en/elasticsearch/reference/8.0/indices-put-template.html#put-index-template-api-request-body), which appears only in Elasticsearch version **8.0.0**. Make sure `index_template` resource can work with both **7.x** and **8.x** versions ([#72](https://github.com/elastic/terraform-provider-elasticstack/pull/72)) - Panic using `elasticstack_elasticsearch_security_user_data_source` when the user absent or there is not enough permissions to fetch users from ES API ([#73](https://github.com/elastic/terraform-provider-elasticstack/issues/73)) - Fix typo in the index alias model ([#78](https://github.com/elastic/terraform-provider-elasticstack/issues/78)) ## [0.3.0] - 2022-02-17 + ### Added + - New resource `elasticstack_elasticsearch_data_stream` to manage Elasticsearch [data streams](https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams.html) ([#45](https://github.com/elastic/terraform-provider-elasticstack/pull/45)) - New resource `elasticstack_elasticsearch_ingest_pipeline` to manage Elasticsearch [ingest pipelines](https://www.elastic.co/guide/en/elasticsearch/reference/7.16/ingest.html) ([#56](https://github.com/elastic/terraform-provider-elasticstack/issues/56)) - New resource `elasticstack_elasticsearch_component_template` to manage Elasticsearch [component templates](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-component-template.html) ([#39](https://github.com/elastic/terraform-provider-elasticstack/pull/39)) - New helper data sources to create [processorts](https://www.elastic.co/guide/en/elasticsearch/reference/current/processors.html) for ingest pipelines ([#67](https://github.com/elastic/terraform-provider-elasticstack/pull/67)) ### Fixed + - Update only changed index settings ([#52](https://github.com/elastic/terraform-provider-elasticstack/issues/52)) - Enable import of index settings ([#53](https://github.com/elastic/terraform-provider-elasticstack/issues/53)) - Handle `allocate` action in ILM policy ([#59](https://github.com/elastic/terraform-provider-elasticstack/issues/59)) - Send only initialized values to Elasticsearch API when managing the users ([#66](https://github.com/elastic/terraform-provider-elasticstack/issues/66)) ## [0.2.0] - 2022-01-27 + ### Added + - New resource to manage Elasticsearch indices ([#38](https://github.com/elastic/terraform-provider-elasticstack/pull/38)) - The `insecure` option to the Elasticsearch provider configuration to ignore certificate verification ([#36](https://github.com/elastic/terraform-provider-elasticstack/pull/36)) - The `ca_file` option to the Elasticsearch provider configuration to provide path to the custom root certificate file ([#35](https://github.com/elastic/terraform-provider-elasticstack/pull/35)) - Documentation templates for all the existing resources ([#32](https://github.com/elastic/terraform-provider-elasticstack/pull/32)) ### Fixed + - Identify missing or deleted resources in the Elasticsearch cluster and make sure Terraform can re-create them ([#40](https://github.com/elastic/terraform-provider-elasticstack/pull/40)) ### Changed + - **[Breaking]** Rename `aliases` configuration option in -`elasticstack_elasticsearch_index_template` resource to singular `alias` + `elasticstack_elasticsearch_index_template` resource to singular `alias` ## [0.1.0] - 2021-12-20 + ### Added + - Initial set of the resources to gather feedback from the community - Initial set of docs - CI integration From e84b7573147cc3c60d808bec06a9252b18edfc69 Mon Sep 17 00:00:00 2001 From: Toby Brain Date: Wed, 31 Jan 2024 12:01:20 +1100 Subject: [PATCH 5/8] Gen docs --- docs/resources/elasticsearch_index_lifecycle.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/resources/elasticsearch_index_lifecycle.md b/docs/resources/elasticsearch_index_lifecycle.md index 21a1ea662..c034a7e7a 100644 --- a/docs/resources/elasticsearch_index_lifecycle.md +++ b/docs/resources/elasticsearch_index_lifecycle.md @@ -81,7 +81,7 @@ resource "elasticstack_elasticsearch_index_lifecycle" "my_ilm" { Optional: - `allocate` (Block List, Max: 1) Updates the index settings to change which nodes are allowed to host the index shards and change the number of replicas. (see [below for nested schema](#nestedblock--cold--allocate)) -- `downsample` (Block List, Max: 2) Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity. (see [below for nested schema](#nestedblock--cold--downsample)) +- `downsample` (Block List, Max: 1) Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity. (see [below for nested schema](#nestedblock--cold--downsample)) - `freeze` (Block List, Max: 1) Freeze the index to minimize its memory footprint. (see [below for nested schema](#nestedblock--cold--freeze)) - `migrate` (Block List, Max: 1) Moves the index to the data tier that corresponds to the current phase by updating the "index.routing.allocation.include._tier_preference" index setting. (see [below for nested schema](#nestedblock--cold--migrate)) - `min_age` (String) ILM moves indices through the lifecycle according to their age. To control the timing of these transitions, you set a minimum age for each phase. @@ -236,7 +236,7 @@ Optional: Optional: -- `downsample` (Block List, Max: 2) Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity. (see [below for nested schema](#nestedblock--hot--downsample)) +- `downsample` (Block List, Max: 1) Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity. (see [below for nested schema](#nestedblock--hot--downsample)) - `forcemerge` (Block List, Max: 1) Force merges the index into the specified maximum number of segments. This action makes the index read-only. (see [below for nested schema](#nestedblock--hot--forcemerge)) - `min_age` (String) ILM moves indices through the lifecycle according to their age. To control the timing of these transitions, you set a minimum age for each phase. - `readonly` (Block List, Max: 1) Makes the index read-only. (see [below for nested schema](#nestedblock--hot--readonly)) @@ -335,7 +335,7 @@ Optional: Optional: - `allocate` (Block List, Max: 1) Updates the index settings to change which nodes are allowed to host the index shards and change the number of replicas. (see [below for nested schema](#nestedblock--warm--allocate)) -- `downsample` (Block List, Max: 2) Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity. (see [below for nested schema](#nestedblock--warm--downsample)) +- `downsample` (Block List, Max: 1) Roll up documents within a fixed interval to a single summary document. Reduces the index footprint by storing time series data at reduced granularity. (see [below for nested schema](#nestedblock--warm--downsample)) - `forcemerge` (Block List, Max: 1) Force merges the index into the specified maximum number of segments. This action makes the index read-only. (see [below for nested schema](#nestedblock--warm--forcemerge)) - `migrate` (Block List, Max: 1) Moves the index to the data tier that corresponds to the current phase by updating the "index.routing.allocation.include._tier_preference" index setting. (see [below for nested schema](#nestedblock--warm--migrate)) - `min_age` (String) ILM moves indices through the lifecycle according to their age. To control the timing of these transitions, you set a minimum age for each phase. From 737956eb9f713a5f6748d641eefee3fcb0949ccf Mon Sep 17 00:00:00 2001 From: Xavier Naveira Date: Fri, 2 Feb 2024 12:13:53 +0100 Subject: [PATCH 6/8] Adjust versions for test cases --- internal/elasticsearch/index/ilm_test.go | 115 +++++++++++++++++++++-- 1 file changed, 108 insertions(+), 7 deletions(-) diff --git a/internal/elasticsearch/index/ilm_test.go b/internal/elasticsearch/index/ilm_test.go index 123c49383..801583206 100644 --- a/internal/elasticsearch/index/ilm_test.go +++ b/internal/elasticsearch/index/ilm_test.go @@ -15,6 +15,8 @@ import ( ) var totalShardsPerNodeVersionLimit = version.Must(version.NewVersion("7.16.0")) +var downsampleNoTimeoutVersionLimit = version.Must(version.NewVersion("8.5.0")) +var downsampleVersionLimit = version.Must(version.NewVersion("8.9.0")) func TestAccResourceILM(t *testing.T) { // generate a random policy name @@ -31,8 +33,6 @@ func TestAccResourceILM(t *testing.T) { resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "name", policyName), resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.min_age", "1h"), resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.set_priority.0.priority", "10"), - resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.downsample.0.fixed_interval", "1d"), - resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.downsample.0.wait_timeout", "1d"), resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.rollover.0.max_age", "1d"), resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.readonly.0.enabled", "true"), resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "delete.0.min_age", "0ms"), @@ -77,6 +77,43 @@ func TestAccResourceILM(t *testing.T) { resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "warm.0.allocate.0.total_shards_per_node", "200"), ), }, + { + SkipFunc: versionutils.CheckIfVersionIsUnsupported(downsampleNoTimeoutVersionLimit), + Config: testAccResourceILMDownsampleNoTimeout(policyName), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "name", policyName), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.min_age", "1h"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.set_priority.0.priority", "10"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.downsample.0.fixed_interval", "1d"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.rollover.0.max_age", "1d"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.readonly.0.enabled", "true"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "delete.0.min_age", "0ms"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.#", "1"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "delete.#", "1"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "warm.#", "0"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "cold.#", "0"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "frozen.#", "0"), + ), + }, + { + SkipFunc: versionutils.CheckIfVersionIsUnsupported(downsampleVersionLimit), + Config: testAccResourceILMDownsample(policyName), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "name", policyName), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.min_age", "1h"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.set_priority.0.priority", "10"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.downsample.0.fixed_interval", "1d"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.downsample.0.wait_timeout", "1d"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.rollover.0.max_age", "1d"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.0.readonly.0.enabled", "true"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "delete.0.min_age", "0ms"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "hot.#", "1"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "delete.#", "1"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "warm.#", "0"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "cold.#", "0"), + resource.TestCheckResourceAttr("elasticstack_elasticsearch_index_lifecycle.test", "frozen.#", "0"), + ), + }, }, }) } @@ -130,11 +167,6 @@ resource "elasticstack_elasticsearch_index_lifecycle" "test" { max_age = "1d" } - downsample { - fixed_interval = "1d" - wait_timeout = "1d" - } - readonly {} } @@ -256,6 +288,75 @@ resource "elasticstack_elasticsearch_index_lifecycle" "test_rollover" { `, name) } +func testAccResourceILMDownsampleNoTimeout(name string) string { + return fmt.Sprintf(` +provider "elasticstack" { + elasticsearch {} +} + +resource "elasticstack_elasticsearch_index_lifecycle" "test" { + name = "%s" + + hot { + min_age = "1h" + + set_priority { + priority = 10 + } + + rollover { + max_age = "1d" + } + + downsample { + fixed_interval = "1d" + } + + readonly {} + } + + delete { + delete {} + } +} + `, name) +} + +func testAccResourceILMDownsample(name string) string { + return fmt.Sprintf(` +provider "elasticstack" { + elasticsearch {} +} + +resource "elasticstack_elasticsearch_index_lifecycle" "test" { + name = "%s" + + hot { + min_age = "1h" + + set_priority { + priority = 10 + } + + rollover { + max_age = "1d" + } + + downsample { + fixed_interval = "1d" + wait_timeout = "1d" + } + + readonly {} + } + + delete { + delete {} + } +} + `, name) +} + func checkResourceILMDestroy(s *terraform.State) error { client, err := clients.NewAcceptanceTestingClient() if err != nil { From 23235e4b064661009bc92472a4c8da1ea6bd2515 Mon Sep 17 00:00:00 2001 From: Toby Brain Date: Mon, 5 Feb 2024 20:33:04 +1100 Subject: [PATCH 7/8] Make wait_timeout optional --- docs/resources/elasticsearch_index_lifecycle.md | 9 +++++++++ internal/elasticsearch/index/ilm.go | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/resources/elasticsearch_index_lifecycle.md b/docs/resources/elasticsearch_index_lifecycle.md index c034a7e7a..a3926b1a1 100644 --- a/docs/resources/elasticsearch_index_lifecycle.md +++ b/docs/resources/elasticsearch_index_lifecycle.md @@ -108,6 +108,9 @@ Optional: Required: - `fixed_interval` (String) Downsampling interval + +Optional: + - `wait_timeout` (String) Downsampling interval @@ -252,6 +255,9 @@ Optional: Required: - `fixed_interval` (String) Downsampling interval + +Optional: + - `wait_timeout` (String) Downsampling interval @@ -362,6 +368,9 @@ Optional: Required: - `fixed_interval` (String) Downsampling interval + +Optional: + - `wait_timeout` (String) Downsampling interval diff --git a/internal/elasticsearch/index/ilm.go b/internal/elasticsearch/index/ilm.go index d69c93036..dae40e015 100644 --- a/internal/elasticsearch/index/ilm.go +++ b/internal/elasticsearch/index/ilm.go @@ -404,7 +404,8 @@ var supportedActions = map[string]*schema.Schema{ "wait_timeout": { Description: "Downsampling interval", Type: schema.TypeString, - Required: true, + Optional: true, + Computed: true, }, }, }, From 9ec10f7ae32120d934cea15ad1c56fd359ccf5d7 Mon Sep 17 00:00:00 2001 From: Toby Brain Date: Tue, 6 Feb 2024 07:19:53 +1100 Subject: [PATCH 8/8] Update wait_timeout version limit --- internal/elasticsearch/index/ilm_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/elasticsearch/index/ilm_test.go b/internal/elasticsearch/index/ilm_test.go index 801583206..177da2195 100644 --- a/internal/elasticsearch/index/ilm_test.go +++ b/internal/elasticsearch/index/ilm_test.go @@ -16,7 +16,7 @@ import ( var totalShardsPerNodeVersionLimit = version.Must(version.NewVersion("7.16.0")) var downsampleNoTimeoutVersionLimit = version.Must(version.NewVersion("8.5.0")) -var downsampleVersionLimit = version.Must(version.NewVersion("8.9.0")) +var downsampleVersionLimit = version.Must(version.NewVersion("8.10.0")) func TestAccResourceILM(t *testing.T) { // generate a random policy name