Bump provider support to ES version 8.x

.ci/Makefile.ci

@@ -4,7 +4,7 @@ GOVERSION ?= 1.17
 
 ELASTICSEARCH_NAME ?= terraform-elasticstack-es
 ELASTICSEARCH_ENDPOINTS ?= http://$(ELASTICSEARCH_NAME):9200
-ELASTICSEARCH_VERSION ?= 7.15.1
+ELASTICSEARCH_VERSION ?= 8.0.0
 ELASTICSEARCH_USERNAME ?= elastic
 ELASTICSEARCH_PASSWORD ?= changeme
 ELASTICSEARCH_NETWORK ?= elasticstack-network

CHANGELOG.md

@@ -1,4 +1,6 @@
 ## [Unreleased]
+### Fixed
+- Add new field `allow_custom_routing` in `data_stream` section of [`index_template`](https://www.elastic.co/guide/en/elasticsearch/reference/8.0/indices-put-template.html#put-index-template-api-request-body), which appears only in Elasticsearch version **8.0.0**. Make sure `index_template` resource can work with both **7.x** and **8.x** versions ([#72](https://github.com/elastic/terraform-provider-elasticstack/pull/72))
 
 ## [0.3.0] - 2022-02-17
 ### Added

docs/resources/elasticsearch_cluster_settings.md

@@ -28,8 +28,8 @@ resource "elasticstack_elasticsearch_cluster_settings" "my_cluster_settings" {
       value = "50mb"
     }
     setting {
-      name  = "indices.breaker.accounting.limit"
-      value = "100%"
+      name  = "indices.breaker.total.limit"
+      value = "65%"
     }
     setting {
       name       = "xpack.security.audit.logfile.events.include"
@@ -39,8 +39,8 @@ resource "elasticstack_elasticsearch_cluster_settings" "my_cluster_settings" {
 
   transient {
     setting {
-      name  = "indices.breaker.accounting.limit"
-      value = "99%"
+      name  = "indices.breaker.total.limit"
+      value = "60%"
     }
   }
 }

docs/resources/elasticsearch_index_template.md

@@ -72,6 +72,7 @@ resource "elasticstack_elasticsearch_index_template" "my_data_stream" {
 
 Optional:
 
+- **allow_custom_routing** (Boolean) If `true`, the data stream supports custom routing. Defaults to `false`. Available only in *8.x*
 - **hidden** (Boolean) If true, the data stream is hidden.
 
 

examples/resources/elasticstack_elasticsearch_cluster_settings/resource.tf

@@ -13,8 +13,8 @@ resource "elasticstack_elasticsearch_cluster_settings" "my_cluster_settings" {
       value = "50mb"
     }
     setting {
-      name  = "indices.breaker.accounting.limit"
-      value = "100%"
+      name  = "indices.breaker.total.limit"
+      value = "65%"
     }
     setting {
       name       = "xpack.security.audit.logfile.events.include"
@@ -24,8 +24,8 @@ resource "elasticstack_elasticsearch_cluster_settings" "my_cluster_settings" {
 
   transient {
     setting {
-      name  = "indices.breaker.accounting.limit"
-      value = "99%"
+      name  = "indices.breaker.total.limit"
+      value = "60%"
     }
   }
 }

internal/elasticsearch/cluster/settings_test.go

@@ -32,13 +32,13 @@ func TestAccResourceClusterSettings(t *testing.T) {
 						}),
 					resource.TestCheckTypeSetElemNestedAttrs("elasticstack_elasticsearch_cluster_settings.test", "persistent.0.setting.*",
 						map[string]string{
-							"name":  "indices.breaker.accounting.limit",
-							"value": "100%",
+							"name":  "indices.breaker.total.limit",
+							"value": "65%",
 						}),
 					resource.TestCheckTypeSetElemNestedAttrs("elasticstack_elasticsearch_cluster_settings.test", "transient.0.setting.*",
 						map[string]string{
-							"name":  "indices.breaker.accounting.limit",
-							"value": "99%",
+							"name":  "indices.breaker.total.limit",
+							"value": "60%",
 						}),
 				),
 			},
@@ -57,8 +57,8 @@ func TestAccResourceClusterSettings(t *testing.T) {
 						}),
 					resource.TestCheckTypeSetElemNestedAttrs("elasticstack_elasticsearch_cluster_settings.test", "persistent.0.setting.*",
 						map[string]string{
-							"name":  "indices.breaker.accounting.limit",
-							"value": "100%",
+							"name":  "indices.breaker.total.limit",
+							"value": "60%",
 						}),
 					resource.TestCheckTypeSetElemAttr("elasticstack_elasticsearch_cluster_settings.test", "persistent.0.setting.*.value_list.*", "ACCESS_DENIED"),
 					resource.TestCheckTypeSetElemAttr("elasticstack_elasticsearch_cluster_settings.test", "persistent.0.setting.*.value_list.*", "ACCESS_GRANTED"),
@@ -86,15 +86,15 @@ resource "elasticstack_elasticsearch_cluster_settings" "test" {
       value = "50mb"
     }
     setting {
-      name  = "indices.breaker.accounting.limit"
-      value = "100%"
+      name  = "indices.breaker.total.limit"
+      value = "65%"
     }
   }
 
   transient {
     setting {
-      name  = "indices.breaker.accounting.limit"
-      value = "99%"
+      name  = "indices.breaker.total.limit"
+      value = "60%"
     }
   }
 }
@@ -118,8 +118,8 @@ resource "elasticstack_elasticsearch_cluster_settings" "test" {
       value = "40mb"
     }
     setting {
-      name  = "indices.breaker.accounting.limit"
-      value = "100%"
+      name  = "indices.breaker.total.limit"
+      value = "60%"
     }
     setting {
       name       = "xpack.security.audit.logfile.events.include"
@@ -136,7 +136,7 @@ func checkResourceClusterSettingsDestroy(s *terraform.State) error {
 	listOfSettings := []string{
 		"indices.lifecycle.poll_interval",
 		"indices.recovery.max_bytes_per_sec",
-		"indices.breaker.accounting.limit",
+		"indices.breaker.total.limit",
 		"xpack.security.audit.logfile.events.include",
 	}
 

internal/elasticsearch/index/template.go

@@ -3,6 +3,7 @@ package index
 import (
 	"context"
 	"encoding/json"
+	"log"
 	"strings"
 
 	"github.com/elastic/terraform-provider-elasticstack/internal/clients"
@@ -48,6 +49,11 @@ func ResourceTemplate() *schema.Resource {
 						Default:     false,
 						Optional:    true,
 					},
+					"allow_custom_routing": {
+						Description: "If `true`, the data stream supports custom routing. Defaults to `false`. Available only in *8.x*",
+						Type:        schema.TypeBool,
+						Optional:    true,
+					},
 				},
 			},
 		},
@@ -195,10 +201,39 @@ func resourceIndexTemplatePut(ctx context.Context, d *schema.ResourceData, meta
 	}
 	indexTemplate.ComposedOf = compsOf
 
-	if v, ok := d.GetOk("data_stream"); ok {
-		// only one definition of stream allowed
-		stream := v.([]interface{})[0].(map[string]interface{})
-		indexTemplate.DataStream = stream
+	if d.HasChange("data_stream") {
+		old, new := d.GetChange("data_stream")
+		log.Printf("[TRACE] old data stream settings <%v> new settings <%v>", old, new)
+
+		// 8.x workaround
+		hasAllowCustomRouting := false
+		//for _, s := range []interface{}{old} {
+		if old != nil && len(old.([]interface{})) == 1 {
+			if old.([]interface{})[0] != nil {
+				setting := old.([]interface{})[0].(map[string]interface{})
+				if _, ok := setting["allow_custom_routing"]; ok {
+					hasAllowCustomRouting = true
+				}
+			}
+		}
+		//	}
+
+		if v, ok := d.GetOk("data_stream"); ok {
+			// only one definition of stream allowed
+			if v.([]interface{})[0] != nil {
+				stream := v.([]interface{})[0].(map[string]interface{})
+				dSettings := &models.DataStreamSettings{}
+				if s, ok := stream["hidden"]; ok {
+					hidden := s.(bool)
+					dSettings.Hidden = &hidden
+				}
+				if s, ok := stream["allow_custom_routing"]; ok && hasAllowCustomRouting && s.(bool) {
+					allow := s.(bool)
+					dSettings.AllowCustomRouting = &allow
+				}
+				indexTemplate.DataStream = dSettings
+			}
+		}
 	}
 
 	if v, ok := d.GetOk("index_patterns"); ok {
@@ -298,9 +333,16 @@ func resourceIndexTemplateRead(ctx context.Context, d *schema.ResourceData, meta
 	if err := d.Set("composed_of", tpl.IndexTemplate.ComposedOf); err != nil {
 		return diag.FromErr(err)
 	}
-	if tpl.IndexTemplate.DataStream != nil {
-		ds := make([]interface{}, 0)
-		ds = append(ds, tpl.IndexTemplate.DataStream)
+	if stream := tpl.IndexTemplate.DataStream; stream != nil {
+		ds := make([]interface{}, 1)
+		dSettings := make(map[string]interface{})
+		if v := stream.Hidden; v != nil {
+			dSettings["hidden"] = *v
+		}
+		if v := stream.AllowCustomRouting; v != nil {
+			dSettings["allow_custom_routing"] = *v
+		}
+		ds[0] = dSettings
 		if err := d.Set("data_stream", ds); err != nil {
 			return diag.FromErr(err)
 		}

internal/models/models.go

@@ -44,14 +44,19 @@ type IndexTemplate struct {
 	Create        bool                   `json:"-"`
 	Timeout       string                 `json:"-"`
 	ComposedOf    []string               `json:"composed_of"`
-	DataStream    map[string]interface{} `json:"data_stream,omitempty"`
+	DataStream    *DataStreamSettings    `json:"data_stream,omitempty"`
 	IndexPatterns []string               `json:"index_patterns"`
 	Meta          map[string]interface{} `json:"_meta,omitempty"`
 	Priority      *int                   `json:"priority,omitempty"`
 	Template      *Template              `json:"template,omitempty"`
 	Version       *int                   `json:"version,omitempty"`
 }
 
+type DataStreamSettings struct {
+	Hidden             *bool `json:"hidden,omitempty"`
+	AllowCustomRouting *bool `json:"allow_custom_routing,omitempty"`
+}
+
 type Template struct {
 	Aliases  map[string]IndexAlias  `json:"aliases,omitempty"`
 	Mappings map[string]interface{} `json:"mappings,omitempty"`