[7.x] [ILM] Remove beat version from default policy name. (#14745)

CHANGELOG.next.asciidoc

@@ -24,6 +24,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Adding new `Enterprise` license type to the licenser. {issue}14246[14246]
 - Fix memory leak in kubernetes autodiscover provider and add_kubernetes_metadata processor happening when pods are terminated without sending a delete event. {pull}14259[14259]
 - Allow Metricbeat's beat module to read monitoring information over a named pipe or unix domain socket. {pull}14558[14558]
+- Remove version information from default ILM policy for improved upgrade experience on custom policies. {pull}14745[14745]
 
 *Auditbeat*
 

libbeat/idxmgmt/ilm/config.go

@@ -111,12 +111,13 @@ func (cfg *Config) Validate() error {
 
 func defaultConfig(info beat.Info) Config {
 	name := info.Beat + "-%{[agent.version]}"
-	nameFmt := fmtstr.MustCompileEvent(name)
+	aliasFmt := fmtstr.MustCompileEvent(name)
+	policyFmt := fmtstr.MustCompileEvent(info.Beat)
 
 	return Config{
 		Mode:          ModeAuto,
-		PolicyName:    *nameFmt,
-		RolloverAlias: *nameFmt,
+		PolicyName:    *policyFmt,
+		RolloverAlias: *aliasFmt,
 		Pattern:       ilmDefaultPattern,
 		PolicyFile:    "",
 		CheckExists:   true,

libbeat/idxmgmt/ilm/ilm_test.go

@@ -264,7 +264,7 @@ func TestDefaultSupport_Manager_EnsureAlias(t *testing.T) {
 
 func TestDefaultSupport_Manager_EnsurePolicy(t *testing.T) {
 	testPolicy := Policy{
-		Name: "test-9.9.9",
+		Name: "test",
 		Body: DefaultPolicy,
 	}
 
@@ -305,11 +305,6 @@ func TestDefaultSupport_Manager_EnsurePolicy(t *testing.T) {
 	for name, test := range cases {
 		test := test
 		t.Run(name, func(t *testing.T) {
-			cfg := test.cfg
-			if cfg == nil {
-				cfg = map[string]interface{}{"name": "test"}
-			}
-
 			h := newMockHandler(test.calls...)
 			m := createManager(t, h, test.cfg)
 			created, err := m.EnsurePolicy(test.overwrite)

libbeat/idxmgmt/std_test.go

@@ -313,11 +313,11 @@ func TestIndexManager_Setup(t *testing.T) {
 				"overwrite":                     "true",
 				"name":                          "test-9.9.9",
 				"pattern":                       "test-9.9.9-*",
-				"settings.index.lifecycle.name": "test-9.9.9",
+				"settings.index.lifecycle.name": "test",
 				"settings.index.lifecycle.rollover_alias": "test-9.9.9",
 			}),
 			alias:  "test-9.9.9",
-			policy: "test-9.9.9",
+			policy: "test",
 		},
 		"template default ilm default with alias and policy changed": {
 			cfg: common.MapStr{
@@ -370,7 +370,7 @@ func TestIndexManager_Setup(t *testing.T) {
 				"setup.template.enabled": false,
 			},
 			alias:  "test-9.9.9",
-			policy: "test-9.9.9",
+			policy: "test",
 		},
 		"template disabled ilm disabled, loadMode Overwrite": {
 			cfg: common.MapStr{
@@ -386,20 +386,20 @@ func TestIndexManager_Setup(t *testing.T) {
 			},
 			loadILM: LoadModeForce,
 			alias:   "test-9.9.9",
-			policy:  "test-9.9.9",
+			policy:  "test",
 		},
 		"template loadmode disabled ilm loadMode enabled": {
 			loadTemplate: LoadModeDisabled,
 			loadILM:      LoadModeEnabled,
 			alias:        "test-9.9.9",
-			policy:       "test-9.9.9",
+			policy:       "test",
 		},
 		"template default ilm loadMode disabled": {
 			loadILM: LoadModeDisabled,
 			tmplCfg: cfgWith(template.DefaultConfig(), map[string]interface{}{
 				"name":                          "test-9.9.9",
 				"pattern":                       "test-9.9.9-*",
-				"settings.index.lifecycle.name": "test-9.9.9",
+				"settings.index.lifecycle.name": "test",
 				"settings.index.lifecycle.rollover_alias": "test-9.9.9",
 			}),
 		},

libbeat/tests/system/idxmgmt.py

@@ -12,15 +12,15 @@ def __init__(self, client, index):
     def needs_init(self, s):
         return s == '' or s == '*'
 
-    def delete(self, indices=[]):
+    def delete(self, indices=[], policies=[]):
         indices = list(filter(lambda x: x != '', indices))
         if not indices:
             indices == [self._index]
         for i in indices:
             self.delete_index_and_alias(i)
             self.delete_template(template=i)
-        for i in indices:
-            self.delete_policy(policy=i)
+        for i in list(filter(lambda x: x != '', policies)):
+            self.delete_policy(i)
 
     def delete_index_and_alias(self, index=""):
         if self.needs_init(index):
@@ -40,10 +40,7 @@ def delete_template(self, template=""):
         except NotFoundError:
             pass
 
-    def delete_policy(self, policy=""):
-        if self.needs_init(policy):
-            policy = self._index
-
+    def delete_policy(self, policy):
         # Delete any existing policy starting with given policy
         policies = self._client.transport.perform_request('GET', "/_ilm/policy")
         for p, _ in policies.items():

libbeat/tests/system/test_cmd_setup_index_management.py

@@ -20,21 +20,24 @@ def setUp(self):
 
         self.cmd = "--index-management"
         # auto-derived default settings, if nothing else is set
-        self.index_name = self.alias_name = self.policy_name = self.beat_name + "-9.9.9"
+        self.policy_name = self.beat_name
+        self.index_name = self.alias_name = self.beat_name + "-9.9.9"
 
         self.custom_alias = self.beat_name + "_foo"
         self.custom_policy = self.beat_name + "_bar"
         self.custom_template = self.beat_name + "_foobar"
 
         self.es = self.es_client()
         self.idxmgmt = IdxMgmt(self.es, self.index_name)
-        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
+        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy],
+                            policies=[self.policy_name, self.custom_policy])
 
         logging.getLogger("urllib3").setLevel(logging.WARNING)
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
     def tearDown(self):
-        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
+        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy],
+                            policies=[self.policy_name, self.custom_policy])
 
     def render_config(self, **kwargs):
         self.render_config_template(
@@ -94,7 +97,7 @@ def test_setup_template_disabled(self):
         assert exit_code == 0
         self.idxmgmt.assert_index_template_not_loaded(self.index_name)
         self.idxmgmt.assert_alias_created(self.index_name)
-        self.idxmgmt.assert_policy_created(self.index_name)
+        self.idxmgmt.assert_policy_created(self.policy_name)
 
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
     @attr('integration')
@@ -239,16 +242,16 @@ def test_setup_overwrite_template_on_ilm_policy_created(self):
                                               "-E", "setup.template.pattern=" + self.custom_alias + "*"])
         assert exit_code == 0
         self.idxmgmt.assert_index_template_loaded(self.custom_alias)
-        self.idxmgmt.assert_policy_not_created(self.index_name)
+        self.idxmgmt.assert_policy_not_created(self.policy_name)
 
         # ensure ilm policy is created, triggering overwriting existing template
         exit_code = self.run_beat(extra_args=["setup", self.cmd,
                                               "-E", "setup.template.overwrite=false",
                                               "-E", "setup.template.settings.index.number_of_shards=2",
                                               "-E", "setup.ilm.rollover_alias=" + self.custom_alias])
         assert exit_code == 0
-        self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias)
-        self.idxmgmt.assert_policy_created(self.index_name)
+        self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias)
+        self.idxmgmt.assert_policy_created(self.policy_name)
         # check that template was overwritten
         resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias)
         assert self.custom_alias in resp

libbeat/tests/system/test_ilm.py

@@ -16,15 +16,18 @@ class TestRunILM(BaseTest):
     def setUp(self):
         super(TestRunILM, self).setUp()
 
-        self.alias_name = self.policy_name = self.index_name = self.beat_name + "-9.9.9"
+        self.alias_name = self.index_name = self.beat_name + "-9.9.9"
+        self.policy_name = self.beat_name
         self.custom_alias = self.beat_name + "_foo"
         self.custom_policy = self.beat_name + "_bar"
         self.es = self.es_client()
         self.idxmgmt = IdxMgmt(self.es, self.index_name)
-        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
+        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy],
+                            policies=[self.policy_name, self.custom_policy])
 
     def tearDown(self):
-        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
+        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy],
+                            policies=[self.policy_name, self.custom_policy])
 
     def render_config(self, **kwargs):
         self.render_config_template(
@@ -160,18 +163,21 @@ def setUp(self):
 
         self.setupCmd = "--ilm-policy"
 
-        self.alias_name = self.policy_name = self.index_name = self.beat_name + "-9.9.9"
+        self.alias_name = self.index_name = self.beat_name + "-9.9.9"
+        self.policy_name = self.beat_name
         self.custom_alias = self.beat_name + "_foo"
         self.custom_policy = self.beat_name + "_bar"
         self.es = self.es_client()
         self.idxmgmt = IdxMgmt(self.es, self.index_name)
-        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
+        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy],
+                            policies=[self.policy_name, self.custom_policy])
 
         logging.getLogger("urllib3").setLevel(logging.WARNING)
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
     def tearDown(self):
-        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
+        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy],
+                            policies=[self.policy_name, self.custom_policy])
 
     def render_config(self, **kwargs):
         self.render_config_template(
@@ -277,7 +283,7 @@ def setUp(self):
         self.config = "libbeat.yml"
         self.output = os.path.join(self.working_dir, self.config)
         shutil.copy(os.path.join(self.beat_path, "fields.yml"), self.output)
-        self.policy_name = self.beat_name + "-9.9.9"
+        self.policy_name = self.beat_name
         self.cmd = "ilm-policy"
 
     def assert_log_contains_policy(self):

libbeat/tests/system/test_template.py

@@ -152,7 +152,7 @@ def test_template_default(self):
         self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
         proc.check_kill_and_wait()
 
-        self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.index_name, self.index_name)
+        self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.beat_name, self.index_name)
         self.idxmgmt.assert_alias_created(self.index_name)
         self.idxmgmt.assert_docs_written_to_alias(self.index_name)
 
@@ -183,15 +183,16 @@ def setUp(self):
         self.setupCmd = "--template"
         self.index_name = self.beat_name + "-9.9.9"
         self.custom_alias = self.beat_name + "_foo"
+        self.policy_name = self.beat_name
 
         self.es = self.es_client()
         self.idxmgmt = IdxMgmt(self.es, self.index_name)
-        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name])
+        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name], policies=[self.policy_name])
         logging.getLogger("urllib3").setLevel(logging.WARNING)
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
     def tearDown(self):
-        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name])
+        self.idxmgmt.delete(indices=[self.custom_alias, self.index_name], policies=[self.policy_name])
 
     def render_config(self, **kwargs):
         self.render_config_template(
@@ -210,9 +211,9 @@ def test_setup(self):
                                   extra_args=["setup", self.setupCmd, "--ilm-policy"])
 
         assert exit_code == 0
-        self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.index_name, self.index_name)
+        self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.policy_name, self.index_name)
         self.idxmgmt.assert_alias_created(self.index_name)
-        self.idxmgmt.assert_policy_created(self.index_name)
+        self.idxmgmt.assert_policy_created(self.policy_name)
 
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
     @attr('integration')
@@ -225,13 +226,13 @@ def test_setup_template_default(self):
                                   extra_args=["setup", self.setupCmd])
 
         assert exit_code == 0
-        self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.index_name, self.index_name)
+        self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.policy_name, self.index_name)
         self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"])
 
         # when running `setup --template`
         # write_alias and rollover_policy related to ILM are also created
         self.idxmgmt.assert_alias_created(self.index_name)
-        self.idxmgmt.assert_policy_created(self.index_name)
+        self.idxmgmt.assert_policy_created(self.policy_name)
 
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
     @attr('integration')
@@ -250,7 +251,7 @@ def test_setup_template_disabled(self):
         # when running `setup --template` and `setup.template.enabled=false`
         # write_alias and rollover_policy related to ILM are still created
         self.idxmgmt.assert_alias_created(self.index_name)
-        self.idxmgmt.assert_policy_created(self.index_name)
+        self.idxmgmt.assert_policy_created(self.policy_name)
 
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
     @attr('integration')
@@ -285,7 +286,7 @@ def test_setup_template_with_ilm_changed_pattern(self):
                                               "-E", "setup.ilm.rollover_alias=" + self.custom_alias])
 
         assert exit_code == 0
-        self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias)
+        self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias)
         self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"])
 
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@@ -304,16 +305,16 @@ def test_template_created_on_ilm_policy_created(self):
                                               "-E", "setup.template.pattern=" + self.custom_alias + "*"])
         assert exit_code == 0
         self.idxmgmt.assert_index_template_loaded(self.custom_alias)
-        self.idxmgmt.assert_policy_not_created(self.index_name)
+        self.idxmgmt.assert_policy_not_created(self.policy_name)
 
         # ensure ilm policy is created, triggering overwriting existing template
         exit_code = self.run_beat(extra_args=["setup", self.setupCmd,
                                               "-E", "setup.template.overwrite=false",
                                               "-E", "setup.template.settings.index.number_of_shards=2",
                                               "-E", "setup.ilm.rollover_alias=" + self.custom_alias])
         assert exit_code == 0
-        self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias)
-        self.idxmgmt.assert_policy_created(self.index_name)
+        self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias)
+        self.idxmgmt.assert_policy_created(self.policy_name)
         # check that template was overwritten
         resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias)
         assert self.custom_alias in resp