From 11a75a347334170c3094e5a4a93a9a27905e6f50 Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Tue, 26 Nov 2019 13:47:10 +0100 Subject: [PATCH] [ILM] Remove beat version from default policy name. (#14745) Prevent breaking customized ILM policies on upgrade, by getting rid of the version in the default policy name. The version can still be manuaylly configured. fixes #14736 --- CHANGELOG.next.asciidoc | 1 + libbeat/idxmgmt/ilm/config.go | 7 +++--- libbeat/idxmgmt/ilm/ilm_test.go | 7 +----- libbeat/idxmgmt/std_test.go | 12 ++++----- libbeat/tests/system/idxmgmt.py | 11 +++----- .../system/test_cmd_setup_index_management.py | 17 +++++++------ libbeat/tests/system/test_ilm.py | 20 +++++++++------ libbeat/tests/system/test_template.py | 25 ++++++++++--------- 8 files changed, 52 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 86febfb17e8f..15d9d39bee9c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -24,6 +24,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Adding new `Enterprise` license type to the licenser. {issue}14246[14246] - Fix memory leak in kubernetes autodiscover provider and add_kubernetes_metadata processor happening when pods are terminated without sending a delete event. {pull}14259[14259] - Allow Metricbeat's beat module to read monitoring information over a named pipe or unix domain socket. {pull}14558[14558] +- Remove version information from default ILM policy for improved upgrade experience on custom policies. {pull}14745[14745] *Auditbeat* diff --git a/libbeat/idxmgmt/ilm/config.go b/libbeat/idxmgmt/ilm/config.go index 1942b7529ffd..3697d808f4e7 100644 --- a/libbeat/idxmgmt/ilm/config.go +++ b/libbeat/idxmgmt/ilm/config.go @@ -111,12 +111,13 @@ func (cfg *Config) Validate() error { func defaultConfig(info beat.Info) Config { name := info.Beat + "-%{[agent.version]}" - nameFmt := fmtstr.MustCompileEvent(name) + aliasFmt := fmtstr.MustCompileEvent(name) + policyFmt := fmtstr.MustCompileEvent(info.Beat) return Config{ Mode: ModeAuto, - PolicyName: *nameFmt, - RolloverAlias: *nameFmt, + PolicyName: *policyFmt, + RolloverAlias: *aliasFmt, Pattern: ilmDefaultPattern, PolicyFile: "", CheckExists: true, diff --git a/libbeat/idxmgmt/ilm/ilm_test.go b/libbeat/idxmgmt/ilm/ilm_test.go index 8a7e960745e0..971f1861b831 100644 --- a/libbeat/idxmgmt/ilm/ilm_test.go +++ b/libbeat/idxmgmt/ilm/ilm_test.go @@ -264,7 +264,7 @@ func TestDefaultSupport_Manager_EnsureAlias(t *testing.T) { func TestDefaultSupport_Manager_EnsurePolicy(t *testing.T) { testPolicy := Policy{ - Name: "test-9.9.9", + Name: "test", Body: DefaultPolicy, } @@ -305,11 +305,6 @@ func TestDefaultSupport_Manager_EnsurePolicy(t *testing.T) { for name, test := range cases { test := test t.Run(name, func(t *testing.T) { - cfg := test.cfg - if cfg == nil { - cfg = map[string]interface{}{"name": "test"} - } - h := newMockHandler(test.calls...) m := createManager(t, h, test.cfg) created, err := m.EnsurePolicy(test.overwrite) diff --git a/libbeat/idxmgmt/std_test.go b/libbeat/idxmgmt/std_test.go index ac0cf47ae8b1..17abcd630868 100644 --- a/libbeat/idxmgmt/std_test.go +++ b/libbeat/idxmgmt/std_test.go @@ -313,11 +313,11 @@ func TestIndexManager_Setup(t *testing.T) { "overwrite": "true", "name": "test-9.9.9", "pattern": "test-9.9.9-*", - "settings.index.lifecycle.name": "test-9.9.9", + "settings.index.lifecycle.name": "test", "settings.index.lifecycle.rollover_alias": "test-9.9.9", }), alias: "test-9.9.9", - policy: "test-9.9.9", + policy: "test", }, "template default ilm default with alias and policy changed": { cfg: common.MapStr{ @@ -370,7 +370,7 @@ func TestIndexManager_Setup(t *testing.T) { "setup.template.enabled": false, }, alias: "test-9.9.9", - policy: "test-9.9.9", + policy: "test", }, "template disabled ilm disabled, loadMode Overwrite": { cfg: common.MapStr{ @@ -386,20 +386,20 @@ func TestIndexManager_Setup(t *testing.T) { }, loadILM: LoadModeForce, alias: "test-9.9.9", - policy: "test-9.9.9", + policy: "test", }, "template loadmode disabled ilm loadMode enabled": { loadTemplate: LoadModeDisabled, loadILM: LoadModeEnabled, alias: "test-9.9.9", - policy: "test-9.9.9", + policy: "test", }, "template default ilm loadMode disabled": { loadILM: LoadModeDisabled, tmplCfg: cfgWith(template.DefaultConfig(), map[string]interface{}{ "name": "test-9.9.9", "pattern": "test-9.9.9-*", - "settings.index.lifecycle.name": "test-9.9.9", + "settings.index.lifecycle.name": "test", "settings.index.lifecycle.rollover_alias": "test-9.9.9", }), }, diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index 836a6d117976..e0598eaea556 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -12,15 +12,15 @@ def __init__(self, client, index): def needs_init(self, s): return s == '' or s == '*' - def delete(self, indices=[]): + def delete(self, indices=[], policies=[]): indices = list(filter(lambda x: x != '', indices)) if not indices: indices == [self._index] for i in indices: self.delete_index_and_alias(i) self.delete_template(template=i) - for i in indices: - self.delete_policy(policy=i) + for i in list(filter(lambda x: x != '', policies)): + self.delete_policy(i) def delete_index_and_alias(self, index=""): if self.needs_init(index): @@ -40,10 +40,7 @@ def delete_template(self, template=""): except NotFoundError: pass - def delete_policy(self, policy=""): - if self.needs_init(policy): - policy = self._index - + def delete_policy(self, policy): # Delete any existing policy starting with given policy policies = self._client.transport.perform_request('GET', "/_ilm/policy") for p, _ in policies.items(): diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 52162ae55211..2a9d42c0e4a2 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -20,7 +20,8 @@ def setUp(self): self.cmd = "--index-management" # auto-derived default settings, if nothing else is set - self.index_name = self.alias_name = self.policy_name = self.beat_name + "-9.9.9" + self.policy_name = self.beat_name + self.index_name = self.alias_name = self.beat_name + "-9.9.9" self.custom_alias = self.beat_name + "_foo" self.custom_policy = self.beat_name + "_bar" @@ -28,13 +29,15 @@ def setUp(self): self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], + policies=[self.policy_name, self.custom_policy]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], + policies=[self.policy_name, self.custom_policy]) def render_config(self, **kwargs): self.render_config_template( @@ -94,7 +97,7 @@ def test_setup_template_disabled(self): assert exit_code == 0 self.idxmgmt.assert_index_template_not_loaded(self.index_name) self.idxmgmt.assert_alias_created(self.index_name) - self.idxmgmt.assert_policy_created(self.index_name) + self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') @@ -239,7 +242,7 @@ def test_setup_overwrite_template_on_ilm_policy_created(self): "-E", "setup.template.pattern=" + self.custom_alias + "*"]) assert exit_code == 0 self.idxmgmt.assert_index_template_loaded(self.custom_alias) - self.idxmgmt.assert_policy_not_created(self.index_name) + self.idxmgmt.assert_policy_not_created(self.policy_name) # ensure ilm policy is created, triggering overwriting existing template exit_code = self.run_beat(extra_args=["setup", self.cmd, @@ -247,8 +250,8 @@ def test_setup_overwrite_template_on_ilm_policy_created(self): "-E", "setup.template.settings.index.number_of_shards=2", "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias) - self.idxmgmt.assert_policy_created(self.index_name) + self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) + self.idxmgmt.assert_policy_created(self.policy_name) # check that template was overwritten resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias) assert self.custom_alias in resp diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index 9195954600aa..62e1fba77937 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -16,15 +16,18 @@ class TestRunILM(BaseTest): def setUp(self): super(TestRunILM, self).setUp() - self.alias_name = self.policy_name = self.index_name = self.beat_name + "-9.9.9" + self.alias_name = self.index_name = self.beat_name + "-9.9.9" + self.policy_name = self.beat_name self.custom_alias = self.beat_name + "_foo" self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], + policies=[self.policy_name, self.custom_policy]) def tearDown(self): - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], + policies=[self.policy_name, self.custom_policy]) def render_config(self, **kwargs): self.render_config_template( @@ -160,18 +163,21 @@ def setUp(self): self.setupCmd = "--ilm-policy" - self.alias_name = self.policy_name = self.index_name = self.beat_name + "-9.9.9" + self.alias_name = self.index_name = self.beat_name + "-9.9.9" + self.policy_name = self.beat_name self.custom_alias = self.beat_name + "_foo" self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], + policies=[self.policy_name, self.custom_policy]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], + policies=[self.policy_name, self.custom_policy]) def render_config(self, **kwargs): self.render_config_template( @@ -277,7 +283,7 @@ def setUp(self): self.config = "libbeat.yml" self.output = os.path.join(self.working_dir, self.config) shutil.copy(os.path.join(self.beat_path, "fields.yml"), self.output) - self.policy_name = self.beat_name + "-9.9.9" + self.policy_name = self.beat_name self.cmd = "ilm-policy" def assert_log_contains_policy(self): diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 084993b88502..18aa194c1b24 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -152,7 +152,7 @@ def test_template_default(self): self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) proc.check_kill_and_wait() - self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.index_name, self.index_name) + self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.beat_name, self.index_name) self.idxmgmt.assert_alias_created(self.index_name) self.idxmgmt.assert_docs_written_to_alias(self.index_name) @@ -183,15 +183,16 @@ def setUp(self): self.setupCmd = "--template" self.index_name = self.beat_name + "-9.9.9" self.custom_alias = self.beat_name + "_foo" + self.policy_name = self.beat_name self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name], policies=[self.policy_name]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name], policies=[self.policy_name]) def render_config(self, **kwargs): self.render_config_template( @@ -210,9 +211,9 @@ def test_setup(self): extra_args=["setup", self.setupCmd, "--ilm-policy"]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.index_name, self.index_name) + self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.policy_name, self.index_name) self.idxmgmt.assert_alias_created(self.index_name) - self.idxmgmt.assert_policy_created(self.index_name) + self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') @@ -225,13 +226,13 @@ def test_setup_template_default(self): extra_args=["setup", self.setupCmd]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.index_name, self.index_name) + self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.policy_name, self.index_name) self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"]) # when running `setup --template` # write_alias and rollover_policy related to ILM are also created self.idxmgmt.assert_alias_created(self.index_name) - self.idxmgmt.assert_policy_created(self.index_name) + self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') @@ -250,7 +251,7 @@ def test_setup_template_disabled(self): # when running `setup --template` and `setup.template.enabled=false` # write_alias and rollover_policy related to ILM are still created self.idxmgmt.assert_alias_created(self.index_name) - self.idxmgmt.assert_policy_created(self.index_name) + self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') @@ -285,7 +286,7 @@ def test_setup_template_with_ilm_changed_pattern(self): "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias) + self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"]) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @@ -304,7 +305,7 @@ def test_template_created_on_ilm_policy_created(self): "-E", "setup.template.pattern=" + self.custom_alias + "*"]) assert exit_code == 0 self.idxmgmt.assert_index_template_loaded(self.custom_alias) - self.idxmgmt.assert_policy_not_created(self.index_name) + self.idxmgmt.assert_policy_not_created(self.policy_name) # ensure ilm policy is created, triggering overwriting existing template exit_code = self.run_beat(extra_args=["setup", self.setupCmd, @@ -312,8 +313,8 @@ def test_template_created_on_ilm_policy_created(self): "-E", "setup.template.settings.index.number_of_shards=2", "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias) - self.idxmgmt.assert_policy_created(self.index_name) + self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) + self.idxmgmt.assert_policy_created(self.policy_name) # check that template was overwritten resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias) assert self.custom_alias in resp