From 6ac18f9f3147bb8274a648e618401a3c7987aca9 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Fri, 15 Nov 2024 18:24:07 +0100 Subject: [PATCH 01/17] Additional stats fields for Elasticsearch --- metricbeat/module/elasticsearch/index/_meta/data.json | 4 ++++ metricbeat/module/elasticsearch/index/_meta/fields.yml | 4 ++++ metricbeat/module/elasticsearch/index/data.go | 8 +++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/_meta/data.json b/metricbeat/module/elasticsearch/index/_meta/data.json index 2fa8eeb4f18..2bae6566c07 100644 --- a/metricbeat/module/elasticsearch/index/_meta/data.json +++ b/metricbeat/module/elasticsearch/index/_meta/data.json @@ -144,6 +144,10 @@ } }, "status": "green", + "tier_preference": [ + "data_content" + ], + "creation_date": 1731657995821, "hidden": true, "shards": { "total": 1, diff --git a/metricbeat/module/elasticsearch/index/_meta/fields.yml b/metricbeat/module/elasticsearch/index/_meta/fields.yml index 8182963959b..9664f6adaca 100644 --- a/metricbeat/module/elasticsearch/index/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/index/_meta/fields.yml @@ -17,6 +17,10 @@ type: keyword - name: status type: keyword + - name: tier_preference + type: keyword + - name: creation_date + type: date - name: name type: keyword description: > diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index d8bec4939d6..9515bf7fafc 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -40,9 +40,11 @@ type Index struct { Primaries primaries `json:"primaries"` Total total `json:"total"` - Index string `json:"index"` - Status string `json:"status"` - Shards shardStats `json:"shards"` + Index string `json:"index"` + Status string `json:"status"` + TierPreference []string `json:"tier_preference"` + CreationDate int `json:"creation_date"` + Shards shardStats `json:"shards"` } type primaries struct { From 0826d9cded82c6cb6e61d8e80fdfb2868a712bad Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Sun, 17 Nov 2024 20:53:42 +0100 Subject: [PATCH 02/17] Populate tier preference and creation date from cluster state --- metricbeat/module/elasticsearch/index/data.go | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index 9515bf7fafc..4e65b6e4ab7 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -251,6 +251,18 @@ func addClusterStateFields(idx *Index, clusterState mapstr.M) error { return fmt.Errorf("failed to get index routing table from cluster state: %w", err) } + indexTierPreference, err := getIndexTierPreference(clusterState, idx.Index) + if err != nil { + return fmt.Errorf("failed to get index tier preference from cluster state: %w", err) + } + idx.TierPreference = indexTierPreference + + indexCreationDate, err := getIndexCreationDate(clusterState, idx.Index) + if err != nil { + return fmt.Errorf("failed to get index creation date from cluster state: %w", err) + } + idx.CreationDate = indexCreationDate + shards, err := getShardsFromRoutingTable(indexRoutingTable) if err != nil { return fmt.Errorf("failed to get shards from routing table: %w", err) @@ -287,6 +299,34 @@ func getClusterStateMetricForIndex(clusterState mapstr.M, index, metricKey strin return mapstr.M(metric), nil } +func getIndexTierPreference(clusterState mapstr.M, index string) ([]string, error) { + fieldKey := "metadata.settings.index.routing.allocation.include._tier_preference" + value, err := clusterState.GetValue(fieldKey) + if err != nil { + return nil, fmt.Errorf("'"+fieldKey+"': %w", err) + } + + tierPreference, ok := value.([]string) + if !ok { + return nil, elastic.MakeErrorForMissingField(fieldKey, elastic.Elasticsearch) + } + return tierPreference, nil +} + +func getIndexCreationDate(clusterState mapstr.M, index string) (int, error) { + fieldKey := "metadata.indices." + index + ".creation_date" + value, err := clusterState.GetValue(fieldKey) + if err != nil { + return 0, fmt.Errorf("'"+fieldKey+"': %w", err) + } + + creationDate, ok := value.(int) + if !ok { + return 0, elastic.MakeErrorForMissingField(fieldKey, elastic.Elasticsearch) + } + return creationDate, nil +} + func getIndexStatus(shards map[string]interface{}) (string, error) { if len(shards) == 0 { // No shards, index is red From 059c13f22a7bdeffce2c93dd02d12c7900b55fac Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Sun, 17 Nov 2024 21:23:35 +0100 Subject: [PATCH 03/17] Mage update --- metricbeat/docs/fields.asciidoc | 14 ++++++++++++++ metricbeat/module/elasticsearch/fields.go | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 09fd5e53245..8317381fdf0 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -32130,6 +32130,20 @@ type: keyword -- +*`elasticsearch.index.tier_preference`*:: ++ +-- +type: keyword + +-- + +*`elasticsearch.index.creation_date`*:: ++ +-- +type: date + +-- + *`elasticsearch.index.name`*:: + -- diff --git a/metricbeat/module/elasticsearch/fields.go b/metricbeat/module/elasticsearch/fields.go index 7fe1827b3ce..828bdf886c6 100644 --- a/metricbeat/module/elasticsearch/fields.go +++ b/metricbeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded zlib format compressed contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzsfV2P3biR9r1/BeGrCWDrRW6NYPIC2WTXC4wxyEz2ZrFQ2BLPObQlUSapdvf++oVIfZASPyVKfey0b5Lpbj31VPGrWCwW34Mv6PkDQBVkHBcMQVrc3gDAMa/QB/D2r+rP374BoESsoLjlmDQfwM9vAABA+xtQk7Kr0BsAKKoQZOgDuMI3ADDEOW6u7AP477eMVW/fgbc3ztu3/9P/7kYozwvSXPD1A7jAivXfXzCqSvZBiHgPGlijD6CoOsYRzUe0bPhBViMOS8hhVmLWVvA57/9efAoAf27Rh17Nb4SWGhxuSvSUU1SQR0SftT+/UtK1w09UJurn7AZpyTLGIeU5xzXKcZPXuKowm/52xIMVhupPW8hvC7Nngk420lFws5rZhZP2ENkDrEM0JxxWB8iecUfhk2AOiy8545Cz6MaCbZ1dSNeUmyiO/UzIzgSPbI04ynpq+98XBc1QAx8qlE6mHXktGz5CXPV/dIB0HXuUXeECNQzFDyQOebet7+g0BwLZAnCU08MmlDLBaQMjWvuW4hpO008cMSExWyKodt2msMTVv9dmzR0DXZmaZ9CGlNuY9h9meD0O1LbYonvT1Q+ImlefTRMQbkpcoHUvV781fa8xIF3Dtd/YFLMpp/fkgZOcco0S1bn+AMEDvK6X2icS2Cs5ecErW7fFKPXzY22UtmRuY69i1fAp71rrIutXJ0alz491NgtUl/4VLVRnNwTbvGOo7Jk9PHN0MDFUE/ospGa91MwscsWwV+h0gjV8UviZJpD4VVJIym+Q3dIs6BxlBshR2iOiDJMmmagl3tzBhUk2z/8mWSZMbU3Muw4ncsq49DYWkMk9GwVo8mZwjRiHdfvGBi1h3/7/6S/fGrujwtyGYaaG9e0UIx0tkGr28M69uUFs67/mZUQDTl9Pczp5kOt+1v+/SHvVVf/V0lwz5IVQVEDG2fDf6oIVJcEOpG95N3swmuMX6OKZja97v3jnQs84oShj+H+Rba6PW/ClGhO3zIc/8ihJYfIM9oq3wE7ao2uNGn6EZAf0KJ2iC0Xs5osH7OcSLGj2Aeh19G7z4zpHoBhtGOHmmsxHlGPa5OW6dQvVbyScuQQtyKR3WH2s3BKnRfNGCecVOpWhT+hEbmHZ+Hnwa4foc17A4oYGfzR5vxcksyhBIzvBXIRpD+UWIWaew752iPEzLBcp6pS5TDKLnMcOnvdHa0XO+cd4ApJMqBfwnczwUql7mt3NjO5lZl+wCxE4e2jDwdqax5b+IGffY/rD8COXiAWNY+2t8wkxtfRYkxmbI1qzfJiok0aSdDUHRztM3LR9IrjhJ7ILlOcKWCdkY4ZXNmP5I6w6dKJ9ImTOcb1T+1eYOG05LXM5Rs4jGSd29i+fUJk/YJ4zxM8jGydWnVbyR1RwQk+eXYKlLmLBeQ3b85jGCNXdk28Uc0TPYxolVTmfOIfdUpAWDiwKmsOOk/xCqop82xgYlGelObnkF4irftxKNNuRZ1D4u6CZwiyTyJmObDs4XPKhqCYc5dr5Sz5sxJLScwrysmVdUSDGLl11hAUH9DATyj9CNJssCMv9FpssMZkLlgoBtVdu64niPDnferZT0GzSe5UFALStr9rMO8xhhJnSaRAsEc2351v0MiRIpoMsW3mnjMloZimDHteKPMAqL26o+CLcyL062QEXkmv4lDP0NW/IXpEGpJUt0+k52dWv6SQ9ga6TWIe2/UfjfIDK3dZ1oo0ySccZh02Jm2vq+UiBXk5KNgZivT+IgsC2cJByH7rLpV80WkQh792k5VYojoUKmk2gIQxs0bAd8nvIRXaHoZu3bd8Ku9IU9L5uBlxJHnOP04m2Iq5kC2iUULQNUDuRkINNhF52ZPn2cpXUXjHKDMnFisRheURPqDhEuoJvTHOevbHEs82M7JpszvS8VLnT8GdC9v5ONgMGiu1nA44SdG8H5ihWzDP7FDVA6L1Zzm6Ju7BcI9z9NvUapXRc1xI19NzU0oeuq0rWtg0NKdHGfcNlTS/m2NWeYbol4Dwl17uiA9bUpSGlqG+suob0ec7Vt2RMuiMNek9OQkgGKGLJzN27XqTLJ7K2ldGLmDoBm312xkQbS/sN7T6uMiG6UIFx6TD+WYjNRm0HoylLlu0IA3hDdGq8AJZ5epZiNU1MVU7a6bnKWXsH2ZT3KMTxQrKO7Uo2sGkTbMnppsSUe+BJfHf8yRlUEmdCGsR7kiGPSr7wnXIHsjckN5guWoStFO68kI2MxHDdcnAbn4kRy1BDjjPbKgctWf8IyBaL0HWZvLZ5/VYy9O5U1XUO4WZltaS6O1XXlPi33Qs+JKEo0Xhdpexsn+HcmU6b+Gya3w7LLrIRCVbNmQoTZuSoRJlYYjP41j4PdqR4xLLV8FMQjkxJiOWrwqegm55hAlKhGVKx1ARuCoLBWXixDCVwCoqxiVyxTDX8FIQj86Ni+arwqegexTMJwbgkrliaCvoWstOdSO3qfPzG/lokXJirSnaPY2JWVekMftiwffgGFUzHw8t/nvb+/Fhn1yKbbZKRqsxmfGc0BwSEnyy0vR5pKv5GBzWU/Ej8mXSruIOOce+tKjT4rtt1pcGmllXcoWQTSkg1igB1x8IWixoTYXqt+Pjqd8QQclTliGXUIlqgLfuiNaG2CN0WaeeIWg5hSOEBNybZF7Au2i5ZP6wILHP4iCi8LkMlbmAXuCrgj8sxM/7ztB1hWdF22cDvmllxfKO2MLHf4Qm0HSwcvWiPrToGryhvYEM2nrT0RhMEsoFmJiAz68lNyEBcd7c02hYXln/tCId5jQuaROWsuLBMYGbdFpWBtkeC7hOpFMv3nO6NKtgOkx0m5Y5VUDdI/7NsgZ10HZ81EIVl8jEqb3Not2mwwE6qgXA/Jmjn8NtOXuFuHYw+4npY5pgxSRpOSZW7+nawAYatXwhm6KCscI25y0XZQlCAWn2VGHpyAk9MT07hsfSmaBQlBWL343BsduYGRcSwinfj+E2kXbSE7Kup8dBVX5LZ4muHOpPX5bGEokvW88kEzq6gP0WfUcGNk3YsmRFq86nKFZmzBF7IwlfE78bAPZfd9l1e73lxC8ubivdi47EA9U4rpz+T3Wvm+eDzLuw8/Ha3ocVx1z3ZWSa33YuZJZtoKy8S7DZmUuew2rfcmgoC2pFsaMCfEeYGdQGvwK0JCiBoY2K/yz2m8trKnjl2RZHpa4czNeWBHJM/b5yH3ZAuWBCcjgIS2XBd+CagnQNTeA7k6Csa/QOMQ1OBKsMFdg0iegody+s2piqq6jsm8p/hSZbxn/40y4ArpGRWqYaQdgKZuLRLlFWU08oVmEapxkrTMQsUXi7PNrIewv2/T6RE4OO/GeUsmj+FJL3lVWGUVJYVd7s0gWkWJyt0G+U9EFIh2MTJ+8gAvyHRtuL/SHzx3382E6hI8UV3VfZTGEHB8DgLIM1E68/r7l+sK0cs+6FD5l8oYez9OL4oaitciKsVYHltR3+Jafzn6uLW+hbA2S2cdyrnTyuyWAOmUJ+nqEAAhKEAyPwVbji6KvrYfRGxjiZzSJZ3nb3arD62XFWOAlpdGHV+bekXsMzRU4Fa090gidKIxrN8vroKCvZ5etN91EN8CsNtZZMIo9XByvIbgabMV3H79cfXU6mD8eMpaxwWavGLdEPDfw3M3QyWIiDx6ik1qJJpd+BgcJbH8lrAhBhTFCxYwGIKvENTmJd4UUcpWT8wBbaBx3UFfl+u//cJ1giQy8DYImn2Zw1Vlxy2iWLyC3zCdVcD1neZpkDDAXxPbhqlo6s5sF2+fqazdVWocpA2NuhYa+N7atKRs6dRjSXgnBaKYzO1Yt9wQhj4hvkNy5Z0c3NWS0nPcBYneaES/DRuOVD5h96zJoL1ZFqpz4WSOrxfiuAVw02B8mErsMFvDtLsd1yjdwA3oGbvgJCos+/FgwvixQ2tlEg9rKKI/7uQAWYZQNyt6oe/bvr7n6qC+doqfQWQnkPC1qJdESj28lsRIO5qSE4oU+DMn+bhiiEM0YP9EQOzSfyhguUTpXY1bAzAIra3eiLPTwasIlPL9+jiMHz9w/e98Y26OIjl25n615cK9kuGZZu+erTOJ1brYWqf6hj46UoRat6BZ9SP9neAovIP5pigibK9K2gyP/WfColYxICzoH4zuTDMWSrHOqaV732Ff7wY5lnAMZ1o+v/e01fmWmHKfnkbZwqLVGMYNlzs7B9IoPeowlf8UKFgAoZCDFvE9zDBMtfPs7r3Svadkrqk2t9M9ai0TPL0X+7YAOe8muENqq5L7oDwsWmIlEu4zePVWwrHO9okgoz09n1HXCbfA2g+dwztyCZfajmkhzYI6N/yTeWoLu5l85vw5u0N5l+aRdUedwkwp9sU4SZF6LW2s7Sew8z+TJUjaA1PvPvpTYtSWB2ZPV30FwEM+rkFXAhVRBpnkeHZ/J3O3nTE/tRi+tyPXle+Q8BMYPB13N7ONObH5//DPlQkFunyiWFbZxfSNXZP0XyYOSM8tbD4Ih8LGJ2YBFjDCWgw0tSsDcXF+oHriD3NXwVCgi0NekJF1+8a85ZUuEhXcdJwsA+CHXsO2bL3uNm4GDmSGlRYj6NhoeQP2U2dWNzU3QVRwKZAla33+vqvMqdDihqe9yqtT5HjKBmOkFUA+90Y900yZR6h3HW72XM6oZxPdE2Dm2vWQGNMyImmJdEKn8w4TAwsfBkk84osM3rGAxMwxEelRMfIFRkE/afI7HikZ8ZvkA8z2HgJh1AGbvARTZyGaKPIVBFN2LWWjBx5apT8zLzoKDXfxtnrs/xFIqsxv/ksd9BnMpbdrdrgJMZGs+20wtrJdlYRvVgtAeJXqRsuy1Xw3z7bOTcJ2zvVDsd+gxc9fmqIkIVFHCOiW8dk433s292Rjpc+Cf4OkqK9B+XRKcyexCl7Tcg0igfWh4ywAHrEhavsciDMDXNnbZhAmBozthVHb9PvqQ3uyHj+x6yjgAwBr5241irPMd+XqELmS09BJ3vGgpwgSedK0Q28pdYiwEKLzEVAxtYFjGQbUcEvAjm0IGQEZHAJxwjMqLqqEbiRNUUjkOOKBEYAx9Zt9UDPa/yFInZT3sUJeIk8CBE9cUSbHjQZdI3oddyLpph87+I+VWBn3evUqYjh9eUdoK6dy3Yve+t6uOlMoCRFJ9Y+oKaM2TfW2xfbbfSkpGiaYU7PhdAa8uGuyQHajPr0PMYcPUG810II9WkQ6W0drZBPHYCboupETOQBFl/6/xXH6hdCQQspx7CqnkHd921UjseRdhsc4/dHe/1BAWz3risIwr1pCDuvd28YAhaE5ZlfikVmxnRZ6XU77oK5ox1lah/ku3LADt0rJnGX/csACF2+1gtyrZzXPzzLrPPBIo5FDRy7/zxi37VjrG02rlxMp3roL7IvPGBbfGB4IPGWEJwSeThsg5x2T/+vG/V/3RQnMqSh7CFIYsQw3yNCdfh4TQ2XtGWEvt4XNaPQtvLT+l9GUUEekVZX9gVOsNMWJ7zgdSETN6ILVUW2lzEFAVlBqoMrrG4tlxzYESjqHbkEnWnH6rJO9wFJTD2ZKNGQlrZK5Vz4XsJ2YG3KJlgP/XXal0ce46Td+iWkfNunluSM+IwrmX0+zlZaVSfgTsjdkaKxENojuXI1zOmhmyoJ/U47BPAi59ssm/H1Iw47dP77qK3AtViZwoZVZNmE22ds+7wamBG7PfdoWJqbXPTycIw585VeEc+ESVpCeQ7Lkq6rMQQMbQmUsrTY7wJSJrBZR4wUeyPMXH95t+AeGQxGAT8VpKtK8IDAx1+nHxIq/qjnY7kXOJBMm4ClktTTsMzjjHS0QAkaegBK2dC/CUh3Qw9i0za0KjhFQw8k0za0StKeb/eIKL70G+6Urqi4pJ+HvfMdME1FQeiO/lC+8mX9/PQ5ja+nrSlPW7dUi/LHqxOcTG44aL1bXRKfsgY+/5yum/nC3CtiQY+BHt0ee89Azgsupqg3Zq+3HQR1VuzYySzNQ5khL0Y5joKUFdhpTA/Y4RFVpaRMgu7jnD8jcKz335K38eaWWaLBx2tSLJcVHXCvuWmH03v1ll69pUh9X72lH9Nb8j9g6Yrl6mDaKewm72ut46sj9+rIRen7XTpyd+B6zcPuihjPW9yiCjf+x0KmUNgD0qqZaRPu37umtyOoEae4YIA0gxwwyhkLjmiVMFwBNWug9BuuymJdU1MvngZrBP4fwOW88mpsEjikmrxfZr17aQLJboF3ysoPq0qpphBWKsw/d4Qv9Z8MTu3AZ1wWMTPbTSXiKVm+jQkqB1zAyUgqnJKYDGXpw51VdqXjIDo4a1HDRy7CY5us1q+O3fWmsxuamuFH4QQSfkN0+iUDBaxMZtNUYKi67NZgG3f0FMd9DoYPHTrZ9s5w+g0C8lJ0I/Qj87lF0neVY9NCVBecc2h382KFw6aUcwS8Ck9fmnzkMe7UpLsL3v6p/+bnD3/i8PrzWytJQktEjec7IK6b3JDEWrKCbYsgnfaR05RWogtusLG+0elTlLcpX2CO8nevF52kVvSmdP8q+0wevL6BY9NVVyYfIjxtLuER8j8a/LVDoK7AZ/JgP0S21mreJPQ/yYOENEu7EIoKyPjwEGlMCaepjUiJZKZpsll2vGNkLhsYkgEgIi9iyKc7CMXNI6xwKash7ijHMM0ZOUUFoeUWrEW7/zpNQyJygx7XkRDVMpmqinETtDWJrJ+558iG1I+JBxcAwmLhhuKG1RBkQtLooJ/a5X+L6l4ys6EhHDwg0ELKUGlIHVnNFkGPDjoUWHx/fH32sBf+BlRzc67rLAduFQxTxX/9Aj42FxLr/G8txB4ULOtJGQ0AVvOFLMLc+wEvFtr7DwRb0DPQonm9Dv5AXmhh6lN0qOHTdhUa0rx8U3wizfsEzTHq8pItMqkS3iqLtSY74CXOy+Rm9tjiZGA0mnFm3v+Qxm9TZWwAH0jHAYLFbUjJawA0P4q7z/0T27N0u0dnEu0L11BKeRvHXv4yAsS6T3JgKEH9qGL6vpY7LBg93oNyqpki9Lsn7LtOkfyO6Fruz6SluzEHYQkzXrL7XvgqRykyeLCzcMKp2RPoqV9M1LDE/WZO3NFx+/d20D4mPkScqe9NQInmBqtqfHQCMoavjYzjiejY6N+cm3cSb97pzQyZVGLTI2WqyVQ255Cl2dN7vSYE3l58Xg5HwlSJfNdMD5LmcWgZEocXPTqjK5xVwOkMXY6vi5GiUyslNnY/E93Pgflyn5tOazm77yV5ZH2mueiOb/H3sB1Eeus0zUuBUufmZNlqVZmTRYtiRCfLlMWkThaqlUg6WbZa6OgFRJ8tU6m+dLBk5RCjlyRi38lCNgdFbEd4S12Q/VPoWaO4cK8hJelszzzNYkxVBZaN2hJSpQvEkeoYq7ufAk1i+eCqd7/Ap5A6gS2CX+6G868Ifgklnd+TsQXxOszingdezyX+D3kQ5AwKPJPuoF3lPTXh63h5HS9pxgvr6CN+NOTJvQ6Zl+f8OmTOJm4bMqqLdy2yglSV3B2ldPNGWFdq0Us/O7QjZCmW5h9MR2MnuaTrFcflT+wpN7jEml54TlFaeFVTKEzngAH+N1whwJ4ZR7VDTLDxTgv0UbTrvl2MrKklTzkEIkdHjp38U2QVBN6sBHEXDSmCZXYMtIjZJsCeDJBuriva7pCeUBFY5vDxmv1xmTisy7jB6pJfKgLX5pgo2ngkWLjaDhYFzzoGryjbW87ZbEs/Ux9bTcaFZV87wmFmvLIQyBgsboJ4kVzUQ+irAlEFW4bKvEUUk9I/GAL1AYskQ6UUwFEiFAnWvhMIrx+bHt+NSMMpqXJfu/pS63XUCteOVOxtmHJobsdcpigXbZetg9KOYHRIEJrf+tUjbwlJV+LnsFxQ+dh+Ap+dos+o4J7hFeB9XggtUC5eM/rXUPiK7Bl0P5Si7ju6P5Sqx2ah3Jeuwp/9cVVd5svlLUWMdTT1zdNjXFp5JyIrSP2AG1TmBSG0xA3kvSawKfOhFP1puQVi2z4Kle2y/emU7ZJPVVc1+gvorIk/VXGK2goX8AV0HiWf3M53MsrmYX9+y4+yX0jlkxt+FAur04KRsq+dKFBs5A56/nKyo7w4mbWoKftBwyFT9zvmldARcP6nCfCfYqcLccMABMMvQP8LFUk9ddpyeZMhynNRSMboIMSXN/goIMEact7MYkIxN79TEy/vVxPc5NiKRxeMknY84pCBvxEK0BOs26pXqOPva9i2y9sJWrQFN7n0D0MfL/LXkcC1uPQiYFc9VORZ7+mSAmDoPLv62GHPEokrM5jJMkf+J4rk9axUxteKeAgm7teRUhas6YULyBDZFFWkkEu7uGLepH2+5jY8aSJKsoou8w2yUSgqwYWSOoxY0jeHgmiBjxzcoOxA6AkWHDBYIyCS8wG/wcZoPHETqyB1Czl+wBXmz6DtaEuYLQVATkL5onQK2LUJM7Siz2RKWLJb2Xr98f8FAAD//02PqlQ=" + return "eJzsfV2P3biR9r1/BeGrCWDrRW6NYPIC2WTXC4wxyEz2ZrFQ2FKdc2hLokxR7e799QuR+qAkfkqU+thp3yTT3XrqqeJXsVgsvkdf4PkDggI3nGQNYJbd3iDECS/gA3r7V/Xnb98glEOTMVJzQqsP6Oc3CCE0+xtU0rwt4A1CDArADXxAV/wGoQY4J9W1+YD++23TFG/fobc3zuu3/9P97kYZTzNaXcj1A7rgoum+vxAo8uaDEPEeVbiEDygr2oYDSwe0pP9BUgLHOeY4yUlTF/g57f5efIoQf67hQ6fmN8ryGRypcnhKGWT0Edjz7M+vjLZ1/xOVifp5c8Msb5KGY8ZTTkpISZWWpChIM/7tgIcLgtWf1pjfFmZPBJ1koKPgJmVjFk7rQ2T3sBbRnHJcHCB7wh2Ej4I5zr6kDce8CW4sXJfJhbZVvoni0M+E7ETwSNaIg6ynuvt9lrEEKvxQQDyZZuS1bPyISdH90QHS59iD7IJkUDUQPpA45u22vjOn2RNIFoCDnA42opQRbjYwgrWvGSnxOP2EERMSkyWCatdtCkvc+fezWXPHQFem5gm0ovk2pt2HCVmPA7UttuheteUDMP3qs2kCIlVOMlj3cvVb3fczBrSt+Ow3JsVMys17cs9JTrlaiepcf4DgHn6ul9onItgrOnnBK1m3xSD182OplbZkbmKvYpX4KW1r4yLrVidEpc+PZTIJVJf+FS0okxvgOm0byDtmD88cDiYGJWXPQmrSSU30IlcMO4VOJ1jiJ4WfbgIJXyWFpPSGm1ucBZ1DooEcpD0Cawitoola4k0dXJhk8/yvk6XDnK2JaduSSE4Zl97GAjK6Z6MAjd4MKaHhuKzfmKAl7Nv/P/7lW213VJibMPTUyHw71dCWZaCa3b9zb24Q0/o/8zKCAcevxzmdPsh1P+n+X6C9yqL7ammuCfJCGWS44U3/3+qCFSTBDDTf8m72YGaOn6eLpzf+3PslOxf6hlMGSUP+F0xzfdiCL9UYuSUu/IFHTjOdZ7BXvAF21B6uJVT8CMkW6EE6gwuD5uaKB+zn4i1o8gHYdfBu0+M6h6eY2TAi1TWajyjHtM7Ltevmq99AOLEJWpCJ77C6WNkljovmjVHOCziVoUvoSG5h2fB58GsL7DnNcHaD3h+N3u8FySRI0MBOMBdh2kO5BYiZ5rCvLTT8DMsFijplLpPMAuexg+f9wVqBc/4xnoAk4+sFfCczvFTqnmZ3PaN7mdkX7HwETh5af7C25rGlP8jZ95j+0P/IJmJB41h7z/n4mFp6rNGMzYGVTdpP1FEjSXM1e0fbT9y4faKk4iey85RnC1hHZKOHVzZj6SMuWjjRPgEyp7jeqf3LT9xsOc1TOUbOIxkmdvIvnyBPHwhPG+DnkQ0Tq04r6SNknLKTZxdvqYtYcFri+jymIULn7sk3Rjiw85gGSVXOJ85htxQ0CwdmGUtxy2l6oUVBv20MDMqz0pRe0gsmRTduJZrpyNMr/J2xRGGWSORkjmw6OFzyYVBSDuns/CXtN2JR6VkFOdk2bZZB01za4ggL9uh+JpR/BCwZLYjz/RYbLTGaC+cKAbVXbuuJ4jw53Xq2k7Fk1HuVBYBmW1+1mXeYQwszptMAzoGl2/MtOhkSJJmDLFt5p4zRaHopvR7Xgj7gIs1ukH0RbuRencyAC8klfkob+JpWdK9IDdLKlvH0HO3q1nSUHkHXUaxF2+6jYT6AfLd1rWiDTNryhuMqJ9U19nykQC8nJRMDsd4fREFgGzhIuQ/t5dItGjUwzDs3abkVCmOhgiYjqA8DUzRsh/wOcpHdoenmdd21wq40hXlf1wOuJA+5x/FEGxFXsgU0RBRtApydSMjBJkIvO7J8O7lKaq8YZZrkYkVivzzCE2SHSFfwtWnOkzcWebaZkG2TzZmelyp3HP6NkL2/k02AnmK72YBDhO5twRzEinlmn6IaiHlvlrNb5C4s1wh7v429Rikd17ZE9T03tvS+66qSZ9uGiuawcd9wWdMLOXY1Z5huCTiPyfW26IAxdalPKeoaqywxe55y9Q0Zk/ZIw7wnRyEkAxShZKbuXS7S5SNZ28joRUwdgc0+OxM6G0v7DW0/rtIh2lCRdunQ/pmPzQZte6MpS5bpCAM5Q3RqvADnaXyWYjWNTFVO2vG5yll7B9mY9yjE8UK0jm1LNjBp423J8abEmHvgSHy3/MkZVCJnQmrEO5Ihj0q+cJ1ye7LXJDfoLlr4rRT2vJCNjMRw3XJwG56JEcpwhhxmtlUOWrT+4ZEtFqDrMnlt8/qtZOjdqarrHMLNys6S6u5UXV3i33Yv+JCEokjjdZWys32Gs2c6beKzaX47LLvIRMRbNWsqjJ+RgxJlQolN4Fv7PNqR4hHKdoYfg3BgSkIoXxU+Bt34DCOQ8s2QCqUmcGMQ9M7CC2UogWNQDE3kCmU6w49BODA/KpSvCh+L7lE8oxAMS+IKpamgbyE73omcXZ0P39hfs4gLc1HI7nFMzKrIrcEPE7YLX6OC7nh4+c/R3p8fy+SaJZNNElrkyYRvjeYgj/CTgbbTI43FX+ug+pIfiD/TdhV3mGPce6sKDb7rdl1psKllFXco2oTiU43CQ92hsMWixoSfXis+rvodIYQsVTlCGdXAMtiyL1oTqjPfbdHsHHGWQ+hTeMCOSfcFrLO6jdYPC4rzFD8Cw9dlqMQObANXBfxxOWaGf462o02S1W3S87smRhzXqM107Hd4AnWLM0sv2mOrtsFXSCtc0Y0nLZ3RBIGkp5kIyMR4cuMzENfdLY622aVJv7aU47QkGYuicpJdmkRgJu0WldFsj4TtJ1Ixlu8p3RsKXPeTHaH5jlVwbpDuZ8kCO+o6PmkgCsukQ1Te5NBu02CBHVUD4X6M0Nbht528wt04GF3E52GZY8YkrTijRWrr294G6Ld+Ppi+g7IgJeE2F2ULQQFq9FVC6MkJPDI9OYWH0hujUYxm0NyPw7HZmesVEcMq3I3jN5F2UVO6r6bGQ1t8iWaLry20Oq/LYQlFl6TjkwicXUF/Bp8h49pJO5TMALX5VOUK+iyBF7LwFfjdGLjjstu+y+s9L25heVPxXmw8FKDeaeX4Z7J7zTwdfN6Fnfvf7ja0OO66JzvL5LZ7MbNkE2zlRYLdxkzqFBf7lltdQUAzkgkNuTPC7KA24BW4MUEBeW1MzHe5h1ReU9kzy64oMH3tcKa6PJBj8ue187Ad0gaLvNNRUCQbrgvfeLSzZwrPgRxdRaN/gHGoK1ClucA+gwieQofyupWuiqr6jon8p3mSZfg3f5qlxxVSEqNUTUg7gkySmyXKKspx5QpMrVRtpemQBYosl2cTWQfh7t8nmgP6+G9aOYvmjyFp3vKqMEYLw4q7XZrA1IuTFbq18h4oLQBXYfI+NojfQLSt+D8SX/z3n/UECpp9mbsq+ykMoKh/nAXRaqT153X3z9aVI5b90CLzL4w2zfthfDGoC5KJqxVoeW1n/hLT8M/WxY31LZC1W1jvVE6fFnSxBoyhPkdRAQ8ITQGQ6StScbgq+ph9EbGORnNIlnedndqsPjZcVQ4CWl0YtX5t6Bc4T+Epg1p3N0iiVKLxDJ+vroKifZ7eeB/1EJ9Cc1tZJ0JrdbSy/EagMfNV3H798fVU6mD8eMpqh4Va/CLe0HBfA7M3g6EISLh6Sg2qaNodOBis5bGcFtAhhhQF8xawmALv0BT6JV7UUYrWD3SBbeRwXZHbl+v+fcIlIHrpGRskTf6spuqSxTZBTH7BT6RsS9R0XabKoD+A78iNo3RwNXu2y9fP5mxtFaospLUNOtTa+J6adODsaFRtCTirhcLYjK3YNZwQhr4RfiOyJe3crNVS4jOcxElekKOfhi0H5H/oPGsqWI+mlfpcGC39+6UIXjWkyiDttwIb/GYvzX4nJbxDpEJl8w4JiXP2nXh0AZ7dYKVE7GEVRPzfhQw0yUDiblU3/Oemv/+pypuvqdKXB+kpJGws2hWAYi6/FQBir4ZkhdIFztxpHrYYQh892B8x0JvEHSpYPlFqVsPEAC1ie6sn8txk0CoytXyPLgzD1T9c32vfqAuDWL6dOf/6UuBuyTBs01eP1rnEznqY2qfaBv10ZQDVO/QM3Wh/hxjkf9DHBHWUzV1hJvNT96mQSEQMOPHqN6ML01hL5RjHtPK9q/CPE0M/C1imk5n+v3f0lblWmLJb3oaZwiBVG4b1Fzv5BxLoPRTkSh4K8CagKcSwRXwH4y1z/Tyrfa9k3impS6r5zVSHSsskT/fljg1w1qsZzqDquuQO8h+bmki5hNs8Xp2lcJyjTSLISG/Xd8Rl8j2A+nNH346s86WWQ7pvA4/+Ld9UDuriTja/CW/e3GDupVlU7bGXALO6TQFuUoBeaztL61nM7M5UOYJW/8S7m964KPnVkdnTRX8RwKibW9CFMkWkdhbpn83f6eyNR+xPNWHP3ei15Tt4zAQaX8fu7Yxjfnj+3+9DRWIWL58Y12VyoW1l9hT1h5kTwlONsy/ysYDBiYmA1Z+AeiONzVoxkq0fuA7Y0/xVIETY0sATZG23a0xrWpAsXsVJzcE+8nbsOW6WvcfOxsbIktSgwjocDQMld8hu7MTipu4uiAxXGRSm3uvqv8qcjhlUPO1UWp8ih1HSHCGrAOa7MfabZMo8wrjtdrPjdEI5n2irilTXpMLamJAVbZZEK3wy7TDRsHBlkEwrsszoGQ5MUB8flRItI1dkEHSfgt7xiM+M3zDvZ7DhEg5lDbrhRxg59dFGkakimrCtDRk58tQo+pl51jKmv42z12f5i0RWY37TWW6vz2gss1u1wUkMjWabafm1k+msInixWgKEr1I3kuer4L95trNuErZ3qh2O/QYvevhUEyHzizgGRLem2RxYWjO4AIMq80/MG4cdA/lQR24KeK5+cUwi4Meuy1kyAePn399BPrbzjD44e9qRs2UuRxlHcc/SlAEWgEeS2So+e8LcCLeWpfGEKUnTbMWZt+n31AZ3ZDz3O9pBQJpY205cY4HpkO9zKEB/38rrUFFbCxRF6VwxuoGzylsAmG99uwDI0JKEgWwDigcGIPvWogyA9K4eGYAZVNI1ADewnGkAclh9wgDg0JKxDuhpjb8waG7Kkzwej6B7IcITB1Z1oNGgS2DXYRscY/K9i6tcnp11r1OnIvqXtreA2jZN273srevhpuOInGatWPuQmq1m3tNvX2y30ZOSgmn6OT0XykrM+2suB2gz6NPxGNIDBfFOCyHUpUGgt3W0Qi51EKmyohXhmAecfen+V5zoXyhDNWac4KJ4RmXXtyEfTkLNNjjG7w/2+r1i5/ZdlxeEfdPglypg3zB4LAjL48YYi8yEabPS63bcBnNHO8rYPsh35YAduleM4i67lwHku3ytF+RSSRV4eJYJ771FLIsaOnb/ecS+a8dY22xcuZiOpdhfZF94wLb4wPBA5C0hOiXycNgGOe6e/l836v+6KY5kSE3FRRTFiH6+R4Dq+PEaGy5qywh9nY95BqFt5TfrfwmDjD7CrKTtCxyex62LeCHrGip2RBuqimyuoIo8EpJUB1dY3Vip2bMjMOgcuQidacfqss40QlFMPZoo0pCWtorlXLge4bZgbcomWA/9dcaZQ17Dab31S8z4tk8NeSHhyV4y8X2YrWYFpWb2WecC70jRWAjtkGy5GvrM1E1FjH5nLSCySDfXy274+v2IHTr/fdBW4BqszHDVFHTZhNtnbPO86pmMuz3tqV+aq1T0cn+MKemWXYEnwiQ1ZTzFec7WhSA8hrYEilnV7HcBKXPnjCNGir3RRl/6ebfgDhn1RkE/ZbQtcvQA6OOv4w8pE3/U8TFcSexJxk3AUknO07D044y2LIMIDd0DxWzo3wSkvaF7sXEbWhUco6F7knEbWiVpzrd7BEYu3YY7pisq6gOkfk+Me0xTQRBzR7+vnPmyfn78nMbX09aYp61bClW549URTiY3HLTerS6RT1k9X56O181cYe4VMa93SI9uj71nIOcFF2OUOjOX+vaCOit2bGUW541On8eqLEdBygpsNaYD7PCIqlLNJkL3sc6fATjGq3fR23hzyyzR8OM1KpbNiha419y0w+m9ekuv3lKgvq/e0o/pLbnfzrTFcudgs1PYTd7XWsdXR+7VkQvS97t05O7A9ZqG3RUantakhoJU7ndKxlDYA8wKqc0m3L+3VWdHVAJnJGsQrXo5aJAz1DqZFeGwBdSMgdJvpMizdTnPed02XAL6f4jk08o7YxPBIZ3J+2XSu5MmkMwWeKes/LgolEIOflXK3HOH/1L/SePU9nyGZZE0erupRBzV0rcxgbzHRZwOpPwpiclQVl3cWeBXOg6igzc1VHzgIjy20Wrd6theb3N2fVM35FE4gZTfgI2/bFCGC53ZZio0UFx2a7CNOzyFcZ+C4X2Hjra905x+I4+8lLkRupH5XIP0XeXYNBCdC045Nrt5ocJxlcs5Al+Fpy9NPvAYdmrS3UVv/9R98/OHP3F8/fmtkSRlOTDt+Q4K6yY3kFhLVriuAbNxHzlOaTlcSEW0pZVOn6KcTfkCc5S7e73oJLWiN6b7F8ln+uD0DSybrrLQ+RD+aXMRj5D/UZGvLaCyQJ/pg/kQ2VgmepPQ/6QPElIv7UIZZLjh/RuoIdWjxjaiOchM02iz7HDHSF+x0CcDQERexJCPdxBKqkdckFwWYtxRjmGcM1IGGWX5FqxFu/86TkMicgOP60iIaplEVUW7CdqaRNbN3FNkQ+rXiLceEBCxcGNxw6oPMoE0OuqmdvnforCYzGyoKEcPgGrMGsg1qSOr2cLrvUOLAovvjy8N7/e4YI+qb851iWfPrYJmqvivX9DH6kJDnf+tNeC9gmUdKa0B0Gq+kPWfOz/gxUJ7/wG4Rh2DWTSv08EdyPOtiX2KDiV+2q5CRauXb4pPtHofoTkGXV6yRUZV/FtlsdYkBzwCehndzA5bnAwMRtPOzPvf8PhtLMqN8ANtOQKc3fqUvAph/Xu8+9w/sT2Lt3u0JtG+cA2lmLdxzJU3A0CM+yQLhhLUD6rj72q5w4LRwz0oq5oxQr97wr7rFMnviK7h/kxcuhtzEJYwwyW774WvcpQigwc7Cyecmj0BT91iooYl7jdz4o6O27+3g/Yh8SHgTH1vAkowN1wUw3sXuGnItZJxPBEdG/ybc/NOws07Ptchk0pMesRMNRnL5hyyNDt6r9OEyNmLz8vhiJgqke6a6VHUPI5ZhsThRY/O6ApnFXA6Q5fj62LE6NRKiY3dL1R3c2C63OfG01rO7ntJHlmfaSq641r8HWx7kc46TdNSoNS5OVm2WlXmZNGiGNHJMmUxqZOFzkoknSxbLXT0AqLPlqlUXzpYsnKI0UkSse9oIZuDIrYDvKEuyP4p9KxRnNnXkJy2phemJjG6qgLLRq0pLeIF4mhxjNXtr5BGsbx31btf8JNPncAa8Je74fwr4C++pNN7MrYgXvpZ3PG27LnE/yEPgqxBgWfaHrSrvKcmfB0vr+MlznhpWvZIHjV5cq9D5uU5vw6Zs4mbhozq4l2zJKNFIXdHMd28AdaWWvTSzw7tCFmKpfkH01HbSS7xesVx+RN7yg0uscbHpWOUFl7VFPLT2WOA/40UgJrnhkNpEeNtvNMCfQx23bcLkTW25CmHQPToyLGVf4ysAs+blSjsoiEDnCfHQIuYbQTs0QDx5rqsbg/pCQXFeYofr8kfl4nDcxk3XFzSS0Hx2hwjRROPCAtX3eIs40nb4Cske8s5623pZupiO5NxaZKvLeU40V5Z8GSMFjdBnEg26j70VYFQ4LqBPK2BEZq7B4OnPmiRZKiUAjhKhCLB2Hc84efHpsd3I1pxRovU1a6u1Po5akFKSyr2Nkw5NLdjLlOUs7pN1kFpSzDaJwjNb93qkdaUxivxc1guqHznP4LPzuAzZNwxvDy8zwtlGaTiNaN/DYWvYM6g+6EUtd/R/aFUPTYL5b50Ff7sj6vqMl8urRk0Tcti3zw9xqWVdyKSjJYPpII8zShlOakw7zTBVZ72pehPyy0Q2/ZBqGyX7U+nbJd8qrqq0V9A55n4UxVnUBckwy+g8yD55Ha+k1E2DfvzW36Q/UIqn9zwg1hcnBaMlH3tRIFiI3fQ85ejHeXFyaSGKu8GDceNut/Rr4SWgPM/dYD/FDtdTKoGYdT/AnW/UJHUU6ctlzcbYDwVhWS0DkJ4eYOPAhKtIafNLKGMcP07NeHyftXBjY6teHRBK2nHIw4J+htlCJ5wWRedQi1/X+K6Xt5OmEVbSJVK/9D38SJ3HQlSiksvAnbVQ0We9Z4uKQD6zrOrjx32LJG4MkMaWebI/USRvJ4Vy/izIh6Cif11pJgFazrhAtJHNoOCZnJpF1fMq7jP19z6J01ESVbRZb7hZhAKObowWvoRi/rmkBct9JGjG5YdCJ5wxlGDS0AiOR/xG660xhM3sTJa1piTB1IQ/ozqltW0MaUAyEkoXZROQbs2YZpWdJlMCUu2K1uvP/6/AAAA//9dbsnr" } From a4636c5a5f1cd22e00d014e02f0e3ccb55faa151 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Sun, 17 Nov 2024 22:09:38 +0100 Subject: [PATCH 04/17] Fixed tier_preference fetch --- metricbeat/module/elasticsearch/index/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index 4e65b6e4ab7..c4403fb0d55 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -300,7 +300,7 @@ func getClusterStateMetricForIndex(clusterState mapstr.M, index, metricKey strin } func getIndexTierPreference(clusterState mapstr.M, index string) ([]string, error) { - fieldKey := "metadata.settings.index.routing.allocation.include._tier_preference" + fieldKey := "metadata.indices." + index + ".settings.index.routing.allocation.include._tier_preference" value, err := clusterState.GetValue(fieldKey) if err != nil { return nil, fmt.Errorf("'"+fieldKey+"': %w", err) From 703b011235ef4974e5b5d381b8262df1972aede5 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Sun, 17 Nov 2024 22:45:23 +0100 Subject: [PATCH 05/17] Need to also fetch metadata --- metricbeat/module/elasticsearch/index/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index c4403fb0d55..3685a1eb490 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -181,7 +181,7 @@ type bulkStats struct { } func eventsMapping(r mb.ReporterV2, httpClient *helper.HTTP, info elasticsearch.Info, content []byte, isXpack bool) error { - clusterStateMetrics := []string{"routing_table"} + clusterStateMetrics := []string{"routing_table", "metadata"} clusterState, err := elasticsearch.GetClusterState(httpClient, httpClient.GetURI(), clusterStateMetrics) if err != nil { return fmt.Errorf("failure retrieving cluster state from Elasticsearch: %w", err) From c75169618889fc8dcd4741bf6d1354900c134c71 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Sun, 17 Nov 2024 23:30:48 +0100 Subject: [PATCH 06/17] Fixed tier_preference / creation_date fetch --- metricbeat/module/elasticsearch/index/data.go | 48 +++++++++++-------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index 3685a1eb490..78b2b22b0a7 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -20,6 +20,7 @@ package index import ( "encoding/json" "fmt" + "strconv" "github.com/joeshaw/multierror" @@ -40,11 +41,11 @@ type Index struct { Primaries primaries `json:"primaries"` Total total `json:"total"` - Index string `json:"index"` - Status string `json:"status"` - TierPreference []string `json:"tier_preference"` - CreationDate int `json:"creation_date"` - Shards shardStats `json:"shards"` + Index string `json:"index"` + Status string `json:"status"` + TierPreference string `json:"tier_preference"` + CreationDate int `json:"creation_date"` + Shards shardStats `json:"shards"` } type primaries struct { @@ -251,15 +252,20 @@ func addClusterStateFields(idx *Index, clusterState mapstr.M) error { return fmt.Errorf("failed to get index routing table from cluster state: %w", err) } - indexTierPreference, err := getIndexTierPreference(clusterState, idx.Index) + indexMetadata, err := getClusterStateMetricForIndex(clusterState, idx.Index, "metadata") if err != nil { - return fmt.Errorf("failed to get index tier preference from cluster state: %w", err) + return fmt.Errorf("failed to get index metadata from cluster state: %w", err) + } + + indexTierPreference, err := getIndexTierPreferenceFromMetadata(indexMetadata) + if err != nil { + return fmt.Errorf("failed to get index tier preference from metadata: %w", err) } idx.TierPreference = indexTierPreference - indexCreationDate, err := getIndexCreationDate(clusterState, idx.Index) + indexCreationDate, err := getIndexCreationDateFromMetadata(indexMetadata) if err != nil { - return fmt.Errorf("failed to get index creation date from cluster state: %w", err) + return fmt.Errorf("failed to get index creation date from metadata: %w", err) } idx.CreationDate = indexCreationDate @@ -299,30 +305,30 @@ func getClusterStateMetricForIndex(clusterState mapstr.M, index, metricKey strin return mapstr.M(metric), nil } -func getIndexTierPreference(clusterState mapstr.M, index string) ([]string, error) { - fieldKey := "metadata.indices." + index + ".settings.index.routing.allocation.include._tier_preference" - value, err := clusterState.GetValue(fieldKey) +func getIndexTierPreferenceFromMetadata(indexMetadata mapstr.M) (string, error) { + fieldKey := "settings.index.routing.allocation.include._tier_preference" + value, err := indexMetadata.GetValue(fieldKey) if err != nil { - return nil, fmt.Errorf("'"+fieldKey+"': %w", err) + return "", fmt.Errorf("'"+fieldKey+"': %w", err) } - tierPreference, ok := value.([]string) + tierPreference, ok := value.(string) if !ok { - return nil, elastic.MakeErrorForMissingField(fieldKey, elastic.Elasticsearch) + return "", elastic.MakeErrorForMissingField(fieldKey, elastic.Elasticsearch) } return tierPreference, nil } -func getIndexCreationDate(clusterState mapstr.M, index string) (int, error) { - fieldKey := "metadata.indices." + index + ".creation_date" - value, err := clusterState.GetValue(fieldKey) +func getIndexCreationDateFromMetadata(indexMetadata mapstr.M) (int, error) { + fieldKey := "settings.index.creation_date" + value, err := indexMetadata.GetValue(fieldKey) if err != nil { return 0, fmt.Errorf("'"+fieldKey+"': %w", err) } - creationDate, ok := value.(int) - if !ok { - return 0, elastic.MakeErrorForMissingField(fieldKey, elastic.Elasticsearch) + creationDate, err := strconv.Atoi(value.(string)) + if err != nil { + return 0, fmt.Errorf("failed to parse index creation date: %w", err) } return creationDate, nil } From 81b4432913e2c465601749fdfb5a8e4070686593 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Mon, 18 Nov 2024 00:16:28 +0100 Subject: [PATCH 07/17] Tier preference is a single value --- metricbeat/module/elasticsearch/index/_meta/data.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/_meta/data.json b/metricbeat/module/elasticsearch/index/_meta/data.json index 2bae6566c07..e18af801c8c 100644 --- a/metricbeat/module/elasticsearch/index/_meta/data.json +++ b/metricbeat/module/elasticsearch/index/_meta/data.json @@ -144,9 +144,7 @@ } }, "status": "green", - "tier_preference": [ - "data_content" - ], + "tier_preference": "data_content", "creation_date": 1731657995821, "hidden": true, "shards": { From 39077a8f9b225974a508b9f6ad1a863bdba9dc71 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Thu, 21 Nov 2024 21:14:11 +0100 Subject: [PATCH 08/17] Added filter_path when cluster state is fetched --- metricbeat/module/elasticsearch/elasticsearch.go | 12 ++++++++++-- metricbeat/module/elasticsearch/index/data.go | 12 +++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 446a2d128bb..6a9ca80670a 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -288,13 +288,21 @@ func GetLicense(http *helper.HTTP, resetURI string) (*License, error) { } // GetClusterState returns cluster state information. -func GetClusterState(http *helper.HTTP, resetURI string, metrics []string) (mapstr.M, error) { +func GetClusterState(http *helper.HTTP, resetURI string, metrics []string, filterPaths []string) (mapstr.M, error) { + var queryParams []string = []string{"local=true"} clusterStateURI := "_cluster/state" if len(metrics) > 0 { clusterStateURI += "/" + strings.Join(metrics, ",") } - content, err := fetchPath(http, resetURI, clusterStateURI, "local=true") + if len(filterPaths) > 0 { + filterPathQueryParam := "filter_path=" + strings.Join(filterPaths, ",") + queryParams = append(queryParams, filterPathQueryParam) + } + + queryString := strings.Join(queryParams, "&") + + content, err := fetchPath(http, resetURI, clusterStateURI, queryString) if err != nil { return nil, err } diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index 78b2b22b0a7..6980887030d 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -183,7 +183,17 @@ type bulkStats struct { func eventsMapping(r mb.ReporterV2, httpClient *helper.HTTP, info elasticsearch.Info, content []byte, isXpack bool) error { clusterStateMetrics := []string{"routing_table", "metadata"} - clusterState, err := elasticsearch.GetClusterState(httpClient, httpClient.GetURI(), clusterStateMetrics) + filterPaths := []string{ + "routing_table", + "**.settings.index.**._tier_preference", + "**.settings.index.**.creation_date", + } + clusterState, err := elasticsearch.GetClusterState( + httpClient, + httpClient.GetURI(), + clusterStateMetrics, + filterPaths, + ) if err != nil { return fmt.Errorf("failure retrieving cluster state from Elasticsearch: %w", err) } From 7e3e633c69bcf9f415c9d30283a0e4ecc01c1d8f Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Thu, 21 Nov 2024 21:18:59 +0100 Subject: [PATCH 09/17] Also consider require._tier_preference key --- metricbeat/module/elasticsearch/index/data.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index 6980887030d..8fb57c46abb 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -319,7 +319,11 @@ func getIndexTierPreferenceFromMetadata(indexMetadata mapstr.M) (string, error) fieldKey := "settings.index.routing.allocation.include._tier_preference" value, err := indexMetadata.GetValue(fieldKey) if err != nil { - return "", fmt.Errorf("'"+fieldKey+"': %w", err) + fieldKey = "settings.index.routing.allocation.require._tier_preference" + value, err = indexMetadata.GetValue(fieldKey) + if err != nil { + return "", fmt.Errorf("'"+fieldKey+"': %w", err) + } } tierPreference, ok := value.(string) From b3aaf8495c44170a5f4bf550b7b776da19296777 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Thu, 21 Nov 2024 21:20:31 +0100 Subject: [PATCH 10/17] Tier preference is optional, don't fail if missing --- metricbeat/module/elasticsearch/index/data.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index 8fb57c46abb..fe2ac8c149b 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -268,10 +268,10 @@ func addClusterStateFields(idx *Index, clusterState mapstr.M) error { } indexTierPreference, err := getIndexTierPreferenceFromMetadata(indexMetadata) - if err != nil { - return fmt.Errorf("failed to get index tier preference from metadata: %w", err) + if err == nil { + // Tier preference is optional, so only set it if it exists + idx.TierPreference = indexTierPreference } - idx.TierPreference = indexTierPreference indexCreationDate, err := getIndexCreationDateFromMetadata(indexMetadata) if err != nil { From 594c13b36ddb22f704a4861b374e27b01c79fd1b Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Thu, 21 Nov 2024 21:39:16 +0100 Subject: [PATCH 11/17] Fixed other cluster state call --- metricbeat/module/elasticsearch/cluster_stats/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/elasticsearch/cluster_stats/data.go b/metricbeat/module/elasticsearch/cluster_stats/data.go index 4fe1d03f6d7..cfd89ce46c0 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/data.go +++ b/metricbeat/module/elasticsearch/cluster_stats/data.go @@ -231,7 +231,7 @@ func eventMapping(r mb.ReporterV2, httpClient *helper.HTTP, info elasticsearch.I } clusterStateMetrics := []string{"version", "master_node", "nodes", "routing_table"} - clusterState, err := elasticsearch.GetClusterState(httpClient, httpClient.GetURI(), clusterStateMetrics) + clusterState, err := elasticsearch.GetClusterState(httpClient, httpClient.GetURI(), clusterStateMetrics, []string{}) if err != nil { return fmt.Errorf("failed to get cluster state from Elasticsearch: %w", err) } From 2d6c9d992bfd96f59f5bd1b46eb2c5060e4bee79 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Thu, 21 Nov 2024 22:03:15 +0100 Subject: [PATCH 12/17] format --- metricbeat/module/elasticsearch/index/data.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index fe2ac8c149b..f3f5994cea4 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -41,11 +41,11 @@ type Index struct { Primaries primaries `json:"primaries"` Total total `json:"total"` - Index string `json:"index"` - Status string `json:"status"` - TierPreference string `json:"tier_preference"` - CreationDate int `json:"creation_date"` - Shards shardStats `json:"shards"` + Index string `json:"index"` + Status string `json:"status"` + TierPreference string `json:"tier_preference"` + CreationDate int `json:"creation_date"` + Shards shardStats `json:"shards"` } type primaries struct { From 5417abb9fe2e913a1cd984bf7cf3f0fb375118ab Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Thu, 21 Nov 2024 23:52:16 +0100 Subject: [PATCH 13/17] Matched test to endpoint --- metricbeat/module/elasticsearch/index/data_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/elasticsearch/index/data_test.go b/metricbeat/module/elasticsearch/index/data_test.go index f755f13f5a4..cd2e71c25a4 100644 --- a/metricbeat/module/elasticsearch/index/data_test.go +++ b/metricbeat/module/elasticsearch/index/data_test.go @@ -118,7 +118,7 @@ func createEsMuxer(esVersion, license string, ccrEnabled bool) *http.ServeMux { w.Write(input) })) - mux.Handle("/_cluster/state/metadata,routing_table", http.HandlerFunc( + mux.Handle("/_cluster/state/routing_table,metadata", http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { input, _ := ioutil.ReadFile("./_meta/test/cluster_state.710.json") w.Write(input) From ba6cc577b85d9fbb7d88a905bdb7ef7646cdcb02 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Fri, 22 Nov 2024 01:49:42 +0100 Subject: [PATCH 14/17] Fix filter path --- metricbeat/module/elasticsearch/index/data.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index f3f5994cea4..1f3f5feb84c 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -185,8 +185,8 @@ func eventsMapping(r mb.ReporterV2, httpClient *helper.HTTP, info elasticsearch. clusterStateMetrics := []string{"routing_table", "metadata"} filterPaths := []string{ "routing_table", - "**.settings.index.**._tier_preference", - "**.settings.index.**.creation_date", + "metadata.indices.**.settings.index.**._tier_preference", + "metadata.indices.**.settings.index.creation_date", } clusterState, err := elasticsearch.GetClusterState( httpClient, From e194958eb95c1e4f0a89fb173d94d6a276243905 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Fri, 22 Nov 2024 02:05:07 +0100 Subject: [PATCH 15/17] lint --- metricbeat/module/elasticsearch/elasticsearch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 6a9ca80670a..a0d4b4a492f 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -289,7 +289,7 @@ func GetLicense(http *helper.HTTP, resetURI string) (*License, error) { // GetClusterState returns cluster state information. func GetClusterState(http *helper.HTTP, resetURI string, metrics []string, filterPaths []string) (mapstr.M, error) { - var queryParams []string = []string{"local=true"} + queryParams := []string{"local=true"} clusterStateURI := "_cluster/state" if len(metrics) > 0 { clusterStateURI += "/" + strings.Join(metrics, ",") From d078d00e1228bca3ba1cacddcdd177b57f2b0327 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Fri, 22 Nov 2024 14:58:12 +0100 Subject: [PATCH 16/17] Fix linting --- metricbeat/module/elasticsearch/index/data.go | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index 1f3f5feb84c..d034d29a77e 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -370,8 +370,15 @@ func getIndexStatus(shards map[string]interface{}) (string, error) { shard := mapstr.M(s) - isPrimary := shard["primary"].(bool) - state := shard["state"].(string) + isPrimary, ok := shard["primary"].(bool) + if !ok { + return "", fmt.Errorf("%v.shards[%v].primary is not a boolean", indexName, shardIdx) + } + + state, ok := shard["state"].(string) + if !ok { + return "", fmt.Errorf("%v.shards[%v].state is not a string", indexName, shardIdx) + } if isPrimary { areAllPrimariesStarted = areAllPrimariesStarted && (state == "STARTED") @@ -419,8 +426,15 @@ func getIndexShardStats(shards mapstr.M) (*shardStats, error) { shard := mapstr.M(s) - isPrimary := shard["primary"].(bool) - state := shard["state"].(string) + isPrimary, ok := shard["primary"].(bool) + if !ok { + return nil, fmt.Errorf("%v.shards[%v].primary is not a boolean", indexName, shardIdx) + } + + state, ok := shard["state"].(string) + if !ok { + return nil, fmt.Errorf("%v.shards[%v].state is not a string", indexName, shardIdx) + } if isPrimary { primaries++ From 3853498c52ecac538608e37f1eca6ce398382d38 Mon Sep 17 00:00:00 2001 From: Alexis Charveriat Date: Mon, 2 Dec 2024 12:38:43 +0100 Subject: [PATCH 17/17] Added entry to CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3b96e629bd7..6657d843dc0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -66,6 +66,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add GCP organization and project details to ECS cloud fields. {pull}40461[40461] - Add support for specifying a custom endpoint for GCP service clients. {issue}40848[40848] {pull}40918[40918] - Fix incorrect handling of types in SQL module. {issue}40090[40090] {pull}41607[41607] +- Added `creation_date` and `tier_preference` fields to the Elasticsearch module. {pull}41652[41652] *Osquerybeat*