Skip to content

Commit 5ad485c

Browse files
3kt3kt
committed
Working tag pull
1 parent bd306e3 commit 5ad485c

File tree

6 files changed

+211
-85
lines changed

6 files changed

+211
-85
lines changed

‎packages/ess_billing/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "1.7.0"
3+
changes:
4+
- description: Added tags for ECH
5+
type: enhancement
6+
link: TODO
27
- version: "1.6.0"
38
changes:
49
- description: Update codeowners

‎packages/ess_billing/data_stream/billing/agent/stream/cel.yml.hbs‎

Lines changed: 193 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -18,113 +18,222 @@ state:
1818
api_key: !!str {{api_key}}
1919
organization_id: !!str {{organization_id}}
2020
lookbehind: !!int {{lookbehind}}
21+
add_tags: !!bool {{add_tags}}
22+
tag_endpoint: !!str "https://cloud.elastic.co/api/v1/deployments/_search"
2123
redact:
2224
fields:
2325
- api_key
2426
program: |-
25-
state.with(
26-
// Determine the 'from' value using the cursor or fallback to
27-
// calculating it based on the lookback days.
28-
state.?cursor.last_to.optMap(ts, timestamp(ts)).orValue(
29-
now - duration(string(int(state.lookbehind) * 24) + "h")
30-
).format(
31-
// Truncate the 'from' time to the UTC day.
32-
time_layout.DateOnly
33-
).parse_time(time_layout.DateOnly).as(from,
34-
// Limit the 'from' value to 2021-01-01 (this is API enforced).
35-
max(from, timestamp("2021-01-01T00:00:00Z"))
36-
).as(from,
27+
(state.add_tags) ?
28+
request(
29+
"POST",
30+
state.tag_endpoint
31+
).with(
3732
{
38-
"from": from,
39-
"to": from + duration("24h"),
33+
"Header": {
34+
"Authorization": ["ApiKey " + state.api_key],
35+
"Content-Type": ["application/json"]
36+
},
37+
"Body": {
38+
"query": {
39+
"bool": {
40+
"filter": [
41+
{
42+
"nested": {
43+
"path": "resources.elasticsearch",
44+
"query": { "exists": { "field": "resources.elasticsearch.id" } }
45+
}
46+
},
47+
{
48+
"nested": {
49+
"path": "resources.elasticsearch",
50+
"query": { "match": { "resources.elasticsearch.info.settings.metadata.organization_id": { "query": state.organization_id } } }
51+
}
52+
}
53+
]
54+
}
55+
},
56+
"size": 100
57+
}.encode_json()
4058
}
41-
).as(req, (req.to > now) ?
42-
// We would fetch data in the future, back off
59+
).do_request().as(deployments_resp, (deployments_resp.StatusCode == 200) ?
60+
bytes(deployments_resp.Body).decode_json().as(deployments_body,
61+
zip(
62+
deployments_body.deployments.map(d,
63+
d.id
64+
),
65+
deployments_body.deployments.map(d, d.metadata.collate("tags"))
66+
)
67+
).as(tags,
68+
state.with(
69+
state.?cursor.last_to.optMap(ts, timestamp(ts)).orValue(
70+
now - duration(string(int(state.lookbehind) * 24) + "h")
71+
).format(
72+
time_layout.DateOnly
73+
).parse_time(time_layout.DateOnly).as(from,
74+
max(from, timestamp("2021-01-01T00:00:00Z"))
75+
).as(from,
76+
{
77+
"from": from,
78+
"to": from + duration("24h"),
79+
}
80+
).as(req, (req.to > now) ?
81+
{
82+
"events": [],
83+
"cursor": { "last_to": req.to },
84+
"want_more": false,
85+
}
86+
:
87+
get_request(
88+
state.url + "?" + {
89+
"from": [req.from.format(time_layout.RFC3339)],
90+
"to": [req.to.format(time_layout.RFC3339)],
91+
}.format_query()
92+
).with(
93+
{ "Header": { "Authorization": ["ApiKey " + state.api_key] } }
94+
).do_request().as(resp, (resp.StatusCode == 200) ?
95+
bytes(resp.Body).decode_json().as(body, (has(body.instances) && size(body.instances) > 0) ?
96+
{
97+
"events": body.instances.map(instance,
98+
instance.product_line_items.map(line_item,
99+
{
100+
"ess": {
101+
"billing": line_item.with(
102+
{
103+
"deployment_name": instance.name,
104+
"deployment_id": instance.id,
105+
"deployment_type": instance.type,
106+
"deployment_tags": tags[?instance.id].orValue([]).map(t,
107+
t.key + ":" + t.value
108+
),
109+
"organization_id": state.organization_id,
110+
"from": req.from.format(time_layout.RFC3339),
111+
"to": req.to.format(time_layout.RFC3339),
112+
"quantities": null
113+
}
114+
),
115+
},
116+
}
117+
)
118+
).flatten(),
119+
"cursor": { "last_to": req.to },
120+
"want_more": req.to < now - duration("24h"),
121+
}
122+
:
123+
{
124+
"events": [ { "fake": true } ],
125+
"cursor": { "last_to": req.to },
126+
"want_more": req.to < now - duration("24h"),
127+
}
128+
)
129+
:
130+
{
131+
"events": { "error": { "code": string(resp.StatusCode), "id": string(resp.Status), "message": "GET " + resp.Request.URL + ": " + ( (size(resp.Body) != 0) ? string(resp.Body) : string(resp.Status) + " (" + string(resp.StatusCode) + ")" ) } },
132+
"want_more": false,
133+
}
134+
)
135+
)
136+
)
137+
)
138+
:
43139
{
44-
"events": [],
45-
"cursor": {
46-
// We don't change the last_to, we're just waiting
47-
"last_to": req.to,
48-
},
140+
"events": { "error": { "code": string(deployments_resp.StatusCode), "id": string(deployments_resp.Status), "message": "POST " + deployments_resp.Request.URL + ": " + ( (size(deployments_resp.Body) != 0) ? string(deployments_resp.Body) : string(deployments_resp.Status) + " (" + string(deployments_resp.StatusCode) + ")" ) } },
49141
"want_more": false,
50142
}
51-
:
52-
get_request(
53-
state.url + "?" + {
54-
"from": [req.from.format(time_layout.RFC3339)],
55-
"to": [req.to.format(time_layout.RFC3339)],
56-
}.format_query()
57-
).with(
143+
)
144+
:
145+
state.with(
146+
state.?cursor.last_to.optMap(ts, timestamp(ts)).orValue(
147+
now - duration(string(int(state.lookbehind) * 24) + "h")
148+
).format(
149+
time_layout.DateOnly
150+
).parse_time(time_layout.DateOnly).as(from,
151+
max(from, timestamp("2021-01-01T00:00:00Z"))
152+
).as(from,
58153
{
59-
"Header": {
60-
"Authorization": ["ApiKey " + state.api_key],
154+
"from": from,
155+
"to": from + duration("24h"),
156+
}
157+
).as(req, (req.to > now) ?
158+
{
159+
"events": [],
160+
"cursor": {
161+
"last_to": req.to,
61162
},
163+
"want_more": false,
62164
}
63-
).do_request().as(resp, (resp.StatusCode == 200) ?
64-
// Response is successful, but did we get any data?
65-
bytes(resp.Body).decode_json().as(body, (has(body.instances) && size(body.instances) > 0) ?
165+
:
166+
get_request(
167+
state.url + "?" + {
168+
"from": [req.from.format(time_layout.RFC3339)],
169+
"to": [req.to.format(time_layout.RFC3339)],
170+
}.format_query()
171+
).with(
66172
{
67-
"events": body.instances.map(instance,
68-
instance.product_line_items.map(line_item,
69-
{
70-
"ess": {
71-
"billing": line_item.with(
72-
{
73-
"deployment_name": instance.name, // Include deployment name
74-
"deployment_id": instance.id, // Include deployment ID
75-
"deployment_type": instance.type, // Include deployment type
76-
"organization_id": state.organization_id, // Include organization ID
77-
"from": req.from.format(time_layout.RFC3339),
78-
"to": req.to.format(time_layout.RFC3339),
79-
"quantities": null // overwrite useless quantities to avoid flatten failure
80-
}
81-
),
82-
},
83-
}
84-
)
85-
).flatten(), // Pass line_items as events, with added info
86-
"cursor": {
87-
"last_to": req.to,
173+
"Header": {
174+
"Authorization": ["ApiKey " + state.api_key],
88175
},
89-
// Are we more than 1 day behind?
90-
"want_more": req.to < now - duration("24h"),
91176
}
177+
).do_request().as(resp, (resp.StatusCode == 200) ?
178+
bytes(resp.Body).decode_json().as(body, (has(body.instances) && size(body.instances) > 0) ?
179+
{
180+
"events": body.instances.map(instance,
181+
instance.product_line_items.map(line_item,
182+
{
183+
"ess": {
184+
"billing": line_item.with(
185+
{
186+
"deployment_name": instance.name,
187+
"deployment_id": instance.id,
188+
"deployment_type": instance.type,
189+
"organization_id": state.organization_id,
190+
"from": req.from.format(time_layout.RFC3339),
191+
"to": req.to.format(time_layout.RFC3339),
192+
"quantities": null,
193+
"deployment_tags": [],
194+
}
195+
),
196+
},
197+
}
198+
)
199+
).flatten(),
200+
"cursor": {
201+
"last_to": req.to,
202+
},
203+
"want_more": req.to < now - duration("24h"),
204+
}
205+
:
206+
{
207+
"events": [
208+
{
209+
"fake": true,
210+
},
211+
],
212+
"cursor": {
213+
"last_to": req.to,
214+
},
215+
"want_more": req.to < now - duration("24h"),
216+
}
217+
)
92218
:
93-
// We don't have any data, but we still need to return an event
94-
// Otherwise the "want_more" logic will not work
95219
{
96-
"events": [
97-
{
98-
"fake": true, // This will be discarded by the drop_event processor.
220+
"events": {
221+
"error": {
222+
"code": string(resp.StatusCode),
223+
"id": string(resp.Status),
224+
"message": "GET " + resp.Request.URL + ": " + (
225+
(size(resp.Body) != 0) ?
226+
string(resp.Body)
227+
:
228+
string(resp.Status) + " (" + string(resp.StatusCode) + ")"
229+
),
99230
},
100-
],
101-
"cursor": {
102-
"last_to": req.to,
103231
},
104-
// Are we more than 1 day behind?
105-
"want_more": req.to < now - duration("24h"),
232+
"want_more": false,
106233
}
107234
)
108-
:
109-
// Response was not successful, return an error event
110-
{
111-
"events": {
112-
"error": {
113-
"code": string(resp.StatusCode),
114-
"id": string(resp.Status),
115-
"message": "GET " + resp.Request.URL + ": " + (
116-
(size(resp.Body) != 0) ?
117-
string(resp.Body)
118-
:
119-
string(resp.Status) + " (" + string(resp.StatusCode) + ")"
120-
),
121-
},
122-
},
123-
"want_more": false,
124-
}
125235
)
126236
)
127-
)
128237
{{#if tags.length}}
129238
tags:
130239
{{else if preserve_original_event}}

‎packages/ess_billing/data_stream/billing/fields/fields.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
- name: deployment_type
2222
type: keyword
2323
description: Type of the Elasticsearch Service deployment.
24+
- name: deployment_tags
25+
type: keyword
26+
description: Tags associated with the Elasticsearch Service deployment.
2427
- name: display_quantity.formatted_value
2528
type: keyword
2629
description: Human-readable representation of the quantity used (e.g., "24 hours").

‎packages/ess_billing/data_stream/billing/manifest.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,14 @@ streams:
1010
template_path: cel.yml.hbs
1111
enabled: true
1212
vars:
13+
- name: add_tags
14+
type: bool
15+
title: Add deployment tags
16+
description: Whether to add deployment tags to each event collected (this will perform an additional API call against the Elasticsearch Service API, for each execution). This only works for commercial cloud (cloud.elastic.co) deployments.
17+
multi: false
18+
required: false
19+
show_user: true
20+
default: false
1321
- name: lookbehind
1422
type: integer
1523
title: Lookbehind

‎packages/ess_billing/docs/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ An example event for `billing` looks as following:
143143
| ess.billing.cloud.service.type | The service type of the serverless project. | keyword |
144144
| ess.billing.deployment_id | ID of the Elasticsearch Service deployment. | keyword |
145145
| ess.billing.deployment_name | Name of the Elasticsearch Service deployment. | keyword |
146+
| ess.billing.deployment_tags | Tags associated with the Elasticsearch Service deployment. | keyword |
146147
| ess.billing.deployment_type | Type of the Elasticsearch Service deployment. | keyword |
147148
| ess.billing.display_quantity.formatted_value | Human-readable representation of the quantity used (e.g., "24 hours"). | keyword |
148149
| ess.billing.display_quantity.type | Type of quantity displayed (default or custom). | keyword |

‎packages/ess_billing/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
format_version: 3.3.2
22
name: ess_billing
33
title: "Elasticsearch Service Billing"
4-
version: "1.6.0"
4+
version: "1.7.0"
55
source:
66
license: "Elastic-2.0"
77
description: "Collects billing metrics from Elasticsearch Service billing API"

0 commit comments

Comments
 (0)