diff --git a/.github/scripts/verify-dashboards-api-example.py b/.github/scripts/verify-dashboards-api-example.py new file mode 100755 index 0000000000..ce6bafed8f --- /dev/null +++ b/.github/scripts/verify-dashboards-api-example.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Verify that the embedded Dashboards API example in the Kibana data +exploration learning tutorial still creates a working dashboard. + +Background +---------- +The tutorial page at ``explore-analyze/kibana-data-exploration-learning-tutorial.md`` +embeds a single ``curl`` example that POSTs to ``/api/dashboards`` and recreates +the dashboard built throughout the tutorial. The Dashboards API is in technical +preview, so its schema can change between minor versions. This script extracts +that JSON payload from the markdown, strips docs-builder ```` callout +markers, posts it to a live Kibana, and asserts the dashboard creates +successfully with the expected number of panels. + +Usage +----- +Set ``KIBANA_URL`` and ``API_KEY`` (a Kibana API key with privileges to create +dashboards) in your environment, then run:: + + python3 .github/scripts/verify-dashboards-api-example.py + +Optional flags:: + + --keep Do not delete the test dashboard after verification. + --markdown F Point at a different markdown file (default: the tutorial). + +Failure modes +------------- +The script exits ``0`` on success and non-zero on any failure, with the +reason printed to stderr. Each failure path is annotated inline below; +this is the index: + +1. **Environment not set** — ``KIBANA_URL`` or ``API_KEY`` missing. Setup + problem, not an API problem. Source your ``.env`` and re-run. +2. **Curl block not found in markdown** — the regex couldn't locate the + ``curl`` POST example. The dropdown was removed, restructured, or the + code fence was reformatted. Open the markdown and confirm the example + is still present in the expected shape. +3. **Extracted JSON is invalid** — a docs edit broke the JSON inside the + curl block (stray comma, unbalanced brace, smart quote). The error + includes the line/column reported by the JSON parser. +4. **Panel count drift in the source** — the payload no longer declares + ``EXPECTED_PANEL_COUNT`` panels. Either the example was edited + intentionally (update the constant) or panels were lost. +5. **API rejected the request (HTTP 4xx/5xx)** — the most informative + failure mode for schema drift. Kibana's response body is printed + verbatim and includes the field path and reason + (for example, ``panels.5.config.layers.0.breakdown_by: field 'fields' is required``). +6. **Non-201 success status** — the request didn't error but didn't + return 201 either. Rare; usually a redirect or proxy quirk. +7. **Panel count mismatch on the server** — POST returned 201 but the + stored dashboard has fewer (or more) panels than we sent. Means the + API silently dropped a panel rather than rejecting the whole request. + The script identifies the missing panels by their ``grid`` position + and prints ``type``/``config.type`` for each so you know which + visualization to look at. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import re +import ssl +import sys +import urllib.error +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_MARKDOWN = ( + REPO_ROOT + / "explore-analyze" + / "kibana-data-exploration-learning-tutorial.md" +) +EXPECTED_PANEL_COUNT = 11 + + +def extract_payload(markdown_path: Path) -> dict: + """Return the parsed JSON payload from the curl example in the markdown.""" + text = markdown_path.read_text(encoding="utf-8") + match = re.search( + r"curl[^\n]*\n(?:[^\n]*\n)*?\s*-d '(\{.*?\})'\n```", + text, + re.DOTALL, + ) + if not match: + # Failure mode 2: the regex couldn't find a curl block matching our + # expected shape. The page has been restructured, the dropdown was + # removed, or the fence/indent changed. Inspect the markdown to + # confirm the example still exists. + raise SystemExit( + f"Could not find a curl POST example in {markdown_path}. " + "Has the page structure changed?" + ) + raw = match.group(1) + cleaned = re.sub(r"\s*<\d+>", "", raw) + try: + return json.loads(cleaned) + except json.JSONDecodeError as exc: + # Failure mode 3: a docs edit produced syntactically invalid JSON + # inside the curl block. The parser reports the line/column to look + # at; also worth checking for smart quotes or trailing commas. + raise SystemExit(f"Extracted JSON is not valid: {exc}") from exc + + +def post_dashboard(kibana_url: str, api_key: str, payload: dict) -> dict: + body = json.dumps(payload).encode("utf-8") + req = urllib.request.Request( + f"{kibana_url.rstrip('/')}/api/dashboards", + data=body, + method="POST", + headers={ + "Authorization": f"ApiKey {api_key}", + "kbn-xsrf": "true", + "Content-Type": "application/json", + }, + ) + ctx = ssl.create_default_context() + try: + with urllib.request.urlopen(req, context=ctx) as resp: + status = resp.status + response_body = resp.read().decode("utf-8") + except urllib.error.HTTPError as exc: + # Failure mode 5: the API rejected the payload. `detail` is Kibana's + # full response body, which for schema rejections includes the field + # path and reason. This is the failure that tells you exactly what + # the API now expects vs. what the example sends. + detail = exc.read().decode("utf-8", errors="replace") + raise SystemExit( + f"POST /api/dashboards failed with HTTP {exc.code}: {detail}" + ) from exc + + if status != 201: + # Failure mode 6: 2xx but not 201. Rare. Usually a proxy or redirect + # in front of Kibana; double-check `KIBANA_URL`. + raise SystemExit(f"Unexpected status code {status}: {response_body}") + return json.loads(response_body) + + +def delete_dashboard(kibana_url: str, api_key: str, dashboard_id: str) -> None: + req = urllib.request.Request( + f"{kibana_url.rstrip('/')}/api/dashboards/{dashboard_id}", + method="DELETE", + headers={ + "Authorization": f"ApiKey {api_key}", + "kbn-xsrf": "true", + }, + ) + ctx = ssl.create_default_context() + try: + urllib.request.urlopen(req, context=ctx) + except urllib.error.HTTPError as exc: + print( + f"Warning: cleanup DELETE failed with HTTP {exc.code}", + file=sys.stderr, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--markdown", + default=str(DEFAULT_MARKDOWN), + help=f"Markdown file to verify (default: {DEFAULT_MARKDOWN})", + ) + parser.add_argument( + "--keep", + action="store_true", + help="Do not delete the dashboard after verification.", + ) + args = parser.parse_args() + + kibana_url = os.environ.get("KIBANA_URL") + api_key = os.environ.get("API_KEY") + if not kibana_url or not api_key: + # Failure mode 1: missing credentials. Source the .env file or + # export both variables before re-running. + raise SystemExit( + "KIBANA_URL and API_KEY must be set in the environment." + ) + + markdown_path = Path(args.markdown) + payload = extract_payload(markdown_path) + + declared_panels = len(payload.get("panels", [])) + if declared_panels != EXPECTED_PANEL_COUNT: + # Failure mode 4: the example no longer matches what this script + # was designed to verify. If the change is intentional (a panel was + # added or removed in the docs), update EXPECTED_PANEL_COUNT above. + raise SystemExit( + f"Payload declares {declared_panels} panels but the verifier " + f"expects {EXPECTED_PANEL_COUNT}. Update EXPECTED_PANEL_COUNT " + "if this change is intentional." + ) + + today = dt.date.today().isoformat() + payload["title"] = f"{today} verify-dashboards-api-example (test run)" + + print( + f"Posting payload extracted from {markdown_path.name} " + f"({declared_panels} panels) to {kibana_url}..." + ) + response = post_dashboard(kibana_url, api_key, payload) + dashboard_id = response.get("id") + created_panels = len(response.get("data", {}).get("panels", [])) + print(f"Created dashboard {dashboard_id} with {created_panels} panels.") + + if created_panels != declared_panels: + # Failure mode 7: the API accepted the request (201) but stored a + # different number of panels than we sent. Identify the missing + # panels by their grid position, which is stable across the + # request/response roundtrip, so the editor knows which panel + # config the API silently dropped. + returned_grids = { + (p.get("grid", {}).get("x"), p.get("grid", {}).get("y")) + for p in response.get("data", {}).get("panels", []) + } + missing = [ + p + for p in payload.get("panels", []) + if (p.get("grid", {}).get("x"), p.get("grid", {}).get("y")) + not in returned_grids + ] + if missing: + missing_lines = "\n".join( + f" - grid x={p['grid']['x']}, y={p['grid']['y']} " + f"(type={p.get('type')}, " + f"config.type={p.get('config', {}).get('type')})" + for p in missing + ) + detail = f"\nMissing panels:\n{missing_lines}" + else: + detail = "" + if not args.keep: + delete_dashboard(kibana_url, api_key, dashboard_id) + raise SystemExit( + f"Server-created panel count ({created_panels}) does not match " + f"the request ({declared_panels}). The schema may have changed." + f"{detail}" + ) + + if args.keep: + print("Skipping cleanup (--keep).") + else: + delete_dashboard(kibana_url, api_key, dashboard_id) + print("Deleted test dashboard.") + + print("OK: example payload still creates a valid dashboard.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/explore-analyze/kibana-data-exploration-learning-tutorial.md b/explore-analyze/kibana-data-exploration-learning-tutorial.md index 4fe51d0923..393ce47c96 100644 --- a/explore-analyze/kibana-data-exploration-learning-tutorial.md +++ b/explore-analyze/kibana-data-exploration-learning-tutorial.md @@ -377,6 +377,7 @@ When you are happy with the layout, select **Save** in the toolbar. Your dashboard now combines multiple panel types built with Lens, and you've seen how inline editing and interactive filtering make the dashboard both customizable and interactive. To learn more, refer to [Dashboards](dashboards.md), [Lens](visualize/lens.md), and [Panels and visualizations](visualize.md). + ## Step 4: Share the dashboard [share-the-dashboard] Once your dashboard is ready, share it with your team: @@ -390,6 +391,329 @@ Users who receive the link need to authenticate and have the appropriate privile For more details on sharing options, access control, and managing dashboard ownership, refer to [Sharing dashboards](dashboards/sharing.md). +## Recreate the dashboard with the API [recreate-dashboard-api] +```{applies_to} +stack: preview 9.4+ +serverless: preview +``` + +Everything you built in this tutorial can also be reproduced in a single API call. The [Dashboards API](dashboards/create-dashboards-programmatically.md) accepts a JSON payload that encodes the complete dashboard, including panel types, data sources, layout, and display options, making it straightforward to version-control dashboards or provision consistent environments programmatically. + + +::::{dropdown} Recreate this dashboard with one API call + +The following `curl` request creates the same dashboard as the one you built in this tutorial, including the optional panels suggestions. + +```bash +curl -X POST "${KIBANA_URL}/api/dashboards" \ + -H "Authorization: ApiKey ${API_KEY}" \ + -H "kbn-xsrf: true" \ + -H "Content-Type: application/json" \ + -d '{ + "title": "Web logs overview", + "description": "Recreates the dashboard built in the Kibana data exploration tutorial.", + "time_range": { "from": "now-90d", "to": "now" }, + "panels": [ + { + "grid": { "x": 0, "y": 0, "w": 12, "h": 5 }, + "type": "vis", + "config": { + "type": "metric", + "data_source": { + "type": "data_view_spec", + "index_pattern": "kibana_sample_data_logs", + "time_field": "timestamp" + }, + "metrics": [ + { + "type": "primary", + "operation": "median", + "field": "bytes", + "label": "Median response size", <1> + "format": { "type": "bytes", "decimals": 2 }, + "background_chart": { "type": "trend" }, + "color": { + "type": "dynamic", + "range": "absolute", + "steps": [ + { "lt": 6000, "color": "#24c292" }, + { "gte": 6000, "lt": 10000, "color": "#fcd883" }, + { "gte": 10000, "color": "#f6726a" } + ] + }, + "apply_color_to": "background" + } + ] + } + }, + { + "grid": { "x": 12, "y": 0, "w": 12, "h": 5 }, + "type": "vis", + "config": { + "type": "metric", + "data_source": { + "type": "data_view_spec", + "index_pattern": "kibana_sample_data_logs", + "time_field": "timestamp" + }, + "metrics": [ + { + "type": "primary", + "operation": "unique_count", + "field": "clientip", + "label": "Unique visitors" <2> + } + ] + } + }, + { + "grid": { "x": 24, "y": 0, "w": 12, "h": 5 }, + "type": "vis", + "config": { + "type": "metric", + "data_source": { + "type": "data_view_spec", + "index_pattern": "kibana_sample_data_logs", + "time_field": "timestamp" + }, + "metrics": [ + { + "type": "primary", + "operation": "count", + "label": "Total requests" <3> + }, + { + "type": "secondary", + "operation": "count", + "label": "Week to week", + "time_shift": "1w", + "compare": { "to": "primary", "palette": "compare_to", "icon": true, "value": true } + } + ] + } + }, + { + "grid": { "x": 36, "y": 0, "w": 12, "h": 5 }, + "type": "vis", + "config": { + "type": "metric", + "data_source": { + "type": "data_view_spec", + "index_pattern": "kibana_sample_data_logs", + "time_field": "timestamp" + }, + "metrics": [ + { + "type": "primary", + "operation": "unique_count", + "field": "request.keyword", + "label": "Unique URLs" <4> + } + ] + } + }, + { + "grid": { "x": 0, "y": 5, "w": 24, "h": 10 }, + "type": "vis", + "config": { + "type": "xy", + "title": "Response size over time, per host", <5> + "layers": [ + { + "type": "line", + "data_source": { + "type": "data_view_spec", + "index_pattern": "kibana_sample_data_logs", + "time_field": "timestamp" + }, + "x": { "operation": "date_histogram", "field": "timestamp" }, + "y": [{ "operation": "median", "field": "bytes" }], + "breakdown_by": { + "operation": "terms", + "fields": ["host.keyword"], + "limit": 9 + } + } + ] + } + }, + { + "grid": { "x": 24, "y": 5, "w": 24, "h": 10 }, + "type": "vis", + "config": { + "type": "xy", + "title": "Log volume over time, per host", <6> + "layers": [ + { + "type": "line", + "data_source": { + "type": "data_view_spec", + "index_pattern": "kibana_sample_data_logs", + "time_field": "timestamp" + }, + "x": { "operation": "date_histogram", "field": "timestamp" }, + "y": [{ "operation": "count" }], + "breakdown_by": { + "operation": "terms", + "fields": ["host.keyword"], + "limit": 9 + } + }, + { + "type": "reference_lines", + "data_source": { + "type": "data_view_spec", + "index_pattern": "kibana_sample_data_logs", + "time_field": "timestamp" + }, + "thresholds": [ + { + "operation": "static_value", + "value": 80, + "label": "High traffic", + "color": { "type": "static", "color": "#f6726a" }, + "icon": "alert" + } + ] + } + ] + } + }, + { + "grid": { "x": 0, "y": 15, "w": 24, "h": 10 }, + "type": "vis", + "config": { + "type": "xy", + "title": "Events by response code", <7> + "layers": [ + { + "type": "bar_stacked", + "data_source": { + "type": "esql", + "query": "FROM kibana_sample_data_logs | WHERE response IS NOT NULL | STATS event_count = COUNT(*) BY response | SORT event_count DESC | LIMIT 50" + }, + "x": { "column": "response" }, + "y": [{ "column": "event_count" }] + } + ] + } + }, + { + "grid": { "x": 24, "y": 15, "w": 24, "h": 10 }, + "type": "vis", + "config": { + "type": "xy", + "title": "Requests by file extension", <8> + "layers": [ + { + "type": "bar_stacked", + "data_source": { + "type": "data_view_spec", + "index_pattern": "kibana_sample_data_logs", + "time_field": "timestamp" + }, + "x": { + "operation": "terms", + "fields": ["extension.keyword"], + "limit": 9, + "includes": { "as_regex": true, "values": [".+"] } + }, + "y": [{ "operation": "count" }] + } + ] + } + }, + { + "grid": { "x": 0, "y": 25, "w": 24, "h": 10 }, + "type": "vis", + "config": { + "type": "pie", + "title": "Traffic distribution by operating system", <9> + "data_source": { + "type": "data_view_spec", + "index_pattern": "kibana_sample_data_logs", + "time_field": "timestamp" + }, + "metrics": [{ "operation": "count" }], + "group_by": [ + { + "operation": "terms", + "fields": ["machine.os.keyword"], + "limit": 9 + } + ] + } + }, + { + "grid": { "x": 24, "y": 25, "w": 24, "h": 10 }, + "type": "vis", + "config": { + "type": "treemap", + "title": "Requests by geography", <10> + "data_source": { + "type": "data_view_spec", + "index_pattern": "kibana_sample_data_logs", + "time_field": "timestamp" + }, + "metrics": [{ "operation": "count" }], + "group_by": [ + { + "operation": "terms", + "fields": ["geo.dest"], + "limit": 9 + } + ] + } + }, + { + "grid": { "x": 0, "y": 35, "w": 48, "h": 17 }, + "type": "vis", + "config": { + "type": "data_table", + "title": "Last 100 events", <11> + "data_source": { + "type": "esql", + "query": "FROM kibana_sample_data_logs | KEEP @timestamp, request, response, bytes | SORT @timestamp DESC | LIMIT 100" + }, + "rows": [ + { "column": "@timestamp" }, + { "column": "request" }, + { "column": "response" } + ], + "metrics": [ + { "column": "bytes" } + ] + } + } + ] +}' +``` + +How each panel maps back to the tutorial: + +1. **Median response size**: Lens metric panel from the [Add a metric panel for median response size](#add-a-metric-panel-for-median-response-size) sub-step, including the bytes `format`, the trend `background_chart` for the sparkline, and the dynamic `color` thresholds. +2. **Unique visitors**: `unique_count` of `clientip`, from the [Optional: add more metrics to build a row](#add-a-metric-panel-for-median-response-size) suggestion. +3. **Total requests**: `count` with a secondary metric configured for week-over-week comparison via `time_shift: "1w"` and `compare`. Same source as the unique-visitors suggestion. +4. **Unique URLs**: `unique_count` of `request.keyword`, also from the optional metrics row. +5. **Response size over time, per host**: line chart of median `bytes` split by host, from the [Optional: add more time series](#add-a-line-chart-of-log-volume-over-time) suggestion in the line-chart sub-step. +6. **Log volume over time, per host**: the line chart from the [Add a line chart of log volume over time](#add-a-line-chart-of-log-volume-over-time) sub-step. The reference line at value `80` is a separate `reference_lines` layer in the same panel. +7. **Events by response code**: the {{esql}} bar chart saved to the dashboard from Discover in [Step 2](#explore-data-in-discover). ES|QL chart layers reference query result columns directly in `x` and `y` (for example, `"x": { "column": "response" }`), instead of the `operation`-based form used by data view layers. +8. **Requests by file extension**: bar chart from the [Add a bar chart of requests by file extension](#add-a-bar-chart-of-requests-by-file-extension) sub-step. The `includes` filter with `as_regex: true` and value `.+` mirrors the regex applied during the inline-edit step. +9. **Traffic distribution by operating system**: pie chart from the [Expand your dashboard](#expand-your-dashboard) sub-step. Pie panels use `config.type: "pie"` with a `metrics` array and a `group_by` array. +10. **Requests by geography**: treemap from the same sub-step. Treemaps use the same `metrics` + `group_by` shape as pies. +11. **Last 100 events**: {{esql}} data table from the [Add a table of recent events with {{esql}}](#add-a-table-of-recent-events-with-esql) sub-step. Categorical columns go in `rows`, numeric columns go in `metrics`. + +For the full request schema, including sections, filter controls, and library-linked panels, refer to the [Dashboards API reference](https://elastic.github.io/dashboards-api-spec/dashboards#tag/Dashboards/operation/post-dashboards). +:::: + + ## Navigate between Discover and dashboards [navigate-between-apps] One of {{kib}}'s strengths is how you can move between exploring raw data and visualizing it. Here are the key navigation paths: