From 5a11735d14c9d761a31d155e7770fb36f894690f Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 09:58:08 +0200 Subject: [PATCH 01/80] Add counter --- tests/integration/queries/counter.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/queries/counter.sql b/tests/integration/queries/counter.sql index b2ab78e2..ff88f2e9 100644 --- a/tests/integration/queries/counter.sql +++ b/tests/integration/queries/counter.sql @@ -1 +1 @@ -SELECT 6217 AS count \ No newline at end of file +SELECT 6217 \ No newline at end of file From ec9b9ded2fe42b851c41cd08b5572c8a90fefa41 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 10:00:03 +0200 Subject: [PATCH 02/80] Rename Dashboards to Dashboard --- .../labs/lsql/{dashboards.py => dashboard.py} | 3 +-- tests/integration/test_dashboards.py | 17 ++--------------- 2 files changed, 3 insertions(+), 17 deletions(-) rename src/databricks/labs/lsql/{dashboards.py => dashboard.py} (99%) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboard.py similarity index 99% rename from src/databricks/labs/lsql/dashboards.py rename to src/databricks/labs/lsql/dashboard.py index a7a18a2f..2bb7f9af 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -29,8 +29,7 @@ class _DataclassInstance(Protocol): __dataclass_fields__: ClassVar[dict] - -class Dashboards: +class Dashboard: def __init__(self, ws: WorkspaceClient): self._ws = ws diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 5cd3fecd..349c9d3f 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -4,24 +4,11 @@ import pytest -from databricks.labs.lsql.dashboards import Dashboards -from databricks.labs.lsql.lakeview.model import CounterSpec, Dashboard - - -@pytest.fixture -def dashboard_id(ws, make_random): - """Clean the lakeview dashboard""" - - dashboard_display_name = f"created_by_lsql_{make_random()}" - dashboard = ws.lakeview.create(dashboard_display_name) - - yield dashboard.dashboard_id - - ws.lakeview.trash(dashboard.dashboard_id) +from databricks.labs.lsql.dashboard import Dashboard def test_load_dashboard(ws): - dashboard = Dashboards(ws) + dashboards = Dashboard(ws) src = "/Workspace/Users/serge.smertin@databricks.com/Trivial Dashboard.lvdash.json" dst = Path(__file__).parent / "sample" dashboard.save_to_folder(src, dst) From a14a81df9db632c45b779225b73d9971e8d21342 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 10:00:28 +0200 Subject: [PATCH 03/80] Remove unused import --- tests/integration/test_dashboards.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 349c9d3f..980e1d42 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -2,8 +2,6 @@ from dataclasses import fields, is_dataclass from pathlib import Path -import pytest - from databricks.labs.lsql.dashboard import Dashboard From 2563c3e448256c2eb999d30c545143a6c17ed8f3 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 10:01:07 +0200 Subject: [PATCH 04/80] Make dashboard singular --- tests/integration/test_dashboards.py | 64 +--------------------------- 1 file changed, 2 insertions(+), 62 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 980e1d42..1e89ef4d 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -6,67 +6,7 @@ def test_load_dashboard(ws): - dashboards = Dashboard(ws) + dashboard = Dashboard(ws) src = "/Workspace/Users/serge.smertin@databricks.com/Trivial Dashboard.lvdash.json" dst = Path(__file__).parent / "sample" - dashboard.save_to_folder(src, dst) - - -def test_dashboard_creates_one_dataset_per_query(ws): - queries = Path(__file__).parent / "queries" - dashboard = Dashboards(ws).create_dashboard(queries) - assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) - - -def test_dashboard_creates_one_counter_widget_per_query(ws): - queries = Path(__file__).parent / "queries" - dashboard = Dashboards(ws).create_dashboard(queries) - - counter_widgets = [] - for page in dashboard.pages: - for layout in page.layout: - if isinstance(layout.widget.spec, CounterSpec): - counter_widgets.append(layout.widget) - - assert len(counter_widgets) == len([query for query in queries.glob("*.sql")]) - - -def replace_recursively(dataklass, replace_fields): - for field in fields(dataklass): - value = getattr(dataklass, field.name) - if is_dataclass(value): - new_value = replace_recursively(value, replace_fields) - elif isinstance(value, list): - new_value = [replace_recursively(v, replace_fields) for v in value] - elif isinstance(value, tuple): - new_value = (replace_recursively(v, replace_fields) for v in value) - else: - new_value = replace_fields.get(field.name, value) - setattr(dataklass, field.name, new_value) - return dataklass - - -def test_dashboard_deploys_dashboard(ws, dashboard_id): - queries = Path(__file__).parent / "queries" - dashboard_client = Dashboards(ws) - lakeview_dashboard = dashboard_client.create_dashboard(queries) - - dashboard = dashboard_client.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) - deployed_lakeview_dashboard = dashboard_client.get_dashboard(dashboard.path) - - replace_name = {"name": "test", "dataset_name": "test"} # Dynamically created names - lakeview_dashboard_wo_name = replace_recursively(lakeview_dashboard, replace_name) - deployed_lakeview_dashboard_wo_name = replace_recursively(deployed_lakeview_dashboard, replace_name) - - assert lakeview_dashboard_wo_name.as_dict() == deployed_lakeview_dashboard_wo_name.as_dict() - - -def test_dashboards_deploys_exported_dashboard_definition(ws, dashboard_id): - dashboard_file = Path(__file__).parent / "dashboards" / "dashboard.json" - with dashboard_file.open("r") as f: - lakeview_dashboard = Dashboard.from_dict(json.load(f)) - - dashboard_client = Dashboards(ws) - dashboard = dashboard_client.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) - - assert ws.lakeview.get(dashboard.dashboard_id) + dashboard.save_to_folder(src, dst) \ No newline at end of file From cef1967a3a3920a28fb7cc144b648e5a597e8110 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 10:04:44 +0200 Subject: [PATCH 05/80] Format --- src/databricks/labs/lsql/dashboard.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboard.py index 2bb7f9af..6187f3b5 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -8,27 +8,16 @@ from databricks.sdk.service.dashboards import Dashboard as SDKDashboard from databricks.sdk.service.workspace import ExportFormat -from databricks.labs.lsql.lakeview import ( - ControlFieldEncoding, - CounterEncodingMap, - CounterFieldEncoding, - CounterSpec, - Dashboard, - Dataset, - Field, - Layout, - NamedQuery, - Page, - Position, - Query, - Widget, -) +from databricks.labs.lsql.lakeview import ControlFieldEncoding +from databricks.labs.lsql.lakeview import Dashboard as LakeviewDashboard +from databricks.labs.lsql.lakeview import NamedQuery, Query @runtime_checkable class _DataclassInstance(Protocol): __dataclass_fields__: ClassVar[dict] + class Dashboard: def __init__(self, ws: WorkspaceClient): self._ws = ws @@ -37,7 +26,7 @@ def get_dashboard(self, dashboard_path: str): with self._ws.workspace.download(dashboard_path, format=ExportFormat.SOURCE) as f: raw = f.read().decode("utf-8") as_dict = json.loads(raw) - return Dashboard.from_dict(as_dict) + return LakeviewDashboard.from_dict(as_dict) def save_to_folder(self, dashboard_path: str, local_path: Path): local_path.mkdir(parents=True, exist_ok=True) From e17b6ea529f725c4b6c5821228f6be8e76caf0ca Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 10:06:37 +0200 Subject: [PATCH 06/80] Add integration test and endpoint to deploy dashboard from code --- src/databricks/labs/lsql/dashboard.py | 42 ++------------------------- tests/integration/test_dashboards.py | 8 ++++- 2 files changed, 9 insertions(+), 41 deletions(-) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboard.py index 6187f3b5..bc96934d 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -46,46 +46,8 @@ def save_to_folder(self, dashboard_path: str, local_path: Path): yaml.safe_dump(page, f) assert True - def create_dashboard(self, dashboard_folder: Path) -> Dashboard: - """Create a dashboard from code, i.e. configuration and queries.""" - datasets, layouts = [], [] - for query_path in dashboard_folder.glob("*.sql"): - with query_path.open("r") as query_file: - raw_query = query_file.read() - dataset = Dataset(name=query_path.stem, display_name=query_path.stem, query=raw_query) - datasets.append(dataset) - - fields = [Field(name="count", expression="`count`")] - query = Query(dataset_name=dataset.name, fields=fields, disaggregated=True) - # As for as testing went, a NamedQuery should always have "main_query" as name - named_query = NamedQuery(name="main_query", query=query) - counter_field_encoding = CounterFieldEncoding(field_name="count", display_name="count") - counter_spec = CounterSpec(CounterEncodingMap(value=counter_field_encoding)) - widget = Widget(name=dataset.name, queries=[named_query], spec=counter_spec) - position = Position(x=0, y=0, width=1, height=3) - layout = Layout(widget=widget, position=position) - layouts.append(layout) - - page = Page(name=dashboard_folder.name, display_name=dashboard_folder.name, layout=layouts) - lakeview_dashboard = Dashboard(datasets=datasets, pages=[page]) - return lakeview_dashboard - - def deploy_dashboard( - self, lakeview_dashboard: Dashboard, *, display_name: str | None = None, dashboard_id: str | None = None - ) -> SDKDashboard: - """Deploy a lakeview dashboard.""" - if (display_name is None and dashboard_id is None) or (display_name is not None and dashboard_id is not None): - raise ValueError("Give either display_name or dashboard_id.") - if display_name is not None: - dashboard = self._ws.lakeview.create( - display_name, serialized_dashboard=json.dumps(lakeview_dashboard.as_dict()) - ) - else: - assert dashboard_id is not None - dashboard = self._ws.lakeview.update( - dashboard_id, serialized_dashboard=json.dumps(lakeview_dashboard.as_dict()) - ) - return dashboard + def deploy(self, dashboard_folder: Path): + """Deploy dashboard from code, i.e. configuration and queries.""" def _format_sql_file(self, sql_query, query_path): with query_path.open("w") as f: diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 1e89ef4d..49140330 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -9,4 +9,10 @@ def test_load_dashboard(ws): dashboard = Dashboard(ws) src = "/Workspace/Users/serge.smertin@databricks.com/Trivial Dashboard.lvdash.json" dst = Path(__file__).parent / "sample" - dashboard.save_to_folder(src, dst) \ No newline at end of file + dashboard.save_to_folder(src, dst) + + +def test_dashboard_deploy_queries(ws): + dashboard = Dashboard(ws) + queries = Path(__file__).parent / "queries" + dashboard.deploy(queries) From 9c3a88e06d16d240bcad496aafdf0145b658989b Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 11:02:15 +0200 Subject: [PATCH 07/80] Handle WidgetSpec v0 table --- src/databricks/labs/lsql/lakeview/model.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/databricks/labs/lsql/lakeview/model.py b/src/databricks/labs/lsql/lakeview/model.py index 3d9fb8fc..94197150 100755 --- a/src/databricks/labs/lsql/lakeview/model.py +++ b/src/databricks/labs/lsql/lakeview/model.py @@ -91,8 +91,6 @@ def as_dict(self) -> Json: def from_dict(cls, d: Json) -> WidgetSpec: if d["version"] == 0 and d["viz_spec"]["viz_type"].lower() == "table": return TableV1Spec.from_dict(json.loads(d["viz_spec"]["serialized_options"])) - if d["version"] == 0 and d["viz_spec"]["viz_type"].lower() == "counter": - return CounterSpec.from_dict(json.loads(d["viz_spec"]["serialized_options"])) if d["version"] == 1 and d["widgetType"] == "details": return DetailsV1Spec.from_dict(d) if d["version"] == 1 and d["widgetType"] == "table": From 1871246d5dd6d2f5c73ddf83bcb9ebc8ed614183 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 11:15:07 +0200 Subject: [PATCH 08/80] Add TODO --- src/databricks/labs/lsql/dashboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboard.py index bc96934d..956dbe6e 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -18,7 +18,7 @@ class _DataclassInstance(Protocol): __dataclass_fields__: ClassVar[dict] -class Dashboard: +class Dashboard: # TODO: Rename, maybe DashboardClient? def __init__(self, ws: WorkspaceClient): self._ws = ws From 858edbd3b88ece6f5858bf5f143ae5dc07fef87c Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 11:16:23 +0200 Subject: [PATCH 09/80] Rewrite test to count queries --- tests/integration/test_dashboards.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 49140330..ddf23858 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -12,7 +12,7 @@ def test_load_dashboard(ws): dashboard.save_to_folder(src, dst) -def test_dashboard_deploy_queries(ws): - dashboard = Dashboard(ws) +def test_dashboard_deploys_one_dataset_per_query(ws): queries = Path(__file__).parent / "queries" - dashboard.deploy(queries) + dashboard = Dashboard(ws).deploy(queries) + assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) \ No newline at end of file From 2f926cce71978e020e50aceb6b318882d25cb62a Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 11:16:49 +0200 Subject: [PATCH 10/80] Create datasets from queries --- src/databricks/labs/lsql/dashboard.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboard.py index 956dbe6e..62025b00 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -8,9 +8,13 @@ from databricks.sdk.service.dashboards import Dashboard as SDKDashboard from databricks.sdk.service.workspace import ExportFormat -from databricks.labs.lsql.lakeview import ControlFieldEncoding -from databricks.labs.lsql.lakeview import Dashboard as LakeviewDashboard -from databricks.labs.lsql.lakeview import NamedQuery, Query +from databricks.labs.lsql.lakeview import ( + ControlFieldEncoding, + Dashboard as LakeviewDashboard, + Dataset, + NamedQuery, + Query, +) @runtime_checkable @@ -46,8 +50,17 @@ def save_to_folder(self, dashboard_path: str, local_path: Path): yaml.safe_dump(page, f) assert True - def deploy(self, dashboard_folder: Path): + @staticmethod + def deploy(dashboard_folder: Path) -> LakeviewDashboard: """Deploy dashboard from code, i.e. configuration and queries.""" + datasets = [] + for query_path in dashboard_folder.glob("*.sql"): + with query_path.open("r") as query_file: + query = query_file.read() + dataset = Dataset(name=query_path.stem, display_name=query_path.stem, query=query) + datasets.append(dataset) + dashboard = LakeviewDashboard(datasets=datasets, pages=[]) + return dashboard def _format_sql_file(self, sql_query, query_path): with query_path.open("w") as f: From 5da22cb9476bef8becdf24820129c15d594b3984 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 11:27:46 +0200 Subject: [PATCH 11/80] Handle counter v0 --- src/databricks/labs/lsql/lakeview/model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/databricks/labs/lsql/lakeview/model.py b/src/databricks/labs/lsql/lakeview/model.py index 94197150..3d9fb8fc 100755 --- a/src/databricks/labs/lsql/lakeview/model.py +++ b/src/databricks/labs/lsql/lakeview/model.py @@ -91,6 +91,8 @@ def as_dict(self) -> Json: def from_dict(cls, d: Json) -> WidgetSpec: if d["version"] == 0 and d["viz_spec"]["viz_type"].lower() == "table": return TableV1Spec.from_dict(json.loads(d["viz_spec"]["serialized_options"])) + if d["version"] == 0 and d["viz_spec"]["viz_type"].lower() == "counter": + return CounterSpec.from_dict(json.loads(d["viz_spec"]["serialized_options"])) if d["version"] == 1 and d["widgetType"] == "details": return DetailsV1Spec.from_dict(d) if d["version"] == 1 and d["widgetType"] == "table": From f424402987c1ddd7bb2407ac9ffda36dcf710af9 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 13:43:47 +0200 Subject: [PATCH 12/80] Add test for creating counter specs --- tests/integration/test_dashboards.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index ddf23858..50be1656 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -3,6 +3,7 @@ from pathlib import Path from databricks.labs.lsql.dashboard import Dashboard +from databricks.labs.lsql.lakeview.model import CounterSpec def test_load_dashboard(ws): @@ -15,4 +16,17 @@ def test_load_dashboard(ws): def test_dashboard_deploys_one_dataset_per_query(ws): queries = Path(__file__).parent / "queries" dashboard = Dashboard(ws).deploy(queries) - assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) \ No newline at end of file + assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) + + +def test_dashboard_deploys_one_counter_widget_per_query(ws): + queries = Path(__file__).parent / "queries" + dashboard = Dashboard(ws).deploy(queries) + + counter_widgets = [] + for page in dashboard.pages: + for layout in page.layout: + if isinstance(layout.widget.spec, CounterSpec): + counter_widgets.append(layout.widget) + + assert len(counter_widgets) == len([query for query in queries.glob("*.sql")]) From f00f5c35e338c774c9708e98d523f12b6327338f Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 13:44:10 +0200 Subject: [PATCH 13/80] Add a counter widget for each query --- src/databricks/labs/lsql/dashboard.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboard.py index 62025b00..7f3fbcd6 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -10,10 +10,17 @@ from databricks.labs.lsql.lakeview import ( ControlFieldEncoding, + CounterSpec, + CounterEncodingMap, Dashboard as LakeviewDashboard, Dataset, + Field, + Layout, NamedQuery, + Page, + Position, Query, + Widget, ) @@ -53,13 +60,23 @@ def save_to_folder(self, dashboard_path: str, local_path: Path): @staticmethod def deploy(dashboard_folder: Path) -> LakeviewDashboard: """Deploy dashboard from code, i.e. configuration and queries.""" - datasets = [] + datasets, layouts = [], [] for query_path in dashboard_folder.glob("*.sql"): with query_path.open("r") as query_file: - query = query_file.read() - dataset = Dataset(name=query_path.stem, display_name=query_path.stem, query=query) + raw_query = query_file.read() + dataset = Dataset(name=query_path.stem, display_name=query_path.stem, query=raw_query) datasets.append(dataset) - dashboard = LakeviewDashboard(datasets=datasets, pages=[]) + + query = Query(dataset_name=dataset.name, fields=[]) + named_query = NamedQuery(name=dataset.name, query=query) + counter_spec = CounterSpec(CounterEncodingMap()) + widget = Widget(name=dataset.name, queries=[named_query], spec=counter_spec) + position = Position(x=0, y=0, width=1, height=1) + layout = Layout(widget=widget, position=position) + layouts.append(layout) + + page = Page(name=dashboard_folder.name, display_name=dashboard_folder.name, layout=layouts) + dashboard = LakeviewDashboard(datasets=datasets, pages=[page]) return dashboard def _format_sql_file(self, sql_query, query_path): From 0002be36d3ad03759f25a317991688f0aad6ed55 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 14:00:06 +0200 Subject: [PATCH 14/80] Deploy lakeview dashboard --- src/databricks/labs/lsql/dashboard.py | 11 +++++++---- tests/integration/test_dashboards.py | 18 +++++++++++++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboard.py index 7f3fbcd6..662d0058 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -57,8 +57,7 @@ def save_to_folder(self, dashboard_path: str, local_path: Path): yaml.safe_dump(page, f) assert True - @staticmethod - def deploy(dashboard_folder: Path) -> LakeviewDashboard: + def deploy(self, display_name: str, dashboard_folder: Path) -> LakeviewDashboard: """Deploy dashboard from code, i.e. configuration and queries.""" datasets, layouts = [], [] for query_path in dashboard_folder.glob("*.sql"): @@ -76,8 +75,12 @@ def deploy(dashboard_folder: Path) -> LakeviewDashboard: layouts.append(layout) page = Page(name=dashboard_folder.name, display_name=dashboard_folder.name, layout=layouts) - dashboard = LakeviewDashboard(datasets=datasets, pages=[page]) - return dashboard + lakeview_dashboard = LakeviewDashboard(datasets=datasets, pages=[page]) + + dashboard = self._ws.lakeview.create(display_name, serialized_dashboard=json.dumps(lakeview_dashboard.as_dict())) + lakeview_dashboard.dashboard = dashboard # TODO: Refactor to get rid of this assignment + + return lakeview_dashboard def _format_sql_file(self, sql_query, query_path): with query_path.open("w") as f: diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 50be1656..ca524d5c 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -14,14 +14,15 @@ def test_load_dashboard(ws): def test_dashboard_deploys_one_dataset_per_query(ws): +def test_dashboard_deploys_one_dataset_per_query(ws, make_random): queries = Path(__file__).parent / "queries" - dashboard = Dashboard(ws).deploy(queries) + dashboard = Dashboard(ws).deploy(f"lsql-D{make_random()}", queries) assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) -def test_dashboard_deploys_one_counter_widget_per_query(ws): +def test_dashboard_deploys_one_counter_widget_per_query(ws, make_random): queries = Path(__file__).parent / "queries" - dashboard = Dashboard(ws).deploy(queries) + dashboard = Dashboard(ws).deploy(f"lsql-D{make_random()}", queries) counter_widgets = [] for page in dashboard.pages: @@ -30,3 +31,14 @@ def test_dashboard_deploys_one_counter_widget_per_query(ws): counter_widgets.append(layout.widget) assert len(counter_widgets) == len([query for query in queries.glob("*.sql")]) + + +def test_dashboard_deploys_dashboard(ws, make_random): + dashboard_display_name = f"lsql-D{make_random()}" + queries = Path(__file__).parent / "queries" + dashboard = Dashboard(ws).deploy(dashboard_display_name, queries) + + dashboard = ws.lakeview.get(dashboard.dashboard.dashboard_id) + + assert dashboard_display_name is not None + assert dashboard.display_name == dashboard_display_name \ No newline at end of file From af41d9c10cdb17e490cdc0f1ee691d140a03911e Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 14:18:03 +0200 Subject: [PATCH 15/80] Remove unused import --- src/databricks/labs/lsql/dashboard.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboard.py index 662d0058..8c86aa32 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -14,7 +14,6 @@ CounterEncodingMap, Dashboard as LakeviewDashboard, Dataset, - Field, Layout, NamedQuery, Page, From b46999c9cd9e28b0e5d9c140e5edc4deb0f40ec6 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 14:22:14 +0200 Subject: [PATCH 16/80] Split dashboard create and deploy --- src/databricks/labs/lsql/dashboard.py | 12 +++++++----- tests/integration/test_dashboards.py | 20 ++++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboard.py index 8c86aa32..4fa66480 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -56,8 +56,9 @@ def save_to_folder(self, dashboard_path: str, local_path: Path): yaml.safe_dump(page, f) assert True - def deploy(self, display_name: str, dashboard_folder: Path) -> LakeviewDashboard: - """Deploy dashboard from code, i.e. configuration and queries.""" + @staticmethod + def create(dashboard_folder: Path) -> LakeviewDashboard: + """Create a dashboard from code, i.e. configuration and queries.""" datasets, layouts = [], [] for query_path in dashboard_folder.glob("*.sql"): with query_path.open("r") as query_file: @@ -75,11 +76,12 @@ def deploy(self, display_name: str, dashboard_folder: Path) -> LakeviewDashboard page = Page(name=dashboard_folder.name, display_name=dashboard_folder.name, layout=layouts) lakeview_dashboard = LakeviewDashboard(datasets=datasets, pages=[page]) + return lakeview_dashboard + def deploy(self, display_name: str, lakeview_dashboard: LakeviewDashboard) -> SDKDashboard: + """Deploy a lakeview dashboard.""" dashboard = self._ws.lakeview.create(display_name, serialized_dashboard=json.dumps(lakeview_dashboard.as_dict())) - lakeview_dashboard.dashboard = dashboard # TODO: Refactor to get rid of this assignment - - return lakeview_dashboard + return dashboard def _format_sql_file(self, sql_query, query_path): with query_path.open("w") as f: diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index ca524d5c..d8038141 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -15,14 +15,15 @@ def test_load_dashboard(ws): def test_dashboard_deploys_one_dataset_per_query(ws): def test_dashboard_deploys_one_dataset_per_query(ws, make_random): +def test_dashboard_creates_one_dataset_per_query(ws, make_random): queries = Path(__file__).parent / "queries" - dashboard = Dashboard(ws).deploy(f"lsql-D{make_random()}", queries) + dashboard = Dashboard(ws).create(queries) assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) -def test_dashboard_deploys_one_counter_widget_per_query(ws, make_random): +def test_dashboard_creates_one_counter_widget_per_query(ws, make_random): queries = Path(__file__).parent / "queries" - dashboard = Dashboard(ws).deploy(f"lsql-D{make_random()}", queries) + dashboard = Dashboard(ws).create(queries) counter_widgets = [] for page in dashboard.pages: @@ -34,11 +35,14 @@ def test_dashboard_deploys_one_counter_widget_per_query(ws, make_random): def test_dashboard_deploys_dashboard(ws, make_random): - dashboard_display_name = f"lsql-D{make_random()}" queries = Path(__file__).parent / "queries" - dashboard = Dashboard(ws).deploy(dashboard_display_name, queries) + dashboard_client = Dashboard(ws) + lakeview_dashboard = dashboard_client.create(queries) + + dashboard_display_name = f"lsql-D{make_random()}" + dashboard = dashboard_client.deploy(dashboard_display_name, lakeview_dashboard) - dashboard = ws.lakeview.get(dashboard.dashboard.dashboard_id) + verify_dashboard = ws.lakeview.get(dashboard.dashboard_id) # To be sure the dashboard is created in the workspace - assert dashboard_display_name is not None - assert dashboard.display_name == dashboard_display_name \ No newline at end of file + assert verify_dashboard.display_name is not None + assert verify_dashboard.display_name == dashboard_display_name From cc7ffb0f94c78fe6cb2d4d4d675c74141e42bea9 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 14:34:02 +0200 Subject: [PATCH 17/80] Add count field --- src/databricks/labs/lsql/dashboard.py | 4 +++- tests/integration/queries/counter.sql | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboard.py index 4fa66480..340a0ab8 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -14,6 +14,7 @@ CounterEncodingMap, Dashboard as LakeviewDashboard, Dataset, + Field, Layout, NamedQuery, Page, @@ -66,7 +67,8 @@ def create(dashboard_folder: Path) -> LakeviewDashboard: dataset = Dataset(name=query_path.stem, display_name=query_path.stem, query=raw_query) datasets.append(dataset) - query = Query(dataset_name=dataset.name, fields=[]) + fields = [Field(name="count", expression="`count`")] + query = Query(dataset_name=dataset.name, fields=fields) named_query = NamedQuery(name=dataset.name, query=query) counter_spec = CounterSpec(CounterEncodingMap()) widget = Widget(name=dataset.name, queries=[named_query], spec=counter_spec) diff --git a/tests/integration/queries/counter.sql b/tests/integration/queries/counter.sql index ff88f2e9..b2ab78e2 100644 --- a/tests/integration/queries/counter.sql +++ b/tests/integration/queries/counter.sql @@ -1 +1 @@ -SELECT 6217 \ No newline at end of file +SELECT 6217 AS count \ No newline at end of file From 1654cb3c1144db00bf63cc3e09ce6b4f82952d16 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 14:46:24 +0200 Subject: [PATCH 18/80] Refactor to clean test dashboards --- src/databricks/labs/lsql/dashboard.py | 22 ++++++++++++++++++---- tests/integration/test_dashboards.py | 26 ++++++++++++++++++++------ 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboard.py index 340a0ab8..910e2c94 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -10,9 +10,11 @@ from databricks.labs.lsql.lakeview import ( ControlFieldEncoding, - CounterSpec, CounterEncodingMap, - Dashboard as LakeviewDashboard, + CounterSpec, +) +from databricks.labs.lsql.lakeview import Dashboard as LakeviewDashboard +from databricks.labs.lsql.lakeview import ( Dataset, Field, Layout, @@ -80,9 +82,21 @@ def create(dashboard_folder: Path) -> LakeviewDashboard: lakeview_dashboard = LakeviewDashboard(datasets=datasets, pages=[page]) return lakeview_dashboard - def deploy(self, display_name: str, lakeview_dashboard: LakeviewDashboard) -> SDKDashboard: + def deploy( + self, lakeview_dashboard: LakeviewDashboard, *, display_name: str | None = None, dashboard_id: str | None = None + ) -> SDKDashboard: """Deploy a lakeview dashboard.""" - dashboard = self._ws.lakeview.create(display_name, serialized_dashboard=json.dumps(lakeview_dashboard.as_dict())) + if (display_name is None and dashboard_id is None) or (display_name is not None and dashboard_id is not None): + raise ValueError("Give either display_name or dashboard_id.") + if display_name is not None: + dashboard = self._ws.lakeview.create( + display_name, serialized_dashboard=json.dumps(lakeview_dashboard.as_dict()) + ) + else: + assert dashboard_id is not None + dashboard = self._ws.lakeview.update( + dashboard_id, serialized_dashboard=json.dumps(lakeview_dashboard.as_dict()) + ) return dashboard def _format_sql_file(self, sql_query, query_path): diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index d8038141..40f99c38 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -2,10 +2,24 @@ from dataclasses import fields, is_dataclass from pathlib import Path +import pytest + from databricks.labs.lsql.dashboard import Dashboard from databricks.labs.lsql.lakeview.model import CounterSpec +@pytest.fixture +def dashboard_id(ws, make_random): + """Clean the lakeview dashboard""" + + dashboard_display_name = f"created_by_lsql_{make_random()}" + dashboard = ws.lakeview.create(dashboard_display_name) + + yield dashboard.dashboard_id + + ws.lakeview.trash(dashboard.dashboard_id) + + def test_load_dashboard(ws): dashboard = Dashboard(ws) src = "/Workspace/Users/serge.smertin@databricks.com/Trivial Dashboard.lvdash.json" @@ -16,12 +30,13 @@ def test_load_dashboard(ws): def test_dashboard_deploys_one_dataset_per_query(ws): def test_dashboard_deploys_one_dataset_per_query(ws, make_random): def test_dashboard_creates_one_dataset_per_query(ws, make_random): +def test_dashboard_creates_one_dataset_per_query(ws): queries = Path(__file__).parent / "queries" dashboard = Dashboard(ws).create(queries) assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) -def test_dashboard_creates_one_counter_widget_per_query(ws, make_random): +def test_dashboard_creates_one_counter_widget_per_query(ws): queries = Path(__file__).parent / "queries" dashboard = Dashboard(ws).create(queries) @@ -34,15 +49,14 @@ def test_dashboard_creates_one_counter_widget_per_query(ws, make_random): assert len(counter_widgets) == len([query for query in queries.glob("*.sql")]) -def test_dashboard_deploys_dashboard(ws, make_random): +def test_dashboard_deploys_dashboard(ws, dashboard_id): queries = Path(__file__).parent / "queries" dashboard_client = Dashboard(ws) lakeview_dashboard = dashboard_client.create(queries) - dashboard_display_name = f"lsql-D{make_random()}" - dashboard = dashboard_client.deploy(dashboard_display_name, lakeview_dashboard) + dashboard = dashboard_client.deploy(lakeview_dashboard, dashboard_id=dashboard_id) verify_dashboard = ws.lakeview.get(dashboard.dashboard_id) # To be sure the dashboard is created in the workspace - assert verify_dashboard.display_name is not None - assert verify_dashboard.display_name == dashboard_display_name + assert verify_dashboard.dashboard_id is not None + assert verify_dashboard.dashboard_id == dashboard_id From 1cd743c84deaffd86c2026b08dc85a5dd670f6e3 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 14:57:20 +0200 Subject: [PATCH 19/80] Test the lakeview dicts --- tests/integration/test_dashboards.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 40f99c38..e55e2d07 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -56,7 +56,4 @@ def test_dashboard_deploys_dashboard(ws, dashboard_id): dashboard = dashboard_client.deploy(lakeview_dashboard, dashboard_id=dashboard_id) - verify_dashboard = ws.lakeview.get(dashboard.dashboard_id) # To be sure the dashboard is created in the workspace - - assert verify_dashboard.dashboard_id is not None - assert verify_dashboard.dashboard_id == dashboard_id + assert dashboard_client.get_dashboard(dashboard.path).as_dict() == lakeview_dashboard.as_dict() From 58726496edf6a9a34beb3fb1ca6ef32395f7f3fa Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 14:58:35 +0200 Subject: [PATCH 20/80] Create random id --- src/databricks/labs/lsql/dashboard.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboard.py index 910e2c94..505adc93 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboard.py @@ -1,4 +1,6 @@ import json +import random +import string from pathlib import Path from typing import ClassVar, Protocol, runtime_checkable @@ -60,20 +62,24 @@ def save_to_folder(self, dashboard_path: str, local_path: Path): assert True @staticmethod - def create(dashboard_folder: Path) -> LakeviewDashboard: + def _create_random_id() -> str: + charset = string.ascii_lowercase + string.digits + return "".join(random.choices(charset, k=8)) + + def create(self, dashboard_folder: Path) -> LakeviewDashboard: """Create a dashboard from code, i.e. configuration and queries.""" datasets, layouts = [], [] for query_path in dashboard_folder.glob("*.sql"): with query_path.open("r") as query_file: raw_query = query_file.read() - dataset = Dataset(name=query_path.stem, display_name=query_path.stem, query=raw_query) + dataset = Dataset(name=self._create_random_id(), display_name=query_path.stem, query=raw_query) datasets.append(dataset) fields = [Field(name="count", expression="`count`")] query = Query(dataset_name=dataset.name, fields=fields) - named_query = NamedQuery(name=dataset.name, query=query) + named_query = NamedQuery(name=self._create_random_id(), query=query) counter_spec = CounterSpec(CounterEncodingMap()) - widget = Widget(name=dataset.name, queries=[named_query], spec=counter_spec) + widget = Widget(name=self._create_random_id(), queries=[named_query], spec=counter_spec) position = Position(x=0, y=0, width=1, height=1) layout = Layout(widget=widget, position=position) layouts.append(layout) From f23b25e1e6d6ae73c58882af623c50ae80d773a3 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 15:07:19 +0200 Subject: [PATCH 21/80] Refactor dashboard to plural --- .../labs/lsql/{dashboard.py => dashboards.py} | 2 +- tests/integration/test_dashboards.py | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) rename src/databricks/labs/lsql/{dashboard.py => dashboards.py} (99%) diff --git a/src/databricks/labs/lsql/dashboard.py b/src/databricks/labs/lsql/dashboards.py similarity index 99% rename from src/databricks/labs/lsql/dashboard.py rename to src/databricks/labs/lsql/dashboards.py index 505adc93..42fe059f 100644 --- a/src/databricks/labs/lsql/dashboard.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -33,7 +33,7 @@ class _DataclassInstance(Protocol): __dataclass_fields__: ClassVar[dict] -class Dashboard: # TODO: Rename, maybe DashboardClient? +class Dashboards: def __init__(self, ws: WorkspaceClient): self._ws = ws diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index e55e2d07..3529b220 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -4,7 +4,7 @@ import pytest -from databricks.labs.lsql.dashboard import Dashboard +from databricks.labs.lsql.dashboards import Dashboards from databricks.labs.lsql.lakeview.model import CounterSpec @@ -21,24 +21,21 @@ def dashboard_id(ws, make_random): def test_load_dashboard(ws): - dashboard = Dashboard(ws) + dashboard = Dashboards(ws) src = "/Workspace/Users/serge.smertin@databricks.com/Trivial Dashboard.lvdash.json" dst = Path(__file__).parent / "sample" dashboard.save_to_folder(src, dst) -def test_dashboard_deploys_one_dataset_per_query(ws): -def test_dashboard_deploys_one_dataset_per_query(ws, make_random): -def test_dashboard_creates_one_dataset_per_query(ws, make_random): def test_dashboard_creates_one_dataset_per_query(ws): queries = Path(__file__).parent / "queries" - dashboard = Dashboard(ws).create(queries) + dashboard = Dashboards(ws).create(queries) assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) def test_dashboard_creates_one_counter_widget_per_query(ws): queries = Path(__file__).parent / "queries" - dashboard = Dashboard(ws).create(queries) + dashboard = Dashboards(ws).create(queries) counter_widgets = [] for page in dashboard.pages: @@ -51,7 +48,7 @@ def test_dashboard_creates_one_counter_widget_per_query(ws): def test_dashboard_deploys_dashboard(ws, dashboard_id): queries = Path(__file__).parent / "queries" - dashboard_client = Dashboard(ws) + dashboard_client = Dashboards(ws) lakeview_dashboard = dashboard_client.create(queries) dashboard = dashboard_client.deploy(lakeview_dashboard, dashboard_id=dashboard_id) From 64bf3d86ca9ded4b7b4a184838c219fea31cdb35 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 15:08:08 +0200 Subject: [PATCH 22/80] Refactor create to create_dashboard --- src/databricks/labs/lsql/dashboards.py | 2 +- tests/integration/test_dashboards.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 42fe059f..fb227915 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -66,7 +66,7 @@ def _create_random_id() -> str: charset = string.ascii_lowercase + string.digits return "".join(random.choices(charset, k=8)) - def create(self, dashboard_folder: Path) -> LakeviewDashboard: + def create_dashboard(self, dashboard_folder: Path) -> LakeviewDashboard: """Create a dashboard from code, i.e. configuration and queries.""" datasets, layouts = [], [] for query_path in dashboard_folder.glob("*.sql"): diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 3529b220..c9dd76a9 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -29,13 +29,13 @@ def test_load_dashboard(ws): def test_dashboard_creates_one_dataset_per_query(ws): queries = Path(__file__).parent / "queries" - dashboard = Dashboards(ws).create(queries) + dashboard = Dashboards(ws).create_dashboard(queries) assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) def test_dashboard_creates_one_counter_widget_per_query(ws): queries = Path(__file__).parent / "queries" - dashboard = Dashboards(ws).create(queries) + dashboard = Dashboards(ws).create_dashboard(queries) counter_widgets = [] for page in dashboard.pages: @@ -49,7 +49,7 @@ def test_dashboard_creates_one_counter_widget_per_query(ws): def test_dashboard_deploys_dashboard(ws, dashboard_id): queries = Path(__file__).parent / "queries" dashboard_client = Dashboards(ws) - lakeview_dashboard = dashboard_client.create(queries) + lakeview_dashboard = dashboard_client.create_dashboard(queries) dashboard = dashboard_client.deploy(lakeview_dashboard, dashboard_id=dashboard_id) From fc1b0e9f19c020ef8fd5d0c62aeee09e86a136d3 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 15:08:33 +0200 Subject: [PATCH 23/80] Refactor deploy to deploy_dashboard --- src/databricks/labs/lsql/dashboards.py | 2 +- tests/integration/test_dashboards.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index fb227915..603c7126 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -88,7 +88,7 @@ def create_dashboard(self, dashboard_folder: Path) -> LakeviewDashboard: lakeview_dashboard = LakeviewDashboard(datasets=datasets, pages=[page]) return lakeview_dashboard - def deploy( + def deploy_dashboard( self, lakeview_dashboard: LakeviewDashboard, *, display_name: str | None = None, dashboard_id: str | None = None ) -> SDKDashboard: """Deploy a lakeview dashboard.""" diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index c9dd76a9..a246a24d 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -51,6 +51,6 @@ def test_dashboard_deploys_dashboard(ws, dashboard_id): dashboard_client = Dashboards(ws) lakeview_dashboard = dashboard_client.create_dashboard(queries) - dashboard = dashboard_client.deploy(lakeview_dashboard, dashboard_id=dashboard_id) + dashboard = dashboard_client.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) assert dashboard_client.get_dashboard(dashboard.path).as_dict() == lakeview_dashboard.as_dict() From 8fe59d2369ff75d6fff1d23407371ca53500f569 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 15:11:03 +0200 Subject: [PATCH 24/80] Remove name collision --- src/databricks/labs/lsql/dashboards.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 603c7126..a19cbd87 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -14,9 +14,7 @@ ControlFieldEncoding, CounterEncodingMap, CounterSpec, -) -from databricks.labs.lsql.lakeview import Dashboard as LakeviewDashboard -from databricks.labs.lsql.lakeview import ( + Dashboard, Dataset, Field, Layout, @@ -41,7 +39,7 @@ def get_dashboard(self, dashboard_path: str): with self._ws.workspace.download(dashboard_path, format=ExportFormat.SOURCE) as f: raw = f.read().decode("utf-8") as_dict = json.loads(raw) - return LakeviewDashboard.from_dict(as_dict) + return Dashboard.from_dict(as_dict) def save_to_folder(self, dashboard_path: str, local_path: Path): local_path.mkdir(parents=True, exist_ok=True) @@ -66,7 +64,7 @@ def _create_random_id() -> str: charset = string.ascii_lowercase + string.digits return "".join(random.choices(charset, k=8)) - def create_dashboard(self, dashboard_folder: Path) -> LakeviewDashboard: + def create_dashboard(self, dashboard_folder: Path) -> Dashboard: """Create a dashboard from code, i.e. configuration and queries.""" datasets, layouts = [], [] for query_path in dashboard_folder.glob("*.sql"): @@ -85,11 +83,11 @@ def create_dashboard(self, dashboard_folder: Path) -> LakeviewDashboard: layouts.append(layout) page = Page(name=dashboard_folder.name, display_name=dashboard_folder.name, layout=layouts) - lakeview_dashboard = LakeviewDashboard(datasets=datasets, pages=[page]) + lakeview_dashboard = Dashboard(datasets=datasets, pages=[page]) return lakeview_dashboard def deploy_dashboard( - self, lakeview_dashboard: LakeviewDashboard, *, display_name: str | None = None, dashboard_id: str | None = None + self, lakeview_dashboard: Dashboard, *, display_name: str | None = None, dashboard_id: str | None = None ) -> SDKDashboard: """Deploy a lakeview dashboard.""" if (display_name is None and dashboard_id is None) or (display_name is not None and dashboard_id is not None): From 89eb08d80a2d0891cbd80ae7bd325d9872e90beb Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Mon, 27 May 2024 16:01:22 +0200 Subject: [PATCH 25/80] Change counter height to three --- src/databricks/labs/lsql/dashboards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index a19cbd87..fd31b7d8 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -78,7 +78,7 @@ def create_dashboard(self, dashboard_folder: Path) -> Dashboard: named_query = NamedQuery(name=self._create_random_id(), query=query) counter_spec = CounterSpec(CounterEncodingMap()) widget = Widget(name=self._create_random_id(), queries=[named_query], spec=counter_spec) - position = Position(x=0, y=0, width=1, height=1) + position = Position(x=0, y=0, width=1, height=3) layout = Layout(widget=widget, position=position) layouts.append(layout) From a6854258b0fcdfaedc57a433dc55486fc12deb63 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 09:13:48 +0200 Subject: [PATCH 26/80] Add working dashboard --- tests/integration/test_dashboards.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index a246a24d..8d4a78c0 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -1,11 +1,10 @@ import json -from dataclasses import fields, is_dataclass from pathlib import Path import pytest from databricks.labs.lsql.dashboards import Dashboards -from databricks.labs.lsql.lakeview.model import CounterSpec +from databricks.labs.lsql.lakeview.model import CounterSpec, Dashboard @pytest.fixture @@ -54,3 +53,14 @@ def test_dashboard_deploys_dashboard(ws, dashboard_id): dashboard = dashboard_client.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) assert dashboard_client.get_dashboard(dashboard.path).as_dict() == lakeview_dashboard.as_dict() + + +def test_dashboards_deploys_exported_dashboard_definition(ws, dashboard_id): + dashboard_file = Path(__file__).parent / "dashboards" / "dashboard.json" + with dashboard_file.open("r") as f: + lakeview_dashboard = Dashboard.from_dict(json.load(f)) + + dashboard_client = Dashboards(ws) + dashboard = dashboard_client.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) + + assert ws.lakeview.get(dashboard.dashboard_id) From 8e9fedef1bc96e0a2f2aea3a08aeb5610db95abb Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 10:00:53 +0200 Subject: [PATCH 27/80] Add disaggregated field --- src/databricks/labs/lsql/dashboards.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index fd31b7d8..94ae1702 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -77,6 +77,7 @@ def create_dashboard(self, dashboard_folder: Path) -> Dashboard: query = Query(dataset_name=dataset.name, fields=fields) named_query = NamedQuery(name=self._create_random_id(), query=query) counter_spec = CounterSpec(CounterEncodingMap()) + query = Query(dataset_name=dataset.name, fields=fields, disaggregated=True) widget = Widget(name=self._create_random_id(), queries=[named_query], spec=counter_spec) position = Position(x=0, y=0, width=1, height=3) layout = Layout(widget=widget, position=position) From b9df7cb3ce71caaf9391275aad470e7e2d4e3b25 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 10:01:14 +0200 Subject: [PATCH 28/80] Add counter field encoding --- src/databricks/labs/lsql/dashboards.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 94ae1702..eaa6935b 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -13,6 +13,7 @@ from databricks.labs.lsql.lakeview import ( ControlFieldEncoding, CounterEncodingMap, + CounterFieldEncoding, CounterSpec, Dashboard, Dataset, @@ -78,6 +79,7 @@ def create_dashboard(self, dashboard_folder: Path) -> Dashboard: named_query = NamedQuery(name=self._create_random_id(), query=query) counter_spec = CounterSpec(CounterEncodingMap()) query = Query(dataset_name=dataset.name, fields=fields, disaggregated=True) + counter_spec = CounterSpec(CounterEncodingMap(value=CounterFieldEncoding(field_name="count", display_name="count"))) widget = Widget(name=self._create_random_id(), queries=[named_query], spec=counter_spec) position = Position(x=0, y=0, width=1, height=3) layout = Layout(widget=widget, position=position) From 871e39b54d92b887328fe4e4a68d6fd2a45f8bfc Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 10:01:25 +0200 Subject: [PATCH 29/80] Name query --- src/databricks/labs/lsql/dashboards.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index eaa6935b..81f29e7e 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -75,10 +75,8 @@ def create_dashboard(self, dashboard_folder: Path) -> Dashboard: datasets.append(dataset) fields = [Field(name="count", expression="`count`")] - query = Query(dataset_name=dataset.name, fields=fields) - named_query = NamedQuery(name=self._create_random_id(), query=query) - counter_spec = CounterSpec(CounterEncodingMap()) query = Query(dataset_name=dataset.name, fields=fields, disaggregated=True) + named_query = NamedQuery(name="main_query", query=query) counter_spec = CounterSpec(CounterEncodingMap(value=CounterFieldEncoding(field_name="count", display_name="count"))) widget = Widget(name=self._create_random_id(), queries=[named_query], spec=counter_spec) position = Position(x=0, y=0, width=1, height=3) From 7bd6ab78eed7b5dced825f470fe52124f1e4707c Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 10:09:00 +0200 Subject: [PATCH 30/80] Format --- src/databricks/labs/lsql/dashboards.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 81f29e7e..8288249b 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -77,7 +77,8 @@ def create_dashboard(self, dashboard_folder: Path) -> Dashboard: fields = [Field(name="count", expression="`count`")] query = Query(dataset_name=dataset.name, fields=fields, disaggregated=True) named_query = NamedQuery(name="main_query", query=query) - counter_spec = CounterSpec(CounterEncodingMap(value=CounterFieldEncoding(field_name="count", display_name="count"))) + counter_field_encoding = CounterFieldEncoding(field_name="count", display_name="count") + counter_spec = CounterSpec(CounterEncodingMap(value=counter_field_encoding)) widget = Widget(name=self._create_random_id(), queries=[named_query], spec=counter_spec) position = Position(x=0, y=0, width=1, height=3) layout = Layout(widget=widget, position=position) From e73f5db1e3ecf07bc954407b7ea19ae1c3b29505 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 10:29:58 +0200 Subject: [PATCH 31/80] Add comment --- src/databricks/labs/lsql/dashboards.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 8288249b..1e4690c3 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -76,6 +76,7 @@ def create_dashboard(self, dashboard_folder: Path) -> Dashboard: fields = [Field(name="count", expression="`count`")] query = Query(dataset_name=dataset.name, fields=fields, disaggregated=True) + # As for as testing went, a NamedQuery should always have "main_query" as name named_query = NamedQuery(name="main_query", query=query) counter_field_encoding = CounterFieldEncoding(field_name="count", display_name="count") counter_spec = CounterSpec(CounterEncodingMap(value=counter_field_encoding)) From d687ce880715df1cac9213beaf1e1a0ece5a47d9 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 10:31:52 +0200 Subject: [PATCH 32/80] Remove create random id --- src/databricks/labs/lsql/dashboards.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 1e4690c3..6cf51f96 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -60,18 +60,13 @@ def save_to_folder(self, dashboard_path: str, local_path: Path): yaml.safe_dump(page, f) assert True - @staticmethod - def _create_random_id() -> str: - charset = string.ascii_lowercase + string.digits - return "".join(random.choices(charset, k=8)) - def create_dashboard(self, dashboard_folder: Path) -> Dashboard: """Create a dashboard from code, i.e. configuration and queries.""" datasets, layouts = [], [] for query_path in dashboard_folder.glob("*.sql"): with query_path.open("r") as query_file: raw_query = query_file.read() - dataset = Dataset(name=self._create_random_id(), display_name=query_path.stem, query=raw_query) + dataset = Dataset(name=query_path.stem, display_name=query_path.stem, query=raw_query) datasets.append(dataset) fields = [Field(name="count", expression="`count`")] @@ -80,7 +75,7 @@ def create_dashboard(self, dashboard_folder: Path) -> Dashboard: named_query = NamedQuery(name="main_query", query=query) counter_field_encoding = CounterFieldEncoding(field_name="count", display_name="count") counter_spec = CounterSpec(CounterEncodingMap(value=counter_field_encoding)) - widget = Widget(name=self._create_random_id(), queries=[named_query], spec=counter_spec) + widget = Widget(name=dataset.name, queries=[named_query], spec=counter_spec) position = Position(x=0, y=0, width=1, height=3) layout = Layout(widget=widget, position=position) layouts.append(layout) From 4219f59c13f00b0cfce0c2089481c45a9d654ddc Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 11:06:40 +0200 Subject: [PATCH 33/80] Replace fields in testing --- tests/integration/test_dashboards.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 8d4a78c0..7d5398ac 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -1,4 +1,5 @@ import json +from dataclasses import fields, is_dataclass, replace from pathlib import Path import pytest @@ -45,14 +46,34 @@ def test_dashboard_creates_one_counter_widget_per_query(ws): assert len(counter_widgets) == len([query for query in queries.glob("*.sql")]) +def replace_recursively(dataklass, replace_fields): + for field in fields(dataklass): + value = getattr(dataklass, field.name) + if is_dataclass(value): + new_value = replace_recursively(value, replace_fields) + elif isinstance(value, list): + new_value = [replace_recursively(v, replace_fields) for v in value] + elif isinstance(value, tuple): + new_value = (replace_recursively(v, replace_fields) for v in value) + else: + new_value = replace_fields.get(field.name, value) + setattr(dataklass, field.name, new_value) + return dataklass + + def test_dashboard_deploys_dashboard(ws, dashboard_id): queries = Path(__file__).parent / "queries" dashboard_client = Dashboards(ws) lakeview_dashboard = dashboard_client.create_dashboard(queries) dashboard = dashboard_client.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) + deployed_lakeview_dashboard = dashboard_client.get_dashboard(dashboard.path) + + replace_name = {"name": "test", "dataset_name": "test"} # Dynamically created names + lakeview_dashboard_wo_name = replace_recursively(lakeview_dashboard, replace_name) + deployed_lakeview_dashboard_wo_name = replace_recursively(deployed_lakeview_dashboard, replace_name) - assert dashboard_client.get_dashboard(dashboard.path).as_dict() == lakeview_dashboard.as_dict() + assert lakeview_dashboard_wo_name.as_dict() == deployed_lakeview_dashboard_wo_name.as_dict() def test_dashboards_deploys_exported_dashboard_definition(ws, dashboard_id): From 98f5368c497c4b97fa7ca70b53c6c9d8f9e3bd61 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 11:22:16 +0200 Subject: [PATCH 34/80] Format --- src/databricks/labs/lsql/dashboards.py | 2 -- tests/integration/test_dashboards.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 6cf51f96..a7a18a2f 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -1,6 +1,4 @@ import json -import random -import string from pathlib import Path from typing import ClassVar, Protocol, runtime_checkable diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 7d5398ac..5cd3fecd 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -1,5 +1,5 @@ import json -from dataclasses import fields, is_dataclass, replace +from dataclasses import fields, is_dataclass from pathlib import Path import pytest From b4d54142d36f28fa9edef16c05974e6df3081b12 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 11:27:06 +0200 Subject: [PATCH 35/80] Raise value error when missing diplay name and dashboard id --- tests/unit/test_dashboards.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/unit/test_dashboards.py diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py new file mode 100644 index 00000000..97f8f466 --- /dev/null +++ b/tests/unit/test_dashboards.py @@ -0,0 +1,16 @@ +import pytest +from unittest.mock import create_autospec + +from databricks.sdk import WorkspaceClient + +from databricks.labs.lsql.dashboards import Dashboards +from databricks.labs.lsql.lakeview import Dashboard + + +def test_dashboard_raises_value_error_with_missing_display_name_and_dashboard_id(): + ws = create_autospec(WorkspaceClient) + dashboards = Dashboards(ws) + lakeview_dashboard = Dashboard([], []) + with pytest.raises(ValueError): + dashboards.deploy_dashboard(lakeview_dashboard) + ws.assert_not_called() \ No newline at end of file From 25d2b947a0be28e2fed6a1adf4b5482d315775f5 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 12:11:26 +0200 Subject: [PATCH 36/80] Raise value error when stating display name and dashboard id --- tests/unit/test_dashboards.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 97f8f466..1eb7515b 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -13,4 +13,13 @@ def test_dashboard_raises_value_error_with_missing_display_name_and_dashboard_id lakeview_dashboard = Dashboard([], []) with pytest.raises(ValueError): dashboards.deploy_dashboard(lakeview_dashboard) - ws.assert_not_called() \ No newline at end of file + ws.assert_not_called() + + +def test_dashboard_raises_value_error_with_both_display_name_and_dashboard_id(): + ws = create_autospec(WorkspaceClient) + dashboards = Dashboards(ws) + lakeview_dashboard = Dashboard([], []) + with pytest.raises(ValueError): + dashboards.deploy_dashboard(lakeview_dashboard, display_name="test", dashboard_id="test") + ws.assert_not_called() From d283c82d67e442fc52772eed6fac3c8f13b9e099 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 12:51:47 +0200 Subject: [PATCH 37/80] Call create when deploy with display name --- tests/unit/test_dashboards.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 1eb7515b..d95a205c 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -23,3 +23,11 @@ def test_dashboard_raises_value_error_with_both_display_name_and_dashboard_id(): with pytest.raises(ValueError): dashboards.deploy_dashboard(lakeview_dashboard, display_name="test", dashboard_id="test") ws.assert_not_called() + + +def test_dashboard_deploy_calls_create_with_display_name(): + ws = create_autospec(WorkspaceClient) + dashboards = Dashboards(ws) + lakeview_dashboard = Dashboard([], []) + dashboards.deploy_dashboard(lakeview_dashboard, display_name="test") + ws.lakeview.create.assert_called_once() From 246e79261ca718316063d5a30df873079b966ad2 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 12:52:44 +0200 Subject: [PATCH 38/80] Call update when deploy with dashboard id --- tests/unit/test_dashboards.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index d95a205c..246108c9 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -31,3 +31,11 @@ def test_dashboard_deploy_calls_create_with_display_name(): lakeview_dashboard = Dashboard([], []) dashboards.deploy_dashboard(lakeview_dashboard, display_name="test") ws.lakeview.create.assert_called_once() + + +def test_dashboard_deploy_calls_update_with_dashboard_id(): + ws = create_autospec(WorkspaceClient) + dashboards = Dashboards(ws) + lakeview_dashboard = Dashboard([], []) + dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id="test") + ws.lakeview.update.assert_called_once() From 5626f82343b509a18cf0e67e5c186ab0d9e4694b Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 12:53:37 +0200 Subject: [PATCH 39/80] Improve tests --- tests/unit/test_dashboards.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 246108c9..05fa1264 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -7,7 +7,7 @@ from databricks.labs.lsql.lakeview import Dashboard -def test_dashboard_raises_value_error_with_missing_display_name_and_dashboard_id(): +def test_dashboard_deploy_raises_value_error_with_missing_display_name_and_dashboard_id(): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) lakeview_dashboard = Dashboard([], []) @@ -16,7 +16,7 @@ def test_dashboard_raises_value_error_with_missing_display_name_and_dashboard_id ws.assert_not_called() -def test_dashboard_raises_value_error_with_both_display_name_and_dashboard_id(): +def test_dashboard_deploy_raises_value_error_with_both_display_name_and_dashboard_id(): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) lakeview_dashboard = Dashboard([], []) @@ -30,7 +30,9 @@ def test_dashboard_deploy_calls_create_with_display_name(): dashboards = Dashboards(ws) lakeview_dashboard = Dashboard([], []) dashboards.deploy_dashboard(lakeview_dashboard, display_name="test") + ws.lakeview.create.assert_called_once() + ws.lakeview.update.assert_not_called() def test_dashboard_deploy_calls_update_with_dashboard_id(): @@ -38,4 +40,6 @@ def test_dashboard_deploy_calls_update_with_dashboard_id(): dashboards = Dashboards(ws) lakeview_dashboard = Dashboard([], []) dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id="test") + + ws.lakeview.create.assert_not_called() ws.lakeview.update.assert_called_once() From 21bbe8d11852ab875d029c686affc34478a7e155 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 13:37:08 +0200 Subject: [PATCH 40/80] Move unit tests to unit test folder --- tests/integration/test_dashboards.py | 21 +-------------------- tests/unit/queries/counter.sql | 1 + tests/unit/test_dashboards.py | 27 +++++++++++++++++++++++++-- 3 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 tests/unit/queries/counter.sql diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 5cd3fecd..da0dcc5c 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -5,7 +5,7 @@ import pytest from databricks.labs.lsql.dashboards import Dashboards -from databricks.labs.lsql.lakeview.model import CounterSpec, Dashboard +from databricks.labs.lsql.lakeview.model import Dashboard @pytest.fixture @@ -27,25 +27,6 @@ def test_load_dashboard(ws): dashboard.save_to_folder(src, dst) -def test_dashboard_creates_one_dataset_per_query(ws): - queries = Path(__file__).parent / "queries" - dashboard = Dashboards(ws).create_dashboard(queries) - assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) - - -def test_dashboard_creates_one_counter_widget_per_query(ws): - queries = Path(__file__).parent / "queries" - dashboard = Dashboards(ws).create_dashboard(queries) - - counter_widgets = [] - for page in dashboard.pages: - for layout in page.layout: - if isinstance(layout.widget.spec, CounterSpec): - counter_widgets.append(layout.widget) - - assert len(counter_widgets) == len([query for query in queries.glob("*.sql")]) - - def replace_recursively(dataklass, replace_fields): for field in fields(dataklass): value = getattr(dataklass, field.name) diff --git a/tests/unit/queries/counter.sql b/tests/unit/queries/counter.sql new file mode 100644 index 00000000..b2ab78e2 --- /dev/null +++ b/tests/unit/queries/counter.sql @@ -0,0 +1 @@ +SELECT 6217 AS count \ No newline at end of file diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 05fa1264..9411501d 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -1,10 +1,33 @@ -import pytest from unittest.mock import create_autospec +from pathlib import Path +import pytest from databricks.sdk import WorkspaceClient from databricks.labs.lsql.dashboards import Dashboards -from databricks.labs.lsql.lakeview import Dashboard +from databricks.labs.lsql.lakeview.model import CounterSpec, Dashboard + + +def test_dashboard_creates_one_dataset_per_query(): + ws = create_autospec(WorkspaceClient) + queries = Path(__file__).parent / "queries" + dashboard = Dashboards(ws).create_dashboard(queries) + assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) + + +def test_dashboard_creates_one_counter_widget_per_query(): + ws = create_autospec(WorkspaceClient) + queries = Path(__file__).parent / "queries" + dashboard = Dashboards(ws).create_dashboard(queries) + + counter_widgets = [] + for page in dashboard.pages: + for layout in page.layout: + if isinstance(layout.widget.spec, CounterSpec): + counter_widgets.append(layout.widget) + + assert len(counter_widgets) == len([query for query in queries.glob("*.sql")]) + def test_dashboard_deploy_raises_value_error_with_missing_display_name_and_dashboard_id(): From 5cbef3055487acb615dc516e6856991cd11fcf2e Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 13:38:15 +0200 Subject: [PATCH 41/80] Move queries into dashboards --- tests/integration/{queries => dashboards/dashboard}/counter.sql | 0 tests/integration/test_dashboards.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/integration/{queries => dashboards/dashboard}/counter.sql (100%) diff --git a/tests/integration/queries/counter.sql b/tests/integration/dashboards/dashboard/counter.sql similarity index 100% rename from tests/integration/queries/counter.sql rename to tests/integration/dashboards/dashboard/counter.sql diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index da0dcc5c..f03e33de 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -43,7 +43,7 @@ def replace_recursively(dataklass, replace_fields): def test_dashboard_deploys_dashboard(ws, dashboard_id): - queries = Path(__file__).parent / "queries" + queries = Path(__file__).parent / "dashboards" / "dashboard" dashboard_client = Dashboards(ws) lakeview_dashboard = dashboard_client.create_dashboard(queries) From 8bfd94932121b87e49ba514641354b6517b09867 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 14:12:08 +0200 Subject: [PATCH 42/80] Rename dashboard_client to dashboards --- tests/integration/test_dashboards.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index f03e33de..e849e5c1 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -44,11 +44,11 @@ def replace_recursively(dataklass, replace_fields): def test_dashboard_deploys_dashboard(ws, dashboard_id): queries = Path(__file__).parent / "dashboards" / "dashboard" - dashboard_client = Dashboards(ws) - lakeview_dashboard = dashboard_client.create_dashboard(queries) + dashboards = Dashboards(ws) + lakeview_dashboard = dashboards.create_dashboard(queries) - dashboard = dashboard_client.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) - deployed_lakeview_dashboard = dashboard_client.get_dashboard(dashboard.path) + dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) + deployed_lakeview_dashboard = dashboards.get_dashboard(dashboard.path) replace_name = {"name": "test", "dataset_name": "test"} # Dynamically created names lakeview_dashboard_wo_name = replace_recursively(lakeview_dashboard, replace_name) @@ -62,7 +62,7 @@ def test_dashboards_deploys_exported_dashboard_definition(ws, dashboard_id): with dashboard_file.open("r") as f: lakeview_dashboard = Dashboard.from_dict(json.load(f)) - dashboard_client = Dashboards(ws) - dashboard = dashboard_client.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) + dashboards = Dashboards(ws) + dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) assert ws.lakeview.get(dashboard.dashboard_id) From 704ea05f63cb859aafb5ca54857f6e9b75ee9484 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 14:24:45 +0200 Subject: [PATCH 43/80] Refactor to separate out better names --- src/databricks/labs/lsql/dashboards.py | 36 ++++++++++++++------------ 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index a7a18a2f..99dec2ae 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -1,6 +1,7 @@ import json +from dataclasses import replace from pathlib import Path -from typing import ClassVar, Protocol, runtime_checkable +from typing import ClassVar, Protocol, runtime_checkable, TypeVar import sqlglot import yaml @@ -25,6 +26,9 @@ ) +T = TypeVar("T") + + @runtime_checkable class _DataclassInstance(Protocol): __dataclass_fields__: ClassVar[dict] @@ -42,21 +46,14 @@ def get_dashboard(self, dashboard_path: str): def save_to_folder(self, dashboard_path: str, local_path: Path): local_path.mkdir(parents=True, exist_ok=True) - dashboard = self.get_dashboard(dashboard_path) - better_names = {} + dashboard = self.with_better_names(self.get_dashboard(dashboard_path)) for dataset in dashboard.datasets: - name = dataset.display_name - better_names[dataset.name] = name - query_path = local_path / f"{name}.sql" - sql_query = dataset.query - self._format_sql_file(sql_query, query_path) - lvdash_yml = local_path / "lvdash.yml" - with lvdash_yml.open("w") as f: - first_page = dashboard.pages[0] - self._replace_names(first_page, better_names) - page = first_page.as_dict() - yaml.safe_dump(page, f) - assert True + query_path = local_path / f"{dataset.display_name}.sql" + self._format_sql_file(dataset.query, query_path) + for page in dashboard.pages: + lvdash_yml = local_path / f"lvdash-{page.display_name}.yml" + with lvdash_yml.open("w") as f: + yaml.safe_dump(page.as_dict(), f) def create_dashboard(self, dashboard_folder: Path) -> Dashboard: """Create a dashboard from code, i.e. configuration and queries.""" @@ -115,8 +112,13 @@ def _format_sql_file(self, sql_query, query_path): except sqlglot.ParseError: f.write(sql_query) - def _replace_names(self, node: _DataclassInstance, better_names: dict[str, str]): - # walk evely dataclass instance recursively and replace names + def with_better_names(self, dashboard: Dashboard) -> Dashboard: + better_names = {dataset.name: dataset.display_name for dataset in dashboard.datasets} + pages = [self._replace_names(page, better_names) for page in dashboard.pages] + return replace(dashboard, pages=pages) + + def _replace_names(self, node: T, better_names: dict[str, str]) -> T: + # walk every dataclass instance recursively and replace names if isinstance(node, _DataclassInstance): for field in node.__dataclass_fields__.values(): value = getattr(node, field.name) From 6e0eb240af8ce326f517c19ec674c0a26ed53100 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 14:25:06 +0200 Subject: [PATCH 44/80] Make create dashboard a static method --- src/databricks/labs/lsql/dashboards.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 99dec2ae..1b31efb3 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -55,7 +55,8 @@ def save_to_folder(self, dashboard_path: str, local_path: Path): with lvdash_yml.open("w") as f: yaml.safe_dump(page.as_dict(), f) - def create_dashboard(self, dashboard_folder: Path) -> Dashboard: + @staticmethod + def create_dashboard(dashboard_folder: Path) -> Dashboard: """Create a dashboard from code, i.e. configuration and queries.""" datasets, layouts = [], [] for query_path in dashboard_folder.glob("*.sql"): From 66c0b04b06aceffb144d1c0ffbae691dec2f8a2b Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 14:25:46 +0200 Subject: [PATCH 45/80] Move deploy integration test up --- tests/integration/test_dashboards.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index e849e5c1..e89b649a 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -20,6 +20,17 @@ def dashboard_id(ws, make_random): ws.lakeview.trash(dashboard.dashboard_id) +def test_dashboards_deploys_exported_dashboard_definition(ws, dashboard_id): + dashboard_file = Path(__file__).parent / "dashboards" / "dashboard.json" + with dashboard_file.open("r") as f: + lakeview_dashboard = Dashboard.from_dict(json.load(f)) + + dashboards = Dashboards(ws) + dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) + + assert ws.lakeview.get(dashboard.dashboard_id) + + def test_load_dashboard(ws): dashboard = Dashboards(ws) src = "/Workspace/Users/serge.smertin@databricks.com/Trivial Dashboard.lvdash.json" @@ -55,14 +66,3 @@ def test_dashboard_deploys_dashboard(ws, dashboard_id): deployed_lakeview_dashboard_wo_name = replace_recursively(deployed_lakeview_dashboard, replace_name) assert lakeview_dashboard_wo_name.as_dict() == deployed_lakeview_dashboard_wo_name.as_dict() - - -def test_dashboards_deploys_exported_dashboard_definition(ws, dashboard_id): - dashboard_file = Path(__file__).parent / "dashboards" / "dashboard.json" - with dashboard_file.open("r") as f: - lakeview_dashboard = Dashboard.from_dict(json.load(f)) - - dashboards = Dashboards(ws) - dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) - - assert ws.lakeview.get(dashboard.dashboard_id) From 64618107796dc3928739297d8105a3de4d8af55d Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 14:32:03 +0200 Subject: [PATCH 46/80] Rewrite integration test to save dashboard to folder --- tests/integration/test_dashboards.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index e89b649a..a25d723e 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -31,11 +31,15 @@ def test_dashboards_deploys_exported_dashboard_definition(ws, dashboard_id): assert ws.lakeview.get(dashboard.dashboard_id) -def test_load_dashboard(ws): - dashboard = Dashboards(ws) - src = "/Workspace/Users/serge.smertin@databricks.com/Trivial Dashboard.lvdash.json" - dst = Path(__file__).parent / "sample" - dashboard.save_to_folder(src, dst) +def test_dashboards_saves_sql_files_to_folder(ws, tmp_path): + dashboard_file = Path(__file__).parent / "dashboards" / "dashboard.json" + with dashboard_file.open("r") as f: + lakeview_dashboard = Dashboard.from_dict(json.load(f)) + + destination = tmp_path / "test" + Dashboards(ws).save_to_folder(lakeview_dashboard, destination) + + assert len(list(destination.glob("*.sql"))) == len(lakeview_dashboard.datasets) def replace_recursively(dataklass, replace_fields): From 722866440ef92fc9fd455afdbc67f479228df79c Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 14:32:18 +0200 Subject: [PATCH 47/80] Add type hint --- src/databricks/labs/lsql/dashboards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 1b31efb3..65b990d7 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -38,7 +38,7 @@ class Dashboards: def __init__(self, ws: WorkspaceClient): self._ws = ws - def get_dashboard(self, dashboard_path: str): + def get_dashboard(self, dashboard_path: str) -> Dashboard: with self._ws.workspace.download(dashboard_path, format=ExportFormat.SOURCE) as f: raw = f.read().decode("utf-8") as_dict = json.loads(raw) From d8b478de246819717370fc00f03dc5def352ebb4 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 14:32:29 +0200 Subject: [PATCH 48/80] Let save to folder expect Dashboard --- src/databricks/labs/lsql/dashboards.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 65b990d7..8ead79aa 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -44,9 +44,10 @@ def get_dashboard(self, dashboard_path: str) -> Dashboard: as_dict = json.loads(raw) return Dashboard.from_dict(as_dict) - def save_to_folder(self, dashboard_path: str, local_path: Path): + def save_to_folder(self, dashboard: Dashboard, local_path: Path): local_path.mkdir(parents=True, exist_ok=True) - dashboard = self.with_better_names(self.get_dashboard(dashboard_path)) + + dashboard = self.with_better_names(dashboard) for dataset in dashboard.datasets: query_path = local_path / f"{dataset.display_name}.sql" self._format_sql_file(dataset.query, query_path) From adb42378ef1b1f1fa06636d08b4d07a02f53ae77 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 14:56:10 +0200 Subject: [PATCH 49/80] Move integration test to unit test --- tests/integration/test_dashboards.py | 10 ---------- tests/unit/test_dashboards.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index a25d723e..d5f731c6 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -31,16 +31,6 @@ def test_dashboards_deploys_exported_dashboard_definition(ws, dashboard_id): assert ws.lakeview.get(dashboard.dashboard_id) -def test_dashboards_saves_sql_files_to_folder(ws, tmp_path): - dashboard_file = Path(__file__).parent / "dashboards" / "dashboard.json" - with dashboard_file.open("r") as f: - lakeview_dashboard = Dashboard.from_dict(json.load(f)) - - destination = tmp_path / "test" - Dashboards(ws).save_to_folder(lakeview_dashboard, destination) - - assert len(list(destination.glob("*.sql"))) == len(lakeview_dashboard.datasets) - def replace_recursively(dataklass, replace_fields): for field in fields(dataklass): diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 9411501d..6452e5bd 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -29,6 +29,16 @@ def test_dashboard_creates_one_counter_widget_per_query(): assert len(counter_widgets) == len([query for query in queries.glob("*.sql")]) +def test_dashboards_saves_sql_files_to_folder(tmp_path): + ws = create_autospec(WorkspaceClient) + queries = Path(__file__).parent / "queries" + dashboard = Dashboards(ws).create_dashboard(queries) + + destination = tmp_path / "test" + Dashboards(ws).save_to_folder(dashboard, destination) + + assert len(list(destination.glob("*.sql"))) == len(dashboard.datasets) + def test_dashboard_deploy_raises_value_error_with_missing_display_name_and_dashboard_id(): ws = create_autospec(WorkspaceClient) From 97d80d2dbb25d6fbfd1246a8af34f69d96e1d115 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:07:07 +0200 Subject: [PATCH 50/80] Fix test names --- tests/unit/test_dashboards.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 6452e5bd..4120332e 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -8,14 +8,14 @@ from databricks.labs.lsql.lakeview.model import CounterSpec, Dashboard -def test_dashboard_creates_one_dataset_per_query(): +def test_dashboards_creates_one_dataset_per_query(): ws = create_autospec(WorkspaceClient) queries = Path(__file__).parent / "queries" dashboard = Dashboards(ws).create_dashboard(queries) assert len(dashboard.datasets) == len([query for query in queries.glob("*.sql")]) -def test_dashboard_creates_one_counter_widget_per_query(): +def test_dashboards_creates_one_counter_widget_per_query(): ws = create_autospec(WorkspaceClient) queries = Path(__file__).parent / "queries" dashboard = Dashboards(ws).create_dashboard(queries) @@ -29,7 +29,7 @@ def test_dashboard_creates_one_counter_widget_per_query(): assert len(counter_widgets) == len([query for query in queries.glob("*.sql")]) -def test_dashboards_saves_sql_files_to_folder(tmp_path): +def test_dashboardss_saves_sql_files_to_folder(tmp_path): ws = create_autospec(WorkspaceClient) queries = Path(__file__).parent / "queries" dashboard = Dashboards(ws).create_dashboard(queries) @@ -38,9 +38,10 @@ def test_dashboards_saves_sql_files_to_folder(tmp_path): Dashboards(ws).save_to_folder(dashboard, destination) assert len(list(destination.glob("*.sql"))) == len(dashboard.datasets) + ws.assert_not_called() -def test_dashboard_deploy_raises_value_error_with_missing_display_name_and_dashboard_id(): +def test_dashboards_deploy_raises_value_error_with_missing_display_name_and_dashboard_id(): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) lakeview_dashboard = Dashboard([], []) @@ -49,7 +50,7 @@ def test_dashboard_deploy_raises_value_error_with_missing_display_name_and_dashb ws.assert_not_called() -def test_dashboard_deploy_raises_value_error_with_both_display_name_and_dashboard_id(): +def test_dashboards_deploy_raises_value_error_with_both_display_name_and_dashboard_id(): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) lakeview_dashboard = Dashboard([], []) @@ -58,7 +59,7 @@ def test_dashboard_deploy_raises_value_error_with_both_display_name_and_dashboar ws.assert_not_called() -def test_dashboard_deploy_calls_create_with_display_name(): +def test_dashboards_deploy_calls_create_with_display_name(): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) lakeview_dashboard = Dashboard([], []) @@ -68,7 +69,7 @@ def test_dashboard_deploy_calls_create_with_display_name(): ws.lakeview.update.assert_not_called() -def test_dashboard_deploy_calls_update_with_dashboard_id(): +def test_dashboards_deploy_calls_update_with_dashboard_id(): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) lakeview_dashboard = Dashboard([], []) From 57c6970fb6f6be9a773d91bccf5a8ca3d372c8dc Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:07:31 +0200 Subject: [PATCH 51/80] Update lvdash yaml name --- src/databricks/labs/lsql/dashboards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 8ead79aa..5c4e88a2 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -52,7 +52,7 @@ def save_to_folder(self, dashboard: Dashboard, local_path: Path): query_path = local_path / f"{dataset.display_name}.sql" self._format_sql_file(dataset.query, query_path) for page in dashboard.pages: - lvdash_yml = local_path / f"lvdash-{page.display_name}.yml" + lvdash_yml = local_path / f"page-{page.display_name}.yml" with lvdash_yml.open("w") as f: yaml.safe_dump(page.as_dict(), f) From 8d8cb5f39cf3e4ccd5ff00d335f5edef500fd8da Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:07:39 +0200 Subject: [PATCH 52/80] Reused with better names --- tests/integration/test_dashboards.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index d5f731c6..f9222113 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -47,16 +47,13 @@ def replace_recursively(dataklass, replace_fields): return dataklass -def test_dashboard_deploys_dashboard(ws, dashboard_id): +def test_dashboard_deploys_dashboard_the_same_as_created_dashboard(ws, dashboard_id): queries = Path(__file__).parent / "dashboards" / "dashboard" dashboards = Dashboards(ws) - lakeview_dashboard = dashboards.create_dashboard(queries) + dashboard = dashboards.create_dashboard(queries) - dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=dashboard_id) - deployed_lakeview_dashboard = dashboards.get_dashboard(dashboard.path) - - replace_name = {"name": "test", "dataset_name": "test"} # Dynamically created names - lakeview_dashboard_wo_name = replace_recursively(lakeview_dashboard, replace_name) - deployed_lakeview_dashboard_wo_name = replace_recursively(deployed_lakeview_dashboard, replace_name) + sdk_dashboard = dashboards.deploy_dashboard(dashboard, dashboard_id=dashboard_id) + deployed_dashboard = dashboards.get_dashboard(sdk_dashboard.path) - assert lakeview_dashboard_wo_name.as_dict() == deployed_lakeview_dashboard_wo_name.as_dict() + assert dashboards.with_better_names(dashboard).as_dict() == dashboards.with_better_names(deployed_dashboard).as_dict() + print(1) \ No newline at end of file From f4517c33432f64da8e2335e95eb80886a3c97a74 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:15:52 +0200 Subject: [PATCH 53/80] Replace name for Widget --- src/databricks/labs/lsql/dashboards.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 5c4e88a2..1ed926fc 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -141,4 +141,6 @@ def _replace_names(self, node: T, better_names: dict[str, str]) -> T: node.name = better_names.get(node.name, node.name) elif isinstance(node, ControlFieldEncoding): node.query_name = better_names.get(node.query_name, node.query_name) + elif isinstance(node, Widget): + node.name = node.spec.as_dict().get("widgetType", node.name) return node From d73edb98a511c41de6042790fba8547361be1cf5 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:19:30 +0200 Subject: [PATCH 54/80] Replace all names --- src/databricks/labs/lsql/dashboards.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 1ed926fc..19088a1e 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -115,9 +115,13 @@ def _format_sql_file(self, sql_query, query_path): f.write(sql_query) def with_better_names(self, dashboard: Dashboard) -> Dashboard: - better_names = {dataset.name: dataset.display_name for dataset in dashboard.datasets} - pages = [self._replace_names(page, better_names) for page in dashboard.pages] - return replace(dashboard, pages=pages) + """Replace names with human-readable names.""" + datasets, better_names = [], {} + for dataset in dashboard.datasets: + datasets.append(replace(dataset, name=dataset.display_name)) + better_names[dataset.name] = dataset.display_name + pages = [replace(self._replace_names(page, better_names), name=page.display_name) for page in dashboard.pages] + return Dashboard(datasets=datasets, pages=pages) def _replace_names(self, node: T, better_names: dict[str, str]) -> T: # walk every dataclass instance recursively and replace names From 240938f80c3c94d5b13c24d60e5ad1805bc9c668 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:21:30 +0200 Subject: [PATCH 55/80] Format tests --- tests/integration/test_dashboards.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index f9222113..96362ecd 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -53,7 +53,6 @@ def test_dashboard_deploys_dashboard_the_same_as_created_dashboard(ws, dashboard dashboard = dashboards.create_dashboard(queries) sdk_dashboard = dashboards.deploy_dashboard(dashboard, dashboard_id=dashboard_id) - deployed_dashboard = dashboards.get_dashboard(sdk_dashboard.path) + new_dashboard = dashboards.get_dashboard(sdk_dashboard.path) - assert dashboards.with_better_names(dashboard).as_dict() == dashboards.with_better_names(deployed_dashboard).as_dict() - print(1) \ No newline at end of file + assert dashboards.with_better_names(dashboard).as_dict() == dashboards.with_better_names(new_dashboard).as_dict() \ No newline at end of file From 42ff42aae217918013b6cf3f4f4e665a1c3567a3 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:21:48 +0200 Subject: [PATCH 56/80] Remove unused function --- tests/integration/test_dashboards.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 96362ecd..60716901 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -31,22 +31,6 @@ def test_dashboards_deploys_exported_dashboard_definition(ws, dashboard_id): assert ws.lakeview.get(dashboard.dashboard_id) - -def replace_recursively(dataklass, replace_fields): - for field in fields(dataklass): - value = getattr(dataklass, field.name) - if is_dataclass(value): - new_value = replace_recursively(value, replace_fields) - elif isinstance(value, list): - new_value = [replace_recursively(v, replace_fields) for v in value] - elif isinstance(value, tuple): - new_value = (replace_recursively(v, replace_fields) for v in value) - else: - new_value = replace_fields.get(field.name, value) - setattr(dataklass, field.name, new_value) - return dataklass - - def test_dashboard_deploys_dashboard_the_same_as_created_dashboard(ws, dashboard_id): queries = Path(__file__).parent / "dashboards" / "dashboard" dashboards = Dashboards(ws) From b05dc7aa473d0e162a0d65926ee8154bff2a1194 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:22:17 +0200 Subject: [PATCH 57/80] Fix typo --- tests/unit/test_dashboards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 4120332e..4ce21416 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -29,7 +29,7 @@ def test_dashboards_creates_one_counter_widget_per_query(): assert len(counter_widgets) == len([query for query in queries.glob("*.sql")]) -def test_dashboardss_saves_sql_files_to_folder(tmp_path): +def test_dashboards_saves_sql_files_to_folder(tmp_path): ws = create_autospec(WorkspaceClient) queries = Path(__file__).parent / "queries" dashboard = Dashboards(ws).create_dashboard(queries) From 163f35128fffb641b3c4f1401d9f115b0d13d289 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:22:49 +0200 Subject: [PATCH 58/80] Move test up --- tests/unit/test_dashboards.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 4ce21416..9ad41ccc 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -8,6 +8,18 @@ from databricks.labs.lsql.lakeview.model import CounterSpec, Dashboard +def test_dashboards_saves_sql_files_to_folder(tmp_path): + ws = create_autospec(WorkspaceClient) + queries = Path(__file__).parent / "queries" + dashboard = Dashboards(ws).create_dashboard(queries) + + destination = tmp_path / "test" + Dashboards(ws).save_to_folder(dashboard, destination) + + assert len(list(destination.glob("*.sql"))) == len(dashboard.datasets) + ws.assert_not_called() + + def test_dashboards_creates_one_dataset_per_query(): ws = create_autospec(WorkspaceClient) queries = Path(__file__).parent / "queries" @@ -29,18 +41,6 @@ def test_dashboards_creates_one_counter_widget_per_query(): assert len(counter_widgets) == len([query for query in queries.glob("*.sql")]) -def test_dashboards_saves_sql_files_to_folder(tmp_path): - ws = create_autospec(WorkspaceClient) - queries = Path(__file__).parent / "queries" - dashboard = Dashboards(ws).create_dashboard(queries) - - destination = tmp_path / "test" - Dashboards(ws).save_to_folder(dashboard, destination) - - assert len(list(destination.glob("*.sql"))) == len(dashboard.datasets) - ws.assert_not_called() - - def test_dashboards_deploy_raises_value_error_with_missing_display_name_and_dashboard_id(): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) From e4b355168ff36ccd2f0971d718a7f4ab7100b36b Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:23:49 +0200 Subject: [PATCH 59/80] Test for yml to be created --- tests/unit/test_dashboards.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 9ad41ccc..56844f91 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -20,6 +20,17 @@ def test_dashboards_saves_sql_files_to_folder(tmp_path): ws.assert_not_called() +def test_dashboards_saves_yml_files_to_folder(tmp_path): + ws = create_autospec(WorkspaceClient) + queries = Path(__file__).parent / "queries" + dashboard = Dashboards(ws).create_dashboard(queries) + + Dashboards(ws).save_to_folder(dashboard, tmp_path) + + assert len(list(tmp_path.glob("*.yml"))) == len(dashboard.pages) + ws.assert_not_called() + + def test_dashboards_creates_one_dataset_per_query(): ws = create_autospec(WorkspaceClient) queries = Path(__file__).parent / "queries" From 5ce4ccefbcec50b7d1ca15204f5c3a2b4956c545 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:24:06 +0200 Subject: [PATCH 60/80] Short test --- tests/unit/test_dashboards.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 56844f91..542781cd 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -13,10 +13,9 @@ def test_dashboards_saves_sql_files_to_folder(tmp_path): queries = Path(__file__).parent / "queries" dashboard = Dashboards(ws).create_dashboard(queries) - destination = tmp_path / "test" - Dashboards(ws).save_to_folder(dashboard, destination) + Dashboards(ws).save_to_folder(dashboard, tmp_path) - assert len(list(destination.glob("*.sql"))) == len(dashboard.datasets) + assert len(list(tmp_path.glob("*.sql"))) == len(dashboard.datasets) ws.assert_not_called() From b6154160dc15e356abdeac61381e3f1a16fb0698 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:28:39 +0200 Subject: [PATCH 61/80] Test dataset names --- tests/unit/test_dashboards.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 542781cd..a6fb8735 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -5,7 +5,7 @@ from databricks.sdk import WorkspaceClient from databricks.labs.lsql.dashboards import Dashboards -from databricks.labs.lsql.lakeview.model import CounterSpec, Dashboard +from databricks.labs.lsql.lakeview.model import CounterSpec, Dataset, Dashboard def test_dashboards_saves_sql_files_to_folder(tmp_path): @@ -87,3 +87,14 @@ def test_dashboards_deploy_calls_update_with_dashboard_id(): ws.lakeview.create.assert_not_called() ws.lakeview.update.assert_called_once() + + +def test_dashboards_with_better_names_replaces_dataset_names_with_display_names(): + ws = create_autospec(WorkspaceClient) + dashboards = Dashboards(ws) + + datasets = [Dataset(name="ugly", query="SELECT 1", display_name="pretty")] + dashboard = dashboards.with_better_names(Dashboard(datasets, [])) + + assert all(dataset.name == "pretty" for dataset in dashboard.datasets) + ws.assert_not_called() From de98f165fe7af08de3c0113bf479e3f5afa8cc2b Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:30:11 +0200 Subject: [PATCH 62/80] Test page names --- tests/unit/test_dashboards.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index a6fb8735..82e8e7c1 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -5,7 +5,7 @@ from databricks.sdk import WorkspaceClient from databricks.labs.lsql.dashboards import Dashboards -from databricks.labs.lsql.lakeview.model import CounterSpec, Dataset, Dashboard +from databricks.labs.lsql.lakeview.model import CounterSpec, Dataset, Dashboard, Page def test_dashboards_saves_sql_files_to_folder(tmp_path): @@ -98,3 +98,14 @@ def test_dashboards_with_better_names_replaces_dataset_names_with_display_names( assert all(dataset.name == "pretty" for dataset in dashboard.datasets) ws.assert_not_called() + + +def test_dashboards_with_better_names_replaces_page_names_with_display_names(): + ws = create_autospec(WorkspaceClient) + dashboards = Dashboards(ws) + + pages = [Page(name="ugly", layout=[], display_name="pretty")] + dashboard = dashboards.with_better_names(Dashboard([], pages)) + + assert all(page.name == "pretty" for page in dashboard.pages) + ws.assert_not_called() From 06f6a1082511eaf7e7e23591426d28a5d6ef208c Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:38:22 +0200 Subject: [PATCH 63/80] Test query names --- tests/unit/test_dashboards.py | 40 ++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 82e8e7c1..5d97e2d0 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -5,7 +5,19 @@ from databricks.sdk import WorkspaceClient from databricks.labs.lsql.dashboards import Dashboards -from databricks.labs.lsql.lakeview.model import CounterSpec, Dataset, Dashboard, Page +from databricks.labs.lsql.lakeview import ( + CounterEncodingMap, + CounterFieldEncoding, + CounterSpec, + Dashboard, + Dataset, + Layout, + NamedQuery, + Page, + Position, + Query, + Widget, +) def test_dashboards_saves_sql_files_to_folder(tmp_path): @@ -109,3 +121,29 @@ def test_dashboards_with_better_names_replaces_page_names_with_display_names(): assert all(page.name == "pretty" for page in dashboard.pages) ws.assert_not_called() + + +def test_dashboards_with_better_names_replaces_query_name_with_dataset_name(): + ws = create_autospec(WorkspaceClient) + dashboards = Dashboards(ws) + + datasets = [Dataset(name="ugly", query="SELECT 1", display_name="pretty")] + + query = Query(dataset_name="ugly", fields=[]) + named_query = NamedQuery(name="main_query", query=query) + counter_spec = CounterSpec(CounterEncodingMap()) + widget = Widget(name="ugly", queries=[named_query], spec=counter_spec) + position = Position(x=0, y=0, width=1, height=1) + layout = Layout(widget=widget, position=position) + pages = [Page(name="ugly", layout=[layout], display_name="pretty")] + + dashboard = dashboards.with_better_names(Dashboard(datasets, pages)) + + queries = [] + for page in dashboard.pages: + for layout in page.layout: + for named_query in layout.widget.queries: + queries.append(named_query.query) + + assert all(query.dataset_name == "pretty" for query in queries) + ws.assert_not_called() From 794ba6d2d0663d0743e850a8a17453cfca477dc6 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:40:08 +0200 Subject: [PATCH 64/80] Move ugly dashboard into fixtures --- tests/unit/test_dashboards.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 5d97e2d0..b63b7c00 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -123,10 +123,8 @@ def test_dashboards_with_better_names_replaces_page_names_with_display_names(): ws.assert_not_called() -def test_dashboards_with_better_names_replaces_query_name_with_dataset_name(): - ws = create_autospec(WorkspaceClient) - dashboards = Dashboards(ws) - +@pytest.fixture +def ugly_dashboard() -> Dashboard: datasets = [Dataset(name="ugly", query="SELECT 1", display_name="pretty")] query = Query(dataset_name="ugly", fields=[]) @@ -137,7 +135,15 @@ def test_dashboards_with_better_names_replaces_query_name_with_dataset_name(): layout = Layout(widget=widget, position=position) pages = [Page(name="ugly", layout=[layout], display_name="pretty")] - dashboard = dashboards.with_better_names(Dashboard(datasets, pages)) + dashboard = Dashboard(datasets, pages) + return dashboard + + +def test_dashboards_with_better_names_replaces_query_name_with_dataset_name(ugly_dashboard): + ws = create_autospec(WorkspaceClient) + dashboards = Dashboards(ws) + + dashboard = dashboards.with_better_names(ugly_dashboard) queries = [] for page in dashboard.pages: From e6f4a186a4617fd814c3d2879d344f2ba8dbee57 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:43:34 +0200 Subject: [PATCH 65/80] Add test for replacing counter names --- tests/unit/test_dashboards.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index b63b7c00..4ca8b119 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -153,3 +153,19 @@ def test_dashboards_with_better_names_replaces_query_name_with_dataset_name(ugly assert all(query.dataset_name == "pretty" for query in queries) ws.assert_not_called() + + +def test_dashboards_with_better_names_replaces_counter_names(ugly_dashboard): + ws = create_autospec(WorkspaceClient) + dashboards = Dashboards(ws) + + dashboard = dashboards.with_better_names(ugly_dashboard) + + counters = [] + for page in dashboard.pages: + for layout in page.layout: + if isinstance(layout.widget.spec, CounterSpec): + counters.append(layout.widget) + + assert all(counter.name == "counter" for counter in counters) + ws.assert_not_called() From 9e883aab7ecb791a2726b942fd926bce958f39cc Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:44:06 +0200 Subject: [PATCH 66/80] Remove unused imports --- tests/unit/test_dashboards.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 4ca8b119..7d4b45a7 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -7,7 +7,6 @@ from databricks.labs.lsql.dashboards import Dashboards from databricks.labs.lsql.lakeview import ( CounterEncodingMap, - CounterFieldEncoding, CounterSpec, Dashboard, Dataset, From 30d3959653ca3330d470ecb32ea2b389dc69008f Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:44:48 +0200 Subject: [PATCH 67/80] Refactor lakeview_dashboard to dashboard --- tests/unit/test_dashboards.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 7d4b45a7..72e1fc4a 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -65,26 +65,26 @@ def test_dashboards_creates_one_counter_widget_per_query(): def test_dashboards_deploy_raises_value_error_with_missing_display_name_and_dashboard_id(): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) - lakeview_dashboard = Dashboard([], []) + dashboard = Dashboard([], []) with pytest.raises(ValueError): - dashboards.deploy_dashboard(lakeview_dashboard) + dashboards.deploy_dashboard(dashboard) ws.assert_not_called() def test_dashboards_deploy_raises_value_error_with_both_display_name_and_dashboard_id(): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) - lakeview_dashboard = Dashboard([], []) + dashboard = Dashboard([], []) with pytest.raises(ValueError): - dashboards.deploy_dashboard(lakeview_dashboard, display_name="test", dashboard_id="test") + dashboards.deploy_dashboard(dashboard, display_name="test", dashboard_id="test") ws.assert_not_called() def test_dashboards_deploy_calls_create_with_display_name(): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) - lakeview_dashboard = Dashboard([], []) - dashboards.deploy_dashboard(lakeview_dashboard, display_name="test") + dashboard = Dashboard([], []) + dashboards.deploy_dashboard(dashboard, display_name="test") ws.lakeview.create.assert_called_once() ws.lakeview.update.assert_not_called() @@ -93,8 +93,8 @@ def test_dashboards_deploy_calls_create_with_display_name(): def test_dashboards_deploy_calls_update_with_dashboard_id(): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) - lakeview_dashboard = Dashboard([], []) - dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id="test") + dashboard = Dashboard([], []) + dashboards.deploy_dashboard(dashboard, dashboard_id="test") ws.lakeview.create.assert_not_called() ws.lakeview.update.assert_called_once() From b2bc6df8e98e67e01a73073a2abbb97bbfecfb82 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:46:46 +0200 Subject: [PATCH 68/80] Reuse dataclasses methods --- src/databricks/labs/lsql/dashboards.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 19088a1e..2915c704 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -1,7 +1,7 @@ import json -from dataclasses import replace +import dataclasses from pathlib import Path -from typing import ClassVar, Protocol, runtime_checkable, TypeVar +from typing import TypeVar import sqlglot import yaml @@ -29,11 +29,6 @@ T = TypeVar("T") -@runtime_checkable -class _DataclassInstance(Protocol): - __dataclass_fields__: ClassVar[dict] - - class Dashboards: def __init__(self, ws: WorkspaceClient): self._ws = ws @@ -118,19 +113,19 @@ def with_better_names(self, dashboard: Dashboard) -> Dashboard: """Replace names with human-readable names.""" datasets, better_names = [], {} for dataset in dashboard.datasets: - datasets.append(replace(dataset, name=dataset.display_name)) + datasets.append(dataclasses.replace(dataset, name=dataset.display_name)) better_names[dataset.name] = dataset.display_name - pages = [replace(self._replace_names(page, better_names), name=page.display_name) for page in dashboard.pages] + pages = [dataclasses.replace(self._replace_names(page, better_names), name=page.display_name) for page in dashboard.pages] return Dashboard(datasets=datasets, pages=pages) def _replace_names(self, node: T, better_names: dict[str, str]) -> T: # walk every dataclass instance recursively and replace names - if isinstance(node, _DataclassInstance): - for field in node.__dataclass_fields__.values(): + if dataclasses.is_dataclass(node): + for field in dataclasses.fields(node): value = getattr(node, field.name) if isinstance(value, list): setattr(node, field.name, [self._replace_names(item, better_names) for item in value]) - elif isinstance(value, _DataclassInstance): + elif dataclasses.is_dataclass(value): setattr(node, field.name, self._replace_names(value, better_names)) if isinstance(node, Query): node.dataset_name = better_names.get(node.dataset_name, node.dataset_name) From d44ea96ea5b5fc96c0ab1e1620f4175d1e247faa Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:51:31 +0200 Subject: [PATCH 69/80] Format --- src/databricks/labs/lsql/dashboards.py | 23 +++++++++++++++-------- tests/integration/test_dashboards.py | 3 +-- tests/unit/test_dashboards.py | 2 +- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 2915c704..f47f0f49 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -1,5 +1,5 @@ -import json import dataclasses +import json from pathlib import Path from typing import TypeVar @@ -25,7 +25,6 @@ Widget, ) - T = TypeVar("T") @@ -113,9 +112,16 @@ def with_better_names(self, dashboard: Dashboard) -> Dashboard: """Replace names with human-readable names.""" datasets, better_names = [], {} for dataset in dashboard.datasets: - datasets.append(dataclasses.replace(dataset, name=dataset.display_name)) - better_names[dataset.name] = dataset.display_name - pages = [dataclasses.replace(self._replace_names(page, better_names), name=page.display_name) for page in dashboard.pages] + if dataset.display_name is not None: + datasets.append(dataclasses.replace(dataset, name=dataset.display_name)) + better_names[dataset.name] = dataset.display_name if dataset.display_name is not None else dataset.name + + pages = [] + for page in dashboard.pages: + better_page = self._replace_names(page, better_names) + if better_page.display_name is not None: + better_page = dataclasses.replace(better_page, name=better_page.display_name) + pages.append(better_page) return Dashboard(datasets=datasets, pages=pages) def _replace_names(self, node: T, better_names: dict[str, str]) -> T: @@ -133,13 +139,14 @@ def _replace_names(self, node: T, better_names: dict[str, str]) -> T: # 'dashboards/01eeb077e38c17e6ba3511036985960c/datasets/01eeb081882017f6a116991d124d3068_...' if node.name.startswith("dashboards/"): parts = [node.query.dataset_name] - for field in node.query.fields: - parts.append(field.name) + for query_field in node.query.fields: + parts.append(query_field.name) new_name = "_".join(parts) better_names[node.name] = new_name node.name = better_names.get(node.name, node.name) elif isinstance(node, ControlFieldEncoding): node.query_name = better_names.get(node.query_name, node.query_name) elif isinstance(node, Widget): - node.name = node.spec.as_dict().get("widgetType", node.name) + if node.spec is not None: + node.name = node.spec.as_dict().get("widgetType", node.name) return node diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 60716901..51467e0a 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -1,5 +1,4 @@ import json -from dataclasses import fields, is_dataclass from pathlib import Path import pytest @@ -39,4 +38,4 @@ def test_dashboard_deploys_dashboard_the_same_as_created_dashboard(ws, dashboard sdk_dashboard = dashboards.deploy_dashboard(dashboard, dashboard_id=dashboard_id) new_dashboard = dashboards.get_dashboard(sdk_dashboard.path) - assert dashboards.with_better_names(dashboard).as_dict() == dashboards.with_better_names(new_dashboard).as_dict() \ No newline at end of file + assert dashboards.with_better_names(dashboard).as_dict() == dashboards.with_better_names(new_dashboard).as_dict() diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 72e1fc4a..be6a0200 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -1,5 +1,5 @@ -from unittest.mock import create_autospec from pathlib import Path +from unittest.mock import create_autospec import pytest from databricks.sdk import WorkspaceClient From 1eecdc034a74260901484ddeec9c3f8362e2a555 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:55:18 +0200 Subject: [PATCH 70/80] Remove redundant if --- src/databricks/labs/lsql/dashboards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index f47f0f49..ff8993b5 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -114,7 +114,7 @@ def with_better_names(self, dashboard: Dashboard) -> Dashboard: for dataset in dashboard.datasets: if dataset.display_name is not None: datasets.append(dataclasses.replace(dataset, name=dataset.display_name)) - better_names[dataset.name] = dataset.display_name if dataset.display_name is not None else dataset.name + better_names[dataset.name] = dataset.display_name pages = [] for page in dashboard.pages: From 8ece444bdcbb72f526261f69550dae1b5b449a4e Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 15:56:36 +0200 Subject: [PATCH 71/80] Remove redundant if --- src/databricks/labs/lsql/dashboards.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index ff8993b5..094ec36f 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -118,10 +118,9 @@ def with_better_names(self, dashboard: Dashboard) -> Dashboard: pages = [] for page in dashboard.pages: - better_page = self._replace_names(page, better_names) - if better_page.display_name is not None: - better_page = dataclasses.replace(better_page, name=better_page.display_name) - pages.append(better_page) + better_page = dataclasses.replace(page, name=page.display_name or page.name) + pages.append(self._replace_names(better_page, better_names)) + return Dashboard(datasets=datasets, pages=pages) def _replace_names(self, node: T, better_names: dict[str, str]) -> T: From 322b0a860675a25d1eebad7f578955bf6bf6c096 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 16:10:38 +0200 Subject: [PATCH 72/80] Refactor format query --- src/databricks/labs/lsql/dashboards.py | 45 +++++++++++++++----------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 094ec36f..f3e17bfb 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -43,13 +43,38 @@ def save_to_folder(self, dashboard: Dashboard, local_path: Path): dashboard = self.with_better_names(dashboard) for dataset in dashboard.datasets: - query_path = local_path / f"{dataset.display_name}.sql" - self._format_sql_file(dataset.query, query_path) + query = self.format_query(dataset.query) + with (local_path / f"{dataset.display_name}.sql").open("w") as f: + f.write(query) for page in dashboard.pages: lvdash_yml = local_path / f"page-{page.display_name}.yml" with lvdash_yml.open("w") as f: yaml.safe_dump(page.as_dict(), f) + @staticmethod + def format_query(query: str) -> str: + try: + parsed_query = sqlglot.parse(query) + except sqlglot.ParseError: + formatted_query = query + else: + statements = [] + for statement in parsed_query: + if statement is None: + continue + # see https://sqlglot.com/sqlglot/generator.html#Generator + statements.append( + statement.sql( + dialect="databricks", + normalize=True, # normalize identifiers to lowercase + pretty=True, # format the produced SQL string + normalize_functions="upper", # normalize function names to uppercase + max_text_width=80, # wrap text at 120 characters + ) + ) + formatted_query = ";\n".join(statements) + return formatted_query + @staticmethod def create_dashboard(dashboard_folder: Path) -> Dashboard: """Create a dashboard from code, i.e. configuration and queries.""" @@ -92,22 +117,6 @@ def deploy_dashboard( ) return dashboard - def _format_sql_file(self, sql_query, query_path): - with query_path.open("w") as f: - try: - for statement in sqlglot.parse(sql_query): - # see https://sqlglot.com/sqlglot/generator.html#Generator - pretty = statement.sql( - dialect="databricks", - normalize=True, # normalize identifiers to lowercase - pretty=True, # format the produced SQL string - normalize_functions="upper", # normalize function names to uppercase - max_text_width=80, # wrap text at 120 characters - ) - f.write(f"{pretty};\n") - except sqlglot.ParseError: - f.write(sql_query) - def with_better_names(self, dashboard: Dashboard) -> Dashboard: """Replace names with human-readable names.""" datasets, better_names = [], {} From 03667bbec1dd3b2a9ed1aa873986033c8ef38939 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 28 May 2024 16:18:03 +0200 Subject: [PATCH 73/80] Save files with names --- src/databricks/labs/lsql/dashboards.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index f3e17bfb..e91d040f 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -42,13 +42,14 @@ def save_to_folder(self, dashboard: Dashboard, local_path: Path): local_path.mkdir(parents=True, exist_ok=True) dashboard = self.with_better_names(dashboard) + for dataset in dashboard.datasets: query = self.format_query(dataset.query) - with (local_path / f"{dataset.display_name}.sql").open("w") as f: + with (local_path / f"{dataset.name}.sql").open("w") as f: f.write(query) + for page in dashboard.pages: - lvdash_yml = local_path / f"page-{page.display_name}.yml" - with lvdash_yml.open("w") as f: + with (local_path / f"{page.name}.yml").open("w") as f: yaml.safe_dump(page.as_dict(), f) @staticmethod From 52f8ad693f6423f2403579c83f71c98aeb3c6faf Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 31 May 2024 10:10:55 +0200 Subject: [PATCH 74/80] Make format query private --- src/databricks/labs/lsql/dashboards.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index e91d040f..ca516b9b 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -44,7 +44,7 @@ def save_to_folder(self, dashboard: Dashboard, local_path: Path): dashboard = self.with_better_names(dashboard) for dataset in dashboard.datasets: - query = self.format_query(dataset.query) + query = self._format_query(dataset.query) with (local_path / f"{dataset.name}.sql").open("w") as f: f.write(query) @@ -53,7 +53,7 @@ def save_to_folder(self, dashboard: Dashboard, local_path: Path): yaml.safe_dump(page.as_dict(), f) @staticmethod - def format_query(query: str) -> str: + def _format_query(query: str) -> str: try: parsed_query = sqlglot.parse(query) except sqlglot.ParseError: From 820bbdeb8d6195ef95d6cd531154f1c9153cb1b4 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 31 May 2024 10:12:09 +0200 Subject: [PATCH 75/80] Return early --- src/databricks/labs/lsql/dashboards.py | 31 +++++++++++++------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index ca516b9b..66fc3f1c 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -57,23 +57,22 @@ def _format_query(query: str) -> str: try: parsed_query = sqlglot.parse(query) except sqlglot.ParseError: - formatted_query = query - else: - statements = [] - for statement in parsed_query: - if statement is None: - continue - # see https://sqlglot.com/sqlglot/generator.html#Generator - statements.append( - statement.sql( - dialect="databricks", - normalize=True, # normalize identifiers to lowercase - pretty=True, # format the produced SQL string - normalize_functions="upper", # normalize function names to uppercase - max_text_width=80, # wrap text at 120 characters - ) + return query + statements = [] + for statement in parsed_query: + if statement is None: + continue + # see https://sqlglot.com/sqlglot/generator.html#Generator + statements.append( + statement.sql( + dialect="databricks", + normalize=True, # normalize identifiers to lowercase + pretty=True, # format the produced SQL string + normalize_functions="upper", # normalize function names to uppercase + max_text_width=80, # wrap text at 120 characters ) - formatted_query = ";\n".join(statements) + ) + formatted_query = ";\n".join(statements) return formatted_query @staticmethod From df37440f16db7657b378667b66c6f09fa89c1cd8 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 31 May 2024 10:12:52 +0200 Subject: [PATCH 76/80] Make with better names private --- src/databricks/labs/lsql/dashboards.py | 4 ++-- tests/integration/test_dashboards.py | 2 +- tests/unit/test_dashboards.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 66fc3f1c..129c9b26 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -41,7 +41,7 @@ def get_dashboard(self, dashboard_path: str) -> Dashboard: def save_to_folder(self, dashboard: Dashboard, local_path: Path): local_path.mkdir(parents=True, exist_ok=True) - dashboard = self.with_better_names(dashboard) + dashboard = self._with_better_names(dashboard) for dataset in dashboard.datasets: query = self._format_query(dataset.query) @@ -117,7 +117,7 @@ def deploy_dashboard( ) return dashboard - def with_better_names(self, dashboard: Dashboard) -> Dashboard: + def _with_better_names(self, dashboard: Dashboard) -> Dashboard: """Replace names with human-readable names.""" datasets, better_names = [], {} for dataset in dashboard.datasets: diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 51467e0a..3e49cbe8 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -38,4 +38,4 @@ def test_dashboard_deploys_dashboard_the_same_as_created_dashboard(ws, dashboard sdk_dashboard = dashboards.deploy_dashboard(dashboard, dashboard_id=dashboard_id) new_dashboard = dashboards.get_dashboard(sdk_dashboard.path) - assert dashboards.with_better_names(dashboard).as_dict() == dashboards.with_better_names(new_dashboard).as_dict() + assert dashboards._with_better_names(dashboard).as_dict() == dashboards._with_better_names(new_dashboard).as_dict() diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index be6a0200..900df76b 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -105,7 +105,7 @@ def test_dashboards_with_better_names_replaces_dataset_names_with_display_names( dashboards = Dashboards(ws) datasets = [Dataset(name="ugly", query="SELECT 1", display_name="pretty")] - dashboard = dashboards.with_better_names(Dashboard(datasets, [])) + dashboard = dashboards._with_better_names(Dashboard(datasets, [])) assert all(dataset.name == "pretty" for dataset in dashboard.datasets) ws.assert_not_called() @@ -116,7 +116,7 @@ def test_dashboards_with_better_names_replaces_page_names_with_display_names(): dashboards = Dashboards(ws) pages = [Page(name="ugly", layout=[], display_name="pretty")] - dashboard = dashboards.with_better_names(Dashboard([], pages)) + dashboard = dashboards._with_better_names(Dashboard([], pages)) assert all(page.name == "pretty" for page in dashboard.pages) ws.assert_not_called() @@ -142,7 +142,7 @@ def test_dashboards_with_better_names_replaces_query_name_with_dataset_name(ugly ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) - dashboard = dashboards.with_better_names(ugly_dashboard) + dashboard = dashboards._with_better_names(ugly_dashboard) queries = [] for page in dashboard.pages: @@ -158,7 +158,7 @@ def test_dashboards_with_better_names_replaces_counter_names(ugly_dashboard): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) - dashboard = dashboards.with_better_names(ugly_dashboard) + dashboard = dashboards._with_better_names(ugly_dashboard) counters = [] for page in dashboard.pages: From 550134a356c9cf51be8e5b7dde510432048285b1 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 31 May 2024 10:14:25 +0200 Subject: [PATCH 77/80] Move dataset replace names into replace names method --- src/databricks/labs/lsql/dashboards.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 129c9b26..47c74bec 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -141,7 +141,9 @@ def _replace_names(self, node: T, better_names: dict[str, str]) -> T: setattr(node, field.name, [self._replace_names(item, better_names) for item in value]) elif dataclasses.is_dataclass(value): setattr(node, field.name, self._replace_names(value, better_names)) - if isinstance(node, Query): + if isinstance(node, Dataset): + node.name = better_names.get(node.name, node.name) + elif isinstance(node, Query): node.dataset_name = better_names.get(node.dataset_name, node.dataset_name) elif isinstance(node, NamedQuery) and node.query: # 'dashboards/01eeb077e38c17e6ba3511036985960c/datasets/01eeb081882017f6a116991d124d3068_...' From a45788e29e5600cc79ca701a69c0664af6994a67 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 31 May 2024 10:17:21 +0200 Subject: [PATCH 78/80] Move dataset and page replace names into replace names method --- src/databricks/labs/lsql/dashboards.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 47c74bec..387d7ebd 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -119,18 +119,14 @@ def deploy_dashboard( def _with_better_names(self, dashboard: Dashboard) -> Dashboard: """Replace names with human-readable names.""" - datasets, better_names = [], {} + better_names = {} for dataset in dashboard.datasets: if dataset.display_name is not None: - datasets.append(dataclasses.replace(dataset, name=dataset.display_name)) better_names[dataset.name] = dataset.display_name - - pages = [] for page in dashboard.pages: - better_page = dataclasses.replace(page, name=page.display_name or page.name) - pages.append(self._replace_names(better_page, better_names)) - - return Dashboard(datasets=datasets, pages=pages) + if page.display_name is not None: + better_names[page.name] = page.display_name + return self._replace_names(dashboard, better_names) def _replace_names(self, node: T, better_names: dict[str, str]) -> T: # walk every dataclass instance recursively and replace names @@ -143,6 +139,8 @@ def _replace_names(self, node: T, better_names: dict[str, str]) -> T: setattr(node, field.name, self._replace_names(value, better_names)) if isinstance(node, Dataset): node.name = better_names.get(node.name, node.name) + elif isinstance(node, Page): + node.name = better_names.get(node.name, node.name) elif isinstance(node, Query): node.dataset_name = better_names.get(node.dataset_name, node.dataset_name) elif isinstance(node, NamedQuery) and node.query: From 4c52d156b52247c8668af007f56c12bb1e29ef9d Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 31 May 2024 10:21:27 +0200 Subject: [PATCH 79/80] Test replace names through public save to folder --- src/databricks/labs/lsql/dashboards.py | 6 ++---- tests/unit/test_dashboards.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/databricks/labs/lsql/dashboards.py b/src/databricks/labs/lsql/dashboards.py index 387d7ebd..51c25c02 100644 --- a/src/databricks/labs/lsql/dashboards.py +++ b/src/databricks/labs/lsql/dashboards.py @@ -38,19 +38,17 @@ def get_dashboard(self, dashboard_path: str) -> Dashboard: as_dict = json.loads(raw) return Dashboard.from_dict(as_dict) - def save_to_folder(self, dashboard: Dashboard, local_path: Path): + def save_to_folder(self, dashboard: Dashboard, local_path: Path) -> Dashboard: local_path.mkdir(parents=True, exist_ok=True) - dashboard = self._with_better_names(dashboard) - for dataset in dashboard.datasets: query = self._format_query(dataset.query) with (local_path / f"{dataset.name}.sql").open("w") as f: f.write(query) - for page in dashboard.pages: with (local_path / f"{page.name}.yml").open("w") as f: yaml.safe_dump(page.as_dict(), f) + return dashboard @staticmethod def _format_query(query: str) -> str: diff --git a/tests/unit/test_dashboards.py b/tests/unit/test_dashboards.py index 900df76b..d2116fab 100644 --- a/tests/unit/test_dashboards.py +++ b/tests/unit/test_dashboards.py @@ -100,23 +100,23 @@ def test_dashboards_deploy_calls_update_with_dashboard_id(): ws.lakeview.update.assert_called_once() -def test_dashboards_with_better_names_replaces_dataset_names_with_display_names(): +def test_dashboards_save_to_folder_replaces_dataset_names_with_display_names(tmp_path): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) datasets = [Dataset(name="ugly", query="SELECT 1", display_name="pretty")] - dashboard = dashboards._with_better_names(Dashboard(datasets, [])) + dashboard = dashboards.save_to_folder(Dashboard(datasets, []), tmp_path) assert all(dataset.name == "pretty" for dataset in dashboard.datasets) ws.assert_not_called() -def test_dashboards_with_better_names_replaces_page_names_with_display_names(): +def test_dashboards_save_to_folder_replaces_page_names_with_display_names(tmp_path): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) pages = [Page(name="ugly", layout=[], display_name="pretty")] - dashboard = dashboards._with_better_names(Dashboard([], pages)) + dashboard = dashboards.save_to_folder(Dashboard([], pages), tmp_path) assert all(page.name == "pretty" for page in dashboard.pages) ws.assert_not_called() @@ -138,11 +138,11 @@ def ugly_dashboard() -> Dashboard: return dashboard -def test_dashboards_with_better_names_replaces_query_name_with_dataset_name(ugly_dashboard): +def test_dashboards_save_to_folder_replaces_query_name_with_dataset_name(ugly_dashboard, tmp_path): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) - dashboard = dashboards._with_better_names(ugly_dashboard) + dashboard = dashboards.save_to_folder(ugly_dashboard, tmp_path) queries = [] for page in dashboard.pages: @@ -154,11 +154,11 @@ def test_dashboards_with_better_names_replaces_query_name_with_dataset_name(ugly ws.assert_not_called() -def test_dashboards_with_better_names_replaces_counter_names(ugly_dashboard): +def test_dashboards_save_to_folder_replaces_counter_names(ugly_dashboard, tmp_path): ws = create_autospec(WorkspaceClient) dashboards = Dashboards(ws) - dashboard = dashboards._with_better_names(ugly_dashboard) + dashboard = dashboards.save_to_folder(ugly_dashboard, tmp_path) counters = [] for page in dashboard.pages: From 14a6352f424b9781654326b09eec614fa1d41779 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 31 May 2024 10:23:38 +0200 Subject: [PATCH 80/80] Update dashboard folder name --- .../dashboards/{dashboard => one_counter}/counter.sql | 0 tests/integration/test_dashboards.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/integration/dashboards/{dashboard => one_counter}/counter.sql (100%) diff --git a/tests/integration/dashboards/dashboard/counter.sql b/tests/integration/dashboards/one_counter/counter.sql similarity index 100% rename from tests/integration/dashboards/dashboard/counter.sql rename to tests/integration/dashboards/one_counter/counter.sql diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index 3e49cbe8..97e2048d 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -31,7 +31,7 @@ def test_dashboards_deploys_exported_dashboard_definition(ws, dashboard_id): def test_dashboard_deploys_dashboard_the_same_as_created_dashboard(ws, dashboard_id): - queries = Path(__file__).parent / "dashboards" / "dashboard" + queries = Path(__file__).parent / "dashboards" / "one_counter" dashboards = Dashboards(ws) dashboard = dashboards.create_dashboard(queries)