From 1439b06545756477d2a192aef8c66e9c47d33dc7 Mon Sep 17 00:00:00 2001 From: ChenyuLi Date: Mon, 21 Mar 2022 11:09:12 -0400 Subject: [PATCH 1/4] add compliation and cache tracking --- core/dbt/task/runnable.py | 12 ++++++++++++ core/dbt/tracking.py | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index 7ddbd3cf0a0..0ec7510418d 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -50,6 +50,7 @@ from dbt.graph import GraphQueue, NodeSelector, SelectionSpec, parse_difference, Graph from dbt.parser.manifest import ManifestLoader +import dbt.tracking import dbt.exceptions from dbt import flags @@ -88,7 +89,12 @@ def compile_manifest(self): def _runtime_initialize(self): self.load_manifest() + + start_compile_manifest = time.perf_counter() self.compile_manifest() + compile_time = time.perf_counter() - start_compile_manifest + if dbt.tracking.active_user is not None: + dbt.tracking.track_runnable_timing({"graph_compilation_elapsed": compile_time}) class GraphRunnableTask(ManifestTask): @@ -391,7 +397,13 @@ def _mark_dependent_errors(self, node_id, result, cause): self._skipped_children[dep_node_id] = cause def populate_adapter_cache(self, adapter): + start_populate_cache = time.perf_counter() adapter.set_relations_cache(self.manifest) + cache_populate_time = time.perf_counter() - start_populate_cache + if dbt.tracking.active_user is not None: + dbt.tracking.track_runnable_timing( + {"adapter_cache_construction_elapsed": cache_populate_time} + ) def before_hooks(self, adapter): pass diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py index c160815075f..52db2dc4cb9 100644 --- a/core/dbt/tracking.py +++ b/core/dbt/tracking.py @@ -44,6 +44,7 @@ RESOURCE_COUNTS = "iglu:com.dbt/resource_counts/jsonschema/1-0-0" EXPERIMENTAL_PARSER = "iglu:com.dbt/experimental_parser/jsonschema/1-0-0" PARTIAL_PARSER = "iglu:com.dbt/partial_parser/jsonschema/1-0-1" +RUNNABLE_TIMING = "iglu:com.dbt/runnable/jsonschema/1-0-0" DBT_INVOCATION_ENV = "DBT_INVOCATION_ENV" @@ -412,6 +413,18 @@ def track_partial_parser(options): ) +def track_runnable_timing(options): + context = [SelfDescribingJson(RUNNABLE_TIMING, options)] + assert active_user is not None, "Cannot track runnable info when active user is None" + track( + active_user, + category="dbt", + action="runnable", + label=get_invocation_id(), + context=context, + ) + + def flush(): fire_event(FlushEvents()) try: From 5cc2519edb747041bc9d8c126ca0cde0f56f23a8 Mon Sep 17 00:00:00 2001 From: ChenyuLi Date: Mon, 21 Mar 2022 14:29:14 -0400 Subject: [PATCH 2/4] add changelog --- .changes/unreleased/Under the Hood-20220321-142854.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20220321-142854.yaml diff --git a/.changes/unreleased/Under the Hood-20220321-142854.yaml b/.changes/unreleased/Under the Hood-20220321-142854.yaml new file mode 100644 index 00000000000..841b82c3686 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20220321-142854.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Add Graph Compilation and Adapter Cache tracking +time: 2022-03-21T14:28:54.160087-04:00 +custom: + Author: ChenyuLInx + Issue: "4625" + PR: "4912" From 49e68d6b8b2307433926e58d4a5c0070d90fc8e6 Mon Sep 17 00:00:00 2001 From: ChenyuLi Date: Mon, 21 Mar 2022 15:14:40 -0400 Subject: [PATCH 3/4] rerun tests From 6d3dc8f9ebf89555bdafd32ee75bb23f25574b99 Mon Sep 17 00:00:00 2001 From: ChenyuLi Date: Wed, 23 Mar 2022 10:24:49 -0400 Subject: [PATCH 4/4] adjust action name --- core/dbt/tracking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py index 52db2dc4cb9..38b04f89035 100644 --- a/core/dbt/tracking.py +++ b/core/dbt/tracking.py @@ -419,7 +419,7 @@ def track_runnable_timing(options): track( active_user, category="dbt", - action="runnable", + action="runnable_timing", label=get_invocation_id(), context=context, )