Add Semantic Models to DAG

.changes/unreleased/Features-20230616-163045.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Added semantic models to the DAG and partial parsing module
+time: 2023-06-16T16:30:45.513314-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "7800"

core/dbt/compilation.py

@@ -179,6 +179,9 @@ def link_node(self, node: GraphMemberNode, manifest: Manifest):
     def link_graph(self, manifest: Manifest):
         for source in manifest.sources.values():
             self.add_node(source.unique_id)
+        for semantic_node in manifest.semantic_nodes.values():
+            self.add_node(semantic_node.unique_id)
+
         for node in manifest.nodes.values():
             self.link_node(node, manifest)
         for exposure in manifest.exposures.values():

core/dbt/contracts/graph/manifest.py

@@ -762,6 +762,9 @@ def build_flat_graph(self):
             "nodes": {k: v.to_dict(omit_none=False) for k, v in self.nodes.items()},
             "sources": {k: v.to_dict(omit_none=False) for k, v in self.sources.items()},
             "public_nodes": {k: v.to_dict(omit_none=False) for k, v in self.public_nodes.items()},
+            "semantic_nodes": {
+                k: v.to_dict(omit_none=False) for k, v in self.semantic_nodes.items()
+            },
         }
 
     def build_disabled_by_file_id(self):
@@ -822,6 +825,7 @@ def get_resource_fqns(self) -> Mapping[str, PathSet]:
             self.nodes.values(),
             self.sources.values(),
             self.metrics.values(),
+            self.semantic_nodes.values(),
         )
         for resource in all_resources:
             resource_type_plural = resource.resource_type.pluralize()
@@ -857,6 +861,8 @@ def deepcopy(self):
             public_nodes={k: _deepcopy(v) for k, v in self.public_nodes.items()},
             files={k: _deepcopy(v) for k, v in self.files.items()},
             state_check=_deepcopy(self.state_check),
+            publications={k: _deepcopy(v) for k, v in self.publications.items()},
+            semantic_nodes={k: _deepcopy(v) for k, v in self.semantic_nodes.items()},
         )
         copy.build_flat_graph()
         return copy
@@ -869,6 +875,7 @@ def build_parent_and_child_maps(self):
                 self.exposures.values(),
                 self.metrics.values(),
                 self.public_nodes.values(),
+                self.semantic_nodes.values(),
             )
         )
         forward_edges, backward_edges = build_node_edges(edge_members)
@@ -933,6 +940,8 @@ def expect(self, unique_id: str) -> GraphMemberNode:
             return self.exposures[unique_id]
         elif unique_id in self.metrics:
             return self.metrics[unique_id]
+        elif unique_id in self.semantic_nodes:
+            return self.semantic_nodes[unique_id]
         else:
             # something terrible has happened
             raise dbt.exceptions.DbtInternalError(
@@ -1002,7 +1011,9 @@ def pydantic_semantic_manifest(self) -> PydanticSemanticManifest:
         return pydantic_semantic_manifest
 
     def resolve_refs(
-        self, source_node: GraphMemberNode, current_project: str
+        self,
+        source_node: ModelNode,
+        current_project: str,  # TODO: ModelNode is overly restrictive typing
     ) -> List[MaybeNonSource]:
         resolved_refs: List[MaybeNonSource] = []
         for ref in source_node.refs:
@@ -1297,6 +1308,8 @@ def __reduce_ex__(self, protocol):
             self.disabled,
             self.env_vars,
             self.public_nodes,
+            self.publications,
+            self.semantic_nodes,
             self._doc_lookup,
             self._source_lookup,
             self._ref_lookup,

core/dbt/contracts/graph/nodes.py

@@ -1478,12 +1478,14 @@ class NodeRelation(dbtClassMixin):
 class SemanticModel(GraphNode):
     model: str
     node_relation: Optional[NodeRelation]
+    depends_on: DependsOn = field(default_factory=DependsOn)
     description: Optional[str] = None
     defaults: Optional[Defaults] = None
     entities: Sequence[Entity] = field(default_factory=list)
     measures: Sequence[Measure] = field(default_factory=list)
     dimensions: Sequence[Dimension] = field(default_factory=list)
     metadata: Optional[SourceFileMetadata] = None
+    created_at: float = field(default_factory=lambda: time.time())  # REVIEW: Needed?
 
     @property
     def entity_references(self) -> List[LinkableElementReference]:
@@ -1534,6 +1536,18 @@ def partition(self) -> Optional[Dimension]:
     def reference(self) -> SemanticModelReference:
         return SemanticModelReference(semantic_model_name=self.name)
 
+    @property
+    def depends_on_nodes(self):
+        return self.depends_on.nodes
+
+    @property
+    def depends_on_public_nodes(self):
+        return self.depends_on.public_nodes
+
+    @property
+    def depends_on_macros(self):
+        return self.depends_on.macros
+
 
 # ====================================
 # Patches
@@ -1641,6 +1655,7 @@ def alias(self):
     ResultNode,
     Exposure,
     Metric,
+    SemanticModel,
 ]
 
 # All "nodes" (or node-like objects) in this file

core/dbt/parser/manifest.py

@@ -586,6 +586,7 @@ def check_for_model_deprecations(self):
 
                 resolved_refs = self.manifest.resolve_refs(node, self.root_project.project_name)
                 resolved_model_refs = [r for r in resolved_refs if isinstance(r, ModelNode)]
+                node.depends_on
                 for resolved_ref in resolved_model_refs:
                     if resolved_ref.deprecation_date:
 
@@ -1198,6 +1199,7 @@ def process_semantic_models(self) -> None:
                             schema_name=refd_node.schema,
                             database=refd_node.database,
                         )
+                        semantic_model.depends_on.add_node(refd_node.unique_id)
 
     # nodes: node and column descriptions
     # sources: source and table descriptions, column descriptions